From 3bef198f1b17d1bb89260bad947ef084c0a2d1a6 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Fri, 18 Dec 2020 12:17:16 -0800 Subject: JFS: more checks for invalid superblock syzbot is feeding invalid superblock data to JFS for mount testing. JFS does not check several of the fields -- just assumes that they are good since the JFS_MAGIC and version fields are good. In this case (syzbot reproducer), we have s_l2bsize == 0xda0c, pad == 0xf045, and s_state == 0x50, all of which are invalid IMO. Having s_l2bsize == 0xda0c causes this UBSAN warning: UBSAN: shift-out-of-bounds in fs/jfs/jfs_mount.c:373:25 shift exponent -9716 is negative s_l2bsize can be tested for correctness. pad can be tested for non-0 and punted. s_state can be tested for its valid values and punted. Do those 3 tests and if any of them fails, report the superblock as invalid/corrupt and let fsck handle it. With this patch, chkSuper() says this when JFS_DEBUG is enabled: jfs_mount: Mount Failure: superblock is corrupt! Mount JFS Failure: -22 jfs_mount failed w/return code = -22 The obvious problem with this method is that next week there could be another syzbot test that uses different fields for invalid values, this making this like a game of whack-a-mole. syzkaller link: https://syzkaller.appspot.com/bug?extid=36315852ece4132ec193 Reported-by: syzbot+36315852ece4132ec193@syzkaller.appspotmail.com Reported-by: kernel test robot # v2 Signed-off-by: Randy Dunlap Signed-off-by: Dave Kleikamp Cc: jfs-discussion@lists.sourceforge.net --- fs/jfs/jfs_filsys.h | 1 + fs/jfs/jfs_mount.c | 10 ++++++++++ 2 files changed, 11 insertions(+) (limited to 'fs') diff --git a/fs/jfs/jfs_filsys.h b/fs/jfs/jfs_filsys.h index 1e899298f7f0..b5d702df7111 100644 --- a/fs/jfs/jfs_filsys.h +++ b/fs/jfs/jfs_filsys.h @@ -268,5 +268,6 @@ * fsck() must be run to repair */ #define FM_EXTENDFS 0x00000008 /* file system extendfs() in progress */ +#define FM_STATE_MAX 0x0000000f /* max value of s_state */ #endif /* _H_JFS_FILSYS */ diff --git a/fs/jfs/jfs_mount.c b/fs/jfs/jfs_mount.c index 2935d4c776ec..5d7d7170c03c 100644 --- a/fs/jfs/jfs_mount.c +++ b/fs/jfs/jfs_mount.c @@ -37,6 +37,7 @@ #include #include #include +#include #include "jfs_incore.h" #include "jfs_filsys.h" @@ -366,6 +367,15 @@ static int chkSuper(struct super_block *sb) sbi->bsize = bsize; sbi->l2bsize = le16_to_cpu(j_sb->s_l2bsize); + /* check some fields for possible corruption */ + if (sbi->l2bsize != ilog2((u32)bsize) || + j_sb->pad != 0 || + le32_to_cpu(j_sb->s_state) > FM_STATE_MAX) { + rc = -EINVAL; + jfs_err("jfs_mount: Mount Failure: superblock is corrupt!"); + goto out; + } + /* * For now, ignore s_pbsize, l2bfactor. All I/O going through buffer * cache. -- cgit v1.2.3 From c77b52c0a137994ad796f44544c802b0b766e496 Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Tue, 22 Dec 2020 14:43:27 -0600 Subject: gfs2: Add common helper for holding and releasing the freeze glock Many places in the gfs2 code queued and dequeued the freeze glock. Almost all of them acquire it in SHARED mode, and need to specify the same LM_FLAG_NOEXP and GL_EXACT flags. This patch adds common helper functions gfs2_freeze_lock and gfs2_freeze_unlock to make the code more readable, and to prepare for the next patch. Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/ops_fstype.c | 6 ++---- fs/gfs2/recovery.c | 8 +++----- fs/gfs2/super.c | 42 ++++++++++++++---------------------------- fs/gfs2/util.c | 25 +++++++++++++++++++++++++ fs/gfs2/util.h | 3 +++ 5 files changed, 47 insertions(+), 37 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index 61fce59cb4d3..4ee56f5e93cb 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -1198,14 +1198,12 @@ static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) if (sb_rdonly(sb)) { struct gfs2_holder freeze_gh; - error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_SHARED, - LM_FLAG_NOEXP | GL_EXACT, - &freeze_gh); + error = gfs2_freeze_lock(sdp, &freeze_gh, 0); if (error) { fs_err(sdp, "can't make FS RO: %d\n", error); goto fail_per_node; } - gfs2_glock_dq_uninit(&freeze_gh); + gfs2_freeze_unlock(&freeze_gh); } else { error = gfs2_make_fs_rw(sdp); if (error) { diff --git a/fs/gfs2/recovery.c b/fs/gfs2/recovery.c index c26c68ebd29d..74ab1fcaf558 100644 --- a/fs/gfs2/recovery.c +++ b/fs/gfs2/recovery.c @@ -470,9 +470,7 @@ void gfs2_recover_func(struct work_struct *work) /* Acquire a shared hold on the freeze lock */ - error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_SHARED, - LM_FLAG_NOEXP | LM_FLAG_PRIORITY | - GL_EXACT, &thaw_gh); + error = gfs2_freeze_lock(sdp, &thaw_gh, LM_FLAG_PRIORITY); if (error) goto fail_gunlock_ji; @@ -522,7 +520,7 @@ void gfs2_recover_func(struct work_struct *work) clean_journal(jd, &head); up_read(&sdp->sd_log_flush_lock); - gfs2_glock_dq_uninit(&thaw_gh); + gfs2_freeze_unlock(&thaw_gh); t_rep = ktime_get(); fs_info(sdp, "jid=%u: Journal replayed in %lldms [jlck:%lldms, " "jhead:%lldms, tlck:%lldms, replay:%lldms]\n", @@ -544,7 +542,7 @@ void gfs2_recover_func(struct work_struct *work) goto done; fail_gunlock_thaw: - gfs2_glock_dq_uninit(&thaw_gh); + gfs2_freeze_unlock(&thaw_gh); fail_gunlock_ji: if (jlocked) { gfs2_glock_dq_uninit(&ji_gh); diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index 2f56acc41c04..ea312a94ce69 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -173,9 +173,7 @@ int gfs2_make_fs_rw(struct gfs2_sbd *sdp) if (error) return error; - error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_SHARED, - LM_FLAG_NOEXP | GL_EXACT, - &freeze_gh); + error = gfs2_freeze_lock(sdp, &freeze_gh, 0); if (error) goto fail_threads; @@ -205,12 +203,12 @@ int gfs2_make_fs_rw(struct gfs2_sbd *sdp) set_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); - gfs2_glock_dq_uninit(&freeze_gh); + gfs2_freeze_unlock(&freeze_gh); return 0; fail: - gfs2_glock_dq_uninit(&freeze_gh); + gfs2_freeze_unlock(&freeze_gh); fail_threads: if (sdp->sd_quotad_process) kthread_stop(sdp->sd_quotad_process); @@ -452,7 +450,7 @@ static int gfs2_lock_fs_check_clean(struct gfs2_sbd *sdp) } if (error) - gfs2_glock_dq_uninit(&sdp->sd_freeze_gh); + gfs2_freeze_unlock(&sdp->sd_freeze_gh); out: while (!list_empty(&list)) { @@ -616,21 +614,12 @@ int gfs2_make_fs_ro(struct gfs2_sbd *sdp) gfs2_holder_mark_uninitialized(&freeze_gh); if (sdp->sd_freeze_gl && !gfs2_glock_is_locked_by_me(sdp->sd_freeze_gl)) { - if (!log_write_allowed) { - error = gfs2_glock_nq_init(sdp->sd_freeze_gl, - LM_ST_SHARED, LM_FLAG_TRY | - LM_FLAG_NOEXP | GL_EXACT, - &freeze_gh); - if (error == GLR_TRYFAILED) - error = 0; - } else { - error = gfs2_glock_nq_init(sdp->sd_freeze_gl, - LM_ST_SHARED, - LM_FLAG_NOEXP | GL_EXACT, - &freeze_gh); - if (error && !gfs2_withdrawn(sdp)) - return error; - } + error = gfs2_freeze_lock(sdp, &freeze_gh, + log_write_allowed ? 0 : LM_FLAG_TRY); + if (error == GLR_TRYFAILED) + error = 0; + if (error && !gfs2_withdrawn(sdp)) + return error; } gfs2_flush_delete_work(sdp); @@ -661,8 +650,7 @@ int gfs2_make_fs_ro(struct gfs2_sbd *sdp) atomic_read(&sdp->sd_reserving_log) == 0, HZ * 5); } - if (gfs2_holder_initialized(&freeze_gh)) - gfs2_glock_dq_uninit(&freeze_gh); + gfs2_freeze_unlock(&freeze_gh); gfs2_quota_cleanup(sdp); @@ -772,10 +760,8 @@ void gfs2_freeze_func(struct work_struct *work) struct super_block *sb = sdp->sd_vfs; atomic_inc(&sb->s_active); - error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_SHARED, - LM_FLAG_NOEXP | GL_EXACT, &freeze_gh); + error = gfs2_freeze_lock(sdp, &freeze_gh, 0); if (error) { - fs_info(sdp, "GFS2: couldn't get freeze lock : %d\n", error); gfs2_assert_withdraw(sdp, 0); } else { atomic_set(&sdp->sd_freeze_state, SFS_UNFROZEN); @@ -785,7 +771,7 @@ void gfs2_freeze_func(struct work_struct *work) error); gfs2_assert_withdraw(sdp, 0); } - gfs2_glock_dq_uninit(&freeze_gh); + gfs2_freeze_unlock(&freeze_gh); } deactivate_super(sb); clear_bit_unlock(SDF_FS_FROZEN, &sdp->sd_flags); @@ -853,7 +839,7 @@ static int gfs2_unfreeze(struct super_block *sb) return 0; } - gfs2_glock_dq_uninit(&sdp->sd_freeze_gh); + gfs2_freeze_unlock(&sdp->sd_freeze_gh); mutex_unlock(&sdp->sd_freeze_mutex); return wait_on_bit(&sdp->sd_flags, SDF_FS_FROZEN, TASK_INTERRUPTIBLE); } diff --git a/fs/gfs2/util.c b/fs/gfs2/util.c index a374397f4273..a115c441e2a1 100644 --- a/fs/gfs2/util.c +++ b/fs/gfs2/util.c @@ -91,6 +91,31 @@ out_unlock: return error; } +/** + * gfs2_freeze_lock - hold the freeze glock + * @sdp: the superblock + * @freeze_gh: pointer to the requested holder + * @caller_flags: any additional flags needed by the caller + */ +int gfs2_freeze_lock(struct gfs2_sbd *sdp, struct gfs2_holder *freeze_gh, + int caller_flags) +{ + int flags = LM_FLAG_NOEXP | GL_EXACT | caller_flags; + int error; + + error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_SHARED, flags, + freeze_gh); + if (error && error != GLR_TRYFAILED) + fs_err(sdp, "can't lock the freeze lock: %d\n", error); + return error; +} + +void gfs2_freeze_unlock(struct gfs2_holder *freeze_gh) +{ + if (gfs2_holder_initialized(freeze_gh)) + gfs2_glock_dq_uninit(freeze_gh); +} + static void signal_our_withdraw(struct gfs2_sbd *sdp) { struct gfs2_glock *gl = sdp->sd_live_gh.gh_gl; diff --git a/fs/gfs2/util.h b/fs/gfs2/util.h index a4443dd8a94b..69e1a0ae5a4d 100644 --- a/fs/gfs2/util.h +++ b/fs/gfs2/util.h @@ -149,6 +149,9 @@ int gfs2_io_error_i(struct gfs2_sbd *sdp, const char *function, extern int check_journal_clean(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, bool verbose); +extern int gfs2_freeze_lock(struct gfs2_sbd *sdp, + struct gfs2_holder *freeze_gh, int caller_flags); +extern void gfs2_freeze_unlock(struct gfs2_holder *freeze_gh); #define gfs2_io_error(sdp) \ gfs2_io_error_i((sdp), __func__, __FILE__, __LINE__) -- cgit v1.2.3 From 96b1454f2e8ede4c619fde405a1bb4e9ba8d218e Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Tue, 22 Dec 2020 14:43:28 -0600 Subject: gfs2: move freeze glock outside the make_fs_rw and _ro functions Before this patch, sister functions gfs2_make_fs_rw and gfs2_make_fs_ro locked (held) the freeze glock by calling gfs2_freeze_lock and gfs2_freeze_unlock. The problem is, not all the callers of gfs2_make_fs_ro should be doing this. The three callers of gfs2_make_fs_ro are: remount (gfs2_reconfigure), signal_our_withdraw, and unmount (gfs2_put_super). But when unmounting the file system we can get into the following circular lock dependency: deactivate_super down_write(&s->s_umount); <-------------------------------------- s_umount deactivate_locked_super gfs2_kill_sb kill_block_super generic_shutdown_super gfs2_put_super gfs2_make_fs_ro gfs2_glock_nq_init sd_freeze_gl freeze_go_sync if (freeze glock in SH) freeze_super (vfs) down_write(&sb->s_umount); <------- s_umount This patch moves the hold of the freeze glock outside the two sister rw/ro functions to their callers, but it doesn't request the glock from gfs2_put_super, thus eliminating the circular dependency. Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/ops_fstype.c | 31 +++++++++++++++++-------------- fs/gfs2/super.c | 23 ----------------------- fs/gfs2/util.c | 18 ++++++++++++++++-- 3 files changed, 33 insertions(+), 39 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index 4ee56f5e93cb..f2c6bbe5cdb8 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -1084,6 +1084,7 @@ static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) int silent = fc->sb_flags & SB_SILENT; struct gfs2_sbd *sdp; struct gfs2_holder mount_gh; + struct gfs2_holder freeze_gh; int error; sdp = init_sbd(sb); @@ -1195,23 +1196,18 @@ static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) goto fail_per_node; } - if (sb_rdonly(sb)) { - struct gfs2_holder freeze_gh; + error = gfs2_freeze_lock(sdp, &freeze_gh, 0); + if (error) + goto fail_per_node; - error = gfs2_freeze_lock(sdp, &freeze_gh, 0); - if (error) { - fs_err(sdp, "can't make FS RO: %d\n", error); - goto fail_per_node; - } - gfs2_freeze_unlock(&freeze_gh); - } else { + if (!sb_rdonly(sb)) error = gfs2_make_fs_rw(sdp); - if (error) { - fs_err(sdp, "can't make FS RW: %d\n", error); - goto fail_per_node; - } - } + gfs2_freeze_unlock(&freeze_gh); + if (error) { + fs_err(sdp, "can't make FS RW: %d\n", error); + goto fail_per_node; + } gfs2_glock_dq_uninit(&mount_gh); gfs2_online_uevent(sdp); return 0; @@ -1512,6 +1508,12 @@ static int gfs2_reconfigure(struct fs_context *fc) fc->sb_flags |= SB_RDONLY; if ((sb->s_flags ^ fc->sb_flags) & SB_RDONLY) { + struct gfs2_holder freeze_gh; + + error = gfs2_freeze_lock(sdp, &freeze_gh, 0); + if (error) + return -EINVAL; + if (fc->sb_flags & SB_RDONLY) { error = gfs2_make_fs_ro(sdp); if (error) @@ -1521,6 +1523,7 @@ static int gfs2_reconfigure(struct fs_context *fc) if (error) errorfc(fc, "unable to remount read-write"); } + gfs2_freeze_unlock(&freeze_gh); } sdp->sd_args = *newargs; diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index ea312a94ce69..754ea2a137b4 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -165,7 +165,6 @@ int gfs2_make_fs_rw(struct gfs2_sbd *sdp) { struct gfs2_inode *ip = GFS2_I(sdp->sd_jdesc->jd_inode); struct gfs2_glock *j_gl = ip->i_gl; - struct gfs2_holder freeze_gh; struct gfs2_log_header_host head; int error; @@ -173,10 +172,6 @@ int gfs2_make_fs_rw(struct gfs2_sbd *sdp) if (error) return error; - error = gfs2_freeze_lock(sdp, &freeze_gh, 0); - if (error) - goto fail_threads; - j_gl->gl_ops->go_inval(j_gl, DIO_METADATA); if (gfs2_withdrawn(sdp)) { error = -EIO; @@ -203,13 +198,9 @@ int gfs2_make_fs_rw(struct gfs2_sbd *sdp) set_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); - gfs2_freeze_unlock(&freeze_gh); - return 0; fail: - gfs2_freeze_unlock(&freeze_gh); -fail_threads: if (sdp->sd_quotad_process) kthread_stop(sdp->sd_quotad_process); sdp->sd_quotad_process = NULL; @@ -607,21 +598,9 @@ out: int gfs2_make_fs_ro(struct gfs2_sbd *sdp) { - struct gfs2_holder freeze_gh; int error = 0; int log_write_allowed = test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); - gfs2_holder_mark_uninitialized(&freeze_gh); - if (sdp->sd_freeze_gl && - !gfs2_glock_is_locked_by_me(sdp->sd_freeze_gl)) { - error = gfs2_freeze_lock(sdp, &freeze_gh, - log_write_allowed ? 0 : LM_FLAG_TRY); - if (error == GLR_TRYFAILED) - error = 0; - if (error && !gfs2_withdrawn(sdp)) - return error; - } - gfs2_flush_delete_work(sdp); if (!log_write_allowed && current == sdp->sd_quotad_process) fs_warn(sdp, "The quotad daemon is withdrawing.\n"); @@ -650,8 +629,6 @@ int gfs2_make_fs_ro(struct gfs2_sbd *sdp) atomic_read(&sdp->sd_reserving_log) == 0, HZ * 5); } - gfs2_freeze_unlock(&freeze_gh); - gfs2_quota_cleanup(sdp); if (!log_write_allowed) diff --git a/fs/gfs2/util.c b/fs/gfs2/util.c index a115c441e2a1..97b1bdc76871 100644 --- a/fs/gfs2/util.c +++ b/fs/gfs2/util.c @@ -122,6 +122,7 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) struct inode *inode = sdp->sd_jdesc->jd_inode; struct gfs2_inode *ip = GFS2_I(inode); u64 no_formal_ino = ip->i_no_formal_ino; + int log_write_allowed = test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); int ret = 0; int tries; @@ -142,8 +143,21 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) * therefore we need to clear SDF_JOURNAL_LIVE manually. */ clear_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); - if (!sb_rdonly(sdp->sd_vfs)) - ret = gfs2_make_fs_ro(sdp); + if (!sb_rdonly(sdp->sd_vfs)) { + struct gfs2_holder freeze_gh; + + gfs2_holder_mark_uninitialized(&freeze_gh); + if (sdp->sd_freeze_gl && + !gfs2_glock_is_locked_by_me(sdp->sd_freeze_gl)) { + ret = gfs2_freeze_lock(sdp, &freeze_gh, + log_write_allowed ? 0 : LM_FLAG_TRY); + if (ret == GLR_TRYFAILED) + ret = 0; + } + if (!ret) + ret = gfs2_make_fs_ro(sdp); + gfs2_freeze_unlock(&freeze_gh); + } if (sdp->sd_lockstruct.ls_ops->lm_lock == NULL) { /* lock_nolock */ if (!ret) -- cgit v1.2.3 From 2a6fe26ccf0bcebf469887bcc7c382d076751f4c Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Tue, 22 Dec 2020 15:37:35 -0500 Subject: gfs2: make gfs2_log_write_page static Function gfs2_log_write_page is only used in lops.c, so make it static. Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/lops.c | 2 +- fs/gfs2/lops.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index 3922b26264f5..9e9dd486bed9 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c @@ -369,7 +369,7 @@ static void gfs2_log_write_bh(struct gfs2_sbd *sdp, struct buffer_head *bh) * the page may be freed at any time. */ -void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page) +static void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page) { struct super_block *sb = sdp->sd_vfs; u64 dblock; diff --git a/fs/gfs2/lops.h b/fs/gfs2/lops.h index fbdbb08dcec6..2280f68862de 100644 --- a/fs/gfs2/lops.h +++ b/fs/gfs2/lops.h @@ -22,7 +22,6 @@ extern void gfs2_log_incr_head(struct gfs2_sbd *sdp); extern u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lbn); extern void gfs2_log_write(struct gfs2_sbd *sdp, struct page *page, unsigned size, unsigned offset, u64 blkno); -extern void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page); extern void gfs2_log_submit_bio(struct bio **biop, int opf); extern void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh); extern int gfs2_find_jhead(struct gfs2_jdesc *jd, -- cgit v1.2.3 From 1e8f44f159b31fe31ad2f40f96575b6ad6df2fe9 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 11 Mar 2020 17:22:19 -0400 Subject: do_tmpfile(): don't mess with finish_open() use vfs_open() instead Signed-off-by: Al Viro --- fs/namei.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/namei.c b/fs/namei.c index 78443a85480a..a3b3ca62ef5c 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -3325,10 +3325,8 @@ static int do_tmpfile(struct nameidata *nd, unsigned flags, audit_inode(nd->name, child, 0); /* Don't check for other permissions, the inode was just created */ error = may_open(&path, 0, op->open_flag); - if (error) - goto out2; - file->f_path.mnt = path.mnt; - error = finish_open(file, child, NULL); + if (!error) + error = vfs_open(&path, file); out2: mnt_drop_write(path.mnt); out: -- cgit v1.2.3 From 26ddb45e13a3e09f5f5517a3c9d6510208181516 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Wed, 9 Dec 2020 17:09:28 -0500 Subject: fs/namei.c: Remove unlikely of status being -ECHILD in lookup_fast() Running my yearly branch profiling code, it detected a 100% wrong branch condition in name.c for lookup_fast(). The code in question has: status = d_revalidate(dentry, nd->flags); if (likely(status > 0)) return dentry; if (unlazy_child(nd, dentry, seq)) return ERR_PTR(-ECHILD); if (unlikely(status == -ECHILD)) /* we'd been told to redo it in non-rcu mode */ status = d_revalidate(dentry, nd->flags); If the status of the d_revalidate() is greater than zero, then the function finishes. Otherwise, if it is an "unlazy_child" it returns with -ECHILD. After the above two checks, the status is compared to -ECHILD, as that is what is returned if the original d_revalidate() needed to be done in a non-rcu mode. Especially this path is called in a condition of: if (nd->flags & LOOKUP_RCU) { And most of the d_revalidate() functions have: if (flags & LOOKUP_RCU) return -ECHILD; It appears that that is the only case that this if statement is triggered on two of my machines, running in production. As it is dependent on what filesystem mix is configured in the running kernel, simply remove the unlikely() from the if statement. Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Al Viro --- fs/namei.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/namei.c b/fs/namei.c index a3b3ca62ef5c..3345a9f38ccb 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -1495,7 +1495,7 @@ static struct dentry *lookup_fast(struct nameidata *nd, return dentry; if (unlazy_child(nd, dentry, seq)) return ERR_PTR(-ECHILD); - if (unlikely(status == -ECHILD)) + if (status == -ECHILD) /* we'd been told to redo it in non-rcu mode */ status = d_revalidate(dentry, nd->flags); } else { -- cgit v1.2.3 From e36cffed20a324e116f329a94061ae30dd26fb51 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 17 Dec 2020 09:19:08 -0700 Subject: fs: make unlazy_walk() error handling consistent Most callers check for non-zero return, and assume it's -ECHILD (which it always will be). One caller uses the actual error return. Clean this up and make it fully consistent, by having unlazy_walk() return a bool instead. Rename it to try_to_unlazy() and return true on success, and failure on error. That's easier to read. No functional changes in this patch. Cc: Al Viro Signed-off-by: Jens Axboe Signed-off-by: Al Viro --- fs/namei.c | 43 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) (limited to 'fs') diff --git a/fs/namei.c b/fs/namei.c index 3345a9f38ccb..21fd06753504 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -669,17 +669,17 @@ static bool legitimize_root(struct nameidata *nd) */ /** - * unlazy_walk - try to switch to ref-walk mode. + * try_to_unlazy - try to switch to ref-walk mode. * @nd: nameidata pathwalk data - * Returns: 0 on success, -ECHILD on failure + * Returns: true on success, false on failure * - * unlazy_walk attempts to legitimize the current nd->path and nd->root + * try_to_unlazy attempts to legitimize the current nd->path and nd->root * for ref-walk mode. * Must be called from rcu-walk context. - * Nothing should touch nameidata between unlazy_walk() failure and + * Nothing should touch nameidata between try_to_unlazy() failure and * terminate_walk(). */ -static int unlazy_walk(struct nameidata *nd) +static bool try_to_unlazy(struct nameidata *nd) { struct dentry *parent = nd->path.dentry; @@ -694,14 +694,14 @@ static int unlazy_walk(struct nameidata *nd) goto out; rcu_read_unlock(); BUG_ON(nd->inode != parent->d_inode); - return 0; + return true; out1: nd->path.mnt = NULL; nd->path.dentry = NULL; out: rcu_read_unlock(); - return -ECHILD; + return false; } /** @@ -792,7 +792,7 @@ static int complete_walk(struct nameidata *nd) */ if (!(nd->flags & (LOOKUP_ROOT | LOOKUP_IS_SCOPED))) nd->root.mnt = NULL; - if (unlikely(unlazy_walk(nd))) + if (!try_to_unlazy(nd)) return -ECHILD; } @@ -1466,7 +1466,7 @@ static struct dentry *lookup_fast(struct nameidata *nd, unsigned seq; dentry = __d_lookup_rcu(parent, &nd->last, &seq); if (unlikely(!dentry)) { - if (unlazy_walk(nd)) + if (!try_to_unlazy(nd)) return ERR_PTR(-ECHILD); return NULL; } @@ -1567,10 +1567,8 @@ static inline int may_lookup(struct nameidata *nd) { if (nd->flags & LOOKUP_RCU) { int err = inode_permission(nd->inode, MAY_EXEC|MAY_NOT_BLOCK); - if (err != -ECHILD) + if (err != -ECHILD || !try_to_unlazy(nd)) return err; - if (unlazy_walk(nd)) - return -ECHILD; } return inode_permission(nd->inode, MAY_EXEC); } @@ -1592,7 +1590,7 @@ static int reserve_stack(struct nameidata *nd, struct path *link, unsigned seq) // unlazy even if we fail to grab the link - cleanup needs it bool grabbed_link = legitimize_path(nd, link, seq); - if (unlazy_walk(nd) != 0 || !grabbed_link) + if (!try_to_unlazy(nd) != 0 || !grabbed_link) return -ECHILD; if (nd_alloc_stack(nd)) @@ -1634,7 +1632,7 @@ static const char *pick_link(struct nameidata *nd, struct path *link, touch_atime(&last->link); cond_resched(); } else if (atime_needs_update(&last->link, inode)) { - if (unlikely(unlazy_walk(nd))) + if (!try_to_unlazy(nd)) return ERR_PTR(-ECHILD); touch_atime(&last->link); } @@ -1651,11 +1649,8 @@ static const char *pick_link(struct nameidata *nd, struct path *link, get = inode->i_op->get_link; if (nd->flags & LOOKUP_RCU) { res = get(NULL, inode, &last->done); - if (res == ERR_PTR(-ECHILD)) { - if (unlikely(unlazy_walk(nd))) - return ERR_PTR(-ECHILD); + if (res == ERR_PTR(-ECHILD) && try_to_unlazy(nd)) res = get(link->dentry, inode, &last->done); - } } else { res = get(link->dentry, inode, &last->done); } @@ -2195,7 +2190,7 @@ OK: } if (unlikely(!d_can_lookup(nd->path.dentry))) { if (nd->flags & LOOKUP_RCU) { - if (unlazy_walk(nd)) + if (!try_to_unlazy(nd)) return -ECHILD; } return -ENOTDIR; @@ -3129,7 +3124,6 @@ static const char *open_last_lookups(struct nameidata *nd, struct inode *inode; struct dentry *dentry; const char *res; - int error; nd->flags |= op->intent; @@ -3153,9 +3147,8 @@ static const char *open_last_lookups(struct nameidata *nd, } else { /* create side of things */ if (nd->flags & LOOKUP_RCU) { - error = unlazy_walk(nd); - if (unlikely(error)) - return ERR_PTR(error); + if (!try_to_unlazy(nd)) + return ERR_PTR(-ECHILD); } audit_inode(nd->name, dir, AUDIT_INODE_PARENT); /* trailing slashes? */ @@ -3164,9 +3157,7 @@ static const char *open_last_lookups(struct nameidata *nd, } if (open_flag & (O_CREAT | O_TRUNC | O_WRONLY | O_RDWR)) { - error = mnt_want_write(nd->path.mnt); - if (!error) - got_write = true; + got_write = !mnt_want_write(nd->path.mnt); /* * do _not_ fail yet - we might not need that or fail with * a different error; let lookup_open() decide; we'll be -- cgit v1.2.3 From ae66db45fd309fd1c6d4e846dfc8414dfec7d6ad Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 4 Jan 2021 00:08:41 -0500 Subject: saner calling conventions for unlazy_child() same as for the previous commit - instead of 0/-ECHILD make it return true/false, rename to try_to_unlazy_child(). Signed-off-by: Al Viro --- fs/namei.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/namei.c b/fs/namei.c index 21fd06753504..2ee219497460 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -705,19 +705,19 @@ out: } /** - * unlazy_child - try to switch to ref-walk mode. + * try_to_unlazy_next - try to switch to ref-walk mode. * @nd: nameidata pathwalk data - * @dentry: child of nd->path.dentry - * @seq: seq number to check dentry against - * Returns: 0 on success, -ECHILD on failure + * @dentry: next dentry to step into + * @seq: seq number to check @dentry against + * Returns: true on success, false on failure * - * unlazy_child attempts to legitimize the current nd->path, nd->root and dentry - * for ref-walk mode. @dentry must be a path found by a do_lookup call on - * @nd. Must be called from rcu-walk context. - * Nothing should touch nameidata between unlazy_child() failure and + * Similar to to try_to_unlazy(), but here we have the next dentry already + * picked by rcu-walk and want to legitimize that in addition to the current + * nd->path and nd->root for ref-walk mode. Must be called from rcu-walk context. + * Nothing should touch nameidata between try_to_unlazy_next() failure and * terminate_walk(). */ -static int unlazy_child(struct nameidata *nd, struct dentry *dentry, unsigned seq) +static bool try_to_unlazy_next(struct nameidata *nd, struct dentry *dentry, unsigned seq) { BUG_ON(!(nd->flags & LOOKUP_RCU)); @@ -747,7 +747,7 @@ static int unlazy_child(struct nameidata *nd, struct dentry *dentry, unsigned se if (unlikely(!legitimize_root(nd))) goto out_dput; rcu_read_unlock(); - return 0; + return true; out2: nd->path.mnt = NULL; @@ -755,11 +755,11 @@ out1: nd->path.dentry = NULL; out: rcu_read_unlock(); - return -ECHILD; + return false; out_dput: rcu_read_unlock(); dput(dentry); - return -ECHILD; + return false; } static inline int d_revalidate(struct dentry *dentry, unsigned int flags) @@ -1372,7 +1372,7 @@ static inline int handle_mounts(struct nameidata *nd, struct dentry *dentry, return -ENOENT; if (likely(__follow_mount_rcu(nd, path, inode, seqp))) return 0; - if (unlazy_child(nd, dentry, seq)) + if (!try_to_unlazy_next(nd, dentry, seq)) return -ECHILD; // *path might've been clobbered by __follow_mount_rcu() path->mnt = nd->path.mnt; @@ -1493,7 +1493,7 @@ static struct dentry *lookup_fast(struct nameidata *nd, status = d_revalidate(dentry, nd->flags); if (likely(status > 0)) return dentry; - if (unlazy_child(nd, dentry, seq)) + if (!try_to_unlazy_next(nd, dentry, seq)) return ERR_PTR(-ECHILD); if (status == -ECHILD) /* we'd been told to redo it in non-rcu mode */ -- cgit v1.2.3 From 6c6ec2b0a3e0381d886d531bd1471dfdb1509237 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 17 Dec 2020 09:19:09 -0700 Subject: fs: add support for LOOKUP_CACHED io_uring always punts opens to async context, since there's no control over whether the lookup blocks or not. Add LOOKUP_CACHED to support just doing the fast RCU based lookups, which we know will not block. If we can do a cached path resolution of the filename, then we don't have to always punt lookups for a worker. During path resolution, we always do LOOKUP_RCU first. If that fails and we terminate LOOKUP_RCU, then fail a LOOKUP_CACHED attempt as well. Cc: Al Viro Signed-off-by: Jens Axboe Signed-off-by: Al Viro --- fs/namei.c | 9 +++++++++ include/linux/namei.h | 1 + 2 files changed, 10 insertions(+) (limited to 'fs') diff --git a/fs/namei.c b/fs/namei.c index 2ee219497460..4cae88733a5c 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -686,6 +686,8 @@ static bool try_to_unlazy(struct nameidata *nd) BUG_ON(!(nd->flags & LOOKUP_RCU)); nd->flags &= ~LOOKUP_RCU; + if (nd->flags & LOOKUP_CACHED) + goto out1; if (unlikely(!legitimize_links(nd))) goto out1; if (unlikely(!legitimize_path(nd, &nd->path, nd->seq))) @@ -722,6 +724,8 @@ static bool try_to_unlazy_next(struct nameidata *nd, struct dentry *dentry, unsi BUG_ON(!(nd->flags & LOOKUP_RCU)); nd->flags &= ~LOOKUP_RCU; + if (nd->flags & LOOKUP_CACHED) + goto out2; if (unlikely(!legitimize_links(nd))) goto out2; if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq))) @@ -792,6 +796,7 @@ static int complete_walk(struct nameidata *nd) */ if (!(nd->flags & (LOOKUP_ROOT | LOOKUP_IS_SCOPED))) nd->root.mnt = NULL; + nd->flags &= ~LOOKUP_CACHED; if (!try_to_unlazy(nd)) return -ECHILD; } @@ -2204,6 +2209,10 @@ static const char *path_init(struct nameidata *nd, unsigned flags) int error; const char *s = nd->name->name; + /* LOOKUP_CACHED requires RCU, ask caller to retry */ + if ((flags & (LOOKUP_RCU | LOOKUP_CACHED)) == LOOKUP_CACHED) + return ERR_PTR(-EAGAIN); + if (!*s) flags &= ~LOOKUP_RCU; if (flags & LOOKUP_RCU) diff --git a/include/linux/namei.h b/include/linux/namei.h index a4bb992623c4..b9605b2b46e7 100644 --- a/include/linux/namei.h +++ b/include/linux/namei.h @@ -46,6 +46,7 @@ enum {LAST_NORM, LAST_ROOT, LAST_DOT, LAST_DOTDOT}; #define LOOKUP_NO_XDEV 0x040000 /* No mountpoint crossing. */ #define LOOKUP_BENEATH 0x080000 /* No escaping from starting point. */ #define LOOKUP_IN_ROOT 0x100000 /* Treat dirfd as fs root. */ +#define LOOKUP_CACHED 0x200000 /* Only do cached lookup */ /* LOOKUP_* flags which do scope-related checks based on the dirfd. */ #define LOOKUP_IS_SCOPED (LOOKUP_BENEATH | LOOKUP_IN_ROOT) -- cgit v1.2.3 From 99668f618062816ca7ba639b007eb145b9d3d41e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 17 Dec 2020 09:19:10 -0700 Subject: fs: expose LOOKUP_CACHED through openat2() RESOLVE_CACHED Now that we support non-blocking path resolution internally, expose it via openat2() in the struct open_how ->resolve flags. This allows applications using openat2() to limit path resolution to the extent that it is already cached. If the lookup cannot be satisfied in a non-blocking manner, openat2(2) will return -1/-EAGAIN. Cc: Al Viro Signed-off-by: Jens Axboe Signed-off-by: Al Viro --- fs/open.c | 6 ++++++ include/linux/fcntl.h | 2 +- include/uapi/linux/openat2.h | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/open.c b/fs/open.c index 1e06e443a565..ca5444733acd 100644 --- a/fs/open.c +++ b/fs/open.c @@ -1091,6 +1091,12 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op) lookup_flags |= LOOKUP_BENEATH; if (how->resolve & RESOLVE_IN_ROOT) lookup_flags |= LOOKUP_IN_ROOT; + if (how->resolve & RESOLVE_CACHED) { + /* Don't bother even trying for create/truncate/tmpfile open */ + if (flags & (O_TRUNC | O_CREAT | O_TMPFILE)) + return -EAGAIN; + lookup_flags |= LOOKUP_CACHED; + } op->lookup_flags = lookup_flags; return 0; diff --git a/include/linux/fcntl.h b/include/linux/fcntl.h index 921e750843e6..766fcd973beb 100644 --- a/include/linux/fcntl.h +++ b/include/linux/fcntl.h @@ -19,7 +19,7 @@ /* List of all valid flags for the how->resolve argument: */ #define VALID_RESOLVE_FLAGS \ (RESOLVE_NO_XDEV | RESOLVE_NO_MAGICLINKS | RESOLVE_NO_SYMLINKS | \ - RESOLVE_BENEATH | RESOLVE_IN_ROOT) + RESOLVE_BENEATH | RESOLVE_IN_ROOT | RESOLVE_CACHED) /* List of all open_how "versions". */ #define OPEN_HOW_SIZE_VER0 24 /* sizeof first published struct */ diff --git a/include/uapi/linux/openat2.h b/include/uapi/linux/openat2.h index 58b1eb711360..a5feb7604948 100644 --- a/include/uapi/linux/openat2.h +++ b/include/uapi/linux/openat2.h @@ -35,5 +35,9 @@ struct open_how { #define RESOLVE_IN_ROOT 0x10 /* Make all jumps to "/" and ".." be scoped inside the dirfd (similar to chroot(2)). */ +#define RESOLVE_CACHED 0x20 /* Only complete if resolution can be + completed through cached lookup. May + return -EAGAIN if that's not + possible. */ #endif /* _UAPI_LINUX_OPENAT2_H */ -- cgit v1.2.3 From 14e43bf435612639cab01541fce7cc41bf7e370b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 22 Sep 2020 09:44:18 -0700 Subject: vfs: don't unnecessarily clone write access for writable fds There's no need for mnt_want_write_file() to increment mnt_writers when the file is already open for writing, provided that mnt_drop_write_file() is changed to conditionally decrement it. We seem to have ended up in the current situation because mnt_want_write_file() used to be paired with mnt_drop_write(), due to mnt_drop_write_file() not having been added yet. So originally mnt_want_write_file() had to always increment mnt_writers. But later mnt_drop_write_file() was added, and all callers of mnt_want_write_file() were paired with it. This makes the compatibility between mnt_want_write_file() and mnt_drop_write() no longer necessary. Therefore, make __mnt_want_write_file() and __mnt_drop_write_file() skip incrementing mnt_writers on files already open for writing. This removes the only caller of mnt_clone_write(), so remove that too. Signed-off-by: Eric Biggers Signed-off-by: Al Viro --- Documentation/filesystems/porting.rst | 7 +++++ fs/namespace.c | 53 +++++++++++++---------------------- include/linux/mount.h | 1 - 3 files changed, 27 insertions(+), 34 deletions(-) (limited to 'fs') diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst index 867036aa90b8..6a6d3e673b48 100644 --- a/Documentation/filesystems/porting.rst +++ b/Documentation/filesystems/porting.rst @@ -865,3 +865,10 @@ no matter what. Everything is handled by the caller. clone_private_mount() returns a longterm mount now, so the proper destructor of its result is kern_unmount() or kern_unmount_array(). + +--- + +**mandatory** + +mnt_want_write_file() can now only be paired with mnt_drop_write_file(), +whereas previously it could be paired with mnt_drop_write() as well. diff --git a/fs/namespace.c b/fs/namespace.c index d2db7dfe232b..9f2d94e0f3e0 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -359,51 +359,37 @@ int mnt_want_write(struct vfsmount *m) } EXPORT_SYMBOL_GPL(mnt_want_write); -/** - * mnt_clone_write - get write access to a mount - * @mnt: the mount on which to take a write - * - * This is effectively like mnt_want_write, except - * it must only be used to take an extra write reference - * on a mountpoint that we already know has a write reference - * on it. This allows some optimisation. - * - * After finished, mnt_drop_write must be called as usual to - * drop the reference. - */ -int mnt_clone_write(struct vfsmount *mnt) -{ - /* superblock may be r/o */ - if (__mnt_is_readonly(mnt)) - return -EROFS; - preempt_disable(); - mnt_inc_writers(real_mount(mnt)); - preempt_enable(); - return 0; -} -EXPORT_SYMBOL_GPL(mnt_clone_write); - /** * __mnt_want_write_file - get write access to a file's mount * @file: the file who's mount on which to take a write * - * This is like __mnt_want_write, but it takes a file and can - * do some optimisations if the file is open for write already + * This is like __mnt_want_write, but if the file is already open for writing it + * skips incrementing mnt_writers (since the open file already has a reference) + * and instead only does the check for emergency r/o remounts. This must be + * paired with __mnt_drop_write_file. */ int __mnt_want_write_file(struct file *file) { - if (!(file->f_mode & FMODE_WRITER)) - return __mnt_want_write(file->f_path.mnt); - else - return mnt_clone_write(file->f_path.mnt); + if (file->f_mode & FMODE_WRITER) { + /* + * Superblock may have become readonly while there are still + * writable fd's, e.g. due to a fs error with errors=remount-ro + */ + if (__mnt_is_readonly(file->f_path.mnt)) + return -EROFS; + return 0; + } + return __mnt_want_write(file->f_path.mnt); } /** * mnt_want_write_file - get write access to a file's mount * @file: the file who's mount on which to take a write * - * This is like mnt_want_write, but it takes a file and can - * do some optimisations if the file is open for write already + * This is like mnt_want_write, but if the file is already open for writing it + * skips incrementing mnt_writers (since the open file already has a reference) + * and instead only does the freeze protection and the check for emergency r/o + * remounts. This must be paired with mnt_drop_write_file. */ int mnt_want_write_file(struct file *file) { @@ -449,7 +435,8 @@ EXPORT_SYMBOL_GPL(mnt_drop_write); void __mnt_drop_write_file(struct file *file) { - __mnt_drop_write(file->f_path.mnt); + if (!(file->f_mode & FMODE_WRITER)) + __mnt_drop_write(file->f_path.mnt); } void mnt_drop_write_file(struct file *file) diff --git a/include/linux/mount.h b/include/linux/mount.h index aaf343b38671..b43191fe6af7 100644 --- a/include/linux/mount.h +++ b/include/linux/mount.h @@ -79,7 +79,6 @@ struct path; extern int mnt_want_write(struct vfsmount *mnt); extern int mnt_want_write_file(struct file *file); -extern int mnt_clone_write(struct vfsmount *mnt); extern void mnt_drop_write(struct vfsmount *mnt); extern void mnt_drop_write_file(struct file *file); extern void mntput(struct vfsmount *mnt); -- cgit v1.2.3 From edbb35cc6bdfc379a2968f17d479567650ddbb16 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 30 Oct 2020 17:44:20 -0700 Subject: fs/inode.c: make inode_init_always() initialize i_ino to 0 Currently inode_init_always() doesn't initialize i_ino to 0. This is unexpected because unlike the other inode fields that aren't initialized by inode_init_always(), i_ino isn't guaranteed to end up back at its initial value after the inode is freed. Only one filesystem (XFS) actually sets set i_ino back to 0 when freeing its inodes. So, callers of new_inode() see some random previous i_ino. Normally that's fine, since normally i_ino isn't accessed before being set. There can be edge cases where that isn't necessarily true, though. The one I've run into is that on ext4, when creating an encrypted file, the new file's encryption key has to be set up prior to the jbd2 transaction, and thus prior to i_ino being set. If something goes wrong, fs/crypto/ may log warning or error messages, which normally include i_ino. So it needs to know whether it is valid to include i_ino yet or not. Also, on some files i_ino needs to be hashed for use in the crypto, so fs/crypto/ needs to know whether that can be done yet or not. There are ways this could be worked around, either in fs/crypto/ or in fs/ext4/. But, it seems there's no reason not to just fix inode_init_always() to do the expected thing and initialize i_ino to 0. So, do that, and also remove the initialization in jfs_fill_super() that becomes redundant. Signed-off-by: Eric Biggers Signed-off-by: Al Viro --- fs/inode.c | 1 + fs/jfs/super.c | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/inode.c b/fs/inode.c index 6442d97d9a4a..6598ea2bb097 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -142,6 +142,7 @@ int inode_init_always(struct super_block *sb, struct inode *inode) atomic_set(&inode->i_count, 1); inode->i_op = &empty_iops; inode->i_fop = &no_open_fops; + inode->i_ino = 0; inode->__i_nlink = 1; inode->i_opflags = 0; if (sb->s_xattr) diff --git a/fs/jfs/super.c b/fs/jfs/super.c index b2dc4d1f9dcc..1f0ffabbde56 100644 --- a/fs/jfs/super.c +++ b/fs/jfs/super.c @@ -551,7 +551,6 @@ static int jfs_fill_super(struct super_block *sb, void *data, int silent) ret = -ENOMEM; goto out_unload; } - inode->i_ino = 0; inode->i_size = i_size_read(sb->s_bdev->bd_inode); inode->i_mapping->a_ops = &jfs_metapage_aops; inode_fake_hash(inode); -- cgit v1.2.3 From 8a00dd0012f383fc0c39b169b694dc15236cec7c Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 22 Jun 2020 20:14:02 -0400 Subject: binfmt_elf: partially sanitize PRSTATUS_SIZE and SET_PR_FPVALID On 64bit architectures that support 32bit processes there are two possible layouts for NT_PRSTATUS note in ELF coredumps. For one thing, several fields are 64bit for native processes and 32bit for compat ones (pr_sigpend, etc.). For another, the register dump is obviously different - the size and number of registers are not going to be the same for 32bit and 64bit variants of processor. Usually that's handled by having two structures - elf_prstatus for native layout and compat_elf_prstatus for 32bit one. 32bit processes are handled by fs/compat_binfmt_elf.c, which defines a macro called 'elf_prstatus' that expands to compat_elf_prstatus. Then it includes fs/binfmt_elf.c, which makes all references to struct elf_prstatus to be textually replaced with struct compat_elf_prstatus. Ugly and somewhat brittle, but it works. However, amd64 is worse - there are _three_ possible layouts. One for native 64bit processes, another for i386 (32bit) processes and yet another for x32 (32bit address space with full 64bit registers). Both i386 and x32 processes are handled by fs/compat_binfmt_elf.c, with usual compat_binfmt_elf.c trickery. However, the layouts for i386 and x32 are not identical - they have the common beginning, but the register dump part (pr_reg) is bigger on x32. Worse, pr_reg is not the last field - it's followed by int pr_fpvalid, so that field ends up at different offsets for i386 and x32 layouts. Fortunately, there's not much code that cares about any of that - it's all encapsulated in fill_thread_core_info(). Since x32 variant is bigger, we define compat_elf_prstatus to match that layout. That way i386 processes have enough space to fit their layout into. Moreover, since these layouts are identical prior to pr_reg, we don't need to distinguish x32 and i386 cases when we are setting the fields prior to pr_reg. Filling pr_reg itself is done by calling ->get() method of appropriate regset, and that method knows what layout (and size) to use. We do need to distinguish x32 and i386 cases only for two things: setting ->pr_fpvalid (offset differs for x32 and i386) and choosing the right size for our note. The way it's done is Not Nice, for the lack of more accurate printable description. There are two macros (PRSTATUS_SIZE and SET_PR_FPVALID), that default essentially to sizeof(struct elf_prstatus) and (S)->pr_fpvalid = 1. On x86 asm/compat.h provides its own variants. Unfortunately, quite a few things go wrong there: * PRSTATUS_SIZE doesn't use the normal test for process being an x32 one; it compares the size reported by regset with the size of pr_reg. * it hardcodes the sizes of x32 and i386 variants (296 and 144 resp.), so if some change in includes leads to asm/compat.h pulled in by fs/binfmt_elf.c we are in trouble - it will end up using the size of x32 variant for 64bit processes. * it's in the wrong place; asm/compat.h couldn't define the structure for i386 layout, since it lacks quite a few types needed for it. Hardcoded sizes are largely due to that. The proper fix would be to have an explicitly defined i386 variant of structure and have PRSTATUS_SIZE/SET_PR_FPVALID check for TIF_X32 to choose the variant that should be used. Unfortunately, that requires some manipulations of headers; we'll do that later in the series, but for now let's go with the minimal variant - rename PRSTATUS_SIZE in asm/compat.h to COMPAT_PRSTATUS_SIZE, have fs/compat_binfmt_elf.c define PRSTATUS_SIZE to COMPAT_PRSTATUS_SIZE and use the normal TIF_X32 check in that macro. The size of i386 variant is kept hardcoded for now. Similar story for SET_PR_FPVALID. Signed-off-by: Al Viro --- arch/x86/include/asm/compat.h | 11 +++++++---- fs/binfmt_elf.c | 13 +++++-------- fs/compat_binfmt_elf.c | 8 ++++++++ 3 files changed, 20 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h index f145e3326c6d..15cf0f831dee 100644 --- a/arch/x86/include/asm/compat.h +++ b/arch/x86/include/asm/compat.h @@ -165,10 +165,13 @@ struct compat_shmid64_ds { typedef struct user_regs_struct compat_elf_gregset_t; /* Full regset -- prstatus on x32, otherwise on ia32 */ -#define PRSTATUS_SIZE(S, R) (R != sizeof(S.pr_reg) ? 144 : 296) -#define SET_PR_FPVALID(S, V, R) \ - do { *(int *) (((void *) &((S)->pr_reg)) + R) = (V); } \ - while (0) +#define COMPAT_PRSTATUS_SIZE (user_64bit_mode(task_pt_regs(current)) \ + ? sizeof(struct compat_elf_prstatus) \ + : 144) +#define COMPAT_SET_PR_FPVALID(S) \ + (*(user_64bit_mode(task_pt_regs(current)) \ + ? &(S)->pr_fpvalid \ + : (int *)((void *)(S) + 140)) = 1) #ifdef CONFIG_X86_X32_ABI #define COMPAT_USE_64BIT_TIME \ diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 950bc177238a..8380478d3d92 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1717,11 +1717,11 @@ static void do_thread_regset_writeback(struct task_struct *task, } #ifndef PRSTATUS_SIZE -#define PRSTATUS_SIZE(S, R) sizeof(S) +#define PRSTATUS_SIZE sizeof(struct elf_prstatus) #endif #ifndef SET_PR_FPVALID -#define SET_PR_FPVALID(S, V, R) ((S)->pr_fpvalid = (V)) +#define SET_PR_FPVALID(S) ((S)->pr_fpvalid = 1) #endif static int fill_thread_core_info(struct elf_thread_core_info *t, @@ -1729,7 +1729,6 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, long signr, size_t *total) { unsigned int i; - int regset0_size; /* * NT_PRSTATUS is the one special case, because the regset data @@ -1738,13 +1737,11 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, * We assume that regset 0 is NT_PRSTATUS. */ fill_prstatus(&t->prstatus, t->task, signr); - regset0_size = regset_get(t->task, &view->regsets[0], + regset_get(t->task, &view->regsets[0], sizeof(t->prstatus.pr_reg), &t->prstatus.pr_reg); - if (regset0_size < 0) - return 0; fill_note(&t->notes[0], "CORE", NT_PRSTATUS, - PRSTATUS_SIZE(t->prstatus, regset0_size), &t->prstatus); + PRSTATUS_SIZE, &t->prstatus); *total += notesize(&t->notes[0]); do_thread_regset_writeback(t->task, &view->regsets[0]); @@ -1772,7 +1769,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, continue; if (is_fpreg) - SET_PR_FPVALID(&t->prstatus, 1, regset0_size); + SET_PR_FPVALID(&t->prstatus); fill_note(&t->notes[i], is_fpreg ? "CORE" : "LINUX", note_type, ret, data); diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c index 2c557229696a..962df845ed51 100644 --- a/fs/compat_binfmt_elf.c +++ b/fs/compat_binfmt_elf.c @@ -95,6 +95,14 @@ #define ELF_EXEC_PAGESIZE COMPAT_ELF_EXEC_PAGESIZE #endif +#ifdef COMPAT_PRSTATUS_SIZE +#define PRSTATUS_SIZE COMPAT_PRSTATUS_SIZE +#endif + +#ifdef COMPAT_SET_PR_FPVALID +#define SET_PR_FPVALID(S) COMPAT_SET_PR_FPVALID(S) +#endif + #ifdef COMPAT_ELF_PLAT_INIT #undef ELF_PLAT_INIT #define ELF_PLAT_INIT COMPAT_ELF_PLAT_INIT -- cgit v1.2.3 From a4db1072e1a3bd7a8d9c356e1902b13ac5deb8ef Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 22 Dec 2020 12:09:53 +0100 Subject: quota: Fix memory leak when handling corrupted quota file When checking corrupted quota file we can bail out and leak allocated info structure. Properly free info structure on error return. Reported-by: syzbot+77779c9b52ab78154b08@syzkaller.appspotmail.com Fixes: 11c514a99bb9 ("quota: Sanity-check quota file headers on load") Signed-off-by: Jan Kara --- fs/quota/quota_v2.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/quota/quota_v2.c b/fs/quota/quota_v2.c index c21106557a37..b1467f3921c2 100644 --- a/fs/quota/quota_v2.c +++ b/fs/quota/quota_v2.c @@ -164,19 +164,24 @@ static int v2_read_file_info(struct super_block *sb, int type) quota_error(sb, "Number of blocks too big for quota file size (%llu > %llu).", (loff_t)qinfo->dqi_blocks << qinfo->dqi_blocksize_bits, i_size_read(sb_dqopt(sb)->files[type])); - goto out; + goto out_free; } if (qinfo->dqi_free_blk >= qinfo->dqi_blocks) { quota_error(sb, "Free block number too big (%u >= %u).", qinfo->dqi_free_blk, qinfo->dqi_blocks); - goto out; + goto out_free; } if (qinfo->dqi_free_entry >= qinfo->dqi_blocks) { quota_error(sb, "Block with free entry too big (%u >= %u).", qinfo->dqi_free_entry, qinfo->dqi_blocks); - goto out; + goto out_free; } ret = 0; +out_free: + if (ret) { + kfree(info->dqi_priv); + info->dqi_priv = NULL; + } out: up_read(&dqopt->dqio_sem); return ret; -- cgit v1.2.3 From ac7b79fd190b02e7151bc7d2b9da692f537657f3 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Sat, 19 Dec 2020 20:46:08 -0800 Subject: inotify, memcg: account inotify instances to kmemcg Currently the fs sysctl inotify/max_user_instances is used to limit the number of inotify instances on the system. For systems running multiple workloads, the per-user namespace sysctl max_inotify_instances can be used to further partition inotify instances. However there is no easy way to set a sensible system level max limit on inotify instances and further partition it between the workloads. It is much easier to charge the underlying resource (i.e. memory) behind the inotify instances to the memcg of the workload and let their memory limits limit the number of inotify instances they can create. With inotify instances charged to memcg, the admin can simply set max_user_instances to INT_MAX and let the memcg limits of the jobs limit their inotify instances. Link: https://lore.kernel.org/r/20201220044608.1258123-1-shakeelb@google.com Reviewed-by: Amir Goldstein Signed-off-by: Shakeel Butt Signed-off-by: Jan Kara --- fs/notify/fanotify/fanotify_user.c | 2 +- fs/notify/group.c | 25 ++++++++++++++++++++----- fs/notify/inotify/inotify_user.c | 4 ++-- include/linux/fsnotify_backend.h | 1 + 4 files changed, 24 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 3e01d8f2ab90..7e7afc2b62e1 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -976,7 +976,7 @@ SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags) f_flags |= O_NONBLOCK; /* fsnotify_alloc_group takes a ref. Dropped in fanotify_release */ - group = fsnotify_alloc_group(&fanotify_fsnotify_ops); + group = fsnotify_alloc_user_group(&fanotify_fsnotify_ops); if (IS_ERR(group)) { free_uid(user); return PTR_ERR(group); diff --git a/fs/notify/group.c b/fs/notify/group.c index a4a4b1c64d32..ffd723ffe46d 100644 --- a/fs/notify/group.c +++ b/fs/notify/group.c @@ -111,14 +111,12 @@ void fsnotify_put_group(struct fsnotify_group *group) } EXPORT_SYMBOL_GPL(fsnotify_put_group); -/* - * Create a new fsnotify_group and hold a reference for the group returned. - */ -struct fsnotify_group *fsnotify_alloc_group(const struct fsnotify_ops *ops) +static struct fsnotify_group *__fsnotify_alloc_group( + const struct fsnotify_ops *ops, gfp_t gfp) { struct fsnotify_group *group; - group = kzalloc(sizeof(struct fsnotify_group), GFP_KERNEL); + group = kzalloc(sizeof(struct fsnotify_group), gfp); if (!group) return ERR_PTR(-ENOMEM); @@ -139,8 +137,25 @@ struct fsnotify_group *fsnotify_alloc_group(const struct fsnotify_ops *ops) return group; } + +/* + * Create a new fsnotify_group and hold a reference for the group returned. + */ +struct fsnotify_group *fsnotify_alloc_group(const struct fsnotify_ops *ops) +{ + return __fsnotify_alloc_group(ops, GFP_KERNEL); +} EXPORT_SYMBOL_GPL(fsnotify_alloc_group); +/* + * Create a new fsnotify_group and hold a reference for the group returned. + */ +struct fsnotify_group *fsnotify_alloc_user_group(const struct fsnotify_ops *ops) +{ + return __fsnotify_alloc_group(ops, GFP_KERNEL_ACCOUNT); +} +EXPORT_SYMBOL_GPL(fsnotify_alloc_user_group); + int fsnotify_fasync(int fd, struct file *file, int on) { struct fsnotify_group *group = file->private_data; diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index 59c177011a0f..266d17e8ecb9 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -632,11 +632,11 @@ static struct fsnotify_group *inotify_new_group(unsigned int max_events) struct fsnotify_group *group; struct inotify_event_info *oevent; - group = fsnotify_alloc_group(&inotify_fsnotify_ops); + group = fsnotify_alloc_user_group(&inotify_fsnotify_ops); if (IS_ERR(group)) return group; - oevent = kmalloc(sizeof(struct inotify_event_info), GFP_KERNEL); + oevent = kmalloc(sizeof(struct inotify_event_info), GFP_KERNEL_ACCOUNT); if (unlikely(!oevent)) { fsnotify_destroy_group(group); return ERR_PTR(-ENOMEM); diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index a2e42d3cd87c..e5409b83e731 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -470,6 +470,7 @@ static inline void fsnotify_update_flags(struct dentry *dentry) /* create a new group */ extern struct fsnotify_group *fsnotify_alloc_group(const struct fsnotify_ops *ops); +extern struct fsnotify_group *fsnotify_alloc_user_group(const struct fsnotify_ops *ops); /* get reference to a group */ extern void fsnotify_get_group(struct fsnotify_group *group); /* drop reference on a group from fsnotify_alloc_group */ -- cgit v1.2.3 From aa7d5c7e307a0b4f77e8edc682dc60d55a4d5555 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 5 Jan 2021 22:57:20 -0500 Subject: cramfs: use %pD instead of messing with file_dentry()->d_name Signed-off-by: Al Viro --- fs/cramfs/inode.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/cramfs/inode.c b/fs/cramfs/inode.c index 4b90cfd1ec36..2be65269a987 100644 --- a/fs/cramfs/inode.c +++ b/fs/cramfs/inode.c @@ -392,8 +392,7 @@ static int cramfs_physmem_mmap(struct file *file, struct vm_area_struct *vma) /* Don't map the last page if it contains some other data */ if (pgoff + pages == max_pages && cramfs_last_page_is_shared(inode)) { - pr_debug("mmap: %s: last page is shared\n", - file_dentry(file)->d_name.name); + pr_debug("mmap: %pD: last page is shared\n", file); pages--; } @@ -430,16 +429,15 @@ static int cramfs_physmem_mmap(struct file *file, struct vm_area_struct *vma) } if (!ret) - pr_debug("mapped %s[%lu] at 0x%08lx (%u/%lu pages) " - "to vma 0x%08lx, page_prot 0x%llx\n", - file_dentry(file)->d_name.name, pgoff, - address, pages, vma_pages(vma), vma->vm_start, + pr_debug("mapped %pD[%lu] at 0x%08lx (%u/%lu pages) " + "to vma 0x%08lx, page_prot 0x%llx\n", file, + pgoff, address, pages, vma_pages(vma), vma->vm_start, (unsigned long long)pgprot_val(vma->vm_page_prot)); return ret; bailout: - pr_debug("%s[%lu]: direct mmap impossible: %s\n", - file_dentry(file)->d_name.name, pgoff, bailout_reason); + pr_debug("%pD[%lu]: direct mmap impossible: %s\n", + file, pgoff, bailout_reason); /* Didn't manage any direct map, but normal paging is still possible */ return 0; } @@ -469,8 +467,8 @@ static unsigned long cramfs_physmem_get_unmapped_area(struct file *file, if (!offset || block_pages != pages) return -ENOSYS; addr = sbi->linear_phys_addr + offset; - pr_debug("get_unmapped for %s ofs %#lx siz %lu at 0x%08lx\n", - file_dentry(file)->d_name.name, pgoff*PAGE_SIZE, len, addr); + pr_debug("get_unmapped for %pD ofs %#lx siz %lu at 0x%08lx\n", + file, pgoff*PAGE_SIZE, len, addr); return addr; } -- cgit v1.2.3 From f2485a2dc9f0f30fbdd013ad5772975100c71360 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 13 Jun 2020 00:08:44 -0400 Subject: elf_prstatus: collect the common part (everything before pr_reg) into a struct Preparations to doing i386 compat elf_prstatus sanely - rather than duplicating the beginning of compat_elf_prstatus, take these fields into a separate structure (compat_elf_prstatus_common), so that it could be reused. Due to the incestous relationship between binfmt_elf.c and compat_binfmt_elf.c we need the same shape change done to native struct elf_prstatus, gathering the fields prior to pr_reg into a new structure (struct elf_prstatus_common). Fortunately, offset of pr_reg is always a multiple of 16 with no padding right before it, so it's possible to turn all the stuff prior to it into a single member without disturbing the layout. [build fix from Geert Uytterhoeven folded in] Signed-off-by: Al Viro --- arch/ia64/kernel/crash.c | 2 +- arch/mips/kernel/binfmt_elfn32.c | 7 ++++++- arch/mips/kernel/binfmt_elfo32.c | 6 +++++- arch/powerpc/platforms/powernv/opal-core.c | 6 +++--- arch/s390/kernel/crash_dump.c | 2 +- fs/binfmt_elf.c | 8 ++++---- fs/binfmt_elf_fdpic.c | 22 +++++----------------- fs/compat_binfmt_elf.c | 1 + include/linux/elfcore-compat.h | 7 ++++++- include/linux/elfcore.h | 7 ++++++- kernel/kexec_core.c | 2 +- 11 files changed, 39 insertions(+), 31 deletions(-) (limited to 'fs') diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c index fec70d662d0c..4f47741005d2 100644 --- a/arch/ia64/kernel/crash.c +++ b/arch/ia64/kernel/crash.c @@ -43,7 +43,7 @@ crash_save_this_cpu(void) elf_greg_t *dst = (elf_greg_t *)&(prstatus->pr_reg); memset(prstatus, 0, sizeof(*prstatus)); - prstatus->pr_pid = current->pid; + prstatus->common.pr_pid = current->pid; ia64_dump_cpu_regs(dst); cfm = dst[43]; diff --git a/arch/mips/kernel/binfmt_elfn32.c b/arch/mips/kernel/binfmt_elfn32.c index 6ee3f7218c67..136dc0c9300d 100644 --- a/arch/mips/kernel/binfmt_elfn32.c +++ b/arch/mips/kernel/binfmt_elfn32.c @@ -44,7 +44,8 @@ typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; #include #define elf_prstatus elf_prstatus32 -struct elf_prstatus32 +#define elf_prstatus_common elf_prstatus32_common +struct elf_prstatus32_common { struct elf_siginfo pr_info; /* Info associated with signal */ short pr_cursig; /* Current signal */ @@ -58,6 +59,10 @@ struct elf_prstatus32 struct old_timeval32 pr_stime; /* System time */ struct old_timeval32 pr_cutime;/* Cumulative user time */ struct old_timeval32 pr_cstime;/* Cumulative system time */ +}; +struct elf_prstatus32 +{ + struct elf_prstatus32_common common: elf_gregset_t pr_reg; /* GP registers */ int pr_fpvalid; /* True if math co-processor being used. */ }; diff --git a/arch/mips/kernel/binfmt_elfo32.c b/arch/mips/kernel/binfmt_elfo32.c index 6dd103d3cebb..b1f4b8f1dee7 100644 --- a/arch/mips/kernel/binfmt_elfo32.c +++ b/arch/mips/kernel/binfmt_elfo32.c @@ -49,7 +49,7 @@ typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; #include #define elf_prstatus elf_prstatus32 -struct elf_prstatus32 +struct elf_prstatus32_common { struct elf_siginfo pr_info; /* Info associated with signal */ short pr_cursig; /* Current signal */ @@ -63,6 +63,10 @@ struct elf_prstatus32 struct old_timeval32 pr_stime; /* System time */ struct old_timeval32 pr_cutime;/* Cumulative user time */ struct old_timeval32 pr_cstime;/* Cumulative system time */ +}; +struct elf_prstatus32 +{ + struct elf_prstatus32_common common: elf_gregset_t pr_reg; /* GP registers */ int pr_fpvalid; /* True if math co-processor being used. */ }; diff --git a/arch/powerpc/platforms/powernv/opal-core.c b/arch/powerpc/platforms/powernv/opal-core.c index 23571f0b555a..0d9ba70f7251 100644 --- a/arch/powerpc/platforms/powernv/opal-core.c +++ b/arch/powerpc/platforms/powernv/opal-core.c @@ -119,8 +119,8 @@ static void fill_prstatus(struct elf_prstatus *prstatus, int pir, * As a PIR value could also be '0', add an offset of '100' * to every PIR to avoid misinterpretations in GDB. */ - prstatus->pr_pid = cpu_to_be32(100 + pir); - prstatus->pr_ppid = cpu_to_be32(1); + prstatus->common.pr_pid = cpu_to_be32(100 + pir); + prstatus->common.pr_ppid = cpu_to_be32(1); /* * Indicate SIGUSR1 for crash initiated from kernel. @@ -130,7 +130,7 @@ static void fill_prstatus(struct elf_prstatus *prstatus, int pir, short sig; sig = kernel_initiated ? SIGUSR1 : SIGTERM; - prstatus->pr_cursig = cpu_to_be16(sig); + prstatus->common.pr_cursig = cpu_to_be16(sig); } } diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 205b2e2648aa..0e36dfc9ccd6 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -365,7 +365,7 @@ static void *fill_cpu_elf_notes(void *ptr, int cpu, struct save_area *sa) memcpy(&nt_prstatus.pr_reg.gprs, sa->gprs, sizeof(sa->gprs)); memcpy(&nt_prstatus.pr_reg.psw, sa->psw, sizeof(sa->psw)); memcpy(&nt_prstatus.pr_reg.acrs, sa->acrs, sizeof(sa->acrs)); - nt_prstatus.pr_pid = cpu; + nt_prstatus.common.pr_pid = cpu; /* Prepare fpregset (floating point) note */ memset(&nt_fpregset, 0, sizeof(nt_fpregset)); memcpy(&nt_fpregset.fpc, &sa->fpc, sizeof(sa->fpc)); diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 8380478d3d92..4c1550b13899 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1495,7 +1495,7 @@ static void fill_note(struct memelfnote *note, const char *name, int type, * fill up all the fields in prstatus from the given task struct, except * registers which need to be filled up separately. */ -static void fill_prstatus(struct elf_prstatus *prstatus, +static void fill_prstatus(struct elf_prstatus_common *prstatus, struct task_struct *p, long signr) { prstatus->pr_info.si_signo = prstatus->pr_cursig = signr; @@ -1736,7 +1736,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, * than being the whole note contents. We fill the reset in here. * We assume that regset 0 is NT_PRSTATUS. */ - fill_prstatus(&t->prstatus, t->task, signr); + fill_prstatus(&t->prstatus.common, t->task, signr); regset_get(t->task, &view->regsets[0], sizeof(t->prstatus.pr_reg), &t->prstatus.pr_reg); @@ -1958,7 +1958,7 @@ static int elf_dump_thread_status(long signr, struct elf_thread_status *t) struct task_struct *p = t->thread; t->num_notes = 0; - fill_prstatus(&t->prstatus, p, signr); + fill_prstatus(&t->prstatus.common, p, signr); elf_core_copy_task_regs(p, &t->prstatus.pr_reg); fill_note(&t->notes[0], "CORE", NT_PRSTATUS, sizeof(t->prstatus), @@ -2037,7 +2037,7 @@ static int fill_note_info(struct elfhdr *elf, int phdrs, } /* now collect the dump for the current */ memset(info->prstatus, 0, sizeof(*info->prstatus)); - fill_prstatus(info->prstatus, current, siginfo->si_signo); + fill_prstatus(&info->prstatus->common, current, siginfo->si_signo); elf_core_copy_regs(&info->prstatus->pr_reg, regs); /* Set up header */ diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index be4062b8ba75..03d81a14bcbf 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -1191,18 +1191,7 @@ static int elf_fdpic_map_file_by_direct_mmap(struct elf_fdpic_params *params, struct elf_prstatus_fdpic { - struct elf_siginfo pr_info; /* Info associated with signal */ - short pr_cursig; /* Current signal */ - unsigned long pr_sigpend; /* Set of pending signals */ - unsigned long pr_sighold; /* Set of held signals */ - pid_t pr_pid; - pid_t pr_ppid; - pid_t pr_pgrp; - pid_t pr_sid; - struct __kernel_old_timeval pr_utime; /* User time */ - struct __kernel_old_timeval pr_stime; /* System time */ - struct __kernel_old_timeval pr_cutime; /* Cumulative user time */ - struct __kernel_old_timeval pr_cstime; /* Cumulative system time */ + struct elf_prstatus_common common; elf_gregset_t pr_reg; /* GP registers */ /* When using FDPIC, the loadmap addresses need to be communicated * to GDB in order for GDB to do the necessary relocations. The @@ -1301,7 +1290,7 @@ static inline void fill_note(struct memelfnote *note, const char *name, int type * fill up all the fields in prstatus from the given task struct, except * registers which need to be filled up separately. */ -static void fill_prstatus(struct elf_prstatus_fdpic *prstatus, +static void fill_prstatus(struct elf_prstatus_common *prstatus, struct task_struct *p, long signr) { prstatus->pr_info.si_signo = prstatus->pr_cursig = signr; @@ -1332,9 +1321,6 @@ static void fill_prstatus(struct elf_prstatus_fdpic *prstatus, } prstatus->pr_cutime = ns_to_kernel_old_timeval(p->signal->cutime); prstatus->pr_cstime = ns_to_kernel_old_timeval(p->signal->cstime); - - prstatus->pr_exec_fdpic_loadmap = p->mm->context.exec_fdpic_loadmap; - prstatus->pr_interp_fdpic_loadmap = p->mm->context.interp_fdpic_loadmap; } static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, @@ -1405,7 +1391,9 @@ static struct elf_thread_status *elf_dump_thread_status(long signr, struct task_ if (!t) return t; - fill_prstatus(&t->prstatus, p, signr); + fill_prstatus(&t->prstatus.common, p, signr); + t->prstatus.pr_exec_fdpic_loadmap = p->mm->context.exec_fdpic_loadmap; + t->prstatus.pr_interp_fdpic_loadmap = p->mm->context.interp_fdpic_loadmap; regset_get(p, &view->regsets[0], sizeof(t->prstatus.pr_reg), &t->prstatus.pr_reg); diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c index 962df845ed51..feb48a5c2d44 100644 --- a/fs/compat_binfmt_elf.c +++ b/fs/compat_binfmt_elf.c @@ -50,6 +50,7 @@ * which requires asm/elf.h to define compat_elf_gregset_t et al. */ #define elf_prstatus compat_elf_prstatus +#define elf_prstatus_common compat_elf_prstatus_common #define elf_prpsinfo compat_elf_prpsinfo #undef ns_to_kernel_old_timeval diff --git a/include/linux/elfcore-compat.h b/include/linux/elfcore-compat.h index 10485f0c9740..4aeda5f1f038 100644 --- a/include/linux/elfcore-compat.h +++ b/include/linux/elfcore-compat.h @@ -17,7 +17,7 @@ struct compat_elf_siginfo compat_int_t si_errno; }; -struct compat_elf_prstatus +struct compat_elf_prstatus_common { struct compat_elf_siginfo pr_info; short pr_cursig; @@ -31,6 +31,11 @@ struct compat_elf_prstatus struct old_timeval32 pr_stime; struct old_timeval32 pr_cutime; struct old_timeval32 pr_cstime; +}; + +struct compat_elf_prstatus +{ + struct compat_elf_prstatus_common common; compat_elf_gregset_t pr_reg; compat_int_t pr_fpvalid; }; diff --git a/include/linux/elfcore.h b/include/linux/elfcore.h index de51c1bef27d..2aaa15779d50 100644 --- a/include/linux/elfcore.h +++ b/include/linux/elfcore.h @@ -29,7 +29,7 @@ struct elf_siginfo * the SVR4 structure, but more Linuxy, with things that Linux does * not support and which gdb doesn't really use excluded. */ -struct elf_prstatus +struct elf_prstatus_common { struct elf_siginfo pr_info; /* Info associated with signal */ short pr_cursig; /* Current signal */ @@ -43,6 +43,11 @@ struct elf_prstatus struct __kernel_old_timeval pr_stime; /* System time */ struct __kernel_old_timeval pr_cutime; /* Cumulative user time */ struct __kernel_old_timeval pr_cstime; /* Cumulative system time */ +}; + +struct elf_prstatus +{ + struct elf_prstatus_common common; elf_gregset_t pr_reg; /* GP registers */ int pr_fpvalid; /* True if math co-processor being used. */ }; diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 4f8efc278aa7..80905e5aa8ae 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1076,7 +1076,7 @@ void crash_save_cpu(struct pt_regs *regs, int cpu) if (!buf) return; memset(&prstatus, 0, sizeof(prstatus)); - prstatus.pr_pid = current->pid; + prstatus.common.pr_pid = current->pid; elf_core_copy_kernel_regs(&prstatus.pr_reg, regs); buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS, &prstatus, sizeof(prstatus)); -- cgit v1.2.3 From 7facdc426f86c67e579e49e100943cbccc43e1c6 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 13 Jun 2020 23:03:25 -0400 Subject: [amd64] clean PRSTATUS_SIZE/SET_PR_FPVALID up properly To get rid of hardcoded size/offset in those macros we need to have a definition of i386 variant of struct elf_prstatus. However, we can't do that in asm/compat.h - the types needed for that are not there and adding an include of asm/user32.h into asm/compat.h would cause a lot of mess. That could be conveniently done in elfcore-compat.h, but currently there is nowhere to put arch-dependent parts of it - no asm/elfcore-compat.h. So we introduce a new file (asm/elfcore-compat.h, present on architectures that have CONFIG_ARCH_HAS_ELFCORE_COMPAT set, currently only on x86), have it pulled by linux/elfcore-compat.h and move the definitions there. As a side benefit, we don't need to worry about accidental inclusion of that file into binfmt_elf.c itself, so we don't need the dance with COMPAT_PRSTATUS_SIZE, etc. - only fs/compat_binfmt_elf.c will see that header. Signed-off-by: Al Viro --- arch/Kconfig | 3 +++ arch/x86/Kconfig | 1 + arch/x86/include/asm/compat.h | 14 -------------- arch/x86/include/asm/elfcore-compat.h | 31 +++++++++++++++++++++++++++++++ fs/compat_binfmt_elf.c | 8 -------- include/linux/elfcore-compat.h | 18 +++++++++++------- 6 files changed, 46 insertions(+), 29 deletions(-) create mode 100644 arch/x86/include/asm/elfcore-compat.h (limited to 'fs') diff --git a/arch/Kconfig b/arch/Kconfig index 78c6f05b10f9..a17ced73b23c 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1105,6 +1105,9 @@ config HAVE_ARCH_PFN_VALID config ARCH_SUPPORTS_DEBUG_PAGEALLOC bool +config ARCH_HAS_ELFCORE_COMPAT + bool + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig" diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 7b6dd10b162a..302a6b453c91 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -31,6 +31,7 @@ config X86_64 select MODULES_USE_ELF_RELA select NEED_DMA_MAP_STATE select SWIOTLB + select ARCH_HAS_ELFCORE_COMPAT config FORCE_DYNAMIC_FTRACE def_bool y diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h index 15cf0f831dee..be09c7eac89f 100644 --- a/arch/x86/include/asm/compat.h +++ b/arch/x86/include/asm/compat.h @@ -159,20 +159,6 @@ struct compat_shmid64_ds { compat_ulong_t __unused5; }; -/* - * The type of struct elf_prstatus.pr_reg in compatible core dumps. - */ -typedef struct user_regs_struct compat_elf_gregset_t; - -/* Full regset -- prstatus on x32, otherwise on ia32 */ -#define COMPAT_PRSTATUS_SIZE (user_64bit_mode(task_pt_regs(current)) \ - ? sizeof(struct compat_elf_prstatus) \ - : 144) -#define COMPAT_SET_PR_FPVALID(S) \ - (*(user_64bit_mode(task_pt_regs(current)) \ - ? &(S)->pr_fpvalid \ - : (int *)((void *)(S) + 140)) = 1) - #ifdef CONFIG_X86_X32_ABI #define COMPAT_USE_64BIT_TIME \ (!!(task_pt_regs(current)->orig_ax & __X32_SYSCALL_BIT)) diff --git a/arch/x86/include/asm/elfcore-compat.h b/arch/x86/include/asm/elfcore-compat.h new file mode 100644 index 000000000000..f1b6c7a8d8fc --- /dev/null +++ b/arch/x86/include/asm/elfcore-compat.h @@ -0,0 +1,31 @@ +#ifndef _ASM_X86_ELFCORE_COMPAT_H +#define _ASM_X86_ELFCORE_COMPAT_H + +#include + +/* + * On amd64 we have two 32bit ABIs - i386 and x32. The latter + * has bigger registers, so we use it for compat_elf_regset_t. + * The former uses i386_elf_prstatus and PRSTATUS_SIZE/SET_PR_FPVALID + * are used to choose the size and location of ->pr_fpvalid of + * the layout actually used. + */ +typedef struct user_regs_struct compat_elf_gregset_t; + +struct i386_elf_prstatus +{ + struct compat_elf_prstatus_common common; + struct user_regs_struct32 pr_reg; + compat_int_t pr_fpvalid; +}; + +#define PRSTATUS_SIZE \ + (user_64bit_mode(task_pt_regs(current)) \ + ? sizeof(struct compat_elf_prstatus) \ + : sizeof(struct i386_elf_prstatus)) +#define SET_PR_FPVALID(S) \ + (*(user_64bit_mode(task_pt_regs(current)) \ + ? &(S)->pr_fpvalid \ + : &((struct i386_elf_prstatus *)(S))->pr_fpvalid) = 1) + +#endif diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c index feb48a5c2d44..a6321415aba0 100644 --- a/fs/compat_binfmt_elf.c +++ b/fs/compat_binfmt_elf.c @@ -96,14 +96,6 @@ #define ELF_EXEC_PAGESIZE COMPAT_ELF_EXEC_PAGESIZE #endif -#ifdef COMPAT_PRSTATUS_SIZE -#define PRSTATUS_SIZE COMPAT_PRSTATUS_SIZE -#endif - -#ifdef COMPAT_SET_PR_FPVALID -#define SET_PR_FPVALID(S) COMPAT_SET_PR_FPVALID(S) -#endif - #ifdef COMPAT_ELF_PLAT_INIT #undef ELF_PLAT_INIT #define ELF_PLAT_INIT COMPAT_ELF_PLAT_INIT diff --git a/include/linux/elfcore-compat.h b/include/linux/elfcore-compat.h index 4aeda5f1f038..e272c3d452ce 100644 --- a/include/linux/elfcore-compat.h +++ b/include/linux/elfcore-compat.h @@ -33,13 +33,6 @@ struct compat_elf_prstatus_common struct old_timeval32 pr_cstime; }; -struct compat_elf_prstatus -{ - struct compat_elf_prstatus_common common; - compat_elf_gregset_t pr_reg; - compat_int_t pr_fpvalid; -}; - struct compat_elf_prpsinfo { char pr_state; @@ -54,4 +47,15 @@ struct compat_elf_prpsinfo char pr_psargs[ELF_PRARGSZ]; }; +#ifdef CONFIG_ARCH_HAS_ELFCORE_COMPAT +#include +#endif + +struct compat_elf_prstatus +{ + struct compat_elf_prstatus_common common; + compat_elf_gregset_t pr_reg; + compat_int_t pr_fpvalid; +}; + #endif /* _LINUX_ELFCORE_COMPAT_H */ -- cgit v1.2.3 From 41026c343540e33627e23c8a91ebb679a7c0f89c Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 2 Dec 2020 23:56:34 -0500 Subject: Kconfig: regularize selection of CONFIG_BINFMT_ELF with mips converted to use of fs/config_binfmt_elf.c, there's no need to keep selects of that thing all over arch/* - we can simply turn into def_bool y if COMPAT && BINFMT_ELF (in fs/Kconfig.binfmt) and get rid of all selects. Several architectures got those selects wrong (e.g. you could end up with sparc64 sans BINFMT_ELF, with select violating dependencies, etc.) Randy Dunlap has spotted some of those; IMO this is simpler than his fix, but it depends upon the stuff that would need to be backported, so we might end up using his variant for -stable. Signed-off-by: Al Viro --- arch/arm64/Kconfig | 1 - arch/mips/Kconfig | 2 -- arch/parisc/Kconfig | 1 - arch/powerpc/Kconfig | 1 - arch/s390/Kconfig | 1 - arch/sparc/Kconfig | 1 - arch/x86/Kconfig | 1 - fs/Kconfig.binfmt | 2 +- 8 files changed, 1 insertion(+), 9 deletions(-) (limited to 'fs') diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 05e17351e4f3..ed48fd42ab33 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1215,7 +1215,6 @@ config ARM64_TAGGED_ADDR_ABI menuconfig COMPAT bool "Kernel support for 32-bit EL0" depends on ARM64_4K_PAGES || EXPERT - select COMPAT_BINFMT_ELF if BINFMT_ELF select HAVE_UID16 select OLD_SIGSUSPEND3 select COMPAT_OLD_SIGACTION diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index a46423f1cabc..f29ec95e3458 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -3278,7 +3278,6 @@ config MIPS32_O32 select ARCH_WANT_OLD_COMPAT_IPC select COMPAT select MIPS32_COMPAT - select COMPAT_BINFMT_ELF select SYSVIPC_COMPAT if SYSVIPC help Select this option if you want to run o32 binaries. These are pure @@ -3292,7 +3291,6 @@ config MIPS32_N32 depends on 64BIT select ARCH_WANT_COMPAT_IPC_PARSE_VERSION select COMPAT - select COMPAT_BINFMT_ELF select MIPS32_COMPAT select SYSVIPC_COMPAT if SYSVIPC help diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 78b17621ee4a..26daf57b9df6 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -336,7 +336,6 @@ source "kernel/Kconfig.hz" config COMPAT def_bool y depends on 64BIT - select COMPAT_BINFMT_ELF if BINFMT_ELF config SYSVIPC_COMPAT def_bool y diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 107bb4319e0e..d26a89cd8908 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -282,7 +282,6 @@ config COMPAT bool "Enable support for 32bit binaries" depends on PPC64 default y if !CPU_LITTLE_ENDIAN - select COMPAT_BINFMT_ELF select ARCH_WANT_OLD_COMPAT_IPC select COMPAT_OLD_SIGACTION diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index e84bdd15150b..73b61fe7231e 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -423,7 +423,6 @@ config 64BIT config COMPAT def_bool y prompt "Kernel support for 31 bit emulation" - select COMPAT_BINFMT_ELF if BINFMT_ELF select ARCH_WANT_OLD_COMPAT_IPC select COMPAT_OLD_SIGACTION select HAVE_UID16 diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index c9c34dc52b7d..1a2b5649d267 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -494,7 +494,6 @@ config COMPAT bool depends on SPARC64 default y - select COMPAT_BINFMT_ELF select HAVE_UID16 select ARCH_WANT_OLD_COMPAT_IPC select COMPAT_OLD_SIGACTION diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index a2182d22b5fa..6d130d1c440b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2899,7 +2899,6 @@ config COMPAT_32 config COMPAT def_bool y depends on IA32_EMULATION || X86_X32 - select COMPAT_BINFMT_ELF if BINFMT_ELF if COMPAT config COMPAT_FOR_U64_ALIGNMENT diff --git a/fs/Kconfig.binfmt b/fs/Kconfig.binfmt index 885da6d983b4..b32f5df68ae9 100644 --- a/fs/Kconfig.binfmt +++ b/fs/Kconfig.binfmt @@ -29,7 +29,7 @@ config BINFMT_ELF latest version). config COMPAT_BINFMT_ELF - bool + def_bool y depends on COMPAT && BINFMT_ELF select ELFCORE -- cgit v1.2.3 From 492ed38192fccb92022b7a6d3b2751a09a3494c6 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Fri, 4 Dec 2020 14:03:08 -0500 Subject: compat_binfmt_elf: don't bother with undef of ELF_ARCH It's not used anywhere downstream (and never had been, AFAICS). Theoretically, fs/binfmt_elf.c does use it, but only in the non-regset coredump handling and all biarch architectures end up with that ifdefed out. Signed-off-by: Al Viro --- fs/compat_binfmt_elf.c | 1 - 1 file changed, 1 deletion(-) (limited to 'fs') diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c index a6321415aba0..e40bdbdc094f 100644 --- a/fs/compat_binfmt_elf.c +++ b/fs/compat_binfmt_elf.c @@ -62,7 +62,6 @@ * differ from the native ones, or omitted when they match. */ -#undef ELF_ARCH #undef elf_check_arch #define elf_check_arch compat_elf_check_arch -- cgit v1.2.3 From e565d89e4aa07e3f20ac5e8757b1da24b5878e69 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Fri, 4 Dec 2020 14:06:04 -0500 Subject: get rid of COMPAT_ELF_EXEC_PAGESIZE not defined by any architecture (and never had been) Signed-off-by: Al Viro --- fs/compat_binfmt_elf.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'fs') diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c index e40bdbdc094f..95e72d271b95 100644 --- a/fs/compat_binfmt_elf.c +++ b/fs/compat_binfmt_elf.c @@ -90,11 +90,6 @@ #define ELF_ET_DYN_BASE COMPAT_ELF_ET_DYN_BASE #endif -#ifdef COMPAT_ELF_EXEC_PAGESIZE -#undef ELF_EXEC_PAGESIZE -#define ELF_EXEC_PAGESIZE COMPAT_ELF_EXEC_PAGESIZE -#endif - #ifdef COMPAT_ELF_PLAT_INIT #undef ELF_PLAT_INIT #define ELF_PLAT_INIT COMPAT_ELF_PLAT_INIT -- cgit v1.2.3 From 181b150f1565ac253ec5313fea76ba5d0c4bddf7 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 5 Jan 2021 23:03:16 -0500 Subject: erofs: use %pd instead of messing with ->d_name Signed-off-by: Al Viro --- fs/erofs/namei.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/erofs/namei.c b/fs/erofs/namei.c index 5f8cc7346c69..3a81e1f7fc06 100644 --- a/fs/erofs/namei.c +++ b/fs/erofs/namei.c @@ -234,8 +234,8 @@ static struct dentry *erofs_lookup(struct inode *dir, } else if (err) { inode = ERR_PTR(err); } else { - erofs_dbg("%s, %s (nid %llu) found, d_type %u", __func__, - dentry->d_name.name, nid, d_type); + erofs_dbg("%s, %pd (nid %llu) found, d_type %u", __func__, + dentry, nid, d_type); inode = erofs_iget(dir->i_sb, nid, d_type == FT_DIR); } return d_splice_alias(inode, dentry); -- cgit v1.2.3 From b9e4666fc1c8f0e259c961cafca81046addb7dfe Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 6 Jan 2021 21:53:52 -0500 Subject: cifs_debug: use %pd instead of messing with ->d_name Signed-off-by: Al Viro --- fs/cifs/cifs_debug.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c index b231dcf1d1f9..aa697ccfa9dc 100644 --- a/fs/cifs/cifs_debug.c +++ b/fs/cifs/cifs_debug.c @@ -197,14 +197,14 @@ static int cifs_debug_files_proc_show(struct seq_file *m, void *v) cfile = list_entry(tmp2, struct cifsFileInfo, tlist); seq_printf(m, - "0x%x 0x%llx 0x%x %d %d %d %s", + "0x%x 0x%llx 0x%x %d %d %d %pd", tcon->tid, cfile->fid.persistent_fid, cfile->f_flags, cfile->count, cfile->pid, from_kuid(&init_user_ns, cfile->uid), - cfile->dentry->d_name.name); + cfile->dentry); #ifdef CONFIG_CIFS_DEBUG2 seq_printf(m, " 0x%llx\n", cfile->fid.mid); #else -- cgit v1.2.3 From d67568410ae1c95004fad85ff1fe78204752f46c Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 6 Jan 2021 21:59:52 -0500 Subject: orangefs_file_mmap(): use %pD ... and no, file can't be NULL there - it's not called that way *and* it would've oopsed a few lines prior on such call anyway. Signed-off-by: Al Viro --- fs/orangefs/file.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/orangefs/file.c b/fs/orangefs/file.c index ec8ae4257975..9b28a7132466 100644 --- a/fs/orangefs/file.c +++ b/fs/orangefs/file.c @@ -487,10 +487,7 @@ static int orangefs_file_mmap(struct file *file, struct vm_area_struct *vma) return ret; gossip_debug(GOSSIP_FILE_DEBUG, - "orangefs_file_mmap: called on %s\n", - (file ? - (char *)file->f_path.dentry->d_name.name : - (char *)"Unknown")); + "orangefs_file_mmap: called on %pD\n", file); /* set the sequential readahead hint */ vma->vm_flags |= VM_SEQ_READ; -- cgit v1.2.3 From 74b30195395c406c787280a77ae55aed82dbbfc7 Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Fri, 27 Nov 2020 17:41:25 +0100 Subject: sysfs: Support zapping of binary attr mmaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want to be able to revoke pci mmaps so that the same access rules applies as for /dev/kmem. Revoke support for devmem was added in 3234ac664a87 ("/dev/mem: Revoke mappings when a driver claims the region"). The simplest way to achieve this is by having the same filp->f_mapping for all mappings, so that unmap_mapping_range can find them all, no matter through which file they've been created. Since this must be set at open time we need sysfs support for this. Add an optional mapping parameter bin_attr, which is only consulted when there's also an mmap callback, since without mmap support allowing to adjust the ->f_mapping makes no sense. Reviewed-by: Greg Kroah-Hartman Signed-off-by: Daniel Vetter Cc: Jason Gunthorpe Cc: Kees Cook Cc: Dan Williams Cc: Andrew Morton Cc: John Hubbard Cc: Jérôme Glisse Cc: Jan Kara Cc: Dan Williams Cc: linux-mm@kvack.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-samsung-soc@vger.kernel.org Cc: linux-media@vger.kernel.org Cc: Bjorn Helgaas Cc: linux-pci@vger.kernel.org Cc: Greg Kroah-Hartman Cc: "Rafael J. Wysocki" Cc: Christian Brauner Cc: "David S. Miller" Cc: Michael Ellerman Cc: Sourabh Jain Cc: Daniel Vetter Cc: Mauro Carvalho Chehab Cc: Nayna Jain Signed-off-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20201127164131.2244124-12-daniel.vetter@ffwll.ch --- fs/sysfs/file.c | 11 +++++++++++ include/linux/sysfs.h | 2 ++ 2 files changed, 13 insertions(+) (limited to 'fs') diff --git a/fs/sysfs/file.c b/fs/sysfs/file.c index 96d0da65e088..9aefa7779b29 100644 --- a/fs/sysfs/file.c +++ b/fs/sysfs/file.c @@ -170,6 +170,16 @@ static int sysfs_kf_bin_mmap(struct kernfs_open_file *of, return battr->mmap(of->file, kobj, battr, vma); } +static int sysfs_kf_bin_open(struct kernfs_open_file *of) +{ + struct bin_attribute *battr = of->kn->priv; + + if (battr->mapping) + of->file->f_mapping = battr->mapping; + + return 0; +} + void sysfs_notify(struct kobject *kobj, const char *dir, const char *attr) { struct kernfs_node *kn = kobj->sd, *tmp; @@ -241,6 +251,7 @@ static const struct kernfs_ops sysfs_bin_kfops_mmap = { .read = sysfs_kf_bin_read, .write = sysfs_kf_bin_write, .mmap = sysfs_kf_bin_mmap, + .open = sysfs_kf_bin_open, }; int sysfs_add_file_mode_ns(struct kernfs_node *parent, diff --git a/include/linux/sysfs.h b/include/linux/sysfs.h index 2caa34c1ca1a..d76a1ddf83a3 100644 --- a/include/linux/sysfs.h +++ b/include/linux/sysfs.h @@ -164,11 +164,13 @@ __ATTRIBUTE_GROUPS(_name) struct file; struct vm_area_struct; +struct address_space; struct bin_attribute { struct attribute attr; size_t size; void *private; + struct address_space *mapping; ssize_t (*read)(struct file *, struct kobject *, struct bin_attribute *, char *, loff_t, size_t); ssize_t (*write)(struct file *, struct kobject *, struct bin_attribute *, -- cgit v1.2.3 From e20b14db050ae25ef7c5a2d9a41dd62b7e5e75c1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 12 Jan 2021 11:02:45 -0800 Subject: fs: only specify I_DIRTY_TIME when needed in generic_update_time() generic_update_time() always passes I_DIRTY_TIME to __mark_inode_dirty(), which doesn't really make sense because (a) generic_update_time() might be asked to do only an i_version update, not also a timestamps update; and (b) I_DIRTY_TIME is only supposed to be set in i_state if the filesystem has lazytime enabled, so using it unconditionally in generic_update_time() is inconsistent. As a result there is a weird edge case where if only an i_version update was requested (not also a timestamps update) but it is no longer needed (i.e. inode_maybe_inc_iversion() returns false), then I_DIRTY_TIME will be set in i_state even if the filesystem isn't mounted with lazytime. Fix this by only passing I_DIRTY_TIME to __mark_inode_dirty() if the timestamps were updated and the filesystem has lazytime enabled. Link: https://lore.kernel.org/r/20210112190253.64307-4-ebiggers@kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Eric Biggers Signed-off-by: Jan Kara --- fs/inode.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) (limited to 'fs') diff --git a/fs/inode.c b/fs/inode.c index 6442d97d9a4a..d0fa43d8e9d5 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -1743,24 +1743,26 @@ static int relatime_need_update(struct vfsmount *mnt, struct inode *inode, int generic_update_time(struct inode *inode, struct timespec64 *time, int flags) { - int iflags = I_DIRTY_TIME; - bool dirty = false; - - if (flags & S_ATIME) - inode->i_atime = *time; - if (flags & S_VERSION) - dirty = inode_maybe_inc_iversion(inode, false); - if (flags & S_CTIME) - inode->i_ctime = *time; - if (flags & S_MTIME) - inode->i_mtime = *time; - if ((flags & (S_ATIME | S_CTIME | S_MTIME)) && - !(inode->i_sb->s_flags & SB_LAZYTIME)) - dirty = true; - - if (dirty) - iflags |= I_DIRTY_SYNC; - __mark_inode_dirty(inode, iflags); + int dirty_flags = 0; + + if (flags & (S_ATIME | S_CTIME | S_MTIME)) { + if (flags & S_ATIME) + inode->i_atime = *time; + if (flags & S_CTIME) + inode->i_ctime = *time; + if (flags & S_MTIME) + inode->i_mtime = *time; + + if (inode->i_sb->s_flags & SB_LAZYTIME) + dirty_flags |= I_DIRTY_TIME; + else + dirty_flags |= I_DIRTY_SYNC; + } + + if ((flags & S_VERSION) && inode_maybe_inc_iversion(inode, false)) + dirty_flags |= I_DIRTY_SYNC; + + __mark_inode_dirty(inode, dirty_flags); return 0; } EXPORT_SYMBOL(generic_update_time); -- cgit v1.2.3 From ff4136e64d129c4b617331d6c84a3e1781dda70d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 12 Jan 2021 11:02:46 -0800 Subject: fat: only specify I_DIRTY_TIME when needed in fat_update_time() As was done for generic_update_time(), only pass I_DIRTY_TIME to __mark_inode_dirty() when the inode's timestamps were actually updated and lazytime is enabled. This avoids a weird edge case where I_DIRTY_TIME could be set in i_state when lazytime isn't enabled. Link: https://lore.kernel.org/r/20210112190253.64307-5-ebiggers@kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Eric Biggers Signed-off-by: Jan Kara --- fs/fat/misc.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/fat/misc.c b/fs/fat/misc.c index f1b2a1fc2a6a..18a50a46b57f 100644 --- a/fs/fat/misc.c +++ b/fs/fat/misc.c @@ -329,22 +329,23 @@ EXPORT_SYMBOL_GPL(fat_truncate_time); int fat_update_time(struct inode *inode, struct timespec64 *now, int flags) { - int iflags = I_DIRTY_TIME; - bool dirty = false; + int dirty_flags = 0; if (inode->i_ino == MSDOS_ROOT_INO) return 0; - fat_truncate_time(inode, now, flags); - if (flags & S_VERSION) - dirty = inode_maybe_inc_iversion(inode, false); - if ((flags & (S_ATIME | S_CTIME | S_MTIME)) && - !(inode->i_sb->s_flags & SB_LAZYTIME)) - dirty = true; + if (flags & (S_ATIME | S_CTIME | S_MTIME)) { + fat_truncate_time(inode, now, flags); + if (inode->i_sb->s_flags & SB_LAZYTIME) + dirty_flags |= I_DIRTY_TIME; + else + dirty_flags |= I_DIRTY_SYNC; + } + + if ((flags & S_VERSION) && inode_maybe_inc_iversion(inode, false)) + dirty_flags |= I_DIRTY_SYNC; - if (dirty) - iflags |= I_DIRTY_SYNC; - __mark_inode_dirty(inode, iflags); + __mark_inode_dirty(inode, dirty_flags); return 0; } EXPORT_SYMBOL_GPL(fat_update_time); -- cgit v1.2.3 From e2728c5621fd9c68c65a6647875a1d1c67b9f257 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 12 Jan 2021 11:02:47 -0800 Subject: fs: don't call ->dirty_inode for lazytime timestamp updates There is no need to call ->dirty_inode for lazytime timestamp updates (i.e. for __mark_inode_dirty(I_DIRTY_TIME)), since by the definition of lazytime, filesystems must ignore these updates. Filesystems only need to care about the updated timestamps when they expire. Therefore, only call ->dirty_inode when I_DIRTY_INODE is set. Based on a patch from Christoph Hellwig: https://lore.kernel.org/r/20200325122825.1086872-4-hch@lst.de Link: https://lore.kernel.org/r/20210112190253.64307-6-ebiggers@kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Eric Biggers Signed-off-by: Jan Kara --- fs/ext4/inode.c | 12 +----------- fs/f2fs/super.c | 3 --- fs/fs-writeback.c | 6 +++--- fs/gfs2/super.c | 2 -- 4 files changed, 4 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 27946882d4ce..4cc6c7834312 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -5933,26 +5933,16 @@ out: * If the inode is marked synchronous, we don't honour that here - doing * so would cause a commit on atime updates, which we don't bother doing. * We handle synchronous inodes at the highest possible level. - * - * If only the I_DIRTY_TIME flag is set, we can skip everything. If - * I_DIRTY_TIME and I_DIRTY_SYNC is set, the only inode fields we need - * to copy into the on-disk inode structure are the timestamp files. */ void ext4_dirty_inode(struct inode *inode, int flags) { handle_t *handle; - if (flags == I_DIRTY_TIME) - return; handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); if (IS_ERR(handle)) - goto out; - + return; ext4_mark_inode_dirty(handle, inode); - ext4_journal_stop(handle); -out: - return; } int ext4_change_inode_journal_flag(struct inode *inode, int val) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index b4a07fe62d1a..cc98dc49f4a2 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -1196,9 +1196,6 @@ static void f2fs_dirty_inode(struct inode *inode, int flags) inode->i_ino == F2FS_META_INO(sbi)) return; - if (flags == I_DIRTY_TIME) - return; - if (is_inode_flag_set(inode, FI_AUTO_RECOVER)) clear_inode_flag(inode, FI_AUTO_RECOVER); diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index c41cb887eb7d..b7616bbd5533 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -2255,16 +2255,16 @@ void __mark_inode_dirty(struct inode *inode, int flags) * Don't do this for I_DIRTY_PAGES - that doesn't actually * dirty the inode itself */ - if (flags & (I_DIRTY_INODE | I_DIRTY_TIME)) { + if (flags & I_DIRTY_INODE) { trace_writeback_dirty_inode_start(inode, flags); if (sb->s_op->dirty_inode) sb->s_op->dirty_inode(inode, flags); trace_writeback_dirty_inode(inode, flags); - } - if (flags & I_DIRTY_INODE) + flags &= ~I_DIRTY_TIME; + } dirtytime = flags & I_DIRTY_TIME; /* diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index 2f56acc41c04..042b94288ff1 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -562,8 +562,6 @@ static void gfs2_dirty_inode(struct inode *inode, int flags) int need_endtrans = 0; int ret; - if (!(flags & I_DIRTY_INODE)) - return; if (unlikely(gfs2_withdrawn(sdp))) return; if (!gfs2_glock_is_locked_by_me(ip->i_gl)) { -- cgit v1.2.3 From a38ed483a72672ee6bdb5d8cf17fc0838377baa0 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 12 Jan 2021 11:02:48 -0800 Subject: fs: pass only I_DIRTY_INODE flags to ->dirty_inode ->dirty_inode is now only called when I_DIRTY_INODE (I_DIRTY_SYNC and/or I_DIRTY_DATASYNC) is set. However it may still be passed other dirty flags at the same time, provided that these other flags happened to be passed to __mark_inode_dirty() at the same time as I_DIRTY_INODE. This doesn't make sense because there is no reason for filesystems to care about these extra flags. Nor are filesystems notified about all updates to these other flags. Therefore, mask the flags before passing them to ->dirty_inode. Also properly document ->dirty_inode in vfs.rst. Link: https://lore.kernel.org/r/20210112190253.64307-7-ebiggers@kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Eric Biggers Signed-off-by: Jan Kara --- Documentation/filesystems/vfs.rst | 5 ++++- fs/fs-writeback.c | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index ca52c82e5bb5..287b80948a40 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -270,7 +270,10 @@ or bottom half). ->alloc_inode. ``dirty_inode`` - this method is called by the VFS to mark an inode dirty. + this method is called by the VFS when an inode is marked dirty. + This is specifically for the inode itself being marked dirty, + not its data. If the update needs to be persisted by fdatasync(), + then I_DIRTY_DATASYNC will be set in the flags argument. ``write_inode`` this method is called when the VFS needs to write an inode to diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index b7616bbd5533..2e6064012f7d 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -2259,7 +2259,7 @@ void __mark_inode_dirty(struct inode *inode, int flags) trace_writeback_dirty_inode_start(inode, flags); if (sb->s_op->dirty_inode) - sb->s_op->dirty_inode(inode, flags); + sb->s_op->dirty_inode(inode, flags & I_DIRTY_INODE); trace_writeback_dirty_inode(inode, flags); -- cgit v1.2.3 From 35d14f278e530ecb635ab00de984065ed90ee12f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 12 Jan 2021 11:02:49 -0800 Subject: fs: clean up __mark_inode_dirty() a bit Improve some comments, and don't bother checking for the I_DIRTY_TIME flag in the case where we just cleared it. Also, warn if I_DIRTY_TIME and I_DIRTY_PAGES are passed to __mark_inode_dirty() at the same time, as this case isn't handled. Link: https://lore.kernel.org/r/20210112190253.64307-8-ebiggers@kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Eric Biggers Signed-off-by: Jan Kara --- fs/fs-writeback.c | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 2e6064012f7d..80ee9816d9df 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -2219,23 +2219,24 @@ static noinline void block_dump___mark_inode_dirty(struct inode *inode) } /** - * __mark_inode_dirty - internal function + * __mark_inode_dirty - internal function to mark an inode dirty * * @inode: inode to mark - * @flags: what kind of dirty (i.e. I_DIRTY_SYNC) + * @flags: what kind of dirty, e.g. I_DIRTY_SYNC. This can be a combination of + * multiple I_DIRTY_* flags, except that I_DIRTY_TIME can't be combined + * with I_DIRTY_PAGES. * - * Mark an inode as dirty. Callers should use mark_inode_dirty or - * mark_inode_dirty_sync. + * Mark an inode as dirty. We notify the filesystem, then update the inode's + * dirty flags. Then, if needed we add the inode to the appropriate dirty list. * - * Put the inode on the super block's dirty list. + * Most callers should use mark_inode_dirty() or mark_inode_dirty_sync() + * instead of calling this directly. * - * CAREFUL! We mark it dirty unconditionally, but move it onto the - * dirty list only if it is hashed or if it refers to a blockdev. - * If it was not hashed, it will never be added to the dirty list - * even if it is later hashed, as it will have been marked dirty already. + * CAREFUL! We only add the inode to the dirty list if it is hashed or if it + * refers to a blockdev. Unhashed inodes will never be added to the dirty list + * even if they are later hashed, as they will have been marked dirty already. * - * In short, make sure you hash any inodes _before_ you start marking - * them dirty. + * In short, ensure you hash any inodes _before_ you start marking them dirty. * * Note that for blockdevs, inode->dirtied_when represents the dirtying time of * the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of @@ -2247,25 +2248,34 @@ static noinline void block_dump___mark_inode_dirty(struct inode *inode) void __mark_inode_dirty(struct inode *inode, int flags) { struct super_block *sb = inode->i_sb; - int dirtytime; + int dirtytime = 0; trace_writeback_mark_inode_dirty(inode, flags); - /* - * Don't do this for I_DIRTY_PAGES - that doesn't actually - * dirty the inode itself - */ if (flags & I_DIRTY_INODE) { + /* + * Notify the filesystem about the inode being dirtied, so that + * (if needed) it can update on-disk fields and journal the + * inode. This is only needed when the inode itself is being + * dirtied now. I.e. it's only needed for I_DIRTY_INODE, not + * for just I_DIRTY_PAGES or I_DIRTY_TIME. + */ trace_writeback_dirty_inode_start(inode, flags); - if (sb->s_op->dirty_inode) sb->s_op->dirty_inode(inode, flags & I_DIRTY_INODE); - trace_writeback_dirty_inode(inode, flags); + /* I_DIRTY_INODE supersedes I_DIRTY_TIME. */ flags &= ~I_DIRTY_TIME; + } else { + /* + * Else it's either I_DIRTY_PAGES, I_DIRTY_TIME, or nothing. + * (We don't support setting both I_DIRTY_PAGES and I_DIRTY_TIME + * in one call to __mark_inode_dirty().) + */ + dirtytime = flags & I_DIRTY_TIME; + WARN_ON_ONCE(dirtytime && flags != I_DIRTY_TIME); } - dirtytime = flags & I_DIRTY_TIME; /* * Paired with smp_mb() in __writeback_single_inode() for the @@ -2288,6 +2298,7 @@ void __mark_inode_dirty(struct inode *inode, int flags) inode_attach_wb(inode, NULL); + /* I_DIRTY_INODE supersedes I_DIRTY_TIME. */ if (flags & I_DIRTY_INODE) inode->i_state &= ~I_DIRTY_TIME; inode->i_state |= flags; -- cgit v1.2.3 From 83dc881d678a8e973212afa0c5e4fb2fe1b06d4e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 12 Jan 2021 11:02:50 -0800 Subject: fs: drop redundant check from __writeback_single_inode() wbc->for_sync implies wbc->sync_mode == WB_SYNC_ALL, so there's no need to check for both. Just check for WB_SYNC_ALL. Link: https://lore.kernel.org/r/20210112190253.64307-9-ebiggers@kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Eric Biggers Signed-off-by: Jan Kara --- fs/fs-writeback.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 80ee9816d9df..cee1df6e3bd4 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -1479,7 +1479,7 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc) * change I_DIRTY_TIME into I_DIRTY_SYNC. */ if ((inode->i_state & I_DIRTY_TIME) && - (wbc->sync_mode == WB_SYNC_ALL || wbc->for_sync || + (wbc->sync_mode == WB_SYNC_ALL || time_after(jiffies, inode->dirtied_time_when + dirtytime_expire_interval * HZ))) { trace_writeback_lazytime(inode); -- cgit v1.2.3 From da0c4c60d8c7c8cc5266b0c5fe7bdf136e5e6415 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 12 Jan 2021 11:02:51 -0800 Subject: fs: improve comments for writeback_single_inode() Some comments for writeback_single_inode() and __writeback_single_inode() are outdated or not very helpful, especially with regards to writeback list handling. Update them. Link: https://lore.kernel.org/r/20210112190253.64307-10-ebiggers@kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Eric Biggers Signed-off-by: Jan Kara --- fs/fs-writeback.c | 57 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 24 deletions(-) (limited to 'fs') diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index cee1df6e3bd4..e91980f49388 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -1442,9 +1442,15 @@ static void requeue_inode(struct inode *inode, struct bdi_writeback *wb, } /* - * Write out an inode and its dirty pages. Do not update the writeback list - * linkage. That is left to the caller. The caller is also responsible for - * setting I_SYNC flag and calling inode_sync_complete() to clear it. + * Write out an inode and its dirty pages (or some of its dirty pages, depending + * on @wbc->nr_to_write), and clear the relevant dirty flags from i_state. + * + * This doesn't remove the inode from the writeback list it is on, except + * potentially to move it from b_dirty_time to b_dirty due to timestamp + * expiration. The caller is otherwise responsible for writeback list handling. + * + * The caller is also responsible for setting the I_SYNC flag beforehand and + * calling inode_sync_complete() to clear it afterwards. */ static int __writeback_single_inode(struct inode *inode, struct writeback_control *wbc) @@ -1487,9 +1493,10 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc) } /* - * Some filesystems may redirty the inode during the writeback - * due to delalloc, clear dirty metadata flags right before - * write_inode() + * Get and clear the dirty flags from i_state. This needs to be done + * after calling writepages because some filesystems may redirty the + * inode during writepages due to delalloc. It also needs to be done + * after handling timestamp expiration, as that may dirty the inode too. */ spin_lock(&inode->i_lock); dirty = inode->i_state & I_DIRTY; @@ -1524,12 +1531,13 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc) } /* - * Write out an inode's dirty pages. Either the caller has an active reference - * on the inode or the inode has I_WILL_FREE set. + * Write out an inode's dirty data and metadata on-demand, i.e. separately from + * the regular batched writeback done by the flusher threads in + * writeback_sb_inodes(). @wbc controls various aspects of the write, such as + * whether it is a data-integrity sync (%WB_SYNC_ALL) or not (%WB_SYNC_NONE). * - * This function is designed to be called for writing back one inode which - * we go e.g. from filesystem. Flusher thread uses __writeback_single_inode() - * and does more profound writeback list handling in writeback_sb_inodes(). + * To prevent the inode from going away, either the caller must have a reference + * to the inode, or the inode must have I_WILL_FREE or I_FREEING set. */ static int writeback_single_inode(struct inode *inode, struct writeback_control *wbc) @@ -1544,23 +1552,23 @@ static int writeback_single_inode(struct inode *inode, WARN_ON(inode->i_state & I_WILL_FREE); if (inode->i_state & I_SYNC) { - if (wbc->sync_mode != WB_SYNC_ALL) - goto out; /* - * It's a data-integrity sync. We must wait. Since callers hold - * inode reference or inode has I_WILL_FREE set, it cannot go - * away under us. + * Writeback is already running on the inode. For WB_SYNC_NONE, + * that's enough and we can just return. For WB_SYNC_ALL, we + * must wait for the existing writeback to complete, then do + * writeback again if there's anything left. */ + if (wbc->sync_mode != WB_SYNC_ALL) + goto out; __inode_wait_for_writeback(inode); } WARN_ON(inode->i_state & I_SYNC); /* - * Skip inode if it is clean and we have no outstanding writeback in - * WB_SYNC_ALL mode. We don't want to mess with writeback lists in this - * function since flusher thread may be doing for example sync in - * parallel and if we move the inode, it could get skipped. So here we - * make sure inode is on some writeback list and leave it there unless - * we have completely cleaned the inode. + * If the inode is already fully clean, then there's nothing to do. + * + * For data-integrity syncs we also need to check whether any pages are + * still under writeback, e.g. due to prior WB_SYNC_NONE writeback. If + * there are any such pages, we'll need to wait for them. */ if (!(inode->i_state & I_DIRTY_ALL) && (wbc->sync_mode != WB_SYNC_ALL || @@ -1576,8 +1584,9 @@ static int writeback_single_inode(struct inode *inode, wb = inode_to_wb_and_lock_list(inode); spin_lock(&inode->i_lock); /* - * If inode is clean, remove it from writeback lists. Otherwise don't - * touch it. See comment above for explanation. + * If the inode is now fully clean, then it can be safely removed from + * its writeback list (if any). Otherwise the flusher threads are + * responsible for the writeback lists. */ if (!(inode->i_state & I_DIRTY_ALL)) inode_io_list_del_locked(inode, wb); -- cgit v1.2.3 From 3aac630b5ce18b0a7e34cb35cdd99fcc508edc92 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 12 Jan 2021 11:02:52 -0800 Subject: gfs2: don't worry about I_DIRTY_TIME in gfs2_fsync() The I_DIRTY_TIME flag is primary used within the VFS, and there's no reason for ->fsync() implementations to do anything with it. This is because when !datasync, the VFS will expire dirty timestamps before calling ->fsync(). (See vfs_fsync_range().) This turns I_DIRTY_TIME into I_DIRTY_SYNC. Therefore, change gfs2_fsync() to not check for I_DIRTY_TIME. Link: https://lore.kernel.org/r/20210112190253.64307-11-ebiggers@kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Eric Biggers Signed-off-by: Jan Kara --- fs/gfs2/file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index b39b339feddc..7fe2497755a3 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -749,7 +749,7 @@ static int gfs2_fsync(struct file *file, loff_t start, loff_t end, { struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; - int sync_state = inode->i_state & I_DIRTY_ALL; + int sync_state = inode->i_state & I_DIRTY; struct gfs2_inode *ip = GFS2_I(inode); int ret = 0, ret1 = 0; @@ -762,7 +762,7 @@ static int gfs2_fsync(struct file *file, loff_t start, loff_t end, if (!gfs2_is_jdata(ip)) sync_state &= ~I_DIRTY_PAGES; if (datasync) - sync_state &= ~(I_DIRTY_SYNC | I_DIRTY_TIME); + sync_state &= ~I_DIRTY_SYNC; if (sync_state) { ret = sync_inode_metadata(inode, 1); -- cgit v1.2.3 From ed296c6c05b0ac52d7c6bf13a90f02b8b8222169 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 12 Jan 2021 11:02:53 -0800 Subject: ext4: simplify i_state checks in __ext4_update_other_inode_time() Since I_DIRTY_TIME and I_DIRTY_INODE are mutually exclusive in i_state, there's no need to check for I_DIRTY_TIME && !I_DIRTY_INODE. Just check for I_DIRTY_TIME. Also introduce a helper function in include/linux/fs.h to do this check. Link: https://lore.kernel.org/r/20210112190253.64307-12-ebiggers@kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Eric Biggers Signed-off-by: Jan Kara --- fs/ext4/inode.c | 8 ++------ include/linux/fs.h | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 4cc6c7834312..d809a06b6ef7 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -4961,15 +4961,11 @@ static void __ext4_update_other_inode_time(struct super_block *sb, if (!inode) return; - if ((inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW | - I_DIRTY_INODE)) || - ((inode->i_state & I_DIRTY_TIME) == 0)) + if (!inode_is_dirtytime_only(inode)) return; spin_lock(&inode->i_lock); - if (((inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW | - I_DIRTY_INODE)) == 0) && - (inode->i_state & I_DIRTY_TIME)) { + if (inode_is_dirtytime_only(inode)) { struct ext4_inode_info *ei = EXT4_I(inode); inode->i_state &= ~I_DIRTY_TIME; diff --git a/include/linux/fs.h b/include/linux/fs.h index 45a0303b2aeb..de0c789104c2 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2194,6 +2194,21 @@ static inline void mark_inode_dirty_sync(struct inode *inode) __mark_inode_dirty(inode, I_DIRTY_SYNC); } +/* + * Returns true if the given inode itself only has dirty timestamps (its pages + * may still be dirty) and isn't currently being allocated or freed. + * Filesystems should call this if when writing an inode when lazytime is + * enabled, they want to opportunistically write the timestamps of other inodes + * located very nearby on-disk, e.g. in the same inode block. This returns true + * if the given inode is in need of such an opportunistic update. Requires + * i_lock, or at least later re-checking under i_lock. + */ +static inline bool inode_is_dirtytime_only(struct inode *inode) +{ + return (inode->i_state & (I_DIRTY_TIME | I_NEW | + I_FREEING | I_WILL_FREE)) == I_DIRTY_TIME; +} + extern void inc_nlink(struct inode *inode); extern void drop_nlink(struct inode *inode); extern void clear_nlink(struct inode *inode); -- cgit v1.2.3 From e7e832ce6fa769f800cd7eaebdb0459ad31e0416 Mon Sep 17 00:00:00 2001 From: Daniel Colascione Date: Fri, 8 Jan 2021 14:22:21 -0800 Subject: fs: add LSM-supporting anon-inode interface This change adds a new function, anon_inode_getfd_secure, that creates anonymous-node file with individual non-S_PRIVATE inode to which security modules can apply policy. Existing callers continue using the original singleton-inode kind of anonymous-inode file. We can transition anonymous inode users to the new kind of anonymous inode in individual patches for the sake of bisection and review. The new function accepts an optional context_inode parameter that callers can use to provide additional contextual information to security modules. For example, in case of userfaultfd, the created inode is a 'logical child' of the context_inode (userfaultfd inode of the parent process) in the sense that it provides the security context required during creation of the child process' userfaultfd inode. Signed-off-by: Daniel Colascione [LG: Delete obsolete comments to alloc_anon_inode()] [LG: Add context_inode description in comments to anon_inode_getfd_secure()] [LG: Remove definition of anon_inode_getfile_secure() as there are no callers] [LG: Make __anon_inode_getfile() static] [LG: Use correct error cast in __anon_inode_getfile()] [LG: Fix error handling in __anon_inode_getfile()] Signed-off-by: Lokesh Gidra Reviewed-by: Eric Biggers Signed-off-by: Paul Moore --- fs/anon_inodes.c | 150 ++++++++++++++++++++++++++++++++------------ fs/libfs.c | 5 -- include/linux/anon_inodes.h | 5 ++ 3 files changed, 115 insertions(+), 45 deletions(-) (limited to 'fs') diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index 89714308c25b..023337d65a03 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -55,61 +55,79 @@ static struct file_system_type anon_inode_fs_type = { .kill_sb = kill_anon_super, }; -/** - * anon_inode_getfile - creates a new file instance by hooking it up to an - * anonymous inode, and a dentry that describe the "class" - * of the file - * - * @name: [in] name of the "class" of the new file - * @fops: [in] file operations for the new file - * @priv: [in] private data for the new file (will be file's private_data) - * @flags: [in] flags - * - * Creates a new file by hooking it on a single inode. This is useful for files - * that do not need to have a full-fledged inode in order to operate correctly. - * All the files created with anon_inode_getfile() will share a single inode, - * hence saving memory and avoiding code duplication for the file/inode/dentry - * setup. Returns the newly created file* or an error pointer. - */ -struct file *anon_inode_getfile(const char *name, - const struct file_operations *fops, - void *priv, int flags) +static struct inode *anon_inode_make_secure_inode( + const char *name, + const struct inode *context_inode) { - struct file *file; + struct inode *inode; + const struct qstr qname = QSTR_INIT(name, strlen(name)); + int error; + + inode = alloc_anon_inode(anon_inode_mnt->mnt_sb); + if (IS_ERR(inode)) + return inode; + inode->i_flags &= ~S_PRIVATE; + error = security_inode_init_security_anon(inode, &qname, context_inode); + if (error) { + iput(inode); + return ERR_PTR(error); + } + return inode; +} - if (IS_ERR(anon_inode_inode)) - return ERR_PTR(-ENODEV); +static struct file *__anon_inode_getfile(const char *name, + const struct file_operations *fops, + void *priv, int flags, + const struct inode *context_inode, + bool secure) +{ + struct inode *inode; + struct file *file; if (fops->owner && !try_module_get(fops->owner)) return ERR_PTR(-ENOENT); - /* - * We know the anon_inode inode count is always greater than zero, - * so ihold() is safe. - */ - ihold(anon_inode_inode); - file = alloc_file_pseudo(anon_inode_inode, anon_inode_mnt, name, + if (secure) { + inode = anon_inode_make_secure_inode(name, context_inode); + if (IS_ERR(inode)) { + file = ERR_CAST(inode); + goto err; + } + } else { + inode = anon_inode_inode; + if (IS_ERR(inode)) { + file = ERR_PTR(-ENODEV); + goto err; + } + /* + * We know the anon_inode inode count is always + * greater than zero, so ihold() is safe. + */ + ihold(inode); + } + + file = alloc_file_pseudo(inode, anon_inode_mnt, name, flags & (O_ACCMODE | O_NONBLOCK), fops); if (IS_ERR(file)) - goto err; + goto err_iput; - file->f_mapping = anon_inode_inode->i_mapping; + file->f_mapping = inode->i_mapping; file->private_data = priv; return file; +err_iput: + iput(inode); err: - iput(anon_inode_inode); module_put(fops->owner); return file; } -EXPORT_SYMBOL_GPL(anon_inode_getfile); /** - * anon_inode_getfd - creates a new file instance by hooking it up to an - * anonymous inode, and a dentry that describe the "class" - * of the file + * anon_inode_getfile - creates a new file instance by hooking it up to an + * anonymous inode, and a dentry that describe the "class" + * of the file * * @name: [in] name of the "class" of the new file * @fops: [in] file operations for the new file @@ -118,12 +136,23 @@ EXPORT_SYMBOL_GPL(anon_inode_getfile); * * Creates a new file by hooking it on a single inode. This is useful for files * that do not need to have a full-fledged inode in order to operate correctly. - * All the files created with anon_inode_getfd() will share a single inode, + * All the files created with anon_inode_getfile() will share a single inode, * hence saving memory and avoiding code duplication for the file/inode/dentry - * setup. Returns new descriptor or an error code. + * setup. Returns the newly created file* or an error pointer. */ -int anon_inode_getfd(const char *name, const struct file_operations *fops, - void *priv, int flags) +struct file *anon_inode_getfile(const char *name, + const struct file_operations *fops, + void *priv, int flags) +{ + return __anon_inode_getfile(name, fops, priv, flags, NULL, false); +} +EXPORT_SYMBOL_GPL(anon_inode_getfile); + +static int __anon_inode_getfd(const char *name, + const struct file_operations *fops, + void *priv, int flags, + const struct inode *context_inode, + bool secure) { int error, fd; struct file *file; @@ -133,7 +162,8 @@ int anon_inode_getfd(const char *name, const struct file_operations *fops, return error; fd = error; - file = anon_inode_getfile(name, fops, priv, flags); + file = __anon_inode_getfile(name, fops, priv, flags, context_inode, + secure); if (IS_ERR(file)) { error = PTR_ERR(file); goto err_put_unused_fd; @@ -146,8 +176,48 @@ err_put_unused_fd: put_unused_fd(fd); return error; } + +/** + * anon_inode_getfd - creates a new file instance by hooking it up to + * an anonymous inode and a dentry that describe + * the "class" of the file + * + * @name: [in] name of the "class" of the new file + * @fops: [in] file operations for the new file + * @priv: [in] private data for the new file (will be file's private_data) + * @flags: [in] flags + * + * Creates a new file by hooking it on a single inode. This is + * useful for files that do not need to have a full-fledged inode in + * order to operate correctly. All the files created with + * anon_inode_getfd() will use the same singleton inode, reducing + * memory use and avoiding code duplication for the file/inode/dentry + * setup. Returns a newly created file descriptor or an error code. + */ +int anon_inode_getfd(const char *name, const struct file_operations *fops, + void *priv, int flags) +{ + return __anon_inode_getfd(name, fops, priv, flags, NULL, false); +} EXPORT_SYMBOL_GPL(anon_inode_getfd); +/** + * Like anon_inode_getfd(), but creates a new !S_PRIVATE anon inode rather than + * reuse the singleton anon inode, and calls the inode_init_security_anon() LSM + * hook. This allows the inode to have its own security context and for a LSM + * to reject creation of the inode. An optional @context_inode argument is + * also added to provide the logical relationship with the new inode. The LSM + * may use @context_inode in inode_init_security_anon(), but a reference to it + * is not held. + */ +int anon_inode_getfd_secure(const char *name, const struct file_operations *fops, + void *priv, int flags, + const struct inode *context_inode) +{ + return __anon_inode_getfd(name, fops, priv, flags, context_inode, true); +} +EXPORT_SYMBOL_GPL(anon_inode_getfd_secure); + static int __init anon_inode_init(void) { anon_inode_mnt = kern_mount(&anon_inode_fs_type); diff --git a/fs/libfs.c b/fs/libfs.c index d1c3bade9f30..e52818fb276a 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -1214,11 +1214,6 @@ static int anon_set_page_dirty(struct page *page) return 0; }; -/* - * A single inode exists for all anon_inode files. Contrary to pipes, - * anon_inode inodes have no associated per-instance data, so we need - * only allocate one of them. - */ struct inode *alloc_anon_inode(struct super_block *s) { static const struct address_space_operations anon_aops = { diff --git a/include/linux/anon_inodes.h b/include/linux/anon_inodes.h index d0d7d96261ad..71881a2b6f78 100644 --- a/include/linux/anon_inodes.h +++ b/include/linux/anon_inodes.h @@ -10,12 +10,17 @@ #define _LINUX_ANON_INODES_H struct file_operations; +struct inode; struct file *anon_inode_getfile(const char *name, const struct file_operations *fops, void *priv, int flags); int anon_inode_getfd(const char *name, const struct file_operations *fops, void *priv, int flags); +int anon_inode_getfd_secure(const char *name, + const struct file_operations *fops, + void *priv, int flags, + const struct inode *context_inode); #endif /* _LINUX_ANON_INODES_H */ -- cgit v1.2.3 From b537900f1598b67bcb8acac20da73c6e26ebbf99 Mon Sep 17 00:00:00 2001 From: Daniel Colascione Date: Fri, 8 Jan 2021 14:22:23 -0800 Subject: userfaultfd: use secure anon inodes for userfaultfd This change gives userfaultfd file descriptors a real security context, allowing policy to act on them. Signed-off-by: Daniel Colascione [LG: Remove owner inode from userfaultfd_ctx] [LG: Use anon_inode_getfd_secure() in userfaultfd syscall] [LG: Use inode of file in userfaultfd_read() in resolve_userfault_fork()] Signed-off-by: Lokesh Gidra Reviewed-by: Eric Biggers Signed-off-by: Paul Moore --- fs/userfaultfd.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 894cc28142e7..0be8cdd4425a 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -979,14 +979,14 @@ static __poll_t userfaultfd_poll(struct file *file, poll_table *wait) static const struct file_operations userfaultfd_fops; -static int resolve_userfault_fork(struct userfaultfd_ctx *ctx, - struct userfaultfd_ctx *new, +static int resolve_userfault_fork(struct userfaultfd_ctx *new, + struct inode *inode, struct uffd_msg *msg) { int fd; - fd = anon_inode_getfd("[userfaultfd]", &userfaultfd_fops, new, - O_RDWR | (new->flags & UFFD_SHARED_FCNTL_FLAGS)); + fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, new, + O_RDWR | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode); if (fd < 0) return fd; @@ -996,7 +996,7 @@ static int resolve_userfault_fork(struct userfaultfd_ctx *ctx, } static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait, - struct uffd_msg *msg) + struct uffd_msg *msg, struct inode *inode) { ssize_t ret; DECLARE_WAITQUEUE(wait, current); @@ -1107,7 +1107,7 @@ static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait, spin_unlock_irq(&ctx->fd_wqh.lock); if (!ret && msg->event == UFFD_EVENT_FORK) { - ret = resolve_userfault_fork(ctx, fork_nctx, msg); + ret = resolve_userfault_fork(fork_nctx, inode, msg); spin_lock_irq(&ctx->event_wqh.lock); if (!list_empty(&fork_event)) { /* @@ -1167,6 +1167,7 @@ static ssize_t userfaultfd_read(struct file *file, char __user *buf, ssize_t _ret, ret = 0; struct uffd_msg msg; int no_wait = file->f_flags & O_NONBLOCK; + struct inode *inode = file_inode(file); if (ctx->state == UFFD_STATE_WAIT_API) return -EINVAL; @@ -1174,7 +1175,7 @@ static ssize_t userfaultfd_read(struct file *file, char __user *buf, for (;;) { if (count < sizeof(msg)) return ret ? ret : -EINVAL; - _ret = userfaultfd_ctx_read(ctx, no_wait, &msg); + _ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode); if (_ret < 0) return ret ? ret : _ret; if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg))) @@ -1999,8 +2000,8 @@ SYSCALL_DEFINE1(userfaultfd, int, flags) /* prevent the mm struct to be freed */ mmgrab(ctx->mm); - fd = anon_inode_getfd("[userfaultfd]", &userfaultfd_fops, ctx, - O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS)); + fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, ctx, + O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL); if (fd < 0) { mmdrop(ctx->mm); kmem_cache_free(userfaultfd_ctx_cachep, ctx); -- cgit v1.2.3 From 365982aba1f264dba26f0908700d62bfa046918c Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Fri, 15 Jan 2021 13:03:42 +0100 Subject: fs: anon_inodes: rephrase to appropriate kernel-doc Commit e7e832ce6fa7 ("fs: add LSM-supporting anon-inode interface") adds more kerneldoc description, but also a few new warnings on anon_inode_getfd_secure() due to missing parameter descriptions. Rephrase to appropriate kernel-doc for anon_inode_getfd_secure(). Signed-off-by: Lukas Bulwahn Signed-off-by: Paul Moore --- fs/anon_inodes.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index 023337d65a03..a280156138ed 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -202,13 +202,20 @@ int anon_inode_getfd(const char *name, const struct file_operations *fops, EXPORT_SYMBOL_GPL(anon_inode_getfd); /** - * Like anon_inode_getfd(), but creates a new !S_PRIVATE anon inode rather than - * reuse the singleton anon inode, and calls the inode_init_security_anon() LSM - * hook. This allows the inode to have its own security context and for a LSM - * to reject creation of the inode. An optional @context_inode argument is - * also added to provide the logical relationship with the new inode. The LSM - * may use @context_inode in inode_init_security_anon(), but a reference to it - * is not held. + * anon_inode_getfd_secure - Like anon_inode_getfd(), but creates a new + * !S_PRIVATE anon inode rather than reuse the singleton anon inode, and calls + * the inode_init_security_anon() LSM hook. This allows the inode to have its + * own security context and for a LSM to reject creation of the inode. + * + * @name: [in] name of the "class" of the new file + * @fops: [in] file operations for the new file + * @priv: [in] private data for the new file (will be file's private_data) + * @flags: [in] flags + * @context_inode: + * [in] the logical relationship with the new inode (optional) + * + * The LSM may use @context_inode in inode_init_security_anon(), but a + * reference to it is not held. */ int anon_inode_getfd_secure(const char *name, const struct file_operations *fops, void *priv, int flags, -- cgit v1.2.3 From bca585d24a1719d9314d5438b0d2804a33d9bbb6 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 5 Jan 2021 14:13:52 -0500 Subject: new helper: d_find_alias_rcu() similar to d_find_alias(inode), except that * the caller must be holding rcu_read_lock() * inode must not be freed until matching rcu_read_unlock() * result is *NOT* pinned and can only be dereferenced until the matching rcu_read_unlock(). Signed-off-by: Al Viro --- fs/dcache.c | 25 +++++++++++++++++++++++++ include/linux/dcache.h | 2 ++ 2 files changed, 27 insertions(+) (limited to 'fs') diff --git a/fs/dcache.c b/fs/dcache.c index 97e81a844a96..843546633457 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1042,6 +1042,31 @@ struct dentry *d_find_alias(struct inode *inode) } EXPORT_SYMBOL(d_find_alias); +/* + * Caller MUST be holding rcu_read_lock() and be guaranteed + * that inode won't get freed until rcu_read_unlock(). + */ +struct dentry *d_find_alias_rcu(struct inode *inode) +{ + struct hlist_head *l = &inode->i_dentry; + struct dentry *de = NULL; + + spin_lock(&inode->i_lock); + // ->i_dentry and ->i_rcu are colocated, but the latter won't be + // used without having I_FREEING set, which means no aliases left + if (likely(!(inode->i_state & I_FREEING) && !hlist_empty(l))) { + if (S_ISDIR(inode->i_mode)) { + de = hlist_entry(l->first, struct dentry, d_u.d_alias); + } else { + hlist_for_each_entry(de, l, d_u.d_alias) + if (!d_unhashed(de)) + break; + } + } + spin_unlock(&inode->i_lock); + return de; +} + /* * Try to kill dentries associated with this inode. * WARNING: you must own a reference to inode. diff --git a/include/linux/dcache.h b/include/linux/dcache.h index d7b369fc15d3..c1e48014106f 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -262,6 +262,8 @@ extern void d_tmpfile(struct dentry *, struct inode *); extern struct dentry *d_find_alias(struct inode *); extern void d_prune_aliases(struct inode *); +extern struct dentry *d_find_alias_rcu(struct inode *); + /* test whether we have any submounts in a subdir tree */ extern int path_has_submounts(const struct path *); -- cgit v1.2.3 From 0eacdd16adc425e61926876f15b58008aa5c4784 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Thu, 8 Oct 2020 00:48:22 +0200 Subject: gfs2: Turn gfs2_rbm_incr into gfs2_rbm_add Change gfs2_rbm_incr to advance an rbm by a given number of blocks. Use that in gfs2_reservation_check_and_update to save a gfs2_rbm_to_block -> gfs2_rbm_from_block round trip. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/rgrp.c | 55 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 22 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index 5e8eef9990e3..47d171745f83 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -277,29 +277,38 @@ static int gfs2_rbm_from_block(struct gfs2_rbm *rbm, u64 block) } /** - * gfs2_rbm_incr - increment an rbm structure + * gfs2_rbm_add - add a number of blocks to an rbm * @rbm: The rbm with rgd already set correctly + * @blocks: The number of blocks to add to rpm * - * This function takes an existing rbm structure and increments it to the next - * viable block offset. - * - * Returns: If incrementing the offset would cause the rbm to go past the - * end of the rgrp, true is returned, otherwise false. + * This function takes an existing rbm structure and adds a number of blocks to + * it. * + * Returns: True if the new rbm would point past the end of the rgrp. */ -static bool gfs2_rbm_incr(struct gfs2_rbm *rbm) +static bool gfs2_rbm_add(struct gfs2_rbm *rbm, u32 blocks) { - if (rbm->offset + 1 < rbm_bi(rbm)->bi_blocks) { /* in the same bitmap */ - rbm->offset++; + struct gfs2_rgrpd *rgd = rbm->rgd; + struct gfs2_bitmap *bi = rgd->rd_bits + rbm->bii; + + if (rbm->offset + blocks < bi->bi_blocks) { + rbm->offset += blocks; return false; } - if (rbm->bii == rbm->rgd->rd_length - 1) /* at the last bitmap */ - return true; + blocks -= bi->bi_blocks - rbm->offset; - rbm->offset = 0; - rbm->bii++; - return false; + for(;;) { + bi++; + if (bi == rgd->rd_bits + rgd->rd_length) + return true; + if (blocks < bi->bi_blocks) { + rbm->offset = blocks; + rbm->bii = bi - rgd->rd_bits; + return false; + } + blocks -= bi->bi_blocks; + } } /** @@ -323,7 +332,7 @@ static bool gfs2_unaligned_extlen(struct gfs2_rbm *rbm, u32 n_unaligned, u32 *le (*len)--; if (*len == 0) return true; - if (gfs2_rbm_incr(rbm)) + if (gfs2_rbm_add(rbm, 1)) return true; } @@ -1639,7 +1648,6 @@ static int gfs2_reservation_check_and_update(struct gfs2_rbm *rbm, u64 block = gfs2_rbm_to_block(rbm); u32 extlen = 1; u64 nblock; - int ret; /* * If we have a minimum extent length, then skip over any extent @@ -1664,12 +1672,15 @@ static int gfs2_reservation_check_and_update(struct gfs2_rbm *rbm, maxext->len = extlen; maxext->rbm = *rbm; } -fail: - nblock = block + extlen; + } else { + u64 len = nblock - block; + if (len >= (u64)1 << 32) + return -E2BIG; + extlen = len; } - ret = gfs2_rbm_from_block(rbm, nblock); - if (ret < 0) - return ret; +fail: + if (gfs2_rbm_add(rbm, extlen)) + return -E2BIG; return 1; } @@ -2205,7 +2216,7 @@ static void rgblk_free(struct gfs2_sbd *sdp, struct gfs2_rgrpd *rgd, bi_prev = bi; } gfs2_setbit(&rbm, false, new_state); - gfs2_rbm_incr(&rbm); + gfs2_rbm_add(&rbm, 1); } } -- cgit v1.2.3 From c65b76b893032a9fa69056656c771f5e2fc9ef0c Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Thu, 11 Oct 2018 18:56:40 +0200 Subject: gfs2: Only use struct gfs2_rbm for bitmap manipulations GFS2 uses struct gfs2_rbm to represent a filesystem block number as a bit position within a resource group. This representation is used in the bitmap manipulation code to prevent excessive conversions between block numbers and bit positions, but also in struct gfs2_blkreserv which is part of struct gfs2_inode, to mark the start of a reservation. In the inode, the bit position representation makes less sense: first, the start position is used as a block number about as often as a bit position; second, the bit position representation makes the code unnecessarily complicated and difficult to read. Therefore, change struct gfs2_blkreserv to represent the start of a reservation as a block number instead of a bit position. (This requires keeping track of the resource group in gfs2_blkreserv separately.) With that change, various things can be slightly simplified, and struct gfs2_rbm can be moved to rgrp.c. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/bmap.c | 2 +- fs/gfs2/incore.h | 30 +--------- fs/gfs2/rgrp.c | 154 ++++++++++++++++++++++++++++++--------------------- fs/gfs2/trace_gfs2.h | 10 ++-- fs/gfs2/trans.h | 2 +- 5 files changed, 101 insertions(+), 97 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index 62d9081d1e26..e2ee3703f7df 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1544,7 +1544,7 @@ more_rgrps: /* Must be done with the rgrp glock held: */ if (gfs2_rs_active(&ip->i_res) && - rgd == ip->i_res.rs_rbm.rgd) + rgd == ip->i_res.rs_rgd) gfs2_rs_deltree(&ip->i_res); } diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index 8e1ab8ed4abc..cca806ff611c 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -125,31 +125,6 @@ struct gfs2_rgrpd { struct rb_root rd_rstree; /* multi-block reservation tree */ }; -struct gfs2_rbm { - struct gfs2_rgrpd *rgd; - u32 offset; /* The offset is bitmap relative */ - int bii; /* Bitmap index */ -}; - -static inline struct gfs2_bitmap *rbm_bi(const struct gfs2_rbm *rbm) -{ - return rbm->rgd->rd_bits + rbm->bii; -} - -static inline u64 gfs2_rbm_to_block(const struct gfs2_rbm *rbm) -{ - BUG_ON(rbm->offset >= rbm->rgd->rd_data); - return rbm->rgd->rd_data0 + (rbm_bi(rbm)->bi_start * GFS2_NBBY) + - rbm->offset; -} - -static inline bool gfs2_rbm_eq(const struct gfs2_rbm *rbm1, - const struct gfs2_rbm *rbm2) -{ - return (rbm1->rgd == rbm2->rgd) && (rbm1->bii == rbm2->bii) && - (rbm1->offset == rbm2->offset); -} - enum gfs2_state_bits { BH_Pinned = BH_PrivateStart, BH_Escaped = BH_PrivateStart + 1, @@ -313,8 +288,9 @@ struct gfs2_qadata { /* quota allocation data */ */ struct gfs2_blkreserv { - struct rb_node rs_node; /* link to other block reservations */ - struct gfs2_rbm rs_rbm; /* Start of reservation */ + struct rb_node rs_node; /* node within rd_rstree */ + struct gfs2_rgrpd *rs_rgd; + u64 rs_start; /* start of reservation */ u32 rs_free; /* how many blocks are still free */ }; diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index 47d171745f83..391186382cee 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -36,6 +36,24 @@ #define BFITNOENT ((u32)~0) #define NO_BLOCK ((u64)~0) +struct gfs2_rbm { + struct gfs2_rgrpd *rgd; + u32 offset; /* The offset is bitmap relative */ + int bii; /* Bitmap index */ +}; + +static inline struct gfs2_bitmap *rbm_bi(const struct gfs2_rbm *rbm) +{ + return rbm->rgd->rd_bits + rbm->bii; +} + +static inline u64 gfs2_rbm_to_block(const struct gfs2_rbm *rbm) +{ + BUG_ON(rbm->offset >= rbm->rgd->rd_data); + return rbm->rgd->rd_data0 + (rbm_bi(rbm)->bi_start * GFS2_NBBY) + + rbm->offset; +} + /* * These routines are used by the resource group routines (rgrp.c) * to keep track of block allocation. Each block is represented by two @@ -175,7 +193,7 @@ static inline u64 gfs2_bit_search(const __le64 *ptr, u64 mask, u8 state) /** * rs_cmp - multi-block reservation range compare - * @blk: absolute file system block number of the new reservation + * @start: start of the new reservation * @len: number of blocks in the new reservation * @rs: existing reservation to compare against * @@ -183,13 +201,11 @@ static inline u64 gfs2_bit_search(const __le64 *ptr, u64 mask, u8 state) * -1 if the block range is before the start of the reservation * 0 if the block range overlaps with the reservation */ -static inline int rs_cmp(u64 blk, u32 len, struct gfs2_blkreserv *rs) +static inline int rs_cmp(u64 start, u32 len, struct gfs2_blkreserv *rs) { - u64 startblk = gfs2_rbm_to_block(&rs->rs_rbm); - - if (blk >= startblk + rs->rs_free) + if (start >= rs->rs_start + rs->rs_free) return 1; - if (blk + len - 1 < startblk) + if (rs->rs_start >= start + len) return -1; return 0; } @@ -311,13 +327,28 @@ static bool gfs2_rbm_add(struct gfs2_rbm *rbm, u32 blocks) } } +static struct gfs2_bitmap *gfs2_block_to_bitmap(struct gfs2_rgrpd *rgd, + u64 block) +{ + unsigned int delta = (sizeof(struct gfs2_rgrp) - + sizeof(struct gfs2_meta_header)) * GFS2_NBBY; + unsigned int rblock, bii; + + if (!rgrp_contains_block(rgd, block)) + return NULL; + rblock = block - rgd->rd_data0; + bii = (rblock + delta) / rgd->rd_sbd->sd_blocks_per_bitmap; + return rgd->rd_bits + bii; +} + /** * gfs2_unaligned_extlen - Look for free blocks which are not byte aligned * @rbm: Position to search (value/result) * @n_unaligned: Number of unaligned blocks to check * @len: Decremented for each block found (terminate on zero) * - * Returns: true if a non-free block is encountered + * Returns: true if a non-free block is encountered or the end of the resource + * group is reached. */ static bool gfs2_unaligned_extlen(struct gfs2_rbm *rbm, u32 n_unaligned, u32 *len) @@ -604,10 +635,11 @@ static void dump_rs(struct seq_file *seq, const struct gfs2_blkreserv *rs, { struct gfs2_inode *ip = container_of(rs, struct gfs2_inode, i_res); - gfs2_print_dbg(seq, "%s B: n:%llu s:%llu b:%u f:%u\n", fs_id_buf, + gfs2_print_dbg(seq, "%s B: n:%llu s:%llu f:%u\n", + fs_id_buf, (unsigned long long)ip->i_no_addr, - (unsigned long long)gfs2_rbm_to_block(&rs->rs_rbm), - rs->rs_rbm.offset, rs->rs_free); + (unsigned long long)rs->rs_start, + rs->rs_free); } /** @@ -622,30 +654,28 @@ static void __rs_deltree(struct gfs2_blkreserv *rs) if (!gfs2_rs_active(rs)) return; - rgd = rs->rs_rbm.rgd; + rgd = rs->rs_rgd; trace_gfs2_rs(rs, TRACE_RS_TREEDEL); rb_erase(&rs->rs_node, &rgd->rd_rstree); RB_CLEAR_NODE(&rs->rs_node); if (rs->rs_free) { - u64 last_block = gfs2_rbm_to_block(&rs->rs_rbm) + - rs->rs_free - 1; - struct gfs2_rbm last_rbm = { .rgd = rs->rs_rbm.rgd, }; + u64 last_block = rs->rs_start + rs->rs_free - 1; struct gfs2_bitmap *start, *last; /* return reserved blocks to the rgrp */ - BUG_ON(rs->rs_rbm.rgd->rd_reserved < rs->rs_free); - rs->rs_rbm.rgd->rd_reserved -= rs->rs_free; + BUG_ON(rs->rs_rgd->rd_reserved < rs->rs_free); + rs->rs_rgd->rd_reserved -= rs->rs_free; /* The rgrp extent failure point is likely not to increase; it will only do so if the freed blocks are somehow contiguous with a span of free blocks that follows. Still, it will force the number to be recalculated later. */ rgd->rd_extfail_pt += rs->rs_free; rs->rs_free = 0; - if (gfs2_rbm_from_block(&last_rbm, last_block)) + start = gfs2_block_to_bitmap(rgd, rs->rs_start); + last = gfs2_block_to_bitmap(rgd, last_block); + if (!start || !last) return; - start = rbm_bi(&rs->rs_rbm); - last = rbm_bi(&last_rbm); do clear_bit(GBF_FULL, &start->bi_flags); while (start++ != last); @@ -661,7 +691,7 @@ void gfs2_rs_deltree(struct gfs2_blkreserv *rs) { struct gfs2_rgrpd *rgd; - rgd = rs->rs_rbm.rgd; + rgd = rs->rs_rgd; if (rgd) { spin_lock(&rgd->rd_rsspin); __rs_deltree(rs); @@ -1467,8 +1497,7 @@ static void rs_insert(struct gfs2_inode *ip) struct rb_node **newn, *parent = NULL; int rc; struct gfs2_blkreserv *rs = &ip->i_res; - struct gfs2_rgrpd *rgd = rs->rs_rbm.rgd; - u64 fsblock = gfs2_rbm_to_block(&rs->rs_rbm); + struct gfs2_rgrpd *rgd = rs->rs_rgd; BUG_ON(gfs2_rs_active(rs)); @@ -1479,7 +1508,7 @@ static void rs_insert(struct gfs2_inode *ip) rb_entry(*newn, struct gfs2_blkreserv, rs_node); parent = *newn; - rc = rs_cmp(fsblock, rs->rs_free, cur); + rc = rs_cmp(rs->rs_start, rs->rs_free, cur); if (rc > 0) newn = &((*newn)->rb_right); else if (rc < 0) @@ -1567,7 +1596,7 @@ static void rg_mblk_search(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip, ret = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &extlen, ip, true); if (ret == 0) { - rs->rs_rbm = rbm; + rs->rs_start = gfs2_rbm_to_block(&rbm); rs->rs_free = extlen; rs_insert(ip); } else { @@ -1612,7 +1641,7 @@ static u64 gfs2_next_unreserved_block(struct gfs2_rgrpd *rgd, u64 block, if (n) { while ((rs_cmp(block, length, rs) == 0) && (&ip->i_res != rs)) { - block = gfs2_rbm_to_block(&rs->rs_rbm) + rs->rs_free; + block = rs->rs_start + rs->rs_free; n = n->rb_right; if (n == NULL) break; @@ -1949,7 +1978,7 @@ static bool gfs2_rgrp_used_recently(const struct gfs2_blkreserv *rs, u64 tdiff; tdiff = ktime_to_ns(ktime_sub(ktime_get_real(), - rs->rs_rbm.rgd->rd_gl->gl_dstamp)); + rs->rs_rgd->rd_gl->gl_dstamp)); return tdiff > (msecs * 1000 * 1000); } @@ -2027,45 +2056,45 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) if (gfs2_assert_warn(sdp, ap->target)) return -EINVAL; if (gfs2_rs_active(rs)) { - begin = rs->rs_rbm.rgd; - } else if (rs->rs_rbm.rgd && - rgrp_contains_block(rs->rs_rbm.rgd, ip->i_goal)) { - begin = rs->rs_rbm.rgd; + begin = rs->rs_rgd; + } else if (rs->rs_rgd && + rgrp_contains_block(rs->rs_rgd, ip->i_goal)) { + begin = rs->rs_rgd; } else { check_and_update_goal(ip); - rs->rs_rbm.rgd = begin = gfs2_blk2rgrpd(sdp, ip->i_goal, 1); + rs->rs_rgd = begin = gfs2_blk2rgrpd(sdp, ip->i_goal, 1); } if (S_ISDIR(ip->i_inode.i_mode) && (ap->aflags & GFS2_AF_ORLOV)) skip = gfs2_orlov_skip(ip); - if (rs->rs_rbm.rgd == NULL) + if (rs->rs_rgd == NULL) return -EBADSLT; while (loops < 3) { rg_locked = 1; - if (!gfs2_glock_is_locked_by_me(rs->rs_rbm.rgd->rd_gl)) { + if (!gfs2_glock_is_locked_by_me(rs->rs_rgd->rd_gl)) { rg_locked = 0; if (skip && skip--) goto next_rgrp; if (!gfs2_rs_active(rs)) { if (loops == 0 && - !fast_to_acquire(rs->rs_rbm.rgd)) + !fast_to_acquire(rs->rs_rgd)) goto next_rgrp; if ((loops < 2) && gfs2_rgrp_used_recently(rs, 1000) && - gfs2_rgrp_congested(rs->rs_rbm.rgd, loops)) + gfs2_rgrp_congested(rs->rs_rgd, loops)) goto next_rgrp; } - error = gfs2_glock_nq_init(rs->rs_rbm.rgd->rd_gl, + error = gfs2_glock_nq_init(rs->rs_rgd->rd_gl, LM_ST_EXCLUSIVE, flags, &ip->i_rgd_gh); if (unlikely(error)) return error; if (!gfs2_rs_active(rs) && (loops < 2) && - gfs2_rgrp_congested(rs->rs_rbm.rgd, loops)) + gfs2_rgrp_congested(rs->rs_rgd, loops)) goto skip_rgrp; if (sdp->sd_args.ar_rgrplvb) { - error = update_rgrp_lvb(rs->rs_rbm.rgd); + error = update_rgrp_lvb(rs->rs_rgd); if (unlikely(error)) { gfs2_glock_dq_uninit(&ip->i_rgd_gh); return error; @@ -2074,24 +2103,24 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) } /* Skip unusable resource groups */ - if ((rs->rs_rbm.rgd->rd_flags & (GFS2_RGF_NOALLOC | + if ((rs->rs_rgd->rd_flags & (GFS2_RGF_NOALLOC | GFS2_RDF_ERROR)) || - (loops == 0 && ap->target > rs->rs_rbm.rgd->rd_extfail_pt)) + (loops == 0 && ap->target > rs->rs_rgd->rd_extfail_pt)) goto skip_rgrp; if (sdp->sd_args.ar_rgrplvb) - gfs2_rgrp_bh_get(rs->rs_rbm.rgd); + gfs2_rgrp_bh_get(rs->rs_rgd); /* Get a reservation if we don't already have one */ if (!gfs2_rs_active(rs)) - rg_mblk_search(rs->rs_rbm.rgd, ip, ap); + rg_mblk_search(rs->rs_rgd, ip, ap); /* Skip rgrps when we can't get a reservation on first pass */ if (!gfs2_rs_active(rs) && (loops < 1)) goto check_rgrp; /* If rgrp has enough free space, use it */ - free_blocks = rgd_free(rs->rs_rbm.rgd, rs); + free_blocks = rgd_free(rs->rs_rgd, rs); if (free_blocks >= ap->target || (loops == 2 && ap->min_target && free_blocks >= ap->min_target)) { @@ -2100,8 +2129,8 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) } check_rgrp: /* Check for unlinked inodes which can be reclaimed */ - if (rs->rs_rbm.rgd->rd_flags & GFS2_RDF_CHECK) - try_rgrp_unlink(rs->rs_rbm.rgd, &last_unlinked, + if (rs->rs_rgd->rd_flags & GFS2_RDF_CHECK) + try_rgrp_unlink(rs->rs_rgd, &last_unlinked, ip->i_no_addr); skip_rgrp: /* Drop reservation, if we couldn't use reserved rgrp */ @@ -2113,7 +2142,7 @@ skip_rgrp: gfs2_glock_dq_uninit(&ip->i_rgd_gh); next_rgrp: /* Find the next rgrp, and continue looking */ - if (gfs2_select_rgrp(&rs->rs_rbm.rgd, begin)) + if (gfs2_select_rgrp(&rs->rs_rgd, begin)) continue; if (skip) continue; @@ -2284,20 +2313,21 @@ static void gfs2_adjust_reservation(struct gfs2_inode *ip, { struct gfs2_blkreserv *rs = &ip->i_res; struct gfs2_rgrpd *rgd = rbm->rgd; - unsigned rlen; - u64 block; - int ret; spin_lock(&rgd->rd_rsspin); if (gfs2_rs_active(rs)) { - if (gfs2_rbm_eq(&rs->rs_rbm, rbm)) { - block = gfs2_rbm_to_block(rbm); - ret = gfs2_rbm_from_block(&rs->rs_rbm, block + len); + u64 start = gfs2_rbm_to_block(rbm); + + if (rs->rs_start == start) { + unsigned int rlen; + + rs->rs_start += len; rlen = min(rs->rs_free, len); rs->rs_free -= rlen; rgd->rd_reserved -= rlen; trace_gfs2_rs(rs, TRACE_RS_CLAIM); - if (rs->rs_free && !ret) + if (rs->rs_start < rgd->rd_data0 + rgd->rd_data && + rs->rs_free) goto out; /* We used up our block reservation, so we should reserve more blocks next time. */ @@ -2326,15 +2356,13 @@ static void gfs2_set_alloc_start(struct gfs2_rbm *rbm, u64 goal; if (gfs2_rs_active(&ip->i_res)) { - *rbm = ip->i_res.rs_rbm; - return; + goal = ip->i_res.rs_start; + } else { + if (!dinode && rgrp_contains_block(rbm->rgd, ip->i_goal)) + goal = ip->i_goal; + else + goal = rbm->rgd->rd_last_alloc + rbm->rgd->rd_data0; } - - if (!dinode && rgrp_contains_block(rbm->rgd, ip->i_goal)) - goal = ip->i_goal; - else - goal = rbm->rgd->rd_last_alloc + rbm->rgd->rd_data0; - if (WARN_ON_ONCE(gfs2_rbm_from_block(rbm, goal))) { rbm->bii = 0; rbm->offset = 0; @@ -2357,7 +2385,7 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, { struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); struct buffer_head *dibh; - struct gfs2_rbm rbm = { .rgd = ip->i_res.rs_rbm.rgd, }; + struct gfs2_rbm rbm = { .rgd = ip->i_res.rs_rgd, }; unsigned int ndata; u64 block; /* block, within the file system scope */ int error; @@ -2589,7 +2617,7 @@ void gfs2_rlist_add(struct gfs2_inode *ip, struct gfs2_rgrp_list *rlist, return; rgd = gfs2_blk2rgrpd(sdp, block, 1); } else { - rgd = ip->i_res.rs_rbm.rgd; + rgd = ip->i_res.rs_rgd; if (!rgd || !rgrp_contains_block(rgd, block)) rgd = gfs2_blk2rgrpd(sdp, block, 1); } diff --git a/fs/gfs2/trace_gfs2.h b/fs/gfs2/trace_gfs2.h index 0b2f858d9a8c..e461a691e5e8 100644 --- a/fs/gfs2/trace_gfs2.h +++ b/fs/gfs2/trace_gfs2.h @@ -603,13 +603,13 @@ TRACE_EVENT(gfs2_rs, ), TP_fast_assign( - __entry->dev = rs->rs_rbm.rgd->rd_sbd->sd_vfs->s_dev; - __entry->rd_addr = rs->rs_rbm.rgd->rd_addr; - __entry->rd_free_clone = rs->rs_rbm.rgd->rd_free_clone; - __entry->rd_reserved = rs->rs_rbm.rgd->rd_reserved; + __entry->dev = rs->rs_rgd->rd_sbd->sd_vfs->s_dev; + __entry->rd_addr = rs->rs_rgd->rd_addr; + __entry->rd_free_clone = rs->rs_rgd->rd_free_clone; + __entry->rd_reserved = rs->rs_rgd->rd_reserved; __entry->inum = container_of(rs, struct gfs2_inode, i_res)->i_no_addr; - __entry->start = gfs2_rbm_to_block(&rs->rs_rbm); + __entry->start = rs->rs_start; __entry->free = rs->rs_free; __entry->func = func; ), diff --git a/fs/gfs2/trans.h b/fs/gfs2/trans.h index 83199ce5a5c5..b812c4958dc4 100644 --- a/fs/gfs2/trans.h +++ b/fs/gfs2/trans.h @@ -27,7 +27,7 @@ struct gfs2_glock; * block, or all of the blocks in the rg, whichever is smaller */ static inline unsigned int gfs2_rg_blocks(const struct gfs2_inode *ip, unsigned requested) { - struct gfs2_rgrpd *rgd = ip->i_res.rs_rbm.rgd; + struct gfs2_rgrpd *rgd = ip->i_res.rs_rgd; if (requested < rgd->rd_length) return requested + 1; -- cgit v1.2.3 From 3ed08befeb09e199c258fc4f3878935f57a2bd9a Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Mon, 16 Mar 2020 15:36:09 +0100 Subject: gfs2: Get rid of unnecessary variable in gfs2_alloc_blocks Variable ndata is only used inside "if (!dinode)", so it can be replaced entirely with *nblocks. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/rgrp.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index 391186382cee..4f3337fd6b23 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -2386,7 +2386,6 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); struct buffer_head *dibh; struct gfs2_rbm rbm = { .rgd = ip->i_res.rs_rgd, }; - unsigned int ndata; u64 block; /* block, within the file system scope */ int error; @@ -2412,12 +2411,8 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, rbm.rgd->rd_last_alloc = block - rbm.rgd->rd_data0; if (gfs2_rs_active(&ip->i_res)) gfs2_adjust_reservation(ip, &rbm, *nblocks); - ndata = *nblocks; - if (dinode) - ndata--; - if (!dinode) { - ip->i_goal = block + ndata - 1; + ip->i_goal = block + *nblocks - 1; error = gfs2_meta_inode_buffer(ip, &dibh); if (error == 0) { struct gfs2_dinode *di = -- cgit v1.2.3 From 4272006d956f49e428f87c476959318301f2d4d2 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Fri, 20 Mar 2020 13:31:41 +0100 Subject: gfs2: Minor gfs2_inplace_reserve cleanup Clean up the reservation size computation logic in gfs2_inplace_reserve a little. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/rgrp.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index 4f3337fd6b23..f759f79d2af7 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -2048,12 +2048,13 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) struct gfs2_blkreserv *rs = &ip->i_res; int error = 0, rg_locked, flags = 0; u64 last_unlinked = NO_BLOCK; + u32 target = ap->target; int loops = 0; u32 free_blocks, skip = 0; if (sdp->sd_args.ar_rgrplvb) flags |= GL_SKIP; - if (gfs2_assert_warn(sdp, ap->target)) + if (gfs2_assert_warn(sdp, target)) return -EINVAL; if (gfs2_rs_active(rs)) { begin = rs->rs_rgd; @@ -2105,7 +2106,7 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) /* Skip unusable resource groups */ if ((rs->rs_rgd->rd_flags & (GFS2_RGF_NOALLOC | GFS2_RDF_ERROR)) || - (loops == 0 && ap->target > rs->rs_rgd->rd_extfail_pt)) + (loops == 0 && target > rs->rs_rgd->rd_extfail_pt)) goto skip_rgrp; if (sdp->sd_args.ar_rgrplvb) @@ -2121,9 +2122,7 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) /* If rgrp has enough free space, use it */ free_blocks = rgd_free(rs->rs_rgd, rs); - if (free_blocks >= ap->target || - (loops == 2 && ap->min_target && - free_blocks >= ap->min_target)) { + if (free_blocks >= target) { ap->allowed = free_blocks; return 0; } @@ -2159,9 +2158,12 @@ next_rgrp: return error; } /* Flushing the log may release space */ - if (loops == 2) + if (loops == 2) { + if (ap->min_target) + target = ap->min_target; gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_INPLACE_RESERVE); + } } return -ENOSPC; -- cgit v1.2.3 From 2fdc2fa21bc72ec06c0c9f0e30b88fe1f2486b75 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Tue, 6 Oct 2020 14:29:04 +0200 Subject: Revert "gfs2: Don't reject a supposedly full bitmap if we have blocks reserved" This reverts commit e79e0e1428188b24c3b57309ffa54a33c4ae40c4. It turns out that we're only setting the GBF_FULL flag of a bitmap if we've been scanning from the beginning of the bitmap until the end and we haven't found a single free block, and we're not skipping reservations in that process, either. This means that in gfs2_rbm_find, we can always skip bitmaps with the GBF_FULL flag set. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/rgrp.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index f759f79d2af7..faf9f21f3e98 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -1754,8 +1754,7 @@ static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext, while(1) { bi = rbm_bi(rbm); - if ((ip == NULL || !gfs2_rs_active(&ip->i_res)) && - test_bit(GBF_FULL, &bi->bi_flags) && + if (test_bit(GBF_FULL, &bi->bi_flags) && (state == GFS2_BLKST_FREE)) goto next_bitmap; -- cgit v1.2.3 From a12c6fa1de8472f0eb56f7e5a60de2ffa4f6c043 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Tue, 6 Oct 2020 15:16:16 +0200 Subject: gfs2: Don't clear GBF_FULL flags in rs_deltree Removing a reservation doesn't make any actual space available, so don't clear the GBF_FULL flags in that case. Otherwise, we'll only spend more time scanning the bitmaps unnecessarily. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/rgrp.c | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index faf9f21f3e98..f585a13cebde 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -327,20 +327,6 @@ static bool gfs2_rbm_add(struct gfs2_rbm *rbm, u32 blocks) } } -static struct gfs2_bitmap *gfs2_block_to_bitmap(struct gfs2_rgrpd *rgd, - u64 block) -{ - unsigned int delta = (sizeof(struct gfs2_rgrp) - - sizeof(struct gfs2_meta_header)) * GFS2_NBBY; - unsigned int rblock, bii; - - if (!rgrp_contains_block(rgd, block)) - return NULL; - rblock = block - rgd->rd_data0; - bii = (rblock + delta) / rgd->rd_sbd->sd_blocks_per_bitmap; - return rgd->rd_bits + bii; -} - /** * gfs2_unaligned_extlen - Look for free blocks which are not byte aligned * @rbm: Position to search (value/result) @@ -660,25 +646,16 @@ static void __rs_deltree(struct gfs2_blkreserv *rs) RB_CLEAR_NODE(&rs->rs_node); if (rs->rs_free) { - u64 last_block = rs->rs_start + rs->rs_free - 1; - struct gfs2_bitmap *start, *last; - /* return reserved blocks to the rgrp */ BUG_ON(rs->rs_rgd->rd_reserved < rs->rs_free); rs->rs_rgd->rd_reserved -= rs->rs_free; + /* The rgrp extent failure point is likely not to increase; it will only do so if the freed blocks are somehow contiguous with a span of free blocks that follows. Still, it will force the number to be recalculated later. */ rgd->rd_extfail_pt += rs->rs_free; rs->rs_free = 0; - start = gfs2_block_to_bitmap(rgd, rs->rs_start); - last = gfs2_block_to_bitmap(rgd, last_block); - if (!start || !last) - return; - do - clear_bit(GBF_FULL, &start->bi_flags); - while (start++ != last); } } -- cgit v1.2.3 From 560b8eba6bf685bb2a29fd24e7b3455e7b8f795a Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Tue, 6 Oct 2020 15:26:22 +0200 Subject: gfs2: Set GBF_FULL flags when reading resource group When reading a resource group from disk or when receiving the resource group statistics from a Lock Value Block (LVB), set/clear the GBF_FULL flags of all bitmaps in that resource group according to whether or not the resource group is full. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/rgrp.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index f585a13cebde..f878d82750f7 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -1165,6 +1165,23 @@ static u32 count_unlinked(struct gfs2_rgrpd *rgd) return count; } +static void rgrp_set_bitmap_flags(struct gfs2_rgrpd *rgd) +{ + struct gfs2_bitmap *bi; + int x; + + if (rgd->rd_free) { + for (x = 0; x < rgd->rd_length; x++) { + bi = rgd->rd_bits + x; + clear_bit(GBF_FULL, &bi->bi_flags); + } + } else { + for (x = 0; x < rgd->rd_length; x++) { + bi = rgd->rd_bits + x; + set_bit(GBF_FULL, &bi->bi_flags); + } + } +} /** * gfs2_rgrp_bh_get - Read in a RG's header and bitmaps @@ -1208,9 +1225,8 @@ static int gfs2_rgrp_bh_get(struct gfs2_rgrpd *rgd) } if (!(rgd->rd_flags & GFS2_RDF_UPTODATE)) { - for (x = 0; x < length; x++) - clear_bit(GBF_FULL, &rgd->rd_bits[x].bi_flags); gfs2_rgrp_in(rgd, (rgd->rd_bits[0].bi_bh)->b_data); + rgrp_set_bitmap_flags(rgd); rgd->rd_flags |= (GFS2_RDF_UPTODATE | GFS2_RDF_CHECK); rgd->rd_free_clone = rgd->rd_free; /* max out the rgrp allocation failure point */ @@ -1260,6 +1276,7 @@ static int update_rgrp_lvb(struct gfs2_rgrpd *rgd) if (rgd->rd_rgl->rl_unlinked == 0) rgd->rd_flags &= ~GFS2_RDF_CHECK; rgd->rd_free = be32_to_cpu(rgd->rd_rgl->rl_free); + rgrp_set_bitmap_flags(rgd); rgd->rd_free_clone = rgd->rd_free; rgd->rd_dinodes = be32_to_cpu(rgd->rd_rgl->rl_dinodes); rgd->rd_igeneration = be64_to_cpu(rgd->rd_rgl->rl_igeneration); -- cgit v1.2.3 From 736b2f778f0a71ce54abe7e6d2c6f7591b95f7b8 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Mon, 7 Dec 2020 00:06:32 +0100 Subject: gfs2: Un-obfuscate function jdesc_find_i Clean up this function to show that it is trivial. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/super.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index 2f56acc41c04..ed7a829e9ffe 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -81,19 +81,12 @@ void gfs2_jindex_free(struct gfs2_sbd *sdp) static struct gfs2_jdesc *jdesc_find_i(struct list_head *head, unsigned int jid) { struct gfs2_jdesc *jd; - int found = 0; list_for_each_entry(jd, head, jd_list) { - if (jd->jd_jid == jid) { - found = 1; - break; - } + if (jd->jd_jid == jid) + return jd; } - - if (!found) - jd = NULL; - - return jd; + return NULL; } struct gfs2_jdesc *gfs2_jdesc_find(struct gfs2_sbd *sdp, unsigned int jid) -- cgit v1.2.3 From 458094c2c6115cda721fd11a56f7e4f27cddd5b1 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sat, 5 Dec 2020 13:21:50 +0100 Subject: gfs2: Simplify the buf_limit and databuf_limit definitions The BUF_OFFSET and DATABUF_OFFSET definitions are only used in buf_limit and databuf_limit, respectively, and the rounding done in those definitions is immediately wiped out by dividing by the element size. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/lops.h | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/lops.h b/fs/gfs2/lops.h index fbdbb08dcec6..3fca5bf239d3 100644 --- a/fs/gfs2/lops.h +++ b/fs/gfs2/lops.h @@ -10,13 +10,6 @@ #include #include "incore.h" -#define BUF_OFFSET \ - ((sizeof(struct gfs2_log_descriptor) + sizeof(__be64) - 1) & \ - ~(sizeof(__be64) - 1)) -#define DATABUF_OFFSET \ - ((sizeof(struct gfs2_log_descriptor) + (2 * sizeof(__be64) - 1)) & \ - ~(2 * sizeof(__be64) - 1)) - extern const struct gfs2_log_operations *gfs2_log_ops[]; extern void gfs2_log_incr_head(struct gfs2_sbd *sdp); extern u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lbn); @@ -29,18 +22,12 @@ extern int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head, bool keep_cache); static inline unsigned int buf_limit(struct gfs2_sbd *sdp) { - unsigned int limit; - - limit = (sdp->sd_sb.sb_bsize - BUF_OFFSET) / sizeof(__be64); - return limit; + return sdp->sd_ldptrs; } static inline unsigned int databuf_limit(struct gfs2_sbd *sdp) { - unsigned int limit; - - limit = (sdp->sd_sb.sb_bsize - DATABUF_OFFSET) / (2 * sizeof(__be64)); - return limit; + return sdp->sd_ldptrs / 2; } static inline void lops_before_commit(struct gfs2_sbd *sdp, -- cgit v1.2.3 From 5a4e9c607e03886815121fa975fcd8e0fa7252fe Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sun, 6 Dec 2020 13:32:54 +0100 Subject: gfs2: Minor gfs2_write_revokes cleanups Clean up the computations in gfs2_write_revokes (no change in functionality). Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 2e9314091c81..ccce17fe605f 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -712,11 +712,13 @@ void gfs2_glock_remove_revoke(struct gfs2_glock *gl) void gfs2_write_revokes(struct gfs2_sbd *sdp) { /* number of revokes we still have room for */ - int max_revokes = (sdp->sd_sb.sb_bsize - sizeof(struct gfs2_log_descriptor)) / sizeof(u64); + unsigned int max_revokes; gfs2_log_lock(sdp); - while (sdp->sd_log_num_revoke > max_revokes) - max_revokes += (sdp->sd_sb.sb_bsize - sizeof(struct gfs2_meta_header)) / sizeof(u64); + max_revokes = sdp->sd_ldptrs; + if (sdp->sd_log_num_revoke > sdp->sd_ldptrs) + max_revokes += roundup(sdp->sd_log_num_revoke - sdp->sd_ldptrs, + sdp->sd_inptrs); max_revokes -= sdp->sd_log_num_revoke; if (!sdp->sd_log_num_revoke) { atomic_dec(&sdp->sd_log_blks_free); -- cgit v1.2.3 From 6188e8777de48c8404b0bcca74a70ac83c37d773 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sun, 6 Dec 2020 13:30:14 +0100 Subject: gfs2: Some documentation updates The calc_reserved description claims that buf_limit is 502 (on 4k filesystems), but it is actually 503. Fix / clarify the entire description. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index ccce17fe605f..12e8280f0806 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -50,10 +50,12 @@ unsigned int gfs2_struct2blk(struct gfs2_sbd *sdp, unsigned int nstruct) unsigned int blks; unsigned int first, second; + /* The initial struct gfs2_log_descriptor block */ blks = 1; first = sdp->sd_ldptrs; if (nstruct > first) { + /* Subsequent struct gfs2_meta_header blocks */ second = sdp->sd_inptrs; blks += DIV_ROUND_UP(nstruct - first, second); } @@ -507,24 +509,20 @@ static inline unsigned int log_distance(struct gfs2_sbd *sdp, unsigned int newer } /** - * calc_reserved - Calculate the number of blocks to reserve when - * refunding a transaction's unused buffers. + * calc_reserved - Calculate the number of blocks to keep reserved * @sdp: The GFS2 superblock * * This is complex. We need to reserve room for all our currently used - * metadata buffers (e.g. normal file I/O rewriting file time stamps) and - * all our journaled data buffers for journaled files (e.g. files in the + * metadata blocks (e.g. normal file I/O rewriting file time stamps) and + * all our journaled data blocks for journaled files (e.g. files in the * meta_fs like rindex, or files for which chattr +j was done.) - * If we don't reserve enough space, gfs2_log_refund and gfs2_log_flush - * will count it as free space (sd_log_blks_free) and corruption will follow. + * If we don't reserve enough space, corruption will follow. * - * We can have metadata bufs and jdata bufs in the same journal. So each - * type gets its own log header, for which we need to reserve a block. - * In fact, each type has the potential for needing more than one header - * in cases where we have more buffers than will fit on a journal page. + * We can have metadata blocks and jdata blocks in the same journal. Each + * type gets its own log descriptor, for which we need to reserve a block. + * In fact, each type has the potential for needing more than one log descriptor + * in cases where we have more blocks than will fit in a log descriptor. * Metadata journal entries take up half the space of journaled buffer entries. - * Thus, metadata entries have buf_limit (502) and journaled buffers have - * databuf_limit (251) before they cause a wrap around. * * Also, we need to reserve blocks for revoke journal entries and one for an * overall header for the lot. @@ -1008,7 +1006,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) if (sdp->sd_log_head != sdp->sd_log_flush_head) { log_flush_wait(sdp); log_write_header(sdp, flags); - } else if (sdp->sd_log_tail != current_tail(sdp) && !sdp->sd_log_idle){ + } else if (sdp->sd_log_tail != current_tail(sdp) && !sdp->sd_log_idle) { atomic_dec(&sdp->sd_log_blks_free); /* Adjust for unreserved buffer */ trace_gfs2_log_blocks(sdp, -1); log_write_header(sdp, flags); -- cgit v1.2.3 From 625a8edd5e00e7c4c70a125a433ec7598d9f0c27 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sun, 6 Dec 2020 20:10:51 +0100 Subject: gfs2: Minor debugging improvement Split the assert in gfs2_trans_end into two parts. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/trans.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index 6d4bf7ea7b3b..7705f04621f4 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -109,8 +109,8 @@ void gfs2_trans_end(struct gfs2_sbd *sdp) nbuf -= tr->tr_num_buf_rm; nbuf -= tr->tr_num_databuf_rm; - if (gfs2_assert_withdraw(sdp, (nbuf <= tr->tr_blocks) && - (tr->tr_num_revoke <= tr->tr_revokes))) + if (gfs2_assert_withdraw(sdp, nbuf <= tr->tr_blocks) || + gfs2_assert_withdraw(sdp, tr->tr_num_revoke <= tr->tr_revokes)) gfs2_print_trans(sdp, tr); gfs2_log_commit(sdp, tr); -- cgit v1.2.3 From e7501bf88cd77ed3a1bc65c451600a847c80485b Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sat, 19 Dec 2020 03:11:51 +0100 Subject: gfs2: Rename gfs2_{write => flush}_revokes Function gfs2_write_revokes doesn't actually write any revokes; instead, it adds revokes to the system transaction during a flush. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 4 ++-- fs/gfs2/log.h | 2 +- fs/gfs2/lops.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 12e8280f0806..7375c007bde5 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -696,7 +696,7 @@ void gfs2_glock_remove_revoke(struct gfs2_glock *gl) } /** - * gfs2_write_revokes - Add as many revokes to the system transaction as we can + * gfs2_flush_revokes - Add as many revokes to the system transaction as we can * @sdp: The GFS2 superblock * * Our usual strategy is to defer writing revokes as much as we can in the hope @@ -707,7 +707,7 @@ void gfs2_glock_remove_revoke(struct gfs2_glock *gl) * been written back. This will basically come at no cost now, and will save * us from having to keep track of those blocks on the AIL2 list later. */ -void gfs2_write_revokes(struct gfs2_sbd *sdp) +void gfs2_flush_revokes(struct gfs2_sbd *sdp) { /* number of revokes we still have room for */ unsigned int max_revokes; diff --git a/fs/gfs2/log.h b/fs/gfs2/log.h index 79f97290146e..a9cdbc990edf 100644 --- a/fs/gfs2/log.h +++ b/fs/gfs2/log.h @@ -78,6 +78,6 @@ extern void log_flush_wait(struct gfs2_sbd *sdp); extern int gfs2_logd(void *data); extern void gfs2_add_revoke(struct gfs2_sbd *sdp, struct gfs2_bufdata *bd); extern void gfs2_glock_remove_revoke(struct gfs2_glock *gl); -extern void gfs2_write_revokes(struct gfs2_sbd *sdp); +extern void gfs2_flush_revokes(struct gfs2_sbd *sdp); #endif /* __LOG_DOT_H__ */ diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index 3922b26264f5..8658ebbcb4a9 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c @@ -845,7 +845,7 @@ static void revoke_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr) struct page *page; unsigned int length; - gfs2_write_revokes(sdp); + gfs2_flush_revokes(sdp); if (!sdp->sd_log_num_revoke) return; -- cgit v1.2.3 From 6e80674af06c0dca6e0153d492d437f3f07fe3e9 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Fri, 11 Dec 2020 14:45:23 +0100 Subject: gfs2: Clean up ail2_empty Clean up the logic in ail2_empty (no functional change). Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 7375c007bde5..e4dc23a24569 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -367,29 +367,33 @@ static void gfs2_ail_empty_tr(struct gfs2_sbd *sdp, struct gfs2_trans *tr, } } +static void __ail2_empty(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + gfs2_ail_empty_tr(sdp, tr, &tr->tr_ail2_list); + list_del(&tr->tr_list); + gfs2_assert_warn(sdp, list_empty(&tr->tr_ail1_list)); + gfs2_assert_warn(sdp, list_empty(&tr->tr_ail2_list)); + gfs2_trans_free(sdp, tr); +} + static void ail2_empty(struct gfs2_sbd *sdp, unsigned int new_tail) { - struct gfs2_trans *tr, *safe; + struct list_head *ail2_list = &sdp->sd_ail2_list; unsigned int old_tail = sdp->sd_log_tail; - int wrap = (new_tail < old_tail); - int a, b, rm; + struct gfs2_trans *tr, *safe; spin_lock(&sdp->sd_ail_lock); - - list_for_each_entry_safe(tr, safe, &sdp->sd_ail2_list, tr_list) { - a = (old_tail <= tr->tr_first); - b = (tr->tr_first < new_tail); - rm = (wrap) ? (a || b) : (a && b); - if (!rm) - continue; - - gfs2_ail_empty_tr(sdp, tr, &tr->tr_ail2_list); - list_del(&tr->tr_list); - gfs2_assert_warn(sdp, list_empty(&tr->tr_ail1_list)); - gfs2_assert_warn(sdp, list_empty(&tr->tr_ail2_list)); - gfs2_trans_free(sdp, tr); + if (old_tail <= new_tail) { + list_for_each_entry_safe(tr, safe, ail2_list, tr_list) { + if (old_tail <= tr->tr_first && tr->tr_first < new_tail) + __ail2_empty(sdp, tr); + } + } else { + list_for_each_entry_safe(tr, safe, ail2_list, tr_list) { + if (old_tail <= tr->tr_first || tr->tr_first < new_tail) + __ail2_empty(sdp, tr); + } } - spin_unlock(&sdp->sd_ail_lock); } -- cgit v1.2.3 From a579fcfa8e49cc77ad59211bb18bc5004133e6a0 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 18 Jan 2021 12:45:46 +0100 Subject: c6x: remove architecture The c6x architecture was added to the kernel in 2011 at a time when running Linux on DSPs was widely seen as the logical evolution. It appears the trend has gone back to running Linux on Arm based SoCs with DSP, using a better supported software ecosystem, and having better real-time behavior for the DSP code. An example of this is TI's own Keystone2 platform. The upstream kernel port appears to no longer have any users. Mark Salter remained avaialable to review patches, but mentioned that he no longer has access to working hardware himself. Without any users, it's best to just remove the code completely to reduce the work for cross-architecture code changes. Many thanks to Mark for maintaining the code for the past ten years. Link: https://lore.kernel.org/lkml/41dc7795afda9f776d8cd0d3075f776cf586e97c.camel@redhat.com/ Signed-off-by: Arnd Bergmann --- Documentation/devicetree/bindings/c6x/clocks.txt | 40 -- Documentation/devicetree/bindings/c6x/dscr.txt | 127 ---- Documentation/devicetree/bindings/c6x/emifa.txt | 62 -- Documentation/devicetree/bindings/c6x/soc.txt | 28 - .../interrupt-controller/ti,c64x+megamod-pic.txt | 103 --- .../devicetree/bindings/timer/ti,c64x+timer64.txt | 25 - MAINTAINERS | 8 - arch/c6x/Kconfig | 113 ---- arch/c6x/Kconfig.debug | 10 - arch/c6x/Makefile | 60 -- arch/c6x/boot/Makefile | 11 - arch/c6x/boot/dts/Makefile | 16 - arch/c6x/boot/dts/dsk6455.dts | 57 -- arch/c6x/boot/dts/evmc6457.dts | 43 -- arch/c6x/boot/dts/evmc6472.dts | 68 -- arch/c6x/boot/dts/evmc6474.dts | 53 -- arch/c6x/boot/dts/evmc6678.dts | 78 --- arch/c6x/boot/dts/tms320c6455.dtsi | 97 --- arch/c6x/boot/dts/tms320c6457.dtsi | 69 -- arch/c6x/boot/dts/tms320c6472.dtsi | 135 ---- arch/c6x/boot/dts/tms320c6474.dtsi | 90 --- arch/c6x/boot/dts/tms320c6678.dtsi | 147 ---- arch/c6x/configs/dsk6455_defconfig | 42 -- arch/c6x/configs/evmc6457_defconfig | 39 -- arch/c6x/configs/evmc6472_defconfig | 40 -- arch/c6x/configs/evmc6474_defconfig | 40 -- arch/c6x/configs/evmc6678_defconfig | 40 -- arch/c6x/include/asm/Kbuild | 5 - arch/c6x/include/asm/asm-offsets.h | 1 - arch/c6x/include/asm/bitops.h | 95 --- arch/c6x/include/asm/bug.h | 20 - arch/c6x/include/asm/cache.h | 94 --- arch/c6x/include/asm/cacheflush.h | 45 -- arch/c6x/include/asm/checksum.h | 34 - arch/c6x/include/asm/clock.h | 145 ---- arch/c6x/include/asm/cmpxchg.h | 63 -- arch/c6x/include/asm/delay.h | 64 -- arch/c6x/include/asm/dscr.h | 30 - arch/c6x/include/asm/elf.h | 117 ---- arch/c6x/include/asm/flat.h | 19 - arch/c6x/include/asm/ftrace.h | 6 - arch/c6x/include/asm/hardirq.h | 17 - arch/c6x/include/asm/irq.h | 50 -- arch/c6x/include/asm/irqflags.h | 68 -- arch/c6x/include/asm/linkage.h | 31 - arch/c6x/include/asm/megamod-pic.h | 10 - arch/c6x/include/asm/mmu_context.h | 6 - arch/c6x/include/asm/module.h | 20 - arch/c6x/include/asm/page.h | 9 - arch/c6x/include/asm/pgtable.h | 66 -- arch/c6x/include/asm/processor.h | 114 ---- arch/c6x/include/asm/procinfo.h | 24 - arch/c6x/include/asm/ptrace.h | 32 - arch/c6x/include/asm/sections.h | 12 - arch/c6x/include/asm/setup.h | 31 - arch/c6x/include/asm/soc.h | 35 - arch/c6x/include/asm/special_insns.h | 60 -- arch/c6x/include/asm/string.h | 18 - arch/c6x/include/asm/switch_to.h | 30 - arch/c6x/include/asm/syscall.h | 75 --- arch/c6x/include/asm/syscalls.h | 46 -- arch/c6x/include/asm/thread_info.h | 94 --- arch/c6x/include/asm/timer64.h | 7 - arch/c6x/include/asm/timex.h | 30 - arch/c6x/include/asm/tlb.h | 7 - arch/c6x/include/asm/traps.h | 33 - arch/c6x/include/asm/uaccess.h | 97 --- arch/c6x/include/asm/unaligned.h | 104 --- arch/c6x/include/asm/vmalloc.h | 4 - arch/c6x/include/uapi/asm/Kbuild | 2 - arch/c6x/include/uapi/asm/byteorder.h | 13 - arch/c6x/include/uapi/asm/ptrace.h | 164 ----- arch/c6x/include/uapi/asm/setup.h | 7 - arch/c6x/include/uapi/asm/sigcontext.h | 81 --- arch/c6x/include/uapi/asm/swab.h | 55 -- arch/c6x/include/uapi/asm/unistd.h | 29 - arch/c6x/kernel/Makefile | 13 - arch/c6x/kernel/asm-offsets.c | 123 ---- arch/c6x/kernel/c6x_ksyms.c | 62 -- arch/c6x/kernel/devicetree.c | 14 - arch/c6x/kernel/entry.S | 736 --------------------- arch/c6x/kernel/head.S | 81 --- arch/c6x/kernel/irq.c | 127 ---- arch/c6x/kernel/module.c | 119 ---- arch/c6x/kernel/process.c | 151 ----- arch/c6x/kernel/ptrace.c | 139 ---- arch/c6x/kernel/setup.c | 476 ------------- arch/c6x/kernel/signal.c | 322 --------- arch/c6x/kernel/soc.c | 87 --- arch/c6x/kernel/switch_to.S | 71 -- arch/c6x/kernel/sys_c6x.c | 71 -- arch/c6x/kernel/time.c | 63 -- arch/c6x/kernel/traps.c | 409 ------------ arch/c6x/kernel/vectors.S | 78 --- arch/c6x/kernel/vmlinux.lds.S | 151 ----- arch/c6x/lib/Makefile | 8 - arch/c6x/lib/checksum.c | 11 - arch/c6x/lib/csum_64plus.S | 414 ------------ arch/c6x/lib/divi.S | 41 -- arch/c6x/lib/divremi.S | 34 - arch/c6x/lib/divremu.S | 75 --- arch/c6x/lib/divu.S | 86 --- arch/c6x/lib/llshl.S | 25 - arch/c6x/lib/llshr.S | 26 - arch/c6x/lib/llshru.S | 26 - arch/c6x/lib/memcpy_64plus.S | 43 -- arch/c6x/lib/mpyll.S | 37 -- arch/c6x/lib/negll.S | 19 - arch/c6x/lib/pop_rts.S | 20 - arch/c6x/lib/push_rts.S | 19 - arch/c6x/lib/remi.S | 52 -- arch/c6x/lib/remu.S | 70 -- arch/c6x/lib/strasgi.S | 77 --- arch/c6x/lib/strasgi_64plus.S | 27 - arch/c6x/mm/Makefile | 6 - arch/c6x/mm/dma-coherent.c | 173 ----- arch/c6x/mm/init.c | 65 -- arch/c6x/platforms/Kconfig | 21 - arch/c6x/platforms/Makefile | 13 - arch/c6x/platforms/cache.c | 444 ------------- arch/c6x/platforms/dscr.c | 595 ----------------- arch/c6x/platforms/emif.c | 84 --- arch/c6x/platforms/megamod-pic.c | 344 ---------- arch/c6x/platforms/pll.c | 440 ------------ arch/c6x/platforms/plldata.c | 467 ------------- arch/c6x/platforms/timer64.c | 241 ------- drivers/bus/Kconfig | 2 +- fs/Kconfig.binfmt | 2 +- include/asm-generic/page.h | 4 - 129 files changed, 2 insertions(+), 11162 deletions(-) delete mode 100644 Documentation/devicetree/bindings/c6x/clocks.txt delete mode 100644 Documentation/devicetree/bindings/c6x/dscr.txt delete mode 100644 Documentation/devicetree/bindings/c6x/emifa.txt delete mode 100644 Documentation/devicetree/bindings/c6x/soc.txt delete mode 100644 Documentation/devicetree/bindings/interrupt-controller/ti,c64x+megamod-pic.txt delete mode 100644 Documentation/devicetree/bindings/timer/ti,c64x+timer64.txt delete mode 100644 arch/c6x/Kconfig delete mode 100644 arch/c6x/Kconfig.debug delete mode 100644 arch/c6x/Makefile delete mode 100644 arch/c6x/boot/Makefile delete mode 100644 arch/c6x/boot/dts/Makefile delete mode 100644 arch/c6x/boot/dts/dsk6455.dts delete mode 100644 arch/c6x/boot/dts/evmc6457.dts delete mode 100644 arch/c6x/boot/dts/evmc6472.dts delete mode 100644 arch/c6x/boot/dts/evmc6474.dts delete mode 100644 arch/c6x/boot/dts/evmc6678.dts delete mode 100644 arch/c6x/boot/dts/tms320c6455.dtsi delete mode 100644 arch/c6x/boot/dts/tms320c6457.dtsi delete mode 100644 arch/c6x/boot/dts/tms320c6472.dtsi delete mode 100644 arch/c6x/boot/dts/tms320c6474.dtsi delete mode 100644 arch/c6x/boot/dts/tms320c6678.dtsi delete mode 100644 arch/c6x/configs/dsk6455_defconfig delete mode 100644 arch/c6x/configs/evmc6457_defconfig delete mode 100644 arch/c6x/configs/evmc6472_defconfig delete mode 100644 arch/c6x/configs/evmc6474_defconfig delete mode 100644 arch/c6x/configs/evmc6678_defconfig delete mode 100644 arch/c6x/include/asm/Kbuild delete mode 100644 arch/c6x/include/asm/asm-offsets.h delete mode 100644 arch/c6x/include/asm/bitops.h delete mode 100644 arch/c6x/include/asm/bug.h delete mode 100644 arch/c6x/include/asm/cache.h delete mode 100644 arch/c6x/include/asm/cacheflush.h delete mode 100644 arch/c6x/include/asm/checksum.h delete mode 100644 arch/c6x/include/asm/clock.h delete mode 100644 arch/c6x/include/asm/cmpxchg.h delete mode 100644 arch/c6x/include/asm/delay.h delete mode 100644 arch/c6x/include/asm/dscr.h delete mode 100644 arch/c6x/include/asm/elf.h delete mode 100644 arch/c6x/include/asm/flat.h delete mode 100644 arch/c6x/include/asm/ftrace.h delete mode 100644 arch/c6x/include/asm/hardirq.h delete mode 100644 arch/c6x/include/asm/irq.h delete mode 100644 arch/c6x/include/asm/irqflags.h delete mode 100644 arch/c6x/include/asm/linkage.h delete mode 100644 arch/c6x/include/asm/megamod-pic.h delete mode 100644 arch/c6x/include/asm/mmu_context.h delete mode 100644 arch/c6x/include/asm/module.h delete mode 100644 arch/c6x/include/asm/page.h delete mode 100644 arch/c6x/include/asm/pgtable.h delete mode 100644 arch/c6x/include/asm/processor.h delete mode 100644 arch/c6x/include/asm/procinfo.h delete mode 100644 arch/c6x/include/asm/ptrace.h delete mode 100644 arch/c6x/include/asm/sections.h delete mode 100644 arch/c6x/include/asm/setup.h delete mode 100644 arch/c6x/include/asm/soc.h delete mode 100644 arch/c6x/include/asm/special_insns.h delete mode 100644 arch/c6x/include/asm/string.h delete mode 100644 arch/c6x/include/asm/switch_to.h delete mode 100644 arch/c6x/include/asm/syscall.h delete mode 100644 arch/c6x/include/asm/syscalls.h delete mode 100644 arch/c6x/include/asm/thread_info.h delete mode 100644 arch/c6x/include/asm/timer64.h delete mode 100644 arch/c6x/include/asm/timex.h delete mode 100644 arch/c6x/include/asm/tlb.h delete mode 100644 arch/c6x/include/asm/traps.h delete mode 100644 arch/c6x/include/asm/uaccess.h delete mode 100644 arch/c6x/include/asm/unaligned.h delete mode 100644 arch/c6x/include/asm/vmalloc.h delete mode 100644 arch/c6x/include/uapi/asm/Kbuild delete mode 100644 arch/c6x/include/uapi/asm/byteorder.h delete mode 100644 arch/c6x/include/uapi/asm/ptrace.h delete mode 100644 arch/c6x/include/uapi/asm/setup.h delete mode 100644 arch/c6x/include/uapi/asm/sigcontext.h delete mode 100644 arch/c6x/include/uapi/asm/swab.h delete mode 100644 arch/c6x/include/uapi/asm/unistd.h delete mode 100644 arch/c6x/kernel/Makefile delete mode 100644 arch/c6x/kernel/asm-offsets.c delete mode 100644 arch/c6x/kernel/c6x_ksyms.c delete mode 100644 arch/c6x/kernel/devicetree.c delete mode 100644 arch/c6x/kernel/entry.S delete mode 100644 arch/c6x/kernel/head.S delete mode 100644 arch/c6x/kernel/irq.c delete mode 100644 arch/c6x/kernel/module.c delete mode 100644 arch/c6x/kernel/process.c delete mode 100644 arch/c6x/kernel/ptrace.c delete mode 100644 arch/c6x/kernel/setup.c delete mode 100644 arch/c6x/kernel/signal.c delete mode 100644 arch/c6x/kernel/soc.c delete mode 100644 arch/c6x/kernel/switch_to.S delete mode 100644 arch/c6x/kernel/sys_c6x.c delete mode 100644 arch/c6x/kernel/time.c delete mode 100644 arch/c6x/kernel/traps.c delete mode 100644 arch/c6x/kernel/vectors.S delete mode 100644 arch/c6x/kernel/vmlinux.lds.S delete mode 100644 arch/c6x/lib/Makefile delete mode 100644 arch/c6x/lib/checksum.c delete mode 100644 arch/c6x/lib/csum_64plus.S delete mode 100644 arch/c6x/lib/divi.S delete mode 100644 arch/c6x/lib/divremi.S delete mode 100644 arch/c6x/lib/divremu.S delete mode 100644 arch/c6x/lib/divu.S delete mode 100644 arch/c6x/lib/llshl.S delete mode 100644 arch/c6x/lib/llshr.S delete mode 100644 arch/c6x/lib/llshru.S delete mode 100644 arch/c6x/lib/memcpy_64plus.S delete mode 100644 arch/c6x/lib/mpyll.S delete mode 100644 arch/c6x/lib/negll.S delete mode 100644 arch/c6x/lib/pop_rts.S delete mode 100644 arch/c6x/lib/push_rts.S delete mode 100644 arch/c6x/lib/remi.S delete mode 100644 arch/c6x/lib/remu.S delete mode 100644 arch/c6x/lib/strasgi.S delete mode 100644 arch/c6x/lib/strasgi_64plus.S delete mode 100644 arch/c6x/mm/Makefile delete mode 100644 arch/c6x/mm/dma-coherent.c delete mode 100644 arch/c6x/mm/init.c delete mode 100644 arch/c6x/platforms/Kconfig delete mode 100644 arch/c6x/platforms/Makefile delete mode 100644 arch/c6x/platforms/cache.c delete mode 100644 arch/c6x/platforms/dscr.c delete mode 100644 arch/c6x/platforms/emif.c delete mode 100644 arch/c6x/platforms/megamod-pic.c delete mode 100644 arch/c6x/platforms/pll.c delete mode 100644 arch/c6x/platforms/plldata.c delete mode 100644 arch/c6x/platforms/timer64.c (limited to 'fs') diff --git a/Documentation/devicetree/bindings/c6x/clocks.txt b/Documentation/devicetree/bindings/c6x/clocks.txt deleted file mode 100644 index a04f5fd30122..000000000000 --- a/Documentation/devicetree/bindings/c6x/clocks.txt +++ /dev/null @@ -1,40 +0,0 @@ -C6X PLL Clock Controllers -------------------------- - -This is a first-cut support for the SoC clock controllers. This is still -under development and will probably change as the common device tree -clock support is added to the kernel. - -Required properties: - -- compatible: "ti,c64x+pll" - May also have SoC-specific value to support SoC-specific initialization - in the driver. One of: - "ti,c6455-pll" - "ti,c6457-pll" - "ti,c6472-pll" - "ti,c6474-pll" - -- reg: base address and size of register area -- clock-frequency: input clock frequency in hz - - -Optional properties: - -- ti,c64x+pll-bypass-delay: CPU cycles to delay when entering bypass mode - -- ti,c64x+pll-reset-delay: CPU cycles to delay after PLL reset - -- ti,c64x+pll-lock-delay: CPU cycles to delay after PLL frequency change - -Example: - - clock-controller@29a0000 { - compatible = "ti,c6472-pll", "ti,c64x+pll"; - reg = <0x029a0000 0x200>; - clock-frequency = <25000000>; - - ti,c64x+pll-bypass-delay = <200>; - ti,c64x+pll-reset-delay = <12000>; - ti,c64x+pll-lock-delay = <80000>; - }; diff --git a/Documentation/devicetree/bindings/c6x/dscr.txt b/Documentation/devicetree/bindings/c6x/dscr.txt deleted file mode 100644 index 92672235de57..000000000000 --- a/Documentation/devicetree/bindings/c6x/dscr.txt +++ /dev/null @@ -1,127 +0,0 @@ -Device State Configuration Registers ------------------------------------- - -TI C6X SoCs contain a region of miscellaneous registers which provide various -function for SoC control or status. Details vary considerably among from SoC -to SoC with no two being alike. - -In general, the Device State Configuration Registers (DSCR) will provide one or -more configuration registers often protected by a lock register where one or -more key values must be written to a lock register in order to unlock the -configuration register for writes. These configuration register may be used to -enable (and disable in some cases) SoC pin drivers, select peripheral clock -sources (internal or pin), etc. In some cases, a configuration register is -write once or the individual bits are write once. In addition to device config, -the DSCR block may provide registers which are used to reset peripherals, -provide device ID information, provide ethernet MAC addresses, as well as other -miscellaneous functions. - -For device state control (enable/disable), each device control is assigned an -id which is used by individual device drivers to control the state as needed. - -Required properties: - -- compatible: must be "ti,c64x+dscr" -- reg: register area base and size - -Optional properties: - - NOTE: These are optional in that not all SoCs will have all properties. For - SoCs which do support a given property, leaving the property out of the - device tree will result in reduced functionality or possibly driver - failure. - -- ti,dscr-devstat - offset of the devstat register - -- ti,dscr-silicon-rev - offset, start bit, and bitsize of silicon revision field - -- ti,dscr-rmii-resets - offset and bitmask of RMII reset field. May have multiple tuples if more - than one ethernet port is available. - -- ti,dscr-locked-regs - possibly multiple tuples describing registers which are write protected by - a lock register. Each tuple consists of the register offset, lock register - offsset, and the key value used to unlock the register. - -- ti,dscr-kick-regs - offset and key values of two "kick" registers used to write protect other - registers in DSCR. On SoCs using kick registers, the first key must be - written to the first kick register and the second key must be written to - the second register before other registers in the area are write-enabled. - -- ti,dscr-mac-fuse-regs - MAC addresses are contained in two registers. Each element of a MAC address - is contained in a single byte. This property has two tuples. Each tuple has - a register offset and four cells representing bytes in the register from - most significant to least. The value of these four cells is the MAC byte - index (1-6) of the byte within the register. A value of 0 means the byte - is unused in the MAC address. - -- ti,dscr-devstate-ctl-regs - This property describes the bitfields used to control the state of devices. - Each tuple describes a range of identical bitfields used to control one or - more devices (one bitfield per device). The layout of each tuple is: - - start_id num_ids reg enable disable start_bit nbits - - Where: - start_id is device id for the first device control in the range - num_ids is the number of device controls in the range - reg is the offset of the register holding the control bits - enable is the value to enable a device - disable is the value to disable a device (0xffffffff if cannot disable) - start_bit is the bit number of the first bit in the range - nbits is the number of bits per device control - -- ti,dscr-devstate-stat-regs - This property describes the bitfields used to provide device state status - for device states controlled by the DSCR. Each tuple describes a range of - identical bitfields used to provide status for one or more devices (one - bitfield per device). The layout of each tuple is: - - start_id num_ids reg enable disable start_bit nbits - - Where: - start_id is device id for the first device status in the range - num_ids is the number of devices covered by the range - reg is the offset of the register holding the status bits - enable is the value indicating device is enabled - disable is the value indicating device is disabled - start_bit is the bit number of the first bit in the range - nbits is the number of bits per device status - -- ti,dscr-privperm - Offset and default value for register used to set access privilege for - some SoC devices. - - -Example: - - device-state-config-regs@2a80000 { - compatible = "ti,c64x+dscr"; - reg = <0x02a80000 0x41000>; - - ti,dscr-devstat = <0>; - ti,dscr-silicon-rev = <8 28 0xf>; - ti,dscr-rmii-resets = <0x40020 0x00040000>; - - ti,dscr-locked-regs = <0x40008 0x40004 0x0f0a0b00>; - ti,dscr-devstate-ctl-regs = - <0 12 0x40008 1 0 0 2 - 12 1 0x40008 3 0 30 2 - 13 2 0x4002c 1 0xffffffff 0 1>; - ti,dscr-devstate-stat-regs = - <0 10 0x40014 1 0 0 3 - 10 2 0x40018 1 0 0 3>; - - ti,dscr-mac-fuse-regs = <0x700 1 2 3 4 - 0x704 5 6 0 0>; - - ti,dscr-privperm = <0x41c 0xaaaaaaaa>; - - ti,dscr-kick-regs = <0x38 0x83E70B13 - 0x3c 0x95A4F1E0>; - }; diff --git a/Documentation/devicetree/bindings/c6x/emifa.txt b/Documentation/devicetree/bindings/c6x/emifa.txt deleted file mode 100644 index 0ff6e9b9a13f..000000000000 --- a/Documentation/devicetree/bindings/c6x/emifa.txt +++ /dev/null @@ -1,62 +0,0 @@ -External Memory Interface -------------------------- - -The emifa node describes a simple external bus controller found on some C6X -SoCs. This interface provides external busses with a number of chip selects. - -Required properties: - -- compatible: must be "ti,c64x+emifa", "simple-bus" -- reg: register area base and size -- #address-cells: must be 2 (chip-select + offset) -- #size-cells: must be 1 -- ranges: mapping from EMIFA space to parent space - - -Optional properties: - -- ti,dscr-dev-enable: Device ID if EMIF is enabled/disabled from DSCR - -- ti,emifa-burst-priority: - Number of memory transfers after which the EMIF will elevate the priority - of the oldest command in the command FIFO. Setting this field to 255 - disables this feature, thereby allowing old commands to stay in the FIFO - indefinitely. - -- ti,emifa-ce-config: - Configuration values for each of the supported chip selects. - -Example: - - emifa@70000000 { - compatible = "ti,c64x+emifa", "simple-bus"; - #address-cells = <2>; - #size-cells = <1>; - reg = <0x70000000 0x100>; - ranges = <0x2 0x0 0xa0000000 0x00000008 - 0x3 0x0 0xb0000000 0x00400000 - 0x4 0x0 0xc0000000 0x10000000 - 0x5 0x0 0xD0000000 0x10000000>; - - ti,dscr-dev-enable = <13>; - ti,emifa-burst-priority = <255>; - ti,emifa-ce-config = <0x00240120 - 0x00240120 - 0x00240122 - 0x00240122>; - - flash@3,0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "cfi-flash"; - reg = <0x3 0x0 0x400000>; - bank-width = <1>; - device-width = <1>; - partition@0 { - reg = <0x0 0x400000>; - label = "NOR"; - }; - }; - }; - -This shows a flash chip attached to chip select 3. diff --git a/Documentation/devicetree/bindings/c6x/soc.txt b/Documentation/devicetree/bindings/c6x/soc.txt deleted file mode 100644 index b1e4973b5769..000000000000 --- a/Documentation/devicetree/bindings/c6x/soc.txt +++ /dev/null @@ -1,28 +0,0 @@ -C6X System-on-Chip ------------------- - -Required properties: - -- compatible: "simple-bus" -- #address-cells: must be 1 -- #size-cells: must be 1 -- ranges - -Optional properties: - -- model: specific SoC model - -- nodes for IP blocks within SoC - - -Example: - - soc { - compatible = "simple-bus"; - model = "tms320c6455"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - ... - }; diff --git a/Documentation/devicetree/bindings/interrupt-controller/ti,c64x+megamod-pic.txt b/Documentation/devicetree/bindings/interrupt-controller/ti,c64x+megamod-pic.txt deleted file mode 100644 index ee3f9c351501..000000000000 --- a/Documentation/devicetree/bindings/interrupt-controller/ti,c64x+megamod-pic.txt +++ /dev/null @@ -1,103 +0,0 @@ -C6X Interrupt Chips -------------------- - -* C64X+ Core Interrupt Controller - - The core interrupt controller provides 16 prioritized interrupts to the - C64X+ core. Priority 0 and 1 are used for reset and NMI respectively. - Priority 2 and 3 are reserved. Priority 4-15 are used for interrupt - sources coming from outside the core. - - Required properties: - -------------------- - - compatible: Should be "ti,c64x+core-pic"; - - #interrupt-cells: <1> - - Interrupt Specifier Definition - ------------------------------ - Single cell specifying the core interrupt priority level (4-15) where - 4 is highest priority and 15 is lowest priority. - - Example - ------- - core_pic: interrupt-controller@0 { - interrupt-controller; - #interrupt-cells = <1>; - compatible = "ti,c64x+core-pic"; - }; - - - -* C64x+ Megamodule Interrupt Controller - - The megamodule PIC consists of four interrupt mupliplexers each of which - combine up to 32 interrupt inputs into a single interrupt output which - may be cascaded into the core interrupt controller. The megamodule PIC - has a total of 12 outputs cascading into the core interrupt controller. - One for each core interrupt priority level. In addition to the combined - interrupt sources, individual megamodule interrupts may be cascaded to - the core interrupt controller. When an individual interrupt is cascaded, - it is no longer handled through a megamodule interrupt combiner and is - considered to have the core interrupt controller as the parent. - - Required properties: - -------------------- - - compatible: "ti,c64x+megamod-pic" - - interrupt-controller - - #interrupt-cells: <1> - - reg: base address and size of register area - - interrupts: This should have four cells; one for each interrupt combiner. - The cells contain the core priority interrupt to which the - corresponding combiner output is wired. - - Optional properties: - -------------------- - - ti,c64x+megamod-pic-mux: Array of 12 cells correspnding to the 12 core - priority interrupts. The first cell corresponds to - core priority 4 and the last cell corresponds to - core priority 15. The value of each cell is the - megamodule interrupt source which is MUXed to - the core interrupt corresponding to the cell - position. Allowed values are 4 - 127. Mapping for - interrupts 0 - 3 (combined interrupt sources) are - ignored. - - Interrupt Specifier Definition - ------------------------------ - Single cell specifying the megamodule interrupt source (4-127). Note that - interrupts mapped directly to the core with "ti,c64x+megamod-pic-mux" will - use the core interrupt controller as their parent and the specifier will - be the core priority level, not the megamodule interrupt number. - - Examples - -------- - megamod_pic: interrupt-controller@1800000 { - compatible = "ti,c64x+megamod-pic"; - interrupt-controller; - #interrupt-cells = <1>; - reg = <0x1800000 0x1000>; - interrupt-parent = <&core_pic>; - interrupts = < 12 13 14 15 >; - }; - - This is a minimal example where all individual interrupts go through a - combiner. Combiner-0 is mapped to core interrupt 12, combiner-1 is mapped - to interrupt 13, etc. - - - megamod_pic: interrupt-controller@1800000 { - compatible = "ti,c64x+megamod-pic"; - interrupt-controller; - #interrupt-cells = <1>; - reg = <0x1800000 0x1000>; - interrupt-parent = <&core_pic>; - interrupts = < 12 13 14 15 >; - ti,c64x+megamod-pic-mux = < 0 0 0 0 - 32 0 0 0 - 0 0 0 0 >; - }; - - This the same as the first example except that megamodule interrupt 32 is - mapped directly to core priority interrupt 8. The node using this interrupt - must set the core controller as its interrupt parent and use 8 in the - interrupt specifier value. diff --git a/Documentation/devicetree/bindings/timer/ti,c64x+timer64.txt b/Documentation/devicetree/bindings/timer/ti,c64x+timer64.txt deleted file mode 100644 index d96c1e283e73..000000000000 --- a/Documentation/devicetree/bindings/timer/ti,c64x+timer64.txt +++ /dev/null @@ -1,25 +0,0 @@ -Timer64 -------- - -The timer64 node describes C6X event timers. - -Required properties: - -- compatible: must be "ti,c64x+timer64" -- reg: base address and size of register region -- interrupts: interrupt id - -Optional properties: - -- ti,dscr-dev-enable: Device ID used to enable timer IP through DSCR interface. - -- ti,core-mask: on multi-core SoCs, bitmask of cores allowed to use this timer. - -Example: - timer0: timer@25e0000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x01 >; - reg = <0x25e0000 0x40>; - interrupt-parent = <&megamod_pic>; - interrupts = < 16 >; - }; diff --git a/MAINTAINERS b/MAINTAINERS index fb971f5f6f0c..7c3eadb185f9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3837,14 +3837,6 @@ F: drivers/irqchip/irq-csky-* N: csky K: csky -C6X ARCHITECTURE -M: Mark Salter -M: Aurelien Jacquiot -L: linux-c6x-dev@linux-c6x.org -S: Maintained -W: http://www.linux-c6x.org/wiki/index.php/Main_Page -F: arch/c6x/ - CA8210 IEEE-802.15.4 RADIO DRIVER M: Harry Morris L: linux-wpan@vger.kernel.org diff --git a/arch/c6x/Kconfig b/arch/c6x/Kconfig deleted file mode 100644 index bdeeac28b1be..000000000000 --- a/arch/c6x/Kconfig +++ /dev/null @@ -1,113 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# For a description of the syntax of this configuration file, -# see Documentation/kbuild/kconfig-language.rst. -# - -config C6X - def_bool y - select ARCH_32BIT_OFF_T - select ARCH_HAS_BINFMT_FLAT - select ARCH_HAS_SYNC_DMA_FOR_CPU - select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select CLKDEV_LOOKUP - select HAVE_LEGACY_CLK - select GENERIC_ATOMIC64 - select GENERIC_IRQ_SHOW - select HAVE_ARCH_TRACEHOOK - select SPARSE_IRQ - select IRQ_DOMAIN - select OF - select OF_EARLY_FLATTREE - select MODULES_USE_ELF_RELA - select MMU_GATHER_NO_RANGE if MMU - select SET_FS - -config MMU - def_bool n - -config FPU - def_bool n - -config GENERIC_CALIBRATE_DELAY - def_bool y - -config GENERIC_HWEIGHT - def_bool y - -config GENERIC_BUG - def_bool y - depends on BUG - -config C6X_BIG_KERNEL - bool "Build a big kernel" - help - The C6X function call instruction has a limited range of +/- 2MiB. - This is sufficient for most kernels, but some kernel configurations - with lots of compiled-in functionality may require a larger range - for function calls. Use this option to have the compiler generate - function calls with 32-bit range. This will make the kernel both - larger and slower. - - If unsure, say N. - -# Use the generic interrupt handling code in kernel/irq/ - -config CMDLINE_BOOL - bool "Default bootloader kernel arguments" - -config CMDLINE - string "Kernel command line" - depends on CMDLINE_BOOL - default "console=ttyS0,57600" - help - On some architectures there is currently no way for the boot loader - to pass arguments to the kernel. For these architectures, you should - supply some command-line options at build time by entering them - here. - -config CMDLINE_FORCE - bool "Force default kernel command string" - depends on CMDLINE_BOOL - default n - help - Set this to have arguments from the default kernel command string - override those passed by the boot loader. - -config CPU_BIG_ENDIAN - bool "Build big-endian kernel" - default n - help - Say Y if you plan on running a kernel in big-endian mode. - Note that your board must be properly built and your board - port must properly enable any big-endian related features - of your chipset/board/processor. - -config FORCE_MAX_ZONEORDER - int "Maximum zone order" - default "13" - help - The kernel memory allocator divides physically contiguous memory - blocks into "zones", where each zone is a power of two number of - pages. This option selects the largest power of two that the kernel - keeps in the memory allocator. If you need to allocate very large - blocks of physically contiguous memory, then you may need to - increase this value. - - This config option is actually maximum order plus one. For example, - a value of 11 means that the largest free memory block is 2^10 pages. - -menu "Processor type and features" - -source "arch/c6x/platforms/Kconfig" - -config KERNEL_RAM_BASE_ADDRESS - hex "Virtual address of memory base" - default 0xe0000000 if SOC_TMS320C6455 - default 0xe0000000 if SOC_TMS320C6457 - default 0xe0000000 if SOC_TMS320C6472 - default 0x80000000 - -source "kernel/Kconfig.hz" - -endmenu diff --git a/arch/c6x/Kconfig.debug b/arch/c6x/Kconfig.debug deleted file mode 100644 index c299e0d8eca3..000000000000 --- a/arch/c6x/Kconfig.debug +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -config ACCESS_CHECK - bool "Check the user pointer address" - default y - help - Usually the pointer transfer from user space is checked to see if its - address is in the kernel space. - - Say N here to disable that check to improve the performance. diff --git a/arch/c6x/Makefile b/arch/c6x/Makefile deleted file mode 100644 index b7aa854f7008..000000000000 --- a/arch/c6x/Makefile +++ /dev/null @@ -1,60 +0,0 @@ -# -# linux/arch/c6x/Makefile -# -# This file is subject to the terms and conditions of the GNU General Public -# License. See the file "COPYING" in the main directory of this archive -# for more details. -# - -KBUILD_DEFCONFIG := dsk6455_defconfig - -cflags-y += -mno-dsbt -msdata=none -D__linux__ - -cflags-$(CONFIG_C6X_BIG_KERNEL) += -mlong-calls - -KBUILD_CFLAGS_MODULE += -mlong-calls -mno-dsbt -msdata=none - -CHECKFLAGS += - -KBUILD_CFLAGS += $(cflags-y) -KBUILD_AFLAGS += $(cflags-y) - -ifdef CONFIG_CPU_BIG_ENDIAN -KBUILD_CFLAGS += -mbig-endian -KBUILD_AFLAGS += -mbig-endian -LINKFLAGS += -mbig-endian -KBUILD_LDFLAGS += -mbig-endian -EB -CHECKFLAGS += -D_BIG_ENDIAN -endif - -head-y := arch/c6x/kernel/head.o -core-y += arch/c6x/kernel/ arch/c6x/mm/ arch/c6x/platforms/ -libs-y += arch/c6x/lib/ - -# Default to vmlinux.bin, override when needed -all: vmlinux.bin - -boot := arch/$(ARCH)/boot - -# Are we making a dtbImage. target? If so, crack out the boardname -DTB:=$(subst dtbImage.,,$(filter dtbImage.%, $(MAKECMDGOALS))) -export DTB - -core-y += $(boot)/dts/ - -# With make 3.82 we cannot mix normal and wildcard targets - -vmlinux.bin: vmlinux - $(Q)$(MAKE) $(build)=$(boot) $(patsubst %,$(boot)/%,$@) - -dtbImage.%: vmlinux - $(Q)$(MAKE) $(build)=$(boot) $(patsubst %,$(boot)/%,$@) - -archclean: - $(Q)$(MAKE) $(clean)=$(boot) - -define archhelp - @echo ' vmlinux.bin - Binary kernel image (arch/$(ARCH)/boot/vmlinux.bin)' - @echo ' dtbImage.
- ELF image with $(arch)/boot/dts/
.dts linked in' - @echo ' - stripped elf with fdt blob' -endef diff --git a/arch/c6x/boot/Makefile b/arch/c6x/boot/Makefile deleted file mode 100644 index 842b7b0bfe80..000000000000 --- a/arch/c6x/boot/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for bootable kernel images -# - -OBJCOPYFLAGS_vmlinux.bin := -O binary -$(obj)/vmlinux.bin: vmlinux FORCE - $(call if_changed,objcopy) - -$(obj)/dtbImage.%: vmlinux - $(call if_changed,objcopy) diff --git a/arch/c6x/boot/dts/Makefile b/arch/c6x/boot/dts/Makefile deleted file mode 100644 index f438285c3640..000000000000 --- a/arch/c6x/boot/dts/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for device trees -# - -DTC_FLAGS ?= -p 1024 - -dtb-$(CONFIG_SOC_TMS320C6455) += dsk6455.dtb -dtb-$(CONFIG_SOC_TMS320C6457) += evmc6457.dtb -dtb-$(CONFIG_SOC_TMS320C6472) += evmc6472.dtb -dtb-$(CONFIG_SOC_TMS320C6474) += evmc6474.dtb -dtb-$(CONFIG_SOC_TMS320C6678) += evmc6678.dtb - -ifneq ($(DTB),) -obj-y += $(DTB).dtb.o -endif diff --git a/arch/c6x/boot/dts/dsk6455.dts b/arch/c6x/boot/dts/dsk6455.dts deleted file mode 100644 index fa904f2916b5..000000000000 --- a/arch/c6x/boot/dts/dsk6455.dts +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * arch/c6x/boot/dts/dsk6455.dts - * - * DSK6455 Evaluation Platform For TMS320C6455 - * Copyright (C) 2011 Texas Instruments Incorporated - * - * Author: Mark Salter - */ - -/dts-v1/; - -/include/ "tms320c6455.dtsi" - -/ { - model = "Spectrum Digital DSK6455"; - compatible = "spectrum-digital,dsk6455"; - - chosen { - bootargs = "root=/dev/nfs ip=dhcp rw"; - }; - - memory { - device_type = "memory"; - reg = <0xE0000000 0x08000000>; - }; - - soc { - megamod_pic: interrupt-controller@1800000 { - interrupts = < 12 13 14 15 >; - }; - - emifa@70000000 { - flash@3,0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "cfi-flash"; - reg = <0x3 0x0 0x400000>; - bank-width = <1>; - device-width = <1>; - partition@0 { - reg = <0x0 0x400000>; - label = "NOR"; - }; - }; - }; - - timer1: timer@2980000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 69 >; - }; - - clock-controller@029a0000 { - clock-frequency = <50000000>; - }; - }; -}; diff --git a/arch/c6x/boot/dts/evmc6457.dts b/arch/c6x/boot/dts/evmc6457.dts deleted file mode 100644 index 73e1d43b51ce..000000000000 --- a/arch/c6x/boot/dts/evmc6457.dts +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * arch/c6x/boot/dts/evmc6457.dts - * - * EVMC6457 Evaluation Platform For TMS320C6457 - * - * Copyright (C) 2011 Texas Instruments Incorporated - * - * Author: Mark Salter - */ - -/dts-v1/; - -/include/ "tms320c6457.dtsi" - -/ { - model = "eInfochips EVMC6457"; - compatible = "einfochips,evmc6457"; - - chosen { - bootargs = "console=hvc root=/dev/nfs ip=dhcp rw"; - }; - - memory { - device_type = "memory"; - reg = <0xE0000000 0x10000000>; - }; - - soc { - megamod_pic: interrupt-controller@1800000 { - interrupts = < 12 13 14 15 >; - }; - - timer0: timer@2940000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 67 >; - }; - - clock-controller@29a0000 { - clock-frequency = <60000000>; - }; - }; -}; diff --git a/arch/c6x/boot/dts/evmc6472.dts b/arch/c6x/boot/dts/evmc6472.dts deleted file mode 100644 index 4878b78919fa..000000000000 --- a/arch/c6x/boot/dts/evmc6472.dts +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * arch/c6x/boot/dts/evmc6472.dts - * - * EVMC6472 Evaluation Platform For TMS320C6472 - * - * Copyright (C) 2011 Texas Instruments Incorporated - * - * Author: Mark Salter - */ - -/dts-v1/; - -/include/ "tms320c6472.dtsi" - -/ { - model = "eInfochips EVMC6472"; - compatible = "einfochips,evmc6472"; - - chosen { - bootargs = "console=hvc root=/dev/nfs ip=dhcp rw"; - }; - - memory { - device_type = "memory"; - reg = <0xE0000000 0x10000000>; - }; - - soc { - megamod_pic: interrupt-controller@1800000 { - interrupts = < 12 13 14 15 >; - }; - - timer0: timer@25e0000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 16 >; - }; - - timer1: timer@25f0000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 16 >; - }; - - timer2: timer@2600000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 16 >; - }; - - timer3: timer@2610000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 16 >; - }; - - timer4: timer@2620000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 16 >; - }; - - timer5: timer@2630000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 16 >; - }; - - clock-controller@29a0000 { - clock-frequency = <25000000>; - }; - }; -}; diff --git a/arch/c6x/boot/dts/evmc6474.dts b/arch/c6x/boot/dts/evmc6474.dts deleted file mode 100644 index d10746453217..000000000000 --- a/arch/c6x/boot/dts/evmc6474.dts +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * arch/c6x/boot/dts/evmc6474.dts - * - * EVMC6474 Evaluation Platform For TMS320C6474 - * - * Copyright (C) 2011 Texas Instruments Incorporated - * - * Author: Mark Salter - */ - -/dts-v1/; - -/include/ "tms320c6474.dtsi" - -/ { - model = "Spectrum Digital EVMC6474"; - compatible = "spectrum-digital,evmc6474"; - - chosen { - bootargs = "console=hvc root=/dev/nfs ip=dhcp rw"; - }; - - memory { - device_type = "memory"; - reg = <0x80000000 0x08000000>; - }; - - soc { - megamod_pic: interrupt-controller@1800000 { - interrupts = < 12 13 14 15 >; - }; - - timer3: timer@2940000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 39 >; - }; - - timer4: timer@2950000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 41 >; - }; - - timer5: timer@2960000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 43 >; - }; - - clock-controller@29a0000 { - clock-frequency = <50000000>; - }; - }; -}; diff --git a/arch/c6x/boot/dts/evmc6678.dts b/arch/c6x/boot/dts/evmc6678.dts deleted file mode 100644 index 5e6c0961e7b2..000000000000 --- a/arch/c6x/boot/dts/evmc6678.dts +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * arch/c6x/boot/dts/evmc6678.dts - * - * EVMC6678 Evaluation Platform For TMS320C6678 - * - * Copyright (C) 2012 Texas Instruments Incorporated - * - * Author: Ken Cox - */ - -/dts-v1/; - -/include/ "tms320c6678.dtsi" - -/ { - model = "Advantech EVMC6678"; - compatible = "advantech,evmc6678"; - - chosen { - bootargs = "root=/dev/nfs ip=dhcp rw"; - }; - - memory { - device_type = "memory"; - reg = <0x80000000 0x20000000>; - }; - - soc { - megamod_pic: interrupt-controller@1800000 { - interrupts = < 12 13 14 15 >; - }; - - timer8: timer@2280000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 66 >; - }; - - timer9: timer@2290000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 68 >; - }; - - timer10: timer@22A0000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 70 >; - }; - - timer11: timer@22B0000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 72 >; - }; - - timer12: timer@22C0000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 74 >; - }; - - timer13: timer@22D0000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 76 >; - }; - - timer14: timer@22E0000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 78 >; - }; - - timer15: timer@22F0000 { - interrupt-parent = <&megamod_pic>; - interrupts = < 80 >; - }; - - clock-controller@2310000 { - clock-frequency = <100000000>; - }; - }; -}; diff --git a/arch/c6x/boot/dts/tms320c6455.dtsi b/arch/c6x/boot/dts/tms320c6455.dtsi deleted file mode 100644 index 0b21cb30343b..000000000000 --- a/arch/c6x/boot/dts/tms320c6455.dtsi +++ /dev/null @@ -1,97 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -/ { - #address-cells = <1>; - #size-cells = <1>; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - cpu@0 { - device_type = "cpu"; - model = "ti,c64x+"; - reg = <0>; - }; - }; - - soc { - compatible = "simple-bus"; - model = "tms320c6455"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - core_pic: interrupt-controller { - interrupt-controller; - #interrupt-cells = <1>; - compatible = "ti,c64x+core-pic"; - }; - - /* - * Megamodule interrupt controller - */ - megamod_pic: interrupt-controller@1800000 { - compatible = "ti,c64x+megamod-pic"; - interrupt-controller; - #interrupt-cells = <1>; - reg = <0x1800000 0x1000>; - interrupt-parent = <&core_pic>; - }; - - cache-controller@1840000 { - compatible = "ti,c64x+cache"; - reg = <0x01840000 0x8400>; - }; - - emifa@70000000 { - compatible = "ti,c64x+emifa", "simple-bus"; - #address-cells = <2>; - #size-cells = <1>; - reg = <0x70000000 0x100>; - ranges = <0x2 0x0 0xa0000000 0x00000008 - 0x3 0x0 0xb0000000 0x00400000 - 0x4 0x0 0xc0000000 0x10000000 - 0x5 0x0 0xD0000000 0x10000000>; - - ti,dscr-dev-enable = <13>; - ti,emifa-burst-priority = <255>; - ti,emifa-ce-config = <0x00240120 - 0x00240120 - 0x00240122 - 0x00240122>; - }; - - timer1: timer@2980000 { - compatible = "ti,c64x+timer64"; - reg = <0x2980000 0x40>; - ti,dscr-dev-enable = <4>; - }; - - clock-controller@029a0000 { - compatible = "ti,c6455-pll", "ti,c64x+pll"; - reg = <0x029a0000 0x200>; - ti,c64x+pll-bypass-delay = <1440>; - ti,c64x+pll-reset-delay = <15360>; - ti,c64x+pll-lock-delay = <24000>; - }; - - device-state-config-regs@2a80000 { - compatible = "ti,c64x+dscr"; - reg = <0x02a80000 0x41000>; - - ti,dscr-devstat = <0>; - ti,dscr-silicon-rev = <8 28 0xf>; - ti,dscr-rmii-resets = <0 0x40020 0x00040000>; - - ti,dscr-locked-regs = <0x40008 0x40004 0x0f0a0b00>; - ti,dscr-devstate-ctl-regs = - <0 12 0x40008 1 0 0 2 - 12 1 0x40008 3 0 30 2 - 13 2 0x4002c 1 0xffffffff 0 1>; - ti,dscr-devstate-stat-regs = - <0 10 0x40014 1 0 0 3 - 10 2 0x40018 1 0 0 3>; - }; - }; -}; diff --git a/arch/c6x/boot/dts/tms320c6457.dtsi b/arch/c6x/boot/dts/tms320c6457.dtsi deleted file mode 100644 index e49f7ae19124..000000000000 --- a/arch/c6x/boot/dts/tms320c6457.dtsi +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -/ { - #address-cells = <1>; - #size-cells = <1>; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - cpu@0 { - device_type = "cpu"; - model = "ti,c64x+"; - reg = <0>; - }; - }; - - soc { - compatible = "simple-bus"; - model = "tms320c6457"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - core_pic: interrupt-controller { - interrupt-controller; - #interrupt-cells = <1>; - compatible = "ti,c64x+core-pic"; - }; - - megamod_pic: interrupt-controller@1800000 { - compatible = "ti,c64x+megamod-pic"; - interrupt-controller; - #interrupt-cells = <1>; - interrupt-parent = <&core_pic>; - reg = <0x1800000 0x1000>; - }; - - cache-controller@1840000 { - compatible = "ti,c64x+cache"; - reg = <0x01840000 0x8400>; - }; - - device-state-controller@2880800 { - compatible = "ti,c64x+dscr"; - reg = <0x02880800 0x400>; - - ti,dscr-devstat = <0x20>; - ti,dscr-silicon-rev = <0x18 28 0xf>; - ti,dscr-mac-fuse-regs = <0x114 3 4 5 6 - 0x118 0 0 1 2>; - ti,dscr-kick-regs = <0x38 0x83E70B13 - 0x3c 0x95A4F1E0>; - }; - - timer0: timer@2940000 { - compatible = "ti,c64x+timer64"; - reg = <0x2940000 0x40>; - }; - - clock-controller@29a0000 { - compatible = "ti,c6457-pll", "ti,c64x+pll"; - reg = <0x029a0000 0x200>; - ti,c64x+pll-bypass-delay = <300>; - ti,c64x+pll-reset-delay = <24000>; - ti,c64x+pll-lock-delay = <50000>; - }; - }; -}; diff --git a/arch/c6x/boot/dts/tms320c6472.dtsi b/arch/c6x/boot/dts/tms320c6472.dtsi deleted file mode 100644 index 9dd4b04e78ef..000000000000 --- a/arch/c6x/boot/dts/tms320c6472.dtsi +++ /dev/null @@ -1,135 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -/ { - #address-cells = <1>; - #size-cells = <1>; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - cpu@0 { - device_type = "cpu"; - reg = <0>; - model = "ti,c64x+"; - }; - cpu@1 { - device_type = "cpu"; - reg = <1>; - model = "ti,c64x+"; - }; - cpu@2 { - device_type = "cpu"; - reg = <2>; - model = "ti,c64x+"; - }; - cpu@3 { - device_type = "cpu"; - reg = <3>; - model = "ti,c64x+"; - }; - cpu@4 { - device_type = "cpu"; - reg = <4>; - model = "ti,c64x+"; - }; - cpu@5 { - device_type = "cpu"; - reg = <5>; - model = "ti,c64x+"; - }; - }; - - soc { - compatible = "simple-bus"; - model = "tms320c6472"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - core_pic: interrupt-controller { - compatible = "ti,c64x+core-pic"; - interrupt-controller; - #interrupt-cells = <1>; - }; - - megamod_pic: interrupt-controller@1800000 { - compatible = "ti,c64x+megamod-pic"; - interrupt-controller; - #interrupt-cells = <1>; - reg = <0x1800000 0x1000>; - interrupt-parent = <&core_pic>; - }; - - cache-controller@1840000 { - compatible = "ti,c64x+cache"; - reg = <0x01840000 0x8400>; - }; - - timer0: timer@25e0000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x01 >; - reg = <0x25e0000 0x40>; - }; - - timer1: timer@25f0000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x02 >; - reg = <0x25f0000 0x40>; - }; - - timer2: timer@2600000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x04 >; - reg = <0x2600000 0x40>; - }; - - timer3: timer@2610000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x08 >; - reg = <0x2610000 0x40>; - }; - - timer4: timer@2620000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x10 >; - reg = <0x2620000 0x40>; - }; - - timer5: timer@2630000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x20 >; - reg = <0x2630000 0x40>; - }; - - clock-controller@29a0000 { - compatible = "ti,c6472-pll", "ti,c64x+pll"; - reg = <0x029a0000 0x200>; - ti,c64x+pll-bypass-delay = <200>; - ti,c64x+pll-reset-delay = <12000>; - ti,c64x+pll-lock-delay = <80000>; - }; - - device-state-controller@2a80000 { - compatible = "ti,c64x+dscr"; - reg = <0x02a80000 0x1000>; - - ti,dscr-devstat = <0>; - ti,dscr-silicon-rev = <0x70c 16 0xff>; - - ti,dscr-mac-fuse-regs = <0x700 1 2 3 4 - 0x704 5 6 0 0>; - - ti,dscr-rmii-resets = <0x208 1 - 0x20c 1>; - - ti,dscr-locked-regs = <0x200 0x204 0x0a1e183a - 0x40c 0x420 0xbea7 - 0x41c 0x420 0xbea7>; - - ti,dscr-privperm = <0x41c 0xaaaaaaaa>; - - ti,dscr-devstate-ctl-regs = <0 13 0x200 1 0 0 1>; - }; - }; -}; diff --git a/arch/c6x/boot/dts/tms320c6474.dtsi b/arch/c6x/boot/dts/tms320c6474.dtsi deleted file mode 100644 index 0ef5333629a6..000000000000 --- a/arch/c6x/boot/dts/tms320c6474.dtsi +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -/ { - #address-cells = <1>; - #size-cells = <1>; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - cpu@0 { - device_type = "cpu"; - reg = <0>; - model = "ti,c64x+"; - }; - cpu@1 { - device_type = "cpu"; - reg = <1>; - model = "ti,c64x+"; - }; - cpu@2 { - device_type = "cpu"; - reg = <2>; - model = "ti,c64x+"; - }; - }; - - soc { - compatible = "simple-bus"; - model = "tms320c6474"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - core_pic: interrupt-controller { - interrupt-controller; - #interrupt-cells = <1>; - compatible = "ti,c64x+core-pic"; - }; - - megamod_pic: interrupt-controller@1800000 { - compatible = "ti,c64x+megamod-pic"; - interrupt-controller; - #interrupt-cells = <1>; - reg = <0x1800000 0x1000>; - interrupt-parent = <&core_pic>; - }; - - cache-controller@1840000 { - compatible = "ti,c64x+cache"; - reg = <0x01840000 0x8400>; - }; - - timer3: timer@2940000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x04 >; - reg = <0x2940000 0x40>; - }; - - timer4: timer@2950000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x02 >; - reg = <0x2950000 0x40>; - }; - - timer5: timer@2960000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x01 >; - reg = <0x2960000 0x40>; - }; - - device-state-controller@2880800 { - compatible = "ti,c64x+dscr"; - reg = <0x02880800 0x400>; - - ti,dscr-devstat = <0x004>; - ti,dscr-silicon-rev = <0x014 28 0xf>; - ti,dscr-mac-fuse-regs = <0x34 3 4 5 6 - 0x38 0 0 1 2>; - }; - - clock-controller@29a0000 { - compatible = "ti,c6474-pll", "ti,c64x+pll"; - reg = <0x029a0000 0x200>; - ti,c64x+pll-bypass-delay = <120>; - ti,c64x+pll-reset-delay = <30000>; - ti,c64x+pll-lock-delay = <60000>; - }; - }; -}; diff --git a/arch/c6x/boot/dts/tms320c6678.dtsi b/arch/c6x/boot/dts/tms320c6678.dtsi deleted file mode 100644 index da1e3f2bf062..000000000000 --- a/arch/c6x/boot/dts/tms320c6678.dtsi +++ /dev/null @@ -1,147 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -/ { - #address-cells = <1>; - #size-cells = <1>; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - cpu@0 { - device_type = "cpu"; - reg = <0>; - model = "ti,c66x"; - }; - cpu@1 { - device_type = "cpu"; - reg = <1>; - model = "ti,c66x"; - }; - cpu@2 { - device_type = "cpu"; - reg = <2>; - model = "ti,c66x"; - }; - cpu@3 { - device_type = "cpu"; - reg = <3>; - model = "ti,c66x"; - }; - cpu@4 { - device_type = "cpu"; - reg = <4>; - model = "ti,c66x"; - }; - cpu@5 { - device_type = "cpu"; - reg = <5>; - model = "ti,c66x"; - }; - cpu@6 { - device_type = "cpu"; - reg = <6>; - model = "ti,c66x"; - }; - cpu@7 { - device_type = "cpu"; - reg = <7>; - model = "ti,c66x"; - }; - }; - - soc { - compatible = "simple-bus"; - model = "tms320c6678"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - core_pic: interrupt-controller { - compatible = "ti,c64x+core-pic"; - interrupt-controller; - #interrupt-cells = <1>; - }; - - megamod_pic: interrupt-controller@1800000 { - compatible = "ti,c64x+megamod-pic"; - interrupt-controller; - #interrupt-cells = <1>; - reg = <0x1800000 0x1000>; - interrupt-parent = <&core_pic>; - }; - - cache-controller@1840000 { - compatible = "ti,c64x+cache"; - reg = <0x01840000 0x8400>; - }; - - timer8: timer@2280000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x01 >; - reg = <0x2280000 0x40>; - }; - - timer9: timer@2290000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x02 >; - reg = <0x2290000 0x40>; - }; - - timer10: timer@22A0000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x04 >; - reg = <0x22A0000 0x40>; - }; - - timer11: timer@22B0000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x08 >; - reg = <0x22B0000 0x40>; - }; - - timer12: timer@22C0000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x10 >; - reg = <0x22C0000 0x40>; - }; - - timer13: timer@22D0000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x20 >; - reg = <0x22D0000 0x40>; - }; - - timer14: timer@22E0000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x40 >; - reg = <0x22E0000 0x40>; - }; - - timer15: timer@22F0000 { - compatible = "ti,c64x+timer64"; - ti,core-mask = < 0x80 >; - reg = <0x22F0000 0x40>; - }; - - clock-controller@2310000 { - compatible = "ti,c6678-pll", "ti,c64x+pll"; - reg = <0x02310000 0x200>; - ti,c64x+pll-bypass-delay = <200>; - ti,c64x+pll-reset-delay = <12000>; - ti,c64x+pll-lock-delay = <80000>; - }; - - device-state-controller@2620000 { - compatible = "ti,c64x+dscr"; - reg = <0x02620000 0x1000>; - - ti,dscr-devstat = <0x20>; - ti,dscr-silicon-rev = <0x18 28 0xf>; - - ti,dscr-mac-fuse-regs = <0x110 1 2 3 4 - 0x114 5 6 0 0>; - - }; - }; -}; diff --git a/arch/c6x/configs/dsk6455_defconfig b/arch/c6x/configs/dsk6455_defconfig deleted file mode 100644 index d764ea4cce7f..000000000000 --- a/arch/c6x/configs/dsk6455_defconfig +++ /dev/null @@ -1,42 +0,0 @@ -CONFIG_SOC_TMS320C6455=y -# CONFIG_LOCALVERSION_AUTO is not set -CONFIG_SYSVIPC=y -CONFIG_SPARSE_IRQ=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_NAMESPACES=y -# CONFIG_UTS_NS is not set -# CONFIG_USER_NS is not set -# CONFIG_PID_NS is not set -CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_SIZE=y -CONFIG_EXPERT=y -# CONFIG_FUTEX is not set -# CONFIG_SLUB_DEBUG is not set -CONFIG_MODULES=y -CONFIG_MODULE_FORCE_LOAD=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="" -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_COUNT=2 -CONFIG_BLK_DEV_RAM_SIZE=17000 -# CONFIG_INPUT is not set -# CONFIG_SERIO is not set -# CONFIG_VT is not set -# CONFIG_HW_RANDOM is not set -# CONFIG_HWMON is not set -# CONFIG_USB_SUPPORT is not set -# CONFIG_IOMMU_SUPPORT is not set -# CONFIG_MISC_FILESYSTEMS is not set -CONFIG_CRC16=y -# CONFIG_ENABLE_MUST_CHECK is not set -# CONFIG_SCHED_DEBUG is not set -# CONFIG_DEBUG_BUGVERBOSE is not set -CONFIG_MTD=y -CONFIG_MTD_CFI=y -CONFIG_MTD_CFI_AMDSTD=y -CONFIG_MTD_PHYSMAP_OF=y diff --git a/arch/c6x/configs/evmc6457_defconfig b/arch/c6x/configs/evmc6457_defconfig deleted file mode 100644 index 05d0b4a25ab1..000000000000 --- a/arch/c6x/configs/evmc6457_defconfig +++ /dev/null @@ -1,39 +0,0 @@ -CONFIG_SOC_TMS320C6457=y -# CONFIG_LOCALVERSION_AUTO is not set -CONFIG_SYSVIPC=y -CONFIG_SPARSE_IRQ=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_NAMESPACES=y -# CONFIG_UTS_NS is not set -# CONFIG_USER_NS is not set -# CONFIG_PID_NS is not set -CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_SIZE=y -CONFIG_EXPERT=y -# CONFIG_FUTEX is not set -# CONFIG_SLUB_DEBUG is not set -CONFIG_MODULES=y -CONFIG_MODULE_FORCE_LOAD=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="" -CONFIG_BOARD_EVM6457=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_COUNT=2 -CONFIG_BLK_DEV_RAM_SIZE=17000 -# CONFIG_INPUT is not set -# CONFIG_SERIO is not set -# CONFIG_VT is not set -# CONFIG_HW_RANDOM is not set -# CONFIG_HWMON is not set -# CONFIG_USB_SUPPORT is not set -# CONFIG_IOMMU_SUPPORT is not set -# CONFIG_MISC_FILESYSTEMS is not set -CONFIG_CRC16=y -# CONFIG_ENABLE_MUST_CHECK is not set -# CONFIG_SCHED_DEBUG is not set -# CONFIG_DEBUG_BUGVERBOSE is not set diff --git a/arch/c6x/configs/evmc6472_defconfig b/arch/c6x/configs/evmc6472_defconfig deleted file mode 100644 index 8d81fcf86b0e..000000000000 --- a/arch/c6x/configs/evmc6472_defconfig +++ /dev/null @@ -1,40 +0,0 @@ -CONFIG_SOC_TMS320C6472=y -# CONFIG_LOCALVERSION_AUTO is not set -CONFIG_SYSVIPC=y -CONFIG_SPARSE_IRQ=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_NAMESPACES=y -# CONFIG_UTS_NS is not set -# CONFIG_USER_NS is not set -# CONFIG_PID_NS is not set -CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_SIZE=y -CONFIG_EXPERT=y -# CONFIG_FUTEX is not set -# CONFIG_SLUB_DEBUG is not set -CONFIG_MODULES=y -CONFIG_MODULE_FORCE_LOAD=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="" -# CONFIG_CMDLINE_FORCE is not set -CONFIG_BOARD_EVM6472=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_COUNT=2 -CONFIG_BLK_DEV_RAM_SIZE=17000 -# CONFIG_INPUT is not set -# CONFIG_SERIO is not set -# CONFIG_VT is not set -# CONFIG_HW_RANDOM is not set -# CONFIG_HWMON is not set -# CONFIG_USB_SUPPORT is not set -# CONFIG_IOMMU_SUPPORT is not set -# CONFIG_MISC_FILESYSTEMS is not set -CONFIG_CRC16=y -# CONFIG_ENABLE_MUST_CHECK is not set -# CONFIG_SCHED_DEBUG is not set -# CONFIG_DEBUG_BUGVERBOSE is not set diff --git a/arch/c6x/configs/evmc6474_defconfig b/arch/c6x/configs/evmc6474_defconfig deleted file mode 100644 index 8156a98f3958..000000000000 --- a/arch/c6x/configs/evmc6474_defconfig +++ /dev/null @@ -1,40 +0,0 @@ -CONFIG_SOC_TMS320C6474=y -# CONFIG_LOCALVERSION_AUTO is not set -CONFIG_SYSVIPC=y -CONFIG_SPARSE_IRQ=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_NAMESPACES=y -# CONFIG_UTS_NS is not set -# CONFIG_USER_NS is not set -# CONFIG_PID_NS is not set -CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_SIZE=y -CONFIG_EXPERT=y -# CONFIG_FUTEX is not set -# CONFIG_SLUB_DEBUG is not set -CONFIG_MODULES=y -CONFIG_MODULE_FORCE_LOAD=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="" -# CONFIG_CMDLINE_FORCE is not set -CONFIG_BOARD_EVM6474=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_COUNT=2 -CONFIG_BLK_DEV_RAM_SIZE=17000 -# CONFIG_INPUT is not set -# CONFIG_SERIO is not set -# CONFIG_VT is not set -# CONFIG_HW_RANDOM is not set -# CONFIG_HWMON is not set -# CONFIG_USB_SUPPORT is not set -# CONFIG_IOMMU_SUPPORT is not set -# CONFIG_MISC_FILESYSTEMS is not set -CONFIG_CRC16=y -# CONFIG_ENABLE_MUST_CHECK is not set -# CONFIG_SCHED_DEBUG is not set -# CONFIG_DEBUG_BUGVERBOSE is not set diff --git a/arch/c6x/configs/evmc6678_defconfig b/arch/c6x/configs/evmc6678_defconfig deleted file mode 100644 index c4f433c25b69..000000000000 --- a/arch/c6x/configs/evmc6678_defconfig +++ /dev/null @@ -1,40 +0,0 @@ -CONFIG_SOC_TMS320C6678=y -# CONFIG_LOCALVERSION_AUTO is not set -CONFIG_SYSVIPC=y -CONFIG_SPARSE_IRQ=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_NAMESPACES=y -# CONFIG_UTS_NS is not set -# CONFIG_USER_NS is not set -# CONFIG_PID_NS is not set -CONFIG_BLK_DEV_INITRD=y -CONFIG_CC_OPTIMIZE_FOR_SIZE=y -CONFIG_EXPERT=y -# CONFIG_FUTEX is not set -# CONFIG_SLUB_DEBUG is not set -CONFIG_MODULES=y -CONFIG_MODULE_FORCE_LOAD=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="" -# CONFIG_CMDLINE_FORCE is not set -CONFIG_BOARD_EVM6678=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_COUNT=2 -CONFIG_BLK_DEV_RAM_SIZE=17000 -# CONFIG_INPUT is not set -# CONFIG_SERIO is not set -# CONFIG_VT is not set -# CONFIG_HW_RANDOM is not set -# CONFIG_HWMON is not set -# CONFIG_USB_SUPPORT is not set -# CONFIG_IOMMU_SUPPORT is not set -# CONFIG_MISC_FILESYSTEMS is not set -CONFIG_CRC16=y -# CONFIG_ENABLE_MUST_CHECK is not set -# CONFIG_SCHED_DEBUG is not set -# CONFIG_DEBUG_BUGVERBOSE is not set diff --git a/arch/c6x/include/asm/Kbuild b/arch/c6x/include/asm/Kbuild deleted file mode 100644 index a4ef93a1f7ae..000000000000 --- a/arch/c6x/include/asm/Kbuild +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -generic-y += extable.h -generic-y += kvm_para.h -generic-y += mcs_spinlock.h -generic-y += user.h diff --git a/arch/c6x/include/asm/asm-offsets.h b/arch/c6x/include/asm/asm-offsets.h deleted file mode 100644 index d370ee36a182..000000000000 --- a/arch/c6x/include/asm/asm-offsets.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/arch/c6x/include/asm/bitops.h b/arch/c6x/include/asm/bitops.h deleted file mode 100644 index 50e618f38a11..000000000000 --- a/arch/c6x/include/asm/bitops.h +++ /dev/null @@ -1,95 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_BITOPS_H -#define _ASM_C6X_BITOPS_H - -#ifdef __KERNEL__ - -#include -#include -#include - -/* - * We are lucky, DSP is perfect for bitops: do it in 3 cycles - */ - -/** - * __ffs - find first bit in word. - * @word: The word to search - * - * Undefined if no bit exists, so code should check against 0 first. - * Note __ffs(0) = undef, __ffs(1) = 0, __ffs(0x80000000) = 31. - * - */ -static inline unsigned long __ffs(unsigned long x) -{ - asm (" bitr .M1 %0,%0\n" - " nop\n" - " lmbd .L1 1,%0,%0\n" - : "+a"(x)); - - return x; -} - -/* - * ffz - find first zero in word. - * @word: The word to search - * - * Undefined if no zero exists, so code should check against ~0UL first. - */ -#define ffz(x) __ffs(~(x)) - -/** - * fls - find last (most-significant) bit set - * @x: the word to search - * - * This is defined the same way as ffs. - * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. - */ -static inline int fls(unsigned int x) -{ - if (!x) - return 0; - - asm (" lmbd .L1 1,%0,%0\n" : "+a"(x)); - - return 32 - x; -} - -/** - * ffs - find first bit set - * @x: the word to search - * - * This is defined the same way as - * the libc and compiler builtin ffs routines, therefore - * differs in spirit from the above ffz (man ffs). - * Note ffs(0) = 0, ffs(1) = 1, ffs(0x80000000) = 32. - */ -static inline int ffs(int x) -{ - if (!x) - return 0; - - return __ffs(x) + 1; -} - -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include - -#endif /* __KERNEL__ */ -#endif /* _ASM_C6X_BITOPS_H */ diff --git a/arch/c6x/include/asm/bug.h b/arch/c6x/include/asm/bug.h deleted file mode 100644 index 1a68676256ee..000000000000 --- a/arch/c6x/include/asm/bug.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_BUG_H -#define _ASM_C6X_BUG_H - -#include -#include - -struct pt_regs; - -extern void die(char *str, struct pt_regs *fp, int nr); -extern asmlinkage int process_exception(struct pt_regs *regs); -extern asmlinkage void enable_exception(void); - -#endif /* _ASM_C6X_BUG_H */ diff --git a/arch/c6x/include/asm/cache.h b/arch/c6x/include/asm/cache.h deleted file mode 100644 index 0fa8bf77c954..000000000000 --- a/arch/c6x/include/asm/cache.h +++ /dev/null @@ -1,94 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2005, 2006, 2009, 2010, 2012 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_CACHE_H -#define _ASM_C6X_CACHE_H - -#include -#include - -/* - * Cache line size - */ -#define L1D_CACHE_SHIFT 6 -#define L1D_CACHE_BYTES (1 << L1D_CACHE_SHIFT) - -#define L1P_CACHE_SHIFT 5 -#define L1P_CACHE_BYTES (1 << L1P_CACHE_SHIFT) - -#define L2_CACHE_SHIFT 7 -#define L2_CACHE_BYTES (1 << L2_CACHE_SHIFT) - -/* - * L2 used as cache - */ -#define L2MODE_SIZE L2MODE_256K_CACHE - -/* - * For practical reasons the L1_CACHE_BYTES defines should not be smaller than - * the L2 line size - */ -#define L1_CACHE_SHIFT L2_CACHE_SHIFT -#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) - -#define L2_CACHE_ALIGN_LOW(x) \ - (((x) & ~(L2_CACHE_BYTES - 1))) -#define L2_CACHE_ALIGN_UP(x) \ - (((x) + (L2_CACHE_BYTES - 1)) & ~(L2_CACHE_BYTES - 1)) -#define L2_CACHE_ALIGN_CNT(x) \ - (((x) + (sizeof(int) - 1)) & ~(sizeof(int) - 1)) - -#define ARCH_DMA_MINALIGN L1_CACHE_BYTES -#define ARCH_SLAB_MINALIGN L1_CACHE_BYTES - -/* - * This is the granularity of hardware cacheability control. - */ -#define CACHEABILITY_ALIGN 0x01000000 - -/* - * Align a physical address to MAR regions - */ -#define CACHE_REGION_START(v) \ - (((u32) (v)) & ~(CACHEABILITY_ALIGN - 1)) -#define CACHE_REGION_END(v) \ - (((u32) (v) + (CACHEABILITY_ALIGN - 1)) & ~(CACHEABILITY_ALIGN - 1)) - -extern void __init c6x_cache_init(void); - -extern void enable_caching(unsigned long start, unsigned long end); -extern void disable_caching(unsigned long start, unsigned long end); - -extern void L1_cache_off(void); -extern void L1_cache_on(void); - -extern void L1P_cache_global_invalidate(void); -extern void L1D_cache_global_invalidate(void); -extern void L1D_cache_global_writeback(void); -extern void L1D_cache_global_writeback_invalidate(void); -extern void L2_cache_set_mode(unsigned int mode); -extern void L2_cache_global_writeback_invalidate(void); -extern void L2_cache_global_writeback(void); - -extern void L1P_cache_block_invalidate(unsigned int start, unsigned int end); -extern void L1D_cache_block_invalidate(unsigned int start, unsigned int end); -extern void L1D_cache_block_writeback_invalidate(unsigned int start, - unsigned int end); -extern void L1D_cache_block_writeback(unsigned int start, unsigned int end); -extern void L2_cache_block_invalidate(unsigned int start, unsigned int end); -extern void L2_cache_block_writeback(unsigned int start, unsigned int end); -extern void L2_cache_block_writeback_invalidate(unsigned int start, - unsigned int end); -extern void L2_cache_block_invalidate_nowait(unsigned int start, - unsigned int end); -extern void L2_cache_block_writeback_nowait(unsigned int start, - unsigned int end); - -extern void L2_cache_block_writeback_invalidate_nowait(unsigned int start, - unsigned int end); - -#endif /* _ASM_C6X_CACHE_H */ diff --git a/arch/c6x/include/asm/cacheflush.h b/arch/c6x/include/asm/cacheflush.h deleted file mode 100644 index 10922d528de6..000000000000 --- a/arch/c6x/include/asm/cacheflush.h +++ /dev/null @@ -1,45 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_CACHEFLUSH_H -#define _ASM_C6X_CACHEFLUSH_H - -#include - -#include -#include -#include -#include -#include - -/* - * physically-indexed cache management - */ -#define flush_icache_range(s, e) \ -do { \ - L1D_cache_block_writeback((s), (e)); \ - L1P_cache_block_invalidate((s), (e)); \ -} while (0) - -#define flush_icache_page(vma, page) \ -do { \ - if ((vma)->vm_flags & PROT_EXEC) \ - L1D_cache_block_writeback_invalidate(page_address(page), \ - (unsigned long) page_address(page) + PAGE_SIZE)); \ - L1P_cache_block_invalidate(page_address(page), \ - (unsigned long) page_address(page) + PAGE_SIZE)); \ -} while (0) - -#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ -do { \ - memcpy(dst, src, len); \ - flush_icache_range((unsigned) (dst), (unsigned) (dst) + (len)); \ -} while (0) - -#include - -#endif /* _ASM_C6X_CACHEFLUSH_H */ diff --git a/arch/c6x/include/asm/checksum.h b/arch/c6x/include/asm/checksum.h deleted file mode 100644 index 934918def632..000000000000 --- a/arch/c6x/include/asm/checksum.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - */ -#ifndef _ASM_C6X_CHECKSUM_H -#define _ASM_C6X_CHECKSUM_H - -static inline __wsum -csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, - __u8 proto, __wsum sum) -{ - unsigned long long tmp; - - asm ("add .d1 %1,%5,%1\n" - "|| addu .l1 %3,%4,%0\n" - "addu .l1 %2,%0,%0\n" -#ifndef CONFIG_CPU_BIG_ENDIAN - "|| shl .s1 %1,8,%1\n" -#endif - "addu .l1 %1,%0,%0\n" - "add .l1 %P0,%p0,%2\n" - : "=&a"(tmp), "+a"(len), "+a"(sum) - : "a" (saddr), "a" (daddr), "a" (proto)); - return sum; -} -#define csum_tcpudp_nofold csum_tcpudp_nofold - -#define _HAVE_ARCH_CSUM_AND_COPY -extern __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len); - -#include - -#endif /* _ASM_C6X_CHECKSUM_H */ diff --git a/arch/c6x/include/asm/clock.h b/arch/c6x/include/asm/clock.h deleted file mode 100644 index 7b6c42a52ec9..000000000000 --- a/arch/c6x/include/asm/clock.h +++ /dev/null @@ -1,145 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * TI C64X clock definitions - * - * Copyright (C) 2010, 2011 Texas Instruments. - * Contributed by: Mark Salter - * - * Copied heavily from arm/mach-davinci/clock.h, so: - * - * Copyright (C) 2006-2007 Texas Instruments. - * Copyright (C) 2008-2009 Deep Root Systems, LLC - */ - -#ifndef _ASM_C6X_CLOCK_H -#define _ASM_C6X_CLOCK_H - -#ifndef __ASSEMBLER__ - -#include - -/* PLL/Reset register offsets */ -#define PLLCTL 0x100 -#define PLLM 0x110 -#define PLLPRE 0x114 -#define PLLDIV1 0x118 -#define PLLDIV2 0x11c -#define PLLDIV3 0x120 -#define PLLPOST 0x128 -#define PLLCMD 0x138 -#define PLLSTAT 0x13c -#define PLLALNCTL 0x140 -#define PLLDCHANGE 0x144 -#define PLLCKEN 0x148 -#define PLLCKSTAT 0x14c -#define PLLSYSTAT 0x150 -#define PLLDIV4 0x160 -#define PLLDIV5 0x164 -#define PLLDIV6 0x168 -#define PLLDIV7 0x16c -#define PLLDIV8 0x170 -#define PLLDIV9 0x174 -#define PLLDIV10 0x178 -#define PLLDIV11 0x17c -#define PLLDIV12 0x180 -#define PLLDIV13 0x184 -#define PLLDIV14 0x188 -#define PLLDIV15 0x18c -#define PLLDIV16 0x190 - -/* PLLM register bits */ -#define PLLM_PLLM_MASK 0xff -#define PLLM_VAL(x) ((x) - 1) - -/* PREDIV register bits */ -#define PLLPREDIV_EN BIT(15) -#define PLLPREDIV_VAL(x) ((x) - 1) - -/* PLLCTL register bits */ -#define PLLCTL_PLLEN BIT(0) -#define PLLCTL_PLLPWRDN BIT(1) -#define PLLCTL_PLLRST BIT(3) -#define PLLCTL_PLLDIS BIT(4) -#define PLLCTL_PLLENSRC BIT(5) -#define PLLCTL_CLKMODE BIT(8) - -/* PLLCMD register bits */ -#define PLLCMD_GOSTAT BIT(0) - -/* PLLSTAT register bits */ -#define PLLSTAT_GOSTAT BIT(0) - -/* PLLDIV register bits */ -#define PLLDIV_EN BIT(15) -#define PLLDIV_RATIO_MASK 0x1f -#define PLLDIV_RATIO(x) ((x) - 1) - -struct pll_data; - -struct clk { - struct list_head node; - struct module *owner; - const char *name; - unsigned long rate; - int usecount; - u32 flags; - struct clk *parent; - struct list_head children; /* list of children */ - struct list_head childnode; /* parent's child list node */ - struct pll_data *pll_data; - u32 div; - unsigned long (*recalc) (struct clk *); - int (*set_rate) (struct clk *clk, unsigned long rate); - int (*round_rate) (struct clk *clk, unsigned long rate); -}; - -/* Clock flags: SoC-specific flags start at BIT(16) */ -#define ALWAYS_ENABLED BIT(1) -#define CLK_PLL BIT(2) /* PLL-derived clock */ -#define PRE_PLL BIT(3) /* source is before PLL mult/div */ -#define FIXED_DIV_PLL BIT(4) /* fixed divisor from PLL */ -#define FIXED_RATE_PLL BIT(5) /* fixed output rate PLL */ - -#define MAX_PLL_SYSCLKS 16 - -struct pll_data { - void __iomem *base; - u32 num; - u32 flags; - u32 input_rate; - u32 bypass_delay; /* in loops */ - u32 reset_delay; /* in loops */ - u32 lock_delay; /* in loops */ - struct clk sysclks[MAX_PLL_SYSCLKS + 1]; -}; - -/* pll_data flag bit */ -#define PLL_HAS_PRE BIT(0) -#define PLL_HAS_MUL BIT(1) -#define PLL_HAS_POST BIT(2) - -#define CLK(dev, con, ck) \ - { \ - .dev_id = dev, \ - .con_id = con, \ - .clk = ck, \ - } \ - -extern void c6x_clks_init(struct clk_lookup *clocks); -extern int clk_register(struct clk *clk); -extern void clk_unregister(struct clk *clk); -extern void c64x_setup_clocks(void); - -extern struct pll_data c6x_soc_pll1; - -extern struct clk clkin1; -extern struct clk c6x_core_clk; -extern struct clk c6x_i2c_clk; -extern struct clk c6x_watchdog_clk; -extern struct clk c6x_mcbsp1_clk; -extern struct clk c6x_mcbsp2_clk; -extern struct clk c6x_mdio_clk; - -#endif - -#endif /* _ASM_C6X_CLOCK_H */ diff --git a/arch/c6x/include/asm/cmpxchg.h b/arch/c6x/include/asm/cmpxchg.h deleted file mode 100644 index 6eed628a9e7f..000000000000 --- a/arch/c6x/include/asm/cmpxchg.h +++ /dev/null @@ -1,63 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_CMPXCHG_H -#define _ASM_C6X_CMPXCHG_H - -#include - -/* - * Misc. functions - */ -static inline unsigned int __xchg(unsigned int x, volatile void *ptr, int size) -{ - unsigned int tmp; - unsigned long flags; - - local_irq_save(flags); - - switch (size) { - case 1: - tmp = 0; - tmp = *((unsigned char *) ptr); - *((unsigned char *) ptr) = (unsigned char) x; - break; - case 2: - tmp = 0; - tmp = *((unsigned short *) ptr); - *((unsigned short *) ptr) = x; - break; - case 4: - tmp = 0; - tmp = *((unsigned int *) ptr); - *((unsigned int *) ptr) = x; - break; - } - local_irq_restore(flags); - return tmp; -} - -#define xchg(ptr, x) \ - ((__typeof__(*(ptr)))__xchg((unsigned int)(x), (void *) (ptr), \ - sizeof(*(ptr)))) - -#include - -/* - * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make - * them available. - */ -#define cmpxchg_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), \ - (unsigned long)(o), \ - (unsigned long)(n), \ - sizeof(*(ptr)))) -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) - -#include - -#endif /* _ASM_C6X_CMPXCHG_H */ diff --git a/arch/c6x/include/asm/delay.h b/arch/c6x/include/asm/delay.h deleted file mode 100644 index 455fc713ae54..000000000000 --- a/arch/c6x/include/asm/delay.h +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_DELAY_H -#define _ASM_C6X_DELAY_H - -#include - -extern unsigned int ticks_per_ns_scaled; - -static inline void __delay(unsigned long loops) -{ - uint32_t tmp; - - /* 6 cycles per loop */ - asm volatile (" mv .s1 %0,%1\n" - "0: [%1] b .s1 0b\n" - " add .l1 -6,%0,%0\n" - " cmplt .l1 1,%0,%1\n" - " nop 3\n" - : "+a"(loops), "=A"(tmp)); -} - -static inline void _c6x_tickdelay(unsigned int x) -{ - uint32_t cnt, endcnt; - - asm volatile (" mvc .s2 TSCL,%0\n" - " add .s2x %0,%1,%2\n" - " || mvk .l2 1,B0\n" - "0: [B0] b .s2 0b\n" - " mvc .s2 TSCL,%0\n" - " sub .s2 %0,%2,%0\n" - " cmpgt .l2 0,%0,B0\n" - " nop 2\n" - : "=b"(cnt), "+a"(x), "=b"(endcnt) : : "B0"); -} - -/* use scaled math to avoid slow division */ -#define C6X_NDELAY_SCALE 10 - -static inline void _ndelay(unsigned int n) -{ - _c6x_tickdelay((ticks_per_ns_scaled * n) >> C6X_NDELAY_SCALE); -} - -static inline void _udelay(unsigned int n) -{ - while (n >= 10) { - _ndelay(10000); - n -= 10; - } - while (n-- > 0) - _ndelay(1000); -} - -#define udelay(x) _udelay((unsigned int)(x)) -#define ndelay(x) _ndelay((unsigned int)(x)) - -#endif /* _ASM_C6X_DELAY_H */ diff --git a/arch/c6x/include/asm/dscr.h b/arch/c6x/include/asm/dscr.h deleted file mode 100644 index f6b095c3d3f5..000000000000 --- a/arch/c6x/include/asm/dscr.h +++ /dev/null @@ -1,30 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - */ -#ifndef _ASM_C6X_DSCR_H -#define _ASM_C6X_DSCR_H - -enum dscr_devstate_t { - DSCR_DEVSTATE_ENABLED, - DSCR_DEVSTATE_DISABLED, -}; - -/* - * Set the device state of the device with the given ID. - * - * Individual drivers should use this to enable or disable the - * hardware device. The devid used to identify the device being - * controlled should be a property in the device's tree node. - */ -extern void dscr_set_devstate(int devid, enum dscr_devstate_t state); - -/* - * Assert or de-assert an RMII reset. - */ -extern void dscr_rmii_reset(int id, int assert); - -extern void dscr_probe(void); - -#endif /* _ASM_C6X_DSCR_H */ diff --git a/arch/c6x/include/asm/elf.h b/arch/c6x/include/asm/elf.h deleted file mode 100644 index ca88acbf560b..000000000000 --- a/arch/c6x/include/asm/elf.h +++ /dev/null @@ -1,117 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_ELF_H -#define _ASM_C6X_ELF_H - -/* - * ELF register definitions.. - */ -#include - -typedef unsigned long elf_greg_t; -typedef unsigned long elf_fpreg_t; - -#define ELF_NGREG 58 -#define ELF_NFPREG 1 - -typedef elf_greg_t elf_gregset_t[ELF_NGREG]; -typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; - -/* - * This is used to ensure we don't load something for the wrong architecture. - */ -#define elf_check_arch(x) ((x)->e_machine == EM_TI_C6000) - -#define elf_check_fdpic(x) (1) -#define elf_check_const_displacement(x) (0) - -#define ELF_FDPIC_PLAT_INIT(_regs, _exec_map, _interp_map, _dynamic_addr) \ -do { \ - _regs->b4 = (_exec_map); \ - _regs->a6 = (_interp_map); \ - _regs->b6 = (_dynamic_addr); \ -} while (0) - -#define ELF_FDPIC_CORE_EFLAGS 0 - -/* - * These are used to set parameters in the core dumps. - */ -#ifdef __LITTLE_ENDIAN__ -#define ELF_DATA ELFDATA2LSB -#else -#define ELF_DATA ELFDATA2MSB -#endif - -#define ELF_CLASS ELFCLASS32 -#define ELF_ARCH EM_TI_C6000 - -/* Nothing for now. Need to setup DP... */ -#define ELF_PLAT_INIT(_r) - -#define ELF_EXEC_PAGESIZE 4096 - -#define ELF_CORE_COPY_REGS(_dest, _regs) \ - memcpy((char *) &_dest, (char *) _regs, \ - sizeof(struct pt_regs)); - -/* This yields a mask that user programs can use to figure out what - instruction set this cpu supports. */ - -#define ELF_HWCAP (0) - -/* This yields a string that ld.so will use to load implementation - specific libraries for optimization. This is more specific in - intent than poking at uname or /proc/cpuinfo. */ - -#define ELF_PLATFORM (NULL) - -/* C6X specific section types */ -#define SHT_C6000_UNWIND 0x70000001 -#define SHT_C6000_PREEMPTMAP 0x70000002 -#define SHT_C6000_ATTRIBUTES 0x70000003 - -/* C6X specific DT_ tags */ -#define DT_C6000_DSBT_BASE 0x70000000 -#define DT_C6000_DSBT_SIZE 0x70000001 -#define DT_C6000_PREEMPTMAP 0x70000002 -#define DT_C6000_DSBT_INDEX 0x70000003 - -/* C6X specific relocs */ -#define R_C6000_NONE 0 -#define R_C6000_ABS32 1 -#define R_C6000_ABS16 2 -#define R_C6000_ABS8 3 -#define R_C6000_PCR_S21 4 -#define R_C6000_PCR_S12 5 -#define R_C6000_PCR_S10 6 -#define R_C6000_PCR_S7 7 -#define R_C6000_ABS_S16 8 -#define R_C6000_ABS_L16 9 -#define R_C6000_ABS_H16 10 -#define R_C6000_SBR_U15_B 11 -#define R_C6000_SBR_U15_H 12 -#define R_C6000_SBR_U15_W 13 -#define R_C6000_SBR_S16 14 -#define R_C6000_SBR_L16_B 15 -#define R_C6000_SBR_L16_H 16 -#define R_C6000_SBR_L16_W 17 -#define R_C6000_SBR_H16_B 18 -#define R_C6000_SBR_H16_H 19 -#define R_C6000_SBR_H16_W 20 -#define R_C6000_SBR_GOT_U15_W 21 -#define R_C6000_SBR_GOT_L16_W 22 -#define R_C6000_SBR_GOT_H16_W 23 -#define R_C6000_DSBT_INDEX 24 -#define R_C6000_PREL31 25 -#define R_C6000_COPY 26 -#define R_C6000_ALIGN 253 -#define R_C6000_FPHEAD 254 -#define R_C6000_NOCMP 255 - -#endif /*_ASM_C6X_ELF_H */ diff --git a/arch/c6x/include/asm/flat.h b/arch/c6x/include/asm/flat.h deleted file mode 100644 index 9e6544b51386..000000000000 --- a/arch/c6x/include/asm/flat.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_C6X_FLAT_H -#define __ASM_C6X_FLAT_H - -#include - -static inline int flat_get_addr_from_rp(u32 __user *rp, u32 relval, u32 flags, - u32 *addr) -{ - *addr = get_unaligned((__force u32 *)rp); - return 0; -} -static inline int flat_put_addr_at_rp(u32 __user *rp, u32 addr, u32 rel) -{ - put_unaligned(addr, (__force u32 *)rp); - return 0; -} - -#endif /* __ASM_C6X_FLAT_H */ diff --git a/arch/c6x/include/asm/ftrace.h b/arch/c6x/include/asm/ftrace.h deleted file mode 100644 index 3701958d3d1c..000000000000 --- a/arch/c6x/include/asm/ftrace.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _ASM_C6X_FTRACE_H -#define _ASM_C6X_FTRACE_H - -/* empty */ - -#endif /* _ASM_C6X_FTRACE_H */ diff --git a/arch/c6x/include/asm/hardirq.h b/arch/c6x/include/asm/hardirq.h deleted file mode 100644 index f37d07d31040..000000000000 --- a/arch/c6x/include/asm/hardirq.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ - -#ifndef _ASM_C6X_HARDIRQ_H -#define _ASM_C6X_HARDIRQ_H - -extern void ack_bad_irq(int irq); -#define ack_bad_irq ack_bad_irq - -#include - -#endif /* _ASM_C6X_HARDIRQ_H */ diff --git a/arch/c6x/include/asm/irq.h b/arch/c6x/include/asm/irq.h deleted file mode 100644 index 9da4d1afd0d7..000000000000 --- a/arch/c6x/include/asm/irq.h +++ /dev/null @@ -1,50 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2006, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * - * Large parts taken directly from powerpc. - */ -#ifndef _ASM_C6X_IRQ_H -#define _ASM_C6X_IRQ_H - -#include -#include -#include -#include -#include - -#define irq_canonicalize(irq) (irq) - -/* - * The C64X+ core has 16 IRQ vectors. One each is used by Reset and NMI. Two - * are reserved. The remaining 12 vectors are used to route SoC interrupts. - * These interrupt vectors are prioritized with IRQ 4 having the highest - * priority and IRQ 15 having the lowest. - * - * The C64x+ megamodule provides a PIC which combines SoC IRQ sources into a - * single core IRQ vector. There are four combined sources, each of which - * feed into one of the 12 general interrupt vectors. The remaining 8 vectors - * can each route a single SoC interrupt directly. - */ -#define NR_PRIORITY_IRQS 16 - -/* Total number of virq in the platform */ -#define NR_IRQS 256 - -/* This number is used when no interrupt has been assigned */ -#define NO_IRQ 0 - -extern void __init init_pic_c64xplus(void); - -extern void init_IRQ(void); - -struct pt_regs; - -extern asmlinkage void c6x_do_IRQ(unsigned int prio, struct pt_regs *regs); - -extern unsigned long irq_err_count; - -#endif /* _ASM_C6X_IRQ_H */ diff --git a/arch/c6x/include/asm/irqflags.h b/arch/c6x/include/asm/irqflags.h deleted file mode 100644 index d6cd71c02629..000000000000 --- a/arch/c6x/include/asm/irqflags.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * C6X IRQ flag handling - * - * Copyright (C) 2010 Texas Instruments Incorporated - * Written by Mark Salter (msalter@redhat.com) - */ - -#ifndef _ASM_IRQFLAGS_H -#define _ASM_IRQFLAGS_H - -#ifndef __ASSEMBLY__ - -/* read interrupt enabled status */ -static inline unsigned long arch_local_save_flags(void) -{ - unsigned long flags; - - asm volatile (" mvc .s2 CSR,%0\n" : "=b"(flags)); - return flags; -} - -/* set interrupt enabled status */ -static inline void arch_local_irq_restore(unsigned long flags) -{ - asm volatile (" mvc .s2 %0,CSR\n" : : "b"(flags) : "memory"); -} - -/* unconditionally enable interrupts */ -static inline void arch_local_irq_enable(void) -{ - unsigned long flags = arch_local_save_flags(); - flags |= 1; - arch_local_irq_restore(flags); -} - -/* unconditionally disable interrupts */ -static inline void arch_local_irq_disable(void) -{ - unsigned long flags = arch_local_save_flags(); - flags &= ~1; - arch_local_irq_restore(flags); -} - -/* get status and disable interrupts */ -static inline unsigned long arch_local_irq_save(void) -{ - unsigned long flags; - - flags = arch_local_save_flags(); - arch_local_irq_restore(flags & ~1); - return flags; -} - -/* test flags */ -static inline int arch_irqs_disabled_flags(unsigned long flags) -{ - return (flags & 1) == 0; -} - -/* test hardware interrupt enable bit */ -static inline int arch_irqs_disabled(void) -{ - return arch_irqs_disabled_flags(arch_local_save_flags()); -} - -#endif /* __ASSEMBLY__ */ -#endif /* __ASM_IRQFLAGS_H */ diff --git a/arch/c6x/include/asm/linkage.h b/arch/c6x/include/asm/linkage.h deleted file mode 100644 index 1ad615da6479..000000000000 --- a/arch/c6x/include/asm/linkage.h +++ /dev/null @@ -1,31 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_C6X_LINKAGE_H -#define _ASM_C6X_LINKAGE_H - -#ifdef __ASSEMBLER__ - -#define __ALIGN .align 2 -#define __ALIGN_STR ".align 2" - -#ifndef __DSBT__ -#define ENTRY(name) \ - .global name @ \ - __ALIGN @ \ -name: -#else -#define ENTRY(name) \ - .global name @ \ - .hidden name @ \ - __ALIGN @ \ -name: -#endif - -#define ENDPROC(name) \ - .type name, @function @ \ - .size name, . - name - -#endif - -#include - -#endif /* _ASM_C6X_LINKAGE_H */ diff --git a/arch/c6x/include/asm/megamod-pic.h b/arch/c6x/include/asm/megamod-pic.h deleted file mode 100644 index a0a6d596bf9b..000000000000 --- a/arch/c6x/include/asm/megamod-pic.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _C6X_MEGAMOD_PIC_H -#define _C6X_MEGAMOD_PIC_H - -#ifdef __KERNEL__ - -extern void __init megamod_pic_init(void); - -#endif /* __KERNEL__ */ -#endif /* _C6X_MEGAMOD_PIC_H */ diff --git a/arch/c6x/include/asm/mmu_context.h b/arch/c6x/include/asm/mmu_context.h deleted file mode 100644 index d2659d0a3297..000000000000 --- a/arch/c6x/include/asm/mmu_context.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _ASM_C6X_MMU_CONTEXT_H -#define _ASM_C6X_MMU_CONTEXT_H - -#include - -#endif /* _ASM_C6X_MMU_CONTEXT_H */ diff --git a/arch/c6x/include/asm/module.h b/arch/c6x/include/asm/module.h deleted file mode 100644 index 9fc9f4a8ecc2..000000000000 --- a/arch/c6x/include/asm/module.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * - * Updated for 2.6.34 by: Mark Salter (msalter@redhat.com) - */ -#ifndef _ASM_C6X_MODULE_H -#define _ASM_C6X_MODULE_H - -#include - -struct loaded_sections { - unsigned int new_vaddr; - unsigned int loaded; -}; - -#endif /* _ASM_C6X_MODULE_H */ diff --git a/arch/c6x/include/asm/page.h b/arch/c6x/include/asm/page.h deleted file mode 100644 index 40079899084d..000000000000 --- a/arch/c6x/include/asm/page.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_C6X_PAGE_H -#define _ASM_C6X_PAGE_H - -#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_TSK_EXEC - -#include - -#endif /* _ASM_C6X_PAGE_H */ diff --git a/arch/c6x/include/asm/pgtable.h b/arch/c6x/include/asm/pgtable.h deleted file mode 100644 index 8a91ceda39fa..000000000000 --- a/arch/c6x/include/asm/pgtable.h +++ /dev/null @@ -1,66 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_PGTABLE_H -#define _ASM_C6X_PGTABLE_H - -#include - -#include -#include - -/* - * All 32bit addresses are effectively valid for vmalloc... - * Sort of meaningless for non-VM targets. - */ -#define VMALLOC_START 0 -#define VMALLOC_END 0xffffffff - -#define pgd_present(pgd) (1) -#define pgd_none(pgd) (0) -#define pgd_bad(pgd) (0) -#define pgd_clear(pgdp) -#define kern_addr_valid(addr) (1) - -#define pmd_none(x) (!pmd_val(x)) -#define pmd_present(x) (pmd_val(x)) -#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0) -#define pmd_bad(x) (pmd_val(x) & ~PAGE_MASK) - -#define PAGE_NONE __pgprot(0) /* these mean nothing to NO_MM */ -#define PAGE_SHARED __pgprot(0) /* these mean nothing to NO_MM */ -#define PAGE_COPY __pgprot(0) /* these mean nothing to NO_MM */ -#define PAGE_READONLY __pgprot(0) /* these mean nothing to NO_MM */ -#define PAGE_KERNEL __pgprot(0) /* these mean nothing to NO_MM */ -#define pgprot_noncached(prot) (prot) - -extern void paging_init(void); - -#define __swp_type(x) (0) -#define __swp_offset(x) (0) -#define __swp_entry(typ, off) ((swp_entry_t) { ((typ) | ((off) << 7)) }) -#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) -#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) - -#define set_pte(pteptr, pteval) (*(pteptr) = pteval) -#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval) - -/* - * ZERO_PAGE is a global shared page that is always zero: used - * for zero-mapped memory areas etc.. - */ -#define ZERO_PAGE(vaddr) virt_to_page(empty_zero_page) -extern unsigned long empty_zero_page; - -#define swapper_pg_dir ((pgd_t *) 0) - -/* - * c6x is !MMU, so define the simpliest implementation - */ -#define pgprot_writecombine pgprot_noncached - -#endif /* _ASM_C6X_PGTABLE_H */ diff --git a/arch/c6x/include/asm/processor.h b/arch/c6x/include/asm/processor.h deleted file mode 100644 index 1456f5e11de3..000000000000 --- a/arch/c6x/include/asm/processor.h +++ /dev/null @@ -1,114 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * - * Updated for 2.6.34: Mark Salter - */ -#ifndef _ASM_C6X_PROCESSOR_H -#define _ASM_C6X_PROCESSOR_H - -#include -#include -#include - -/* - * User space process size. This is mostly meaningless for NOMMU - * but some C6X processors may have RAM addresses up to 0xFFFFFFFF. - * Since calls like mmap() can return an address or an error, we - * have to allow room for error returns when code does something - * like: - * - * addr = do_mmap(...) - * if ((unsigned long)addr >= TASK_SIZE) - * ... its an error code, not an address ... - * - * Here, we allow for 4096 error codes which means we really can't - * use the last 4K page on systems with RAM extending all the way - * to the end of the 32-bit address space. - */ -#define TASK_SIZE 0xFFFFF000 - -/* - * This decides where the kernel will search for a free chunk of vm - * space during mmap's. We won't be using it - */ -#define TASK_UNMAPPED_BASE 0 - -struct thread_struct { - unsigned long long b15_14; - unsigned long long a15_14; - unsigned long long b13_12; - unsigned long long a13_12; - unsigned long long b11_10; - unsigned long long a11_10; - unsigned long long ricl_icl; - unsigned long usp; /* user stack pointer */ - unsigned long pc; /* kernel pc */ - unsigned long wchan; -}; - -#define INIT_THREAD \ -{ \ - .usp = 0, \ - .wchan = 0, \ -} - -#define INIT_MMAP { \ - &init_mm, 0, 0, NULL, PAGE_SHARED, VM_READ | VM_WRITE | VM_EXEC, 1, \ - NULL, NULL } - -#define task_pt_regs(task) \ - ((struct pt_regs *)(THREAD_START_SP + task_stack_page(task)) - 1) - -#define alloc_kernel_stack() __get_free_page(GFP_KERNEL) -#define free_kernel_stack(page) free_page((page)) - - -/* Forward declaration, a strange C thing */ -struct task_struct; - -extern void start_thread(struct pt_regs *regs, unsigned int pc, - unsigned long usp); - -/* Free all resources held by a thread. */ -static inline void release_thread(struct task_struct *dead_task) -{ -} - -/* - * saved kernel SP and DP of a blocked thread. - */ -#ifdef _BIG_ENDIAN -#define thread_saved_ksp(tsk) \ - (*(unsigned long *)&(tsk)->thread.b15_14) -#define thread_saved_dp(tsk) \ - (*(((unsigned long *)&(tsk)->thread.b15_14) + 1)) -#else -#define thread_saved_ksp(tsk) \ - (*(((unsigned long *)&(tsk)->thread.b15_14) + 1)) -#define thread_saved_dp(tsk) \ - (*(unsigned long *)&(tsk)->thread.b15_14) -#endif - -extern unsigned long get_wchan(struct task_struct *p); - -#define KSTK_EIP(task) (task_pt_regs(task)->pc) -#define KSTK_ESP(task) (task_pt_regs(task)->sp) - -#define cpu_relax() do { } while (0) - -extern const struct seq_operations cpuinfo_op; - -/* Reset the board */ -#define HARD_RESET_NOW() - -extern unsigned int c6x_core_freq; - - -extern void (*c6x_restart)(void); -extern void (*c6x_halt)(void); - -#endif /* ASM_C6X_PROCESSOR_H */ diff --git a/arch/c6x/include/asm/procinfo.h b/arch/c6x/include/asm/procinfo.h deleted file mode 100644 index aaa3cb902c43..000000000000 --- a/arch/c6x/include/asm/procinfo.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2010 Texas Instruments Incorporated - * Author: Mark Salter (msalter@redhat.com) - */ -#ifndef _ASM_C6X_PROCINFO_H -#define _ASM_C6X_PROCINFO_H - -#ifdef __KERNEL__ - -struct proc_info_list { - unsigned int cpu_val; - unsigned int cpu_mask; - const char *arch_name; - const char *elf_name; - unsigned int elf_hwcap; -}; - -#else /* __KERNEL__ */ -#include -#warning "Please include asm/elf.h instead" -#endif /* __KERNEL__ */ - -#endif /* _ASM_C6X_PROCINFO_H */ diff --git a/arch/c6x/include/asm/ptrace.h b/arch/c6x/include/asm/ptrace.h deleted file mode 100644 index 7cbae382cf37..000000000000 --- a/arch/c6x/include/asm/ptrace.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2004, 2006, 2009, 2010 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * - * Updated for 2.6.34: Mark Salter - */ -#ifndef _ASM_C6X_PTRACE_H -#define _ASM_C6X_PTRACE_H - -#include - -#ifndef __ASSEMBLY__ -#ifdef _BIG_ENDIAN -#else -#endif - -#include - -#define user_mode(regs) ((((regs)->tsr) & 0x40) != 0) - -#define instruction_pointer(regs) ((regs)->pc) -#define profile_pc(regs) instruction_pointer(regs) -#define user_stack_pointer(regs) ((regs)->sp) - -extern void show_regs(struct pt_regs *); - -extern asmlinkage unsigned long syscall_trace_entry(struct pt_regs *regs); -extern asmlinkage void syscall_trace_exit(struct pt_regs *regs); - -#endif /* __ASSEMBLY__ */ -#endif /* _ASM_C6X_PTRACE_H */ diff --git a/arch/c6x/include/asm/sections.h b/arch/c6x/include/asm/sections.h deleted file mode 100644 index dc2f15eb3bde..000000000000 --- a/arch/c6x/include/asm/sections.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_C6X_SECTIONS_H -#define _ASM_C6X_SECTIONS_H - -#include - -extern char _vectors_start[]; -extern char _vectors_end[]; - -extern char _data_lma[]; - -#endif /* _ASM_C6X_SECTIONS_H */ diff --git a/arch/c6x/include/asm/setup.h b/arch/c6x/include/asm/setup.h deleted file mode 100644 index 5496bccecaa0..000000000000 --- a/arch/c6x/include/asm/setup.h +++ /dev/null @@ -1,31 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_SETUP_H -#define _ASM_C6X_SETUP_H - -#include -#include - -#ifndef __ASSEMBLY__ -extern int c6x_add_memory(phys_addr_t start, unsigned long size); - -extern unsigned long ram_start; -extern unsigned long ram_end; - -extern int c6x_num_cores; -extern unsigned int c6x_silicon_rev; -extern unsigned int c6x_devstat; -extern unsigned char c6x_fuse_mac[6]; - -extern void machine_init(unsigned long dt_ptr); -extern void time_init(void); - -extern void coherent_mem_init(u32 start, u32 size); - -#endif /* !__ASSEMBLY__ */ -#endif /* _ASM_C6X_SETUP_H */ diff --git a/arch/c6x/include/asm/soc.h b/arch/c6x/include/asm/soc.h deleted file mode 100644 index 43f50159e59b..000000000000 --- a/arch/c6x/include/asm/soc.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Miscellaneous SoC-specific hooks. - * - * Copyright (C) 2011 Texas Instruments Incorporated - * - * Author: Mark Salter - * - * This file is licensed under the terms of the GNU General Public License - * version 2. This program is licensed "as is" without any warranty of any - * kind, whether express or implied. - */ -#ifndef _ASM_C6X_SOC_H -#define _ASM_C6X_SOC_H - -struct soc_ops { - /* Return active exception event or -1 if none */ - int (*get_exception)(void); - - /* Assert an event */ - void (*assert_event)(unsigned int evt); -}; - -extern struct soc_ops soc_ops; - -extern int soc_get_exception(void); -extern void soc_assert_event(unsigned int event); -extern int soc_mac_addr(unsigned int index, u8 *addr); - -/* - * for mmio on SoC devices. regs are always same byte order as cpu. - */ -#define soc_readl(addr) __raw_readl(addr) -#define soc_writel(b, addr) __raw_writel((b), (addr)) - -#endif /* _ASM_C6X_SOC_H */ diff --git a/arch/c6x/include/asm/special_insns.h b/arch/c6x/include/asm/special_insns.h deleted file mode 100644 index d233160aefd4..000000000000 --- a/arch/c6x/include/asm/special_insns.h +++ /dev/null @@ -1,60 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_SPECIAL_INSNS_H -#define _ASM_C6X_SPECIAL_INSNS_H - - -#define get_creg(reg) \ - ({ unsigned int __x; \ - asm volatile ("mvc .s2 " #reg ",%0\n" : "=b"(__x)); __x; }) - -#define set_creg(reg, v) \ - do { unsigned int __x = (unsigned int)(v); \ - asm volatile ("mvc .s2 %0," #reg "\n" : : "b"(__x)); \ - } while (0) - -#define or_creg(reg, n) \ - do { unsigned __x, __n = (unsigned)(n); \ - asm volatile ("mvc .s2 " #reg ",%0\n" \ - "or .l2 %1,%0,%0\n" \ - "mvc .s2 %0," #reg "\n" \ - "nop\n" \ - : "=&b"(__x) : "b"(__n)); \ - } while (0) - -#define and_creg(reg, n) \ - do { unsigned __x, __n = (unsigned)(n); \ - asm volatile ("mvc .s2 " #reg ",%0\n" \ - "and .l2 %1,%0,%0\n" \ - "mvc .s2 %0," #reg "\n" \ - "nop\n" \ - : "=&b"(__x) : "b"(__n)); \ - } while (0) - -#define get_coreid() (get_creg(DNUM) & 0xff) - -/* Set/get IST */ -#define set_ist(x) set_creg(ISTP, x) -#define get_ist() get_creg(ISTP) - -/* - * Exception management - */ -#define disable_exception() -#define get_except_type() get_creg(EFR) -#define ack_exception(type) set_creg(ECR, 1 << (type)) -#define get_iexcept() get_creg(IERR) -#define set_iexcept(mask) set_creg(IERR, (mask)) - -#define _extu(x, s, e) \ - ({ unsigned int __x; \ - asm volatile ("extu .S2 %3,%1,%2,%0\n" : \ - "=b"(__x) : "n"(s), "n"(e), "b"(x)); \ - __x; }) - -#endif /* _ASM_C6X_SPECIAL_INSNS_H */ diff --git a/arch/c6x/include/asm/string.h b/arch/c6x/include/asm/string.h deleted file mode 100644 index b290ead40f68..000000000000 --- a/arch/c6x/include/asm/string.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_STRING_H -#define _ASM_C6X_STRING_H - -#include -#include - -asmlinkage extern void *memcpy(void *to, const void *from, size_t n); - -#define __HAVE_ARCH_MEMCPY - -#endif /* _ASM_C6X_STRING_H */ diff --git a/arch/c6x/include/asm/switch_to.h b/arch/c6x/include/asm/switch_to.h deleted file mode 100644 index 36c5332fadae..000000000000 --- a/arch/c6x/include/asm/switch_to.h +++ /dev/null @@ -1,30 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_SWITCH_TO_H -#define _ASM_C6X_SWITCH_TO_H - -#include - -#define prepare_to_switch() do { } while (0) - -struct task_struct; -struct thread_struct; -asmlinkage void *__switch_to(struct thread_struct *prev, - struct thread_struct *next, - struct task_struct *tsk); - -#define switch_to(prev, next, last) \ - do { \ - current->thread.wchan = (u_long) __builtin_return_address(0); \ - (last) = __switch_to(&(prev)->thread, \ - &(next)->thread, (prev)); \ - mb(); \ - current->thread.wchan = 0; \ - } while (0) - -#endif /* _ASM_C6X_SWITCH_TO_H */ diff --git a/arch/c6x/include/asm/syscall.h b/arch/c6x/include/asm/syscall.h deleted file mode 100644 index 38f3e2284ecd..000000000000 --- a/arch/c6x/include/asm/syscall.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - */ - -#ifndef __ASM_C6X_SYSCALL_H -#define __ASM_C6X_SYSCALL_H - -#include -#include -#include - -static inline int syscall_get_nr(struct task_struct *task, - struct pt_regs *regs) -{ - return regs->b0; -} - -static inline void syscall_rollback(struct task_struct *task, - struct pt_regs *regs) -{ - /* do nothing */ -} - -static inline long syscall_get_error(struct task_struct *task, - struct pt_regs *regs) -{ - return IS_ERR_VALUE(regs->a4) ? regs->a4 : 0; -} - -static inline long syscall_get_return_value(struct task_struct *task, - struct pt_regs *regs) -{ - return regs->a4; -} - -static inline void syscall_set_return_value(struct task_struct *task, - struct pt_regs *regs, - int error, long val) -{ - regs->a4 = error ?: val; -} - -static inline void syscall_get_arguments(struct task_struct *task, - struct pt_regs *regs, - unsigned long *args) -{ - *args++ = regs->a4; - *args++ = regs->b4; - *args++ = regs->a6; - *args++ = regs->b6; - *args++ = regs->a8; - *args = regs->b8; -} - -static inline void syscall_set_arguments(struct task_struct *task, - struct pt_regs *regs, - const unsigned long *args) -{ - regs->a4 = *args++; - regs->b4 = *args++; - regs->a6 = *args++; - regs->b6 = *args++; - regs->a8 = *args++; - regs->a9 = *args; -} - -static inline int syscall_get_arch(struct task_struct *task) -{ - return IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) - ? AUDIT_ARCH_C6XBE : AUDIT_ARCH_C6X; -} - -#endif /* __ASM_C6X_SYSCALLS_H */ diff --git a/arch/c6x/include/asm/syscalls.h b/arch/c6x/include/asm/syscalls.h deleted file mode 100644 index df3d05feb153..000000000000 --- a/arch/c6x/include/asm/syscalls.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, version 2. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or - * NON INFRINGEMENT. See the GNU General Public License for - * more details. - */ - -#ifndef __ASM_C6X_SYSCALLS_H -#define __ASM_C6X_SYSCALLS_H - -#include -#include -#include - -/* The array of function pointers for syscalls. */ -extern void *sys_call_table[]; - -/* The following are trampolines in entry.S to handle 64-bit arguments */ -extern long sys_pread_c6x(unsigned int fd, char __user *buf, - size_t count, off_t pos_low, off_t pos_high); -extern long sys_pwrite_c6x(unsigned int fd, const char __user *buf, - size_t count, off_t pos_low, off_t pos_high); -extern long sys_truncate64_c6x(const char __user *path, - off_t length_low, off_t length_high); -extern long sys_ftruncate64_c6x(unsigned int fd, - off_t length_low, off_t length_high); -extern long sys_fadvise64_c6x(int fd, u32 offset_lo, u32 offset_hi, - u32 len, int advice); -extern long sys_fadvise64_64_c6x(int fd, u32 offset_lo, u32 offset_hi, - u32 len_lo, u32 len_hi, int advice); -extern long sys_fallocate_c6x(int fd, int mode, - u32 offset_lo, u32 offset_hi, - u32 len_lo, u32 len_hi); -extern int sys_cache_sync(unsigned long s, unsigned long e); - -#include - -#endif /* __ASM_C6X_SYSCALLS_H */ diff --git a/arch/c6x/include/asm/thread_info.h b/arch/c6x/include/asm/thread_info.h deleted file mode 100644 index dd8913d57189..000000000000 --- a/arch/c6x/include/asm/thread_info.h +++ /dev/null @@ -1,94 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * - * Updated for 2.6.3x: Mark Salter - */ -#ifndef _ASM_C6X_THREAD_INFO_H -#define _ASM_C6X_THREAD_INFO_H - -#ifdef __KERNEL__ - -#include - -#ifdef CONFIG_4KSTACKS -#define THREAD_SIZE 4096 -#define THREAD_SHIFT 12 -#define THREAD_SIZE_ORDER 0 -#else -#define THREAD_SIZE 8192 -#define THREAD_SHIFT 13 -#define THREAD_SIZE_ORDER 1 -#endif - -#define THREAD_START_SP (THREAD_SIZE - 8) - -#ifndef __ASSEMBLY__ - -typedef struct { - unsigned long seg; -} mm_segment_t; - -/* - * low level task data. - */ -struct thread_info { - struct task_struct *task; /* main task structure */ - unsigned long flags; /* low level flags */ - int cpu; /* cpu we're on */ - int preempt_count; /* 0 = preemptable, <0 = BUG */ - mm_segment_t addr_limit; /* thread address space */ -}; - -/* - * macros/functions for gaining access to the thread information structure - * - * preempt_count needs to be 1 initially, until the scheduler is functional. - */ -#define INIT_THREAD_INFO(tsk) \ -{ \ - .task = &tsk, \ - .flags = 0, \ - .cpu = 0, \ - .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ -} - -/* get the thread information struct of current task */ -static inline __attribute__((const)) -struct thread_info *current_thread_info(void) -{ - struct thread_info *ti; - asm volatile (" clr .s2 B15,0,%1,%0\n" - : "=b" (ti) - : "Iu5" (THREAD_SHIFT - 1)); - return ti; -} - -#define get_thread_info(ti) get_task_struct((ti)->task) -#define put_thread_info(ti) put_task_struct((ti)->task) -#endif /* __ASSEMBLY__ */ - -/* - * thread information flag bit numbers - * - pending work-to-be-done flags are in LSW - * - other flags in MSW - */ -#define TIF_SYSCALL_TRACE 0 /* syscall trace active */ -#define TIF_NOTIFY_RESUME 1 /* resumption notification requested */ -#define TIF_SIGPENDING 2 /* signal pending */ -#define TIF_NEED_RESCHED 3 /* rescheduling necessary */ -#define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */ -#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ - -#define TIF_MEMDIE 17 /* OOM killer killed process */ - -#define TIF_WORK_MASK 0x00007FFE /* work on irq/exception return */ -#define TIF_ALLWORK_MASK 0x00007FFF /* work on any return to u-space */ - -#endif /* __KERNEL__ */ - -#endif /* _ASM_C6X_THREAD_INFO_H */ diff --git a/arch/c6x/include/asm/timer64.h b/arch/c6x/include/asm/timer64.h deleted file mode 100644 index b850dfef1f79..000000000000 --- a/arch/c6x/include/asm/timer64.h +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _C6X_TIMER64_H -#define _C6X_TIMER64_H - -extern void __init timer64_init(void); - -#endif /* _C6X_TIMER64_H */ diff --git a/arch/c6x/include/asm/timex.h b/arch/c6x/include/asm/timex.h deleted file mode 100644 index f946ce297e13..000000000000 --- a/arch/c6x/include/asm/timex.h +++ /dev/null @@ -1,30 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * - * Modified for 2.6.34: Mark Salter - */ -#ifndef _ASM_C6X_TIMEX_H -#define _ASM_C6X_TIMEX_H - -#define CLOCK_TICK_RATE ((1000 * 1000000UL) / 6) - -/* 64-bit timestamp */ -typedef unsigned long long cycles_t; - -static inline cycles_t get_cycles(void) -{ - unsigned l, h; - - asm volatile (" dint\n" - " mvc .s2 TSCL,%0\n" - " mvc .s2 TSCH,%1\n" - " rint\n" - : "=b"(l), "=b"(h)); - return ((cycles_t)h << 32) | l; -} - -#endif /* _ASM_C6X_TIMEX_H */ diff --git a/arch/c6x/include/asm/tlb.h b/arch/c6x/include/asm/tlb.h deleted file mode 100644 index 240ba0febb57..000000000000 --- a/arch/c6x/include/asm/tlb.h +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_C6X_TLB_H -#define _ASM_C6X_TLB_H - -#include - -#endif /* _ASM_C6X_TLB_H */ diff --git a/arch/c6x/include/asm/traps.h b/arch/c6x/include/asm/traps.h deleted file mode 100644 index 7e1d31c47680..000000000000 --- a/arch/c6x/include/asm/traps.h +++ /dev/null @@ -1,33 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#ifndef _ASM_C6X_TRAPS_H -#define _ASM_C6X_TRAPS_H - -#define EXCEPT_TYPE_NXF 31 /* NMI */ -#define EXCEPT_TYPE_EXC 30 /* external exception */ -#define EXCEPT_TYPE_IXF 1 /* internal exception */ -#define EXCEPT_TYPE_SXF 0 /* software exception */ - -#define EXCEPT_CAUSE_LBX (1 << 7) /* loop buffer exception */ -#define EXCEPT_CAUSE_PRX (1 << 6) /* privilege exception */ -#define EXCEPT_CAUSE_RAX (1 << 5) /* resource access exception */ -#define EXCEPT_CAUSE_RCX (1 << 4) /* resource conflict exception */ -#define EXCEPT_CAUSE_OPX (1 << 3) /* opcode exception */ -#define EXCEPT_CAUSE_EPX (1 << 2) /* execute packet exception */ -#define EXCEPT_CAUSE_FPX (1 << 1) /* fetch packet exception */ -#define EXCEPT_CAUSE_IFX (1 << 0) /* instruction fetch exception */ - -struct exception_info { - char *kernel_str; - int signo; - int code; -}; - -extern int (*c6x_nmi_handler)(struct pt_regs *regs); - -#endif /* _ASM_C6X_TRAPS_H */ diff --git a/arch/c6x/include/asm/uaccess.h b/arch/c6x/include/asm/uaccess.h deleted file mode 100644 index 585adf9201b7..000000000000 --- a/arch/c6x/include/asm/uaccess.h +++ /dev/null @@ -1,97 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - */ -#ifndef _ASM_C6X_UACCESS_H -#define _ASM_C6X_UACCESS_H - -#include -#include -#include - -/* - * C6X supports unaligned 32 and 64 bit loads and stores. - */ -static inline __must_check unsigned long -raw_copy_from_user(void *to, const void __user *from, unsigned long n) -{ - u32 tmp32; - u64 tmp64; - - if (__builtin_constant_p(n)) { - switch (n) { - case 1: - *(u8 *)to = *(u8 __force *)from; - return 0; - case 4: - asm volatile ("ldnw .d1t1 *%2,%0\n" - "nop 4\n" - "stnw .d1t1 %0,*%1\n" - : "=&a"(tmp32) - : "A"(to), "a"(from) - : "memory"); - return 0; - case 8: - asm volatile ("ldndw .d1t1 *%2,%0\n" - "nop 4\n" - "stndw .d1t1 %0,*%1\n" - : "=&a"(tmp64) - : "a"(to), "a"(from) - : "memory"); - return 0; - default: - break; - } - } - - memcpy(to, (const void __force *)from, n); - return 0; -} - -static inline __must_check unsigned long -raw_copy_to_user(void __user *to, const void *from, unsigned long n) -{ - u32 tmp32; - u64 tmp64; - - if (__builtin_constant_p(n)) { - switch (n) { - case 1: - *(u8 __force *)to = *(u8 *)from; - return 0; - case 4: - asm volatile ("ldnw .d1t1 *%2,%0\n" - "nop 4\n" - "stnw .d1t1 %0,*%1\n" - : "=&a"(tmp32) - : "a"(to), "a"(from) - : "memory"); - return 0; - case 8: - asm volatile ("ldndw .d1t1 *%2,%0\n" - "nop 4\n" - "stndw .d1t1 %0,*%1\n" - : "=&a"(tmp64) - : "a"(to), "a"(from) - : "memory"); - return 0; - default: - break; - } - } - - memcpy((void __force *)to, from, n); - return 0; -} -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER - -extern int _access_ok(unsigned long addr, unsigned long size); -#ifdef CONFIG_ACCESS_CHECK -#define __access_ok _access_ok -#endif - -#include - -#endif /* _ASM_C6X_UACCESS_H */ diff --git a/arch/c6x/include/asm/unaligned.h b/arch/c6x/include/asm/unaligned.h deleted file mode 100644 index d628cc170564..000000000000 --- a/arch/c6x/include/asm/unaligned.h +++ /dev/null @@ -1,104 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * Rewritten for 2.6.3x: Mark Salter - */ -#ifndef _ASM_C6X_UNALIGNED_H -#define _ASM_C6X_UNALIGNED_H - -#include -#include - -/* - * The C64x+ can do unaligned word and dword accesses in hardware - * using special load/store instructions. - */ - -static inline u16 get_unaligned_le16(const void *p) -{ - const u8 *_p = p; - return _p[0] | _p[1] << 8; -} - -static inline u16 get_unaligned_be16(const void *p) -{ - const u8 *_p = p; - return _p[0] << 8 | _p[1]; -} - -static inline void put_unaligned_le16(u16 val, void *p) -{ - u8 *_p = p; - _p[0] = val; - _p[1] = val >> 8; -} - -static inline void put_unaligned_be16(u16 val, void *p) -{ - u8 *_p = p; - _p[0] = val >> 8; - _p[1] = val; -} - -static inline u32 get_unaligned32(const void *p) -{ - u32 val = (u32) p; - asm (" ldnw .d1t1 *%0,%0\n" - " nop 4\n" - : "+a"(val)); - return val; -} - -static inline void put_unaligned32(u32 val, void *p) -{ - asm volatile (" stnw .d2t1 %0,*%1\n" - : : "a"(val), "b"(p) : "memory"); -} - -static inline u64 get_unaligned64(const void *p) -{ - u64 val; - asm volatile (" ldndw .d1t1 *%1,%0\n" - " nop 4\n" - : "=a"(val) : "a"(p)); - return val; -} - -static inline void put_unaligned64(u64 val, const void *p) -{ - asm volatile (" stndw .d2t1 %0,*%1\n" - : : "a"(val), "b"(p) : "memory"); -} - -#ifdef CONFIG_CPU_BIG_ENDIAN - -#define get_unaligned_le32(p) __swab32(get_unaligned32(p)) -#define get_unaligned_le64(p) __swab64(get_unaligned64(p)) -#define get_unaligned_be32(p) get_unaligned32(p) -#define get_unaligned_be64(p) get_unaligned64(p) -#define put_unaligned_le32(v, p) put_unaligned32(__swab32(v), (p)) -#define put_unaligned_le64(v, p) put_unaligned64(__swab64(v), (p)) -#define put_unaligned_be32(v, p) put_unaligned32((v), (p)) -#define put_unaligned_be64(v, p) put_unaligned64((v), (p)) -#define get_unaligned __get_unaligned_be -#define put_unaligned __put_unaligned_be - -#else - -#define get_unaligned_le32(p) get_unaligned32(p) -#define get_unaligned_le64(p) get_unaligned64(p) -#define get_unaligned_be32(p) __swab32(get_unaligned32(p)) -#define get_unaligned_be64(p) __swab64(get_unaligned64(p)) -#define put_unaligned_le32(v, p) put_unaligned32((v), (p)) -#define put_unaligned_le64(v, p) put_unaligned64((v), (p)) -#define put_unaligned_be32(v, p) put_unaligned32(__swab32(v), (p)) -#define put_unaligned_be64(v, p) put_unaligned64(__swab64(v), (p)) -#define get_unaligned __get_unaligned_le -#define put_unaligned __put_unaligned_le - -#endif - -#endif /* _ASM_C6X_UNALIGNED_H */ diff --git a/arch/c6x/include/asm/vmalloc.h b/arch/c6x/include/asm/vmalloc.h deleted file mode 100644 index 26c6c6696bbd..000000000000 --- a/arch/c6x/include/asm/vmalloc.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef _ASM_C6X_VMALLOC_H -#define _ASM_C6X_VMALLOC_H - -#endif /* _ASM_C6X_VMALLOC_H */ diff --git a/arch/c6x/include/uapi/asm/Kbuild b/arch/c6x/include/uapi/asm/Kbuild deleted file mode 100644 index e78470141932..000000000000 --- a/arch/c6x/include/uapi/asm/Kbuild +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -generic-y += ucontext.h diff --git a/arch/c6x/include/uapi/asm/byteorder.h b/arch/c6x/include/uapi/asm/byteorder.h deleted file mode 100644 index ab61f867391c..000000000000 --- a/arch/c6x/include/uapi/asm/byteorder.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _ASM_C6X_BYTEORDER_H -#define _ASM_C6X_BYTEORDER_H - -#include - -#ifdef _BIG_ENDIAN -#include -#else /* _BIG_ENDIAN */ -#include -#endif /* _BIG_ENDIAN */ - -#endif /* _ASM_BYTEORDER_H */ diff --git a/arch/c6x/include/uapi/asm/ptrace.h b/arch/c6x/include/uapi/asm/ptrace.h deleted file mode 100644 index 9b51110a0842..000000000000 --- a/arch/c6x/include/uapi/asm/ptrace.h +++ /dev/null @@ -1,164 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * Copyright (C) 2004, 2006, 2009, 2010 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * - * Updated for 2.6.34: Mark Salter - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ -#ifndef _UAPI_ASM_C6X_PTRACE_H -#define _UAPI_ASM_C6X_PTRACE_H - -#define BKPT_OPCODE 0x56454314 /* illegal opcode */ - -#ifdef _BIG_ENDIAN -#define PT_LO(odd, even) odd -#define PT_HI(odd, even) even -#else -#define PT_LO(odd, even) even -#define PT_HI(odd, even) odd -#endif - -#define PT_A4_ORG PT_LO(1, 0) -#define PT_TSR PT_HI(1, 0) -#define PT_ILC PT_LO(3, 2) -#define PT_RILC PT_HI(3, 2) -#define PT_CSR PT_LO(5, 4) -#define PT_PC PT_HI(5, 4) -#define PT_B16 PT_LO(7, 6) -#define PT_B17 PT_HI(7, 6) -#define PT_B18 PT_LO(9, 8) -#define PT_B19 PT_HI(9, 8) -#define PT_B20 PT_LO(11, 10) -#define PT_B21 PT_HI(11, 10) -#define PT_B22 PT_LO(13, 12) -#define PT_B23 PT_HI(13, 12) -#define PT_B24 PT_LO(15, 14) -#define PT_B25 PT_HI(15, 14) -#define PT_B26 PT_LO(17, 16) -#define PT_B27 PT_HI(17, 16) -#define PT_B28 PT_LO(19, 18) -#define PT_B29 PT_HI(19, 18) -#define PT_B30 PT_LO(21, 20) -#define PT_B31 PT_HI(21, 20) -#define PT_B0 PT_LO(23, 22) -#define PT_B1 PT_HI(23, 22) -#define PT_B2 PT_LO(25, 24) -#define PT_B3 PT_HI(25, 24) -#define PT_B4 PT_LO(27, 26) -#define PT_B5 PT_HI(27, 26) -#define PT_B6 PT_LO(29, 28) -#define PT_B7 PT_HI(29, 28) -#define PT_B8 PT_LO(31, 30) -#define PT_B9 PT_HI(31, 30) -#define PT_B10 PT_LO(33, 32) -#define PT_B11 PT_HI(33, 32) -#define PT_B12 PT_LO(35, 34) -#define PT_B13 PT_HI(35, 34) -#define PT_A16 PT_LO(37, 36) -#define PT_A17 PT_HI(37, 36) -#define PT_A18 PT_LO(39, 38) -#define PT_A19 PT_HI(39, 38) -#define PT_A20 PT_LO(41, 40) -#define PT_A21 PT_HI(41, 40) -#define PT_A22 PT_LO(43, 42) -#define PT_A23 PT_HI(43, 42) -#define PT_A24 PT_LO(45, 44) -#define PT_A25 PT_HI(45, 44) -#define PT_A26 PT_LO(47, 46) -#define PT_A27 PT_HI(47, 46) -#define PT_A28 PT_LO(49, 48) -#define PT_A29 PT_HI(49, 48) -#define PT_A30 PT_LO(51, 50) -#define PT_A31 PT_HI(51, 50) -#define PT_A0 PT_LO(53, 52) -#define PT_A1 PT_HI(53, 52) -#define PT_A2 PT_LO(55, 54) -#define PT_A3 PT_HI(55, 54) -#define PT_A4 PT_LO(57, 56) -#define PT_A5 PT_HI(57, 56) -#define PT_A6 PT_LO(59, 58) -#define PT_A7 PT_HI(59, 58) -#define PT_A8 PT_LO(61, 60) -#define PT_A9 PT_HI(61, 60) -#define PT_A10 PT_LO(63, 62) -#define PT_A11 PT_HI(63, 62) -#define PT_A12 PT_LO(65, 64) -#define PT_A13 PT_HI(65, 64) -#define PT_A14 PT_LO(67, 66) -#define PT_A15 PT_HI(67, 66) -#define PT_B14 PT_LO(69, 68) -#define PT_B15 PT_HI(69, 68) - -#define NR_PTREGS 70 - -#define PT_DP PT_B14 /* Data Segment Pointer (B14) */ -#define PT_SP PT_B15 /* Stack Pointer (B15) */ - -#define PTRACE_GETFDPIC 31 /* get the ELF fdpic loadmap address */ - -#define PTRACE_GETFDPIC_EXEC 0 /* [addr] request the executable loadmap */ -#define PTRACE_GETFDPIC_INTERP 1 /* [addr] request the interpreter loadmap */ - -#ifndef __ASSEMBLY__ - -#ifdef _BIG_ENDIAN -#define REG_PAIR(odd, even) unsigned long odd; unsigned long even -#else -#define REG_PAIR(odd, even) unsigned long even; unsigned long odd -#endif - -/* - * this struct defines the way the registers are stored on the - * stack during a system call. fields defined with REG_PAIR - * are saved and restored using double-word memory operations - * which means the word ordering of the pair depends on endianess. - */ -struct pt_regs { - REG_PAIR(tsr, orig_a4); - REG_PAIR(rilc, ilc); - REG_PAIR(pc, csr); - - REG_PAIR(b17, b16); - REG_PAIR(b19, b18); - REG_PAIR(b21, b20); - REG_PAIR(b23, b22); - REG_PAIR(b25, b24); - REG_PAIR(b27, b26); - REG_PAIR(b29, b28); - REG_PAIR(b31, b30); - - REG_PAIR(b1, b0); - REG_PAIR(b3, b2); - REG_PAIR(b5, b4); - REG_PAIR(b7, b6); - REG_PAIR(b9, b8); - REG_PAIR(b11, b10); - REG_PAIR(b13, b12); - - REG_PAIR(a17, a16); - REG_PAIR(a19, a18); - REG_PAIR(a21, a20); - REG_PAIR(a23, a22); - REG_PAIR(a25, a24); - REG_PAIR(a27, a26); - REG_PAIR(a29, a28); - REG_PAIR(a31, a30); - - REG_PAIR(a1, a0); - REG_PAIR(a3, a2); - REG_PAIR(a5, a4); - REG_PAIR(a7, a6); - REG_PAIR(a9, a8); - REG_PAIR(a11, a10); - REG_PAIR(a13, a12); - - REG_PAIR(a15, a14); - REG_PAIR(sp, dp); -}; - -#endif /* __ASSEMBLY__ */ -#endif /* _UAPI_ASM_C6X_PTRACE_H */ diff --git a/arch/c6x/include/uapi/asm/setup.h b/arch/c6x/include/uapi/asm/setup.h deleted file mode 100644 index e90548cebec3..000000000000 --- a/arch/c6x/include/uapi/asm/setup.h +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI_ASM_C6X_SETUP_H -#define _UAPI_ASM_C6X_SETUP_H - -#define COMMAND_LINE_SIZE 1024 - -#endif /* _UAPI_ASM_C6X_SETUP_H */ diff --git a/arch/c6x/include/uapi/asm/sigcontext.h b/arch/c6x/include/uapi/asm/sigcontext.h deleted file mode 100644 index 4e5a9a260861..000000000000 --- a/arch/c6x/include/uapi/asm/sigcontext.h +++ /dev/null @@ -1,81 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ -#ifndef _ASM_C6X_SIGCONTEXT_H -#define _ASM_C6X_SIGCONTEXT_H - - -struct sigcontext { - unsigned long sc_mask; /* old sigmask */ - unsigned long sc_sp; /* old user stack pointer */ - - unsigned long sc_a4; - unsigned long sc_b4; - unsigned long sc_a6; - unsigned long sc_b6; - unsigned long sc_a8; - unsigned long sc_b8; - - unsigned long sc_a0; - unsigned long sc_a1; - unsigned long sc_a2; - unsigned long sc_a3; - unsigned long sc_a5; - unsigned long sc_a7; - unsigned long sc_a9; - - unsigned long sc_b0; - unsigned long sc_b1; - unsigned long sc_b2; - unsigned long sc_b3; - unsigned long sc_b5; - unsigned long sc_b7; - unsigned long sc_b9; - - unsigned long sc_a16; - unsigned long sc_a17; - unsigned long sc_a18; - unsigned long sc_a19; - unsigned long sc_a20; - unsigned long sc_a21; - unsigned long sc_a22; - unsigned long sc_a23; - unsigned long sc_a24; - unsigned long sc_a25; - unsigned long sc_a26; - unsigned long sc_a27; - unsigned long sc_a28; - unsigned long sc_a29; - unsigned long sc_a30; - unsigned long sc_a31; - - unsigned long sc_b16; - unsigned long sc_b17; - unsigned long sc_b18; - unsigned long sc_b19; - unsigned long sc_b20; - unsigned long sc_b21; - unsigned long sc_b22; - unsigned long sc_b23; - unsigned long sc_b24; - unsigned long sc_b25; - unsigned long sc_b26; - unsigned long sc_b27; - unsigned long sc_b28; - unsigned long sc_b29; - unsigned long sc_b30; - unsigned long sc_b31; - - unsigned long sc_csr; - unsigned long sc_pc; -}; - -#endif /* _ASM_C6X_SIGCONTEXT_H */ diff --git a/arch/c6x/include/uapi/asm/swab.h b/arch/c6x/include/uapi/asm/swab.h deleted file mode 100644 index c407c0497718..000000000000 --- a/arch/c6x/include/uapi/asm/swab.h +++ /dev/null @@ -1,55 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ -#ifndef _ASM_C6X_SWAB_H -#define _ASM_C6X_SWAB_H - -static inline __attribute_const__ __u16 __c6x_swab16(__u16 val) -{ - asm("swap4 .l1 %0,%0\n" : "+a"(val)); - return val; -} - -static inline __attribute_const__ __u32 __c6x_swab32(__u32 val) -{ - asm("swap4 .l1 %0,%0\n" - "swap2 .l1 %0,%0\n" - : "+a"(val)); - return val; -} - -static inline __attribute_const__ __u64 __c6x_swab64(__u64 val) -{ - asm(" swap2 .s1 %p0,%P0\n" - "|| swap2 .l1 %P0,%p0\n" - " swap4 .l1 %p0,%p0\n" - " swap4 .l1 %P0,%P0\n" - : "+a"(val)); - return val; -} - -static inline __attribute_const__ __u32 __c6x_swahw32(__u32 val) -{ - asm("swap2 .l1 %0,%0\n" : "+a"(val)); - return val; -} - -static inline __attribute_const__ __u32 __c6x_swahb32(__u32 val) -{ - asm("swap4 .l1 %0,%0\n" : "+a"(val)); - return val; -} - -#define __arch_swab16 __c6x_swab16 -#define __arch_swab32 __c6x_swab32 -#define __arch_swab64 __c6x_swab64 -#define __arch_swahw32 __c6x_swahw32 -#define __arch_swahb32 __c6x_swahb32 - -#endif /* _ASM_C6X_SWAB_H */ diff --git a/arch/c6x/include/uapi/asm/unistd.h b/arch/c6x/include/uapi/asm/unistd.h deleted file mode 100644 index 79b724c39d9b..000000000000 --- a/arch/c6x/include/uapi/asm/unistd.h +++ /dev/null @@ -1,29 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * Copyright (C) 2011 Texas Instruments Incorporated - * - * Based on arch/tile version. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, version 2. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or - * NON INFRINGEMENT. See the GNU General Public License for - * more details. - */ - -#define __ARCH_WANT_RENAMEAT -#define __ARCH_WANT_STAT64 -#define __ARCH_WANT_SET_GET_RLIMIT -#define __ARCH_WANT_SYS_CLONE -#define __ARCH_WANT_TIME32_SYSCALLS - -/* Use the standard ABI for syscalls. */ -#include - -/* C6X-specific syscalls. */ -#define __NR_cache_sync (__NR_arch_specific_syscall + 0) -__SYSCALL(__NR_cache_sync, sys_cache_sync) diff --git a/arch/c6x/kernel/Makefile b/arch/c6x/kernel/Makefile deleted file mode 100644 index fbe74174de87..000000000000 --- a/arch/c6x/kernel/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for arch/c6x/kernel/ -# - -extra-y := head.o vmlinux.lds - -obj-y := process.o traps.o irq.o signal.o ptrace.o -obj-y += setup.o sys_c6x.o time.o devicetree.o -obj-y += switch_to.o entry.o vectors.o c6x_ksyms.o -obj-y += soc.o - -obj-$(CONFIG_MODULES) += module.o diff --git a/arch/c6x/kernel/asm-offsets.c b/arch/c6x/kernel/asm-offsets.c deleted file mode 100644 index 4a264ef87dcb..000000000000 --- a/arch/c6x/kernel/asm-offsets.c +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Generate definitions needed by assembly language modules. - * This code generates raw asm output which is post-processed - * to extract and format the required data. - */ - -#include -#include -#include -#include -#include - -void foo(void) -{ - OFFSET(REGS_A16, pt_regs, a16); - OFFSET(REGS_A17, pt_regs, a17); - OFFSET(REGS_A18, pt_regs, a18); - OFFSET(REGS_A19, pt_regs, a19); - OFFSET(REGS_A20, pt_regs, a20); - OFFSET(REGS_A21, pt_regs, a21); - OFFSET(REGS_A22, pt_regs, a22); - OFFSET(REGS_A23, pt_regs, a23); - OFFSET(REGS_A24, pt_regs, a24); - OFFSET(REGS_A25, pt_regs, a25); - OFFSET(REGS_A26, pt_regs, a26); - OFFSET(REGS_A27, pt_regs, a27); - OFFSET(REGS_A28, pt_regs, a28); - OFFSET(REGS_A29, pt_regs, a29); - OFFSET(REGS_A30, pt_regs, a30); - OFFSET(REGS_A31, pt_regs, a31); - - OFFSET(REGS_B16, pt_regs, b16); - OFFSET(REGS_B17, pt_regs, b17); - OFFSET(REGS_B18, pt_regs, b18); - OFFSET(REGS_B19, pt_regs, b19); - OFFSET(REGS_B20, pt_regs, b20); - OFFSET(REGS_B21, pt_regs, b21); - OFFSET(REGS_B22, pt_regs, b22); - OFFSET(REGS_B23, pt_regs, b23); - OFFSET(REGS_B24, pt_regs, b24); - OFFSET(REGS_B25, pt_regs, b25); - OFFSET(REGS_B26, pt_regs, b26); - OFFSET(REGS_B27, pt_regs, b27); - OFFSET(REGS_B28, pt_regs, b28); - OFFSET(REGS_B29, pt_regs, b29); - OFFSET(REGS_B30, pt_regs, b30); - OFFSET(REGS_B31, pt_regs, b31); - - OFFSET(REGS_A0, pt_regs, a0); - OFFSET(REGS_A1, pt_regs, a1); - OFFSET(REGS_A2, pt_regs, a2); - OFFSET(REGS_A3, pt_regs, a3); - OFFSET(REGS_A4, pt_regs, a4); - OFFSET(REGS_A5, pt_regs, a5); - OFFSET(REGS_A6, pt_regs, a6); - OFFSET(REGS_A7, pt_regs, a7); - OFFSET(REGS_A8, pt_regs, a8); - OFFSET(REGS_A9, pt_regs, a9); - OFFSET(REGS_A10, pt_regs, a10); - OFFSET(REGS_A11, pt_regs, a11); - OFFSET(REGS_A12, pt_regs, a12); - OFFSET(REGS_A13, pt_regs, a13); - OFFSET(REGS_A14, pt_regs, a14); - OFFSET(REGS_A15, pt_regs, a15); - - OFFSET(REGS_B0, pt_regs, b0); - OFFSET(REGS_B1, pt_regs, b1); - OFFSET(REGS_B2, pt_regs, b2); - OFFSET(REGS_B3, pt_regs, b3); - OFFSET(REGS_B4, pt_regs, b4); - OFFSET(REGS_B5, pt_regs, b5); - OFFSET(REGS_B6, pt_regs, b6); - OFFSET(REGS_B7, pt_regs, b7); - OFFSET(REGS_B8, pt_regs, b8); - OFFSET(REGS_B9, pt_regs, b9); - OFFSET(REGS_B10, pt_regs, b10); - OFFSET(REGS_B11, pt_regs, b11); - OFFSET(REGS_B12, pt_regs, b12); - OFFSET(REGS_B13, pt_regs, b13); - OFFSET(REGS_DP, pt_regs, dp); - OFFSET(REGS_SP, pt_regs, sp); - - OFFSET(REGS_TSR, pt_regs, tsr); - OFFSET(REGS_ORIG_A4, pt_regs, orig_a4); - - DEFINE(REGS__END, sizeof(struct pt_regs)); - BLANK(); - - OFFSET(THREAD_PC, thread_struct, pc); - OFFSET(THREAD_B15_14, thread_struct, b15_14); - OFFSET(THREAD_A15_14, thread_struct, a15_14); - OFFSET(THREAD_B13_12, thread_struct, b13_12); - OFFSET(THREAD_A13_12, thread_struct, a13_12); - OFFSET(THREAD_B11_10, thread_struct, b11_10); - OFFSET(THREAD_A11_10, thread_struct, a11_10); - OFFSET(THREAD_RICL_ICL, thread_struct, ricl_icl); - BLANK(); - - OFFSET(TASK_STATE, task_struct, state); - BLANK(); - - OFFSET(THREAD_INFO_FLAGS, thread_info, flags); - OFFSET(THREAD_INFO_PREEMPT_COUNT, thread_info, preempt_count); - BLANK(); - - /* These would be unneccessary if we ran asm files - * through the preprocessor. - */ - DEFINE(KTHREAD_SHIFT, THREAD_SHIFT); - DEFINE(KTHREAD_START_SP, THREAD_START_SP); - DEFINE(ENOSYS_, ENOSYS); - DEFINE(NR_SYSCALLS_, __NR_syscalls); - - DEFINE(_TIF_SYSCALL_TRACE, (1< -#include -#include - -/* - * libgcc functions - used internally by the compiler... - */ -extern int __c6xabi_divi(int dividend, int divisor); -EXPORT_SYMBOL(__c6xabi_divi); - -extern unsigned __c6xabi_divu(unsigned dividend, unsigned divisor); -EXPORT_SYMBOL(__c6xabi_divu); - -extern int __c6xabi_remi(int dividend, int divisor); -EXPORT_SYMBOL(__c6xabi_remi); - -extern unsigned __c6xabi_remu(unsigned dividend, unsigned divisor); -EXPORT_SYMBOL(__c6xabi_remu); - -extern int __c6xabi_divremi(int dividend, int divisor); -EXPORT_SYMBOL(__c6xabi_divremi); - -extern unsigned __c6xabi_divremu(unsigned dividend, unsigned divisor); -EXPORT_SYMBOL(__c6xabi_divremu); - -extern unsigned long long __c6xabi_mpyll(unsigned long long src1, - unsigned long long src2); -EXPORT_SYMBOL(__c6xabi_mpyll); - -extern long long __c6xabi_negll(long long src); -EXPORT_SYMBOL(__c6xabi_negll); - -extern unsigned long long __c6xabi_llshl(unsigned long long src1, uint src2); -EXPORT_SYMBOL(__c6xabi_llshl); - -extern long long __c6xabi_llshr(long long src1, uint src2); -EXPORT_SYMBOL(__c6xabi_llshr); - -extern unsigned long long __c6xabi_llshru(unsigned long long src1, uint src2); -EXPORT_SYMBOL(__c6xabi_llshru); - -extern void __c6xabi_strasgi(int *dst, const int *src, unsigned cnt); -EXPORT_SYMBOL(__c6xabi_strasgi); - -extern void __c6xabi_push_rts(void); -EXPORT_SYMBOL(__c6xabi_push_rts); - -extern void __c6xabi_pop_rts(void); -EXPORT_SYMBOL(__c6xabi_pop_rts); - -extern void __c6xabi_strasgi_64plus(int *dst, const int *src, unsigned cnt); -EXPORT_SYMBOL(__c6xabi_strasgi_64plus); - -/* lib functions */ -EXPORT_SYMBOL(memcpy); diff --git a/arch/c6x/kernel/devicetree.c b/arch/c6x/kernel/devicetree.c deleted file mode 100644 index a0c73f0545b2..000000000000 --- a/arch/c6x/kernel/devicetree.c +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Architecture specific OF callbacks. - * - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - */ -#include -#include - -void __init early_init_dt_add_memory_arch(u64 base, u64 size) -{ - c6x_add_memory(base, size); -} diff --git a/arch/c6x/kernel/entry.S b/arch/c6x/kernel/entry.S deleted file mode 100644 index fb154d19625b..000000000000 --- a/arch/c6x/kernel/entry.S +++ /dev/null @@ -1,736 +0,0 @@ -; SPDX-License-Identifier: GPL-2.0-only -; -; Port on Texas Instruments TMS320C6x architecture -; -; Copyright (C) 2004-2011 Texas Instruments Incorporated -; Author: Aurelien Jacquiot (aurelien.jacquiot@virtuallogix.com) -; Updated for 2.6.34: Mark Salter -; - -#include -#include -#include -#include -#include -#include - -; Registers naming -#define DP B14 -#define SP B15 - -#ifndef CONFIG_PREEMPTION -#define resume_kernel restore_all -#endif - - .altmacro - - .macro MASK_INT reg - MVC .S2 CSR,reg - CLR .S2 reg,0,0,reg - MVC .S2 reg,CSR - .endm - - .macro UNMASK_INT reg - MVC .S2 CSR,reg - SET .S2 reg,0,0,reg - MVC .S2 reg,CSR - .endm - - .macro GET_THREAD_INFO reg - SHR .S1X SP,THREAD_SHIFT,reg - SHL .S1 reg,THREAD_SHIFT,reg - .endm - - ;; - ;; This defines the normal kernel pt_regs layout. - ;; - .macro SAVE_ALL __rp __tsr - STW .D2T2 B0,*SP--[2] ; save original B0 - MVKL .S2 current_ksp,B0 - MVKH .S2 current_ksp,B0 - LDW .D2T2 *B0,B1 ; KSP - - NOP 3 - STW .D2T2 B1,*+SP[1] ; save original B1 - XOR .D2 SP,B1,B0 ; (SP ^ KSP) - LDW .D2T2 *+SP[1],B1 ; restore B0/B1 - LDW .D2T2 *++SP[2],B0 - SHR .S2 B0,THREAD_SHIFT,B0 ; 0 if already using kstack - [B0] STDW .D2T2 SP:DP,*--B1[1] ; user: save user sp/dp kstack - [B0] MV .S2 B1,SP ; and switch to kstack -||[!B0] STDW .D2T2 SP:DP,*--SP[1] ; kernel: save on current stack - - SUBAW .D2 SP,2,SP - - ADD .D1X SP,-8,A15 - || STDW .D2T1 A15:A14,*SP--[16] ; save A15:A14 - - STDW .D2T2 B13:B12,*SP--[1] - || STDW .D1T1 A13:A12,*A15--[1] - || MVC .S2 __rp,B13 - - STDW .D2T2 B11:B10,*SP--[1] - || STDW .D1T1 A11:A10,*A15--[1] - || MVC .S2 CSR,B12 - - STDW .D2T2 B9:B8,*SP--[1] - || STDW .D1T1 A9:A8,*A15--[1] - || MVC .S2 RILC,B11 - STDW .D2T2 B7:B6,*SP--[1] - || STDW .D1T1 A7:A6,*A15--[1] - || MVC .S2 ILC,B10 - - STDW .D2T2 B5:B4,*SP--[1] - || STDW .D1T1 A5:A4,*A15--[1] - - STDW .D2T2 B3:B2,*SP--[1] - || STDW .D1T1 A3:A2,*A15--[1] - || MVC .S2 __tsr,B5 - - STDW .D2T2 B1:B0,*SP--[1] - || STDW .D1T1 A1:A0,*A15--[1] - || MV .S1X B5,A5 - - STDW .D2T2 B31:B30,*SP--[1] - || STDW .D1T1 A31:A30,*A15--[1] - STDW .D2T2 B29:B28,*SP--[1] - || STDW .D1T1 A29:A28,*A15--[1] - STDW .D2T2 B27:B26,*SP--[1] - || STDW .D1T1 A27:A26,*A15--[1] - STDW .D2T2 B25:B24,*SP--[1] - || STDW .D1T1 A25:A24,*A15--[1] - STDW .D2T2 B23:B22,*SP--[1] - || STDW .D1T1 A23:A22,*A15--[1] - STDW .D2T2 B21:B20,*SP--[1] - || STDW .D1T1 A21:A20,*A15--[1] - STDW .D2T2 B19:B18,*SP--[1] - || STDW .D1T1 A19:A18,*A15--[1] - STDW .D2T2 B17:B16,*SP--[1] - || STDW .D1T1 A17:A16,*A15--[1] - - STDW .D2T2 B13:B12,*SP--[1] ; save PC and CSR - - STDW .D2T2 B11:B10,*SP--[1] ; save RILC and ILC - STDW .D2T1 A5:A4,*SP--[1] ; save TSR and orig A4 - - ;; We left an unused word on the stack just above pt_regs. - ;; It is used to save whether or not this frame is due to - ;; a syscall. It is cleared here, but the syscall handler - ;; sets it to a non-zero value. - MVK .L2 0,B1 - STW .D2T2 B1,*+SP(REGS__END+8) ; clear syscall flag - .endm - - .macro RESTORE_ALL __rp __tsr - LDDW .D2T2 *++SP[1],B9:B8 ; get TSR (B9) - LDDW .D2T2 *++SP[1],B11:B10 ; get RILC (B11) and ILC (B10) - LDDW .D2T2 *++SP[1],B13:B12 ; get PC (B13) and CSR (B12) - - ADDAW .D1X SP,30,A15 - - LDDW .D1T1 *++A15[1],A17:A16 - || LDDW .D2T2 *++SP[1],B17:B16 - LDDW .D1T1 *++A15[1],A19:A18 - || LDDW .D2T2 *++SP[1],B19:B18 - LDDW .D1T1 *++A15[1],A21:A20 - || LDDW .D2T2 *++SP[1],B21:B20 - LDDW .D1T1 *++A15[1],A23:A22 - || LDDW .D2T2 *++SP[1],B23:B22 - LDDW .D1T1 *++A15[1],A25:A24 - || LDDW .D2T2 *++SP[1],B25:B24 - LDDW .D1T1 *++A15[1],A27:A26 - || LDDW .D2T2 *++SP[1],B27:B26 - LDDW .D1T1 *++A15[1],A29:A28 - || LDDW .D2T2 *++SP[1],B29:B28 - LDDW .D1T1 *++A15[1],A31:A30 - || LDDW .D2T2 *++SP[1],B31:B30 - - LDDW .D1T1 *++A15[1],A1:A0 - || LDDW .D2T2 *++SP[1],B1:B0 - - LDDW .D1T1 *++A15[1],A3:A2 - || LDDW .D2T2 *++SP[1],B3:B2 - || MVC .S2 B9,__tsr - LDDW .D1T1 *++A15[1],A5:A4 - || LDDW .D2T2 *++SP[1],B5:B4 - || MVC .S2 B11,RILC - LDDW .D1T1 *++A15[1],A7:A6 - || LDDW .D2T2 *++SP[1],B7:B6 - || MVC .S2 B10,ILC - - LDDW .D1T1 *++A15[1],A9:A8 - || LDDW .D2T2 *++SP[1],B9:B8 - || MVC .S2 B13,__rp - - LDDW .D1T1 *++A15[1],A11:A10 - || LDDW .D2T2 *++SP[1],B11:B10 - || MVC .S2 B12,CSR - - LDDW .D1T1 *++A15[1],A13:A12 - || LDDW .D2T2 *++SP[1],B13:B12 - - MV .D2X A15,SP - || MVKL .S1 current_ksp,A15 - MVKH .S1 current_ksp,A15 - || ADDAW .D1X SP,6,A14 - STW .D1T1 A14,*A15 ; save kernel stack pointer - - LDDW .D2T1 *++SP[1],A15:A14 - - B .S2 __rp ; return from interruption - LDDW .D2T2 *+SP[1],SP:DP - NOP 4 - .endm - - .section .text - - ;; - ;; Jump to schedule() then return to ret_from_exception - ;; -_reschedule: -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 schedule,A0 - MVKH .S1 schedule,A0 - B .S2X A0 -#else - B .S1 schedule -#endif - ADDKPC .S2 ret_from_exception,B3,4 - - ;; - ;; Called before syscall handler when process is being debugged - ;; -tracesys_on: -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 syscall_trace_entry,A0 - MVKH .S1 syscall_trace_entry,A0 - B .S2X A0 -#else - B .S1 syscall_trace_entry -#endif - ADDKPC .S2 ret_from_syscall_trace,B3,3 - ADD .S1X 8,SP,A4 - -ret_from_syscall_trace: - ;; tracing returns (possibly new) syscall number - MV .D2X A4,B0 - || MVK .S2 __NR_syscalls,B1 - CMPLTU .L2 B0,B1,B1 - - [!B1] BNOP .S2 ret_from_syscall_function,5 - || MVK .S1 -ENOSYS,A4 - - ;; reload syscall args from (possibly modified) stack frame - ;; and get syscall handler addr from sys_call_table: - LDW .D2T2 *+SP(REGS_B4+8),B4 - || MVKL .S2 sys_call_table,B1 - LDW .D2T1 *+SP(REGS_A6+8),A6 - || MVKH .S2 sys_call_table,B1 - LDW .D2T2 *+B1[B0],B0 - || MVKL .S2 ret_from_syscall_function,B3 - LDW .D2T2 *+SP(REGS_B6+8),B6 - || MVKH .S2 ret_from_syscall_function,B3 - LDW .D2T1 *+SP(REGS_A8+8),A8 - LDW .D2T2 *+SP(REGS_B8+8),B8 - NOP - ; B0 = sys_call_table[__NR_*] - BNOP .S2 B0,5 ; branch to syscall handler - || LDW .D2T1 *+SP(REGS_ORIG_A4+8),A4 - -syscall_exit_work: - AND .D1 _TIF_SYSCALL_TRACE,A2,A0 - [!A0] BNOP .S1 work_pending,5 - [A0] B .S2 syscall_trace_exit - ADDKPC .S2 resume_userspace,B3,1 - MVC .S2 CSR,B1 - SET .S2 B1,0,0,B1 - MVC .S2 B1,CSR ; enable ints - -work_pending: - AND .D1 _TIF_NEED_RESCHED,A2,A0 - [!A0] BNOP .S1 work_notifysig,5 - -work_resched: -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 schedule,A1 - MVKH .S1 schedule,A1 - B .S2X A1 -#else - B .S2 schedule -#endif - ADDKPC .S2 work_rescheduled,B3,4 -work_rescheduled: - ;; make sure we don't miss an interrupt setting need_resched or - ;; sigpending between sampling and the rti - MASK_INT B2 - GET_THREAD_INFO A12 - LDW .D1T1 *+A12(THREAD_INFO_FLAGS),A2 - MVK .S1 _TIF_WORK_MASK,A1 - MVK .S1 _TIF_NEED_RESCHED,A3 - NOP 2 - AND .D1 A1,A2,A0 - || AND .S1 A3,A2,A1 - [!A0] BNOP .S1 restore_all,5 - [A1] BNOP .S1 work_resched,5 - -work_notifysig: - ;; enable interrupts for do_notify_resume() - UNMASK_INT B2 - B .S2 do_notify_resume - LDW .D2T1 *+SP(REGS__END+8),A6 ; syscall flag - ADDKPC .S2 resume_userspace,B3,1 - ADD .S1X 8,SP,A4 ; pt_regs pointer is first arg - MV .D2X A2,B4 ; thread_info flags is second arg - - ;; - ;; On C64x+, the return way from exception and interrupt - ;; is a little bit different - ;; -ENTRY(ret_from_exception) -#ifdef CONFIG_PREEMPTION - MASK_INT B2 -#endif - -ENTRY(ret_from_interrupt) - ;; - ;; Check if we are comming from user mode. - ;; - LDW .D2T2 *+SP(REGS_TSR+8),B0 - MVK .S2 0x40,B1 - NOP 3 - AND .D2 B0,B1,B0 - [!B0] BNOP .S2 resume_kernel,5 - -resume_userspace: - ;; make sure we don't miss an interrupt setting need_resched or - ;; sigpending between sampling and the rti - MASK_INT B2 - GET_THREAD_INFO A12 - LDW .D1T1 *+A12(THREAD_INFO_FLAGS),A2 - MVK .S1 _TIF_WORK_MASK,A1 - MVK .S1 _TIF_NEED_RESCHED,A3 - NOP 2 - AND .D1 A1,A2,A0 - [A0] BNOP .S1 work_pending,5 - BNOP .S1 restore_all,5 - - ;; - ;; System call handling - ;; B0 = syscall number (in sys_call_table) - ;; A4,B4,A6,B6,A8,B8 = arguments of the syscall function - ;; A4 is the return value register - ;; -system_call_saved: - MVK .L2 1,B2 - STW .D2T2 B2,*+SP(REGS__END+8) ; set syscall flag - MVC .S2 B2,ECR ; ack the software exception - - UNMASK_INT B2 ; re-enable global IT - -system_call_saved_noack: - ;; Check system call number - MVK .S2 __NR_syscalls,B1 -#ifdef CONFIG_C6X_BIG_KERNEL - || MVKL .S1 sys_ni_syscall,A0 -#endif - CMPLTU .L2 B0,B1,B1 -#ifdef CONFIG_C6X_BIG_KERNEL - || MVKH .S1 sys_ni_syscall,A0 -#endif - - ;; Check for ptrace - GET_THREAD_INFO A12 - -#ifdef CONFIG_C6X_BIG_KERNEL - [!B1] B .S2X A0 -#else - [!B1] B .S2 sys_ni_syscall -#endif - [!B1] ADDKPC .S2 ret_from_syscall_function,B3,4 - - ;; Get syscall handler addr from sys_call_table - ;; call tracesys_on or call syscall handler - LDW .D1T1 *+A12(THREAD_INFO_FLAGS),A2 - || MVKL .S2 sys_call_table,B1 - MVKH .S2 sys_call_table,B1 - LDW .D2T2 *+B1[B0],B0 - NOP 2 - ; A2 = thread_info flags - AND .D1 _TIF_SYSCALL_TRACE,A2,A2 - [A2] BNOP .S1 tracesys_on,5 - ;; B0 = _sys_call_table[__NR_*] - B .S2 B0 - ADDKPC .S2 ret_from_syscall_function,B3,4 - -ret_from_syscall_function: - STW .D2T1 A4,*+SP(REGS_A4+8) ; save return value in A4 - ; original A4 is in orig_A4 -syscall_exit: - ;; make sure we don't miss an interrupt setting need_resched or - ;; sigpending between sampling and the rti - MASK_INT B2 - LDW .D1T1 *+A12(THREAD_INFO_FLAGS),A2 - MVK .S1 _TIF_ALLWORK_MASK,A1 - NOP 3 - AND .D1 A1,A2,A2 ; check for work to do - [A2] BNOP .S1 syscall_exit_work,5 - -restore_all: - RESTORE_ALL NRP,NTSR - - ;; - ;; After a fork we jump here directly from resume, - ;; so that A4 contains the previous task structure. - ;; -ENTRY(ret_from_fork) -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 schedule_tail,A0 - MVKH .S1 schedule_tail,A0 - B .S2X A0 -#else - B .S2 schedule_tail -#endif - ADDKPC .S2 ret_from_fork_2,B3,4 -ret_from_fork_2: - ;; return 0 in A4 for child process - GET_THREAD_INFO A12 - BNOP .S2 syscall_exit,3 - MVK .L2 0,B0 - STW .D2T2 B0,*+SP(REGS_A4+8) -ENDPROC(ret_from_fork) - -ENTRY(ret_from_kernel_thread) -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 schedule_tail,A0 - MVKH .S1 schedule_tail,A0 - B .S2X A0 -#else - B .S2 schedule_tail -#endif - LDW .D2T2 *+SP(REGS_A0+8),B10 /* get fn */ - ADDKPC .S2 0f,B3,3 -0: - B .S2 B10 /* call fn */ - LDW .D2T1 *+SP(REGS_A1+8),A4 /* get arg */ - ADDKPC .S2 ret_from_fork_2,B3,3 -ENDPROC(ret_from_kernel_thread) - - ;; - ;; These are the interrupt handlers, responsible for calling c6x_do_IRQ() - ;; - .macro SAVE_ALL_INT - SAVE_ALL IRP,ITSR - .endm - - .macro CALL_INT int -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 c6x_do_IRQ,A0 - MVKH .S1 c6x_do_IRQ,A0 - BNOP .S2X A0,1 - MVK .S1 int,A4 - ADDAW .D2 SP,2,B4 - MVKL .S2 ret_from_interrupt,B3 - MVKH .S2 ret_from_interrupt,B3 -#else - CALLP .S2 c6x_do_IRQ,B3 - || MVK .S1 int,A4 - || ADDAW .D2 SP,2,B4 - B .S1 ret_from_interrupt - NOP 5 -#endif - .endm - -ENTRY(_int4_handler) - SAVE_ALL_INT - CALL_INT 4 -ENDPROC(_int4_handler) - -ENTRY(_int5_handler) - SAVE_ALL_INT - CALL_INT 5 -ENDPROC(_int5_handler) - -ENTRY(_int6_handler) - SAVE_ALL_INT - CALL_INT 6 -ENDPROC(_int6_handler) - -ENTRY(_int7_handler) - SAVE_ALL_INT - CALL_INT 7 -ENDPROC(_int7_handler) - -ENTRY(_int8_handler) - SAVE_ALL_INT - CALL_INT 8 -ENDPROC(_int8_handler) - -ENTRY(_int9_handler) - SAVE_ALL_INT - CALL_INT 9 -ENDPROC(_int9_handler) - -ENTRY(_int10_handler) - SAVE_ALL_INT - CALL_INT 10 -ENDPROC(_int10_handler) - -ENTRY(_int11_handler) - SAVE_ALL_INT - CALL_INT 11 -ENDPROC(_int11_handler) - -ENTRY(_int12_handler) - SAVE_ALL_INT - CALL_INT 12 -ENDPROC(_int12_handler) - -ENTRY(_int13_handler) - SAVE_ALL_INT - CALL_INT 13 -ENDPROC(_int13_handler) - -ENTRY(_int14_handler) - SAVE_ALL_INT - CALL_INT 14 -ENDPROC(_int14_handler) - -ENTRY(_int15_handler) - SAVE_ALL_INT - CALL_INT 15 -ENDPROC(_int15_handler) - - ;; - ;; Handler for uninitialized and spurious interrupts - ;; -ENTRY(_bad_interrupt) - B .S2 IRP - NOP 5 -ENDPROC(_bad_interrupt) - - ;; - ;; Entry for NMI/exceptions/syscall - ;; -ENTRY(_nmi_handler) - SAVE_ALL NRP,NTSR - - MVC .S2 EFR,B2 - CMPEQ .L2 1,B2,B2 - || MVC .S2 TSR,B1 - CLR .S2 B1,10,10,B1 - MVC .S2 B1,TSR -#ifdef CONFIG_C6X_BIG_KERNEL - [!B2] MVKL .S1 process_exception,A0 - [!B2] MVKH .S1 process_exception,A0 - [!B2] B .S2X A0 -#else - [!B2] B .S2 process_exception -#endif - [B2] B .S2 system_call_saved - [!B2] ADDAW .D2 SP,2,B1 - [!B2] MV .D1X B1,A4 - ADDKPC .S2 ret_from_trap,B3,2 - -ret_from_trap: - MV .D2X A4,B0 - [!B0] BNOP .S2 ret_from_exception,5 - -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S2 system_call_saved_noack,B3 - MVKH .S2 system_call_saved_noack,B3 -#endif - LDW .D2T2 *+SP(REGS_B0+8),B0 - LDW .D2T1 *+SP(REGS_A4+8),A4 - LDW .D2T2 *+SP(REGS_B4+8),B4 - LDW .D2T1 *+SP(REGS_A6+8),A6 - LDW .D2T2 *+SP(REGS_B6+8),B6 - LDW .D2T1 *+SP(REGS_A8+8),A8 -#ifdef CONFIG_C6X_BIG_KERNEL - || B .S2 B3 -#else - || B .S2 system_call_saved_noack -#endif - LDW .D2T2 *+SP(REGS_B8+8),B8 - NOP 4 -ENDPROC(_nmi_handler) - - ;; - ;; Jump to schedule() then return to ret_from_isr - ;; -#ifdef CONFIG_PREEMPTION -resume_kernel: - GET_THREAD_INFO A12 - LDW .D1T1 *+A12(THREAD_INFO_PREEMPT_COUNT),A1 - NOP 4 - [A1] BNOP .S2 restore_all,5 - -preempt_schedule: - GET_THREAD_INFO A2 - LDW .D1T1 *+A2(THREAD_INFO_FLAGS),A1 -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S2 preempt_schedule_irq,B0 - MVKH .S2 preempt_schedule_irq,B0 - NOP 2 -#else - NOP 4 -#endif - AND .D1 _TIF_NEED_RESCHED,A1,A1 - [!A1] BNOP .S2 restore_all,5 -#ifdef CONFIG_C6X_BIG_KERNEL - B .S2 B0 -#else - B .S2 preempt_schedule_irq -#endif - ADDKPC .S2 preempt_schedule,B3,4 -#endif /* CONFIG_PREEMPTION */ - -ENTRY(enable_exception) - DINT - MVC .S2 TSR,B0 - MVC .S2 B3,NRP - MVK .L2 0xc,B1 - OR .D2 B0,B1,B0 - MVC .S2 B0,TSR ; Set GEE and XEN in TSR - B .S2 NRP - NOP 5 -ENDPROC(enable_exception) - - ;; - ;; Special system calls - ;; return address is in B3 - ;; -ENTRY(sys_rt_sigreturn) - ADD .D1X SP,8,A4 -#ifdef CONFIG_C6X_BIG_KERNEL - || MVKL .S1 do_rt_sigreturn,A0 - MVKH .S1 do_rt_sigreturn,A0 - BNOP .S2X A0,5 -#else - || B .S2 do_rt_sigreturn - NOP 5 -#endif -ENDPROC(sys_rt_sigreturn) - -ENTRY(sys_pread_c6x) - MV .D2X A8,B7 -#ifdef CONFIG_C6X_BIG_KERNEL - || MVKL .S1 sys_pread64,A0 - MVKH .S1 sys_pread64,A0 - BNOP .S2X A0,5 -#else - || B .S2 sys_pread64 - NOP 5 -#endif -ENDPROC(sys_pread_c6x) - -ENTRY(sys_pwrite_c6x) - MV .D2X A8,B7 -#ifdef CONFIG_C6X_BIG_KERNEL - || MVKL .S1 sys_pwrite64,A0 - MVKH .S1 sys_pwrite64,A0 - BNOP .S2X A0,5 -#else - || B .S2 sys_pwrite64 - NOP 5 -#endif -ENDPROC(sys_pwrite_c6x) - -;; On Entry -;; A4 - path -;; B4 - offset_lo (LE), offset_hi (BE) -;; A6 - offset_lo (BE), offset_hi (LE) -ENTRY(sys_truncate64_c6x) -#ifdef CONFIG_CPU_BIG_ENDIAN - MV .S2 B4,B5 - MV .D2X A6,B4 -#else - MV .D2X A6,B5 -#endif -#ifdef CONFIG_C6X_BIG_KERNEL - || MVKL .S1 sys_truncate64,A0 - MVKH .S1 sys_truncate64,A0 - BNOP .S2X A0,5 -#else - || B .S2 sys_truncate64 - NOP 5 -#endif -ENDPROC(sys_truncate64_c6x) - -;; On Entry -;; A4 - fd -;; B4 - offset_lo (LE), offset_hi (BE) -;; A6 - offset_lo (BE), offset_hi (LE) -ENTRY(sys_ftruncate64_c6x) -#ifdef CONFIG_CPU_BIG_ENDIAN - MV .S2 B4,B5 - MV .D2X A6,B4 -#else - MV .D2X A6,B5 -#endif -#ifdef CONFIG_C6X_BIG_KERNEL - || MVKL .S1 sys_ftruncate64,A0 - MVKH .S1 sys_ftruncate64,A0 - BNOP .S2X A0,5 -#else - || B .S2 sys_ftruncate64 - NOP 5 -#endif -ENDPROC(sys_ftruncate64_c6x) - -;; On Entry -;; A4 - fd -;; B4 - offset_lo (LE), offset_hi (BE) -;; A6 - offset_lo (BE), offset_hi (LE) -;; B6 - len_lo (LE), len_hi (BE) -;; A8 - len_lo (BE), len_hi (LE) -;; B8 - advice -ENTRY(sys_fadvise64_64_c6x) -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 sys_fadvise64_64,A0 - MVKH .S1 sys_fadvise64_64,A0 - BNOP .S2X A0,2 -#else - B .S2 sys_fadvise64_64 - NOP 2 -#endif -#ifdef CONFIG_CPU_BIG_ENDIAN - MV .L2 B4,B5 - || MV .D2X A6,B4 - MV .L1 A8,A6 - || MV .D1X B6,A7 -#else - MV .D2X A6,B5 - MV .L1 A8,A7 - || MV .D1X B6,A6 -#endif - MV .L2 B8,B6 -ENDPROC(sys_fadvise64_64_c6x) - -;; On Entry -;; A4 - fd -;; B4 - mode -;; A6 - offset_hi -;; B6 - offset_lo -;; A8 - len_hi -;; B8 - len_lo -ENTRY(sys_fallocate_c6x) -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 sys_fallocate,A0 - MVKH .S1 sys_fallocate,A0 - BNOP .S2X A0,1 -#else - B .S2 sys_fallocate - NOP -#endif - MV .D1 A6,A7 - MV .D1X B6,A6 - MV .D2X A8,B7 - MV .D2 B8,B6 -ENDPROC(sys_fallocate_c6x) - - ;; put this in .neardata for faster access when using DSBT mode - .section .neardata,"aw",@progbits - .global current_ksp - .hidden current_ksp -current_ksp: - .word init_thread_union + THREAD_START_SP diff --git a/arch/c6x/kernel/head.S b/arch/c6x/kernel/head.S deleted file mode 100644 index fecbeef827bc..000000000000 --- a/arch/c6x/kernel/head.S +++ /dev/null @@ -1,81 +0,0 @@ -; SPDX-License-Identifier: GPL-2.0-only -; -; Port on Texas Instruments TMS320C6x architecture -; -; Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated -; Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) -; -#include -#include -#include - - __HEAD -ENTRY(_c_int00) - ;; Save magic and pointer - MV .S1 A4,A10 - MV .S2 B4,B10 - MVKL .S2 __bss_start,B5 - MVKH .S2 __bss_start,B5 - MVKL .S2 __bss_stop,B6 - MVKH .S2 __bss_stop,B6 - SUB .L2 B6,B5,B6 ; bss size - - ;; Set the stack pointer - MVKL .S2 current_ksp,B0 - MVKH .S2 current_ksp,B0 - LDW .D2T2 *B0,B15 - - ;; clear bss - SHR .S2 B6,3,B0 ; number of dwords to clear - ZERO .L2 B13 - ZERO .L2 B12 -bss_loop: - BDEC .S2 bss_loop,B0 - NOP 3 - CMPLT .L2 B0,0,B1 - [!B1] STDW .D2T2 B13:B12,*B5++[1] - - NOP 4 - AND .D2 ~7,B15,B15 - - ;; Clear GIE and PGIE - MVC .S2 CSR,B2 - CLR .S2 B2,0,1,B2 - MVC .S2 B2,CSR - MVC .S2 TSR,B2 - CLR .S2 B2,0,1,B2 - MVC .S2 B2,TSR - MVC .S2 ITSR,B2 - CLR .S2 B2,0,1,B2 - MVC .S2 B2,ITSR - MVC .S2 NTSR,B2 - CLR .S2 B2,0,1,B2 - MVC .S2 B2,NTSR - - ;; pass DTB pointer to machine_init (or zero if none) - MVKL .S1 OF_DT_HEADER,A0 - MVKH .S1 OF_DT_HEADER,A0 - CMPEQ .L1 A10,A0,A0 - [A0] MV .S1X B10,A4 - [!A0] MVK .S1 0,A4 - -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 machine_init,A0 - MVKH .S1 machine_init,A0 - B .S2X A0 - ADDKPC .S2 0f,B3,4 -0: -#else - CALLP .S2 machine_init,B3 -#endif - - ;; Jump to Linux init -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 start_kernel,A0 - MVKH .S1 start_kernel,A0 - B .S2X A0 -#else - B .S2 start_kernel -#endif - NOP 5 -L1: BNOP .S2 L1,5 diff --git a/arch/c6x/kernel/irq.c b/arch/c6x/kernel/irq.c deleted file mode 100644 index e4c53d185b62..000000000000 --- a/arch/c6x/kernel/irq.c +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2011-2012 Texas Instruments Incorporated - * - * This borrows heavily from powerpc version, which is: - * - * Derived from arch/i386/kernel/irq.c - * Copyright (C) 1992 Linus Torvalds - * Adapted from arch/i386 by Gary Thomas - * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) - * Updated and modified by Cort Dougan - * Copyright (C) 1996-2001 Cort Dougan - * Adapted for Power Macintosh by Paul Mackerras - * Copyright (C) 1996 Paul Mackerras (paulus@cs.anu.edu.au) - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -unsigned long irq_err_count; - -static DEFINE_RAW_SPINLOCK(core_irq_lock); - -static void mask_core_irq(struct irq_data *data) -{ - unsigned int prio = data->hwirq; - - raw_spin_lock(&core_irq_lock); - and_creg(IER, ~(1 << prio)); - raw_spin_unlock(&core_irq_lock); -} - -static void unmask_core_irq(struct irq_data *data) -{ - unsigned int prio = data->hwirq; - - raw_spin_lock(&core_irq_lock); - or_creg(IER, 1 << prio); - raw_spin_unlock(&core_irq_lock); -} - -static struct irq_chip core_chip = { - .name = "core", - .irq_mask = mask_core_irq, - .irq_unmask = unmask_core_irq, -}; - -static int prio_to_virq[NR_PRIORITY_IRQS]; - -asmlinkage void c6x_do_IRQ(unsigned int prio, struct pt_regs *regs) -{ - struct pt_regs *old_regs = set_irq_regs(regs); - - irq_enter(); - - generic_handle_irq(prio_to_virq[prio]); - - irq_exit(); - - set_irq_regs(old_regs); -} - -static struct irq_domain *core_domain; - -static int core_domain_map(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hw) -{ - if (hw < 4 || hw >= NR_PRIORITY_IRQS) - return -EINVAL; - - prio_to_virq[hw] = virq; - - irq_set_status_flags(virq, IRQ_LEVEL); - irq_set_chip_and_handler(virq, &core_chip, handle_level_irq); - return 0; -} - -static const struct irq_domain_ops core_domain_ops = { - .map = core_domain_map, - .xlate = irq_domain_xlate_onecell, -}; - -void __init init_IRQ(void) -{ - struct device_node *np; - - /* Mask all priority IRQs */ - and_creg(IER, ~0xfff0); - - np = of_find_compatible_node(NULL, NULL, "ti,c64x+core-pic"); - if (np != NULL) { - /* create the core host */ - core_domain = irq_domain_add_linear(np, NR_PRIORITY_IRQS, - &core_domain_ops, NULL); - if (core_domain) - irq_set_default_host(core_domain); - of_node_put(np); - } - - printk(KERN_INFO "Core interrupt controller initialized\n"); - - /* now we're ready for other SoC controllers */ - megamod_pic_init(); - - /* Clear all general IRQ flags */ - set_creg(ICR, 0xfff0); -} - -void ack_bad_irq(int irq) -{ - printk(KERN_ERR "IRQ: spurious interrupt %d\n", irq); - irq_err_count++; -} - -int arch_show_interrupts(struct seq_file *p, int prec) -{ - seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count); - return 0; -} diff --git a/arch/c6x/kernel/module.c b/arch/c6x/kernel/module.c deleted file mode 100644 index 09b4c6bfe877..000000000000 --- a/arch/c6x/kernel/module.c +++ /dev/null @@ -1,119 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2005, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Thomas Charleux (thomas.charleux@jaluna.com) - */ -#include -#include -#include -#include - -static inline int fixup_pcr(u32 *ip, Elf32_Addr dest, u32 maskbits, int shift) -{ - u32 opcode; - long ep = (long)ip & ~31; - long delta = ((long)dest - ep) >> 2; - long mask = (1 << maskbits) - 1; - - if ((delta >> (maskbits - 1)) == 0 || - (delta >> (maskbits - 1)) == -1) { - opcode = *ip; - opcode &= ~(mask << shift); - opcode |= ((delta & mask) << shift); - *ip = opcode; - - pr_debug("REL PCR_S%d[%p] dest[%p] opcode[%08x]\n", - maskbits, ip, (void *)dest, opcode); - - return 0; - } - pr_err("PCR_S%d reloc %p -> %p out of range!\n", - maskbits, ip, (void *)dest); - - return -1; -} - -/* - * apply a RELA relocation - */ -int apply_relocate_add(Elf32_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - Elf32_Rela *rel = (void *) sechdrs[relsec].sh_addr; - Elf_Sym *sym; - u32 *location, opcode; - unsigned int i; - Elf32_Addr v; - Elf_Addr offset = 0; - - pr_debug("Applying relocate section %u to %u with offset 0x%x\n", - relsec, sechdrs[relsec].sh_info, offset); - - for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { - /* This is where to make the change */ - location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr - + rel[i].r_offset - offset; - - /* This is the symbol it is referring to. Note that all - undefined symbols have been resolved. */ - sym = (Elf_Sym *)sechdrs[symindex].sh_addr - + ELF32_R_SYM(rel[i].r_info); - - /* this is the adjustment to be made */ - v = sym->st_value + rel[i].r_addend; - - switch (ELF32_R_TYPE(rel[i].r_info)) { - case R_C6000_ABS32: - pr_debug("RELA ABS32: [%p] = 0x%x\n", location, v); - *location = v; - break; - case R_C6000_ABS16: - pr_debug("RELA ABS16: [%p] = 0x%x\n", location, v); - *(u16 *)location = v; - break; - case R_C6000_ABS8: - pr_debug("RELA ABS8: [%p] = 0x%x\n", location, v); - *(u8 *)location = v; - break; - case R_C6000_ABS_L16: - opcode = *location; - opcode &= ~0x7fff80; - opcode |= ((v & 0xffff) << 7); - pr_debug("RELA ABS_L16[%p] v[0x%x] opcode[0x%x]\n", - location, v, opcode); - *location = opcode; - break; - case R_C6000_ABS_H16: - opcode = *location; - opcode &= ~0x7fff80; - opcode |= ((v >> 9) & 0x7fff80); - pr_debug("RELA ABS_H16[%p] v[0x%x] opcode[0x%x]\n", - location, v, opcode); - *location = opcode; - break; - case R_C6000_PCR_S21: - if (fixup_pcr(location, v, 21, 7)) - return -ENOEXEC; - break; - case R_C6000_PCR_S12: - if (fixup_pcr(location, v, 12, 16)) - return -ENOEXEC; - break; - case R_C6000_PCR_S10: - if (fixup_pcr(location, v, 10, 13)) - return -ENOEXEC; - break; - default: - pr_err("module %s: Unknown RELA relocation: %u\n", - me->name, ELF32_R_TYPE(rel[i].r_info)); - return -ENOEXEC; - } - } - - return 0; -} diff --git a/arch/c6x/kernel/process.c b/arch/c6x/kernel/process.c deleted file mode 100644 index 9f4fd6a40a10..000000000000 --- a/arch/c6x/kernel/process.c +++ /dev/null @@ -1,151 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2006, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -/* hooks for board specific support */ -void (*c6x_restart)(void); -void (*c6x_halt)(void); - -extern asmlinkage void ret_from_fork(void); -extern asmlinkage void ret_from_kernel_thread(void); - -/* - * power off function, if any - */ -void (*pm_power_off)(void); -EXPORT_SYMBOL(pm_power_off); - -void arch_cpu_idle(void) -{ - unsigned long tmp; - - /* - * Put local_irq_enable and idle in same execute packet - * to make them atomic and avoid race to idle with - * interrupts enabled. - */ - asm volatile (" mvc .s2 CSR,%0\n" - " or .d2 1,%0,%0\n" - " mvc .s2 %0,CSR\n" - "|| idle\n" - : "=b"(tmp)); -} - -static void halt_loop(void) -{ - printk(KERN_EMERG "System Halted, OK to turn off power\n"); - local_irq_disable(); - while (1) - asm volatile("idle\n"); -} - -void machine_restart(char *__unused) -{ - if (c6x_restart) - c6x_restart(); - halt_loop(); -} - -void machine_halt(void) -{ - if (c6x_halt) - c6x_halt(); - halt_loop(); -} - -void machine_power_off(void) -{ - if (pm_power_off) - pm_power_off(); - halt_loop(); -} - -void flush_thread(void) -{ -} - -/* - * Do necessary setup to start up a newly executed thread. - */ -void start_thread(struct pt_regs *regs, unsigned int pc, unsigned long usp) -{ - /* - * The binfmt loader will setup a "full" stack, but the C6X - * operates an "empty" stack. So we adjust the usp so that - * argc doesn't get destroyed if an interrupt is taken before - * it is read from the stack. - * - * NB: Library startup code needs to match this. - */ - usp -= 8; - - regs->pc = pc; - regs->sp = usp; - regs->tsr |= 0x40; /* set user mode */ - current->thread.usp = usp; -} - -/* - * Copy a new thread context in its stack. - */ -int copy_thread(unsigned long clone_flags, unsigned long usp, - unsigned long ustk_size, struct task_struct *p, - unsigned long tls) -{ - struct pt_regs *childregs; - - childregs = task_pt_regs(p); - - if (unlikely(p->flags & PF_KTHREAD)) { - /* case of __kernel_thread: we return to supervisor space */ - memset(childregs, 0, sizeof(struct pt_regs)); - childregs->sp = (unsigned long)(childregs + 1); - p->thread.pc = (unsigned long) ret_from_kernel_thread; - childregs->a0 = usp; /* function */ - childregs->a1 = ustk_size; /* argument */ - } else { - /* Otherwise use the given stack */ - *childregs = *current_pt_regs(); - if (usp) - childregs->sp = usp; - p->thread.pc = (unsigned long) ret_from_fork; - } - - /* Set usp/ksp */ - p->thread.usp = childregs->sp; - thread_saved_ksp(p) = (unsigned long)childregs - 8; - p->thread.wchan = p->thread.pc; -#ifdef __DSBT__ - { - unsigned long dp; - - asm volatile ("mv .S2 b14,%0\n" : "=b"(dp)); - - thread_saved_dp(p) = dp; - if (usp == -1) - childregs->dp = dp; - } -#endif - return 0; -} - -unsigned long get_wchan(struct task_struct *p) -{ - return p->thread.wchan; -} diff --git a/arch/c6x/kernel/ptrace.c b/arch/c6x/kernel/ptrace.c deleted file mode 100644 index 3cdaa8cf0ed6..000000000000 --- a/arch/c6x/kernel/ptrace.c +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2006, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * - * Updated for 2.6.34: Mark Salter - */ -#include -#include -#include -#include -#include - -#include - -#define PT_REG_SIZE (sizeof(struct pt_regs)) - -/* - * Called by kernel/ptrace.c when detaching. - */ -void ptrace_disable(struct task_struct *child) -{ - /* nothing to do */ -} - -/* - * Get a register number from live pt_regs for the specified task. - */ -static inline long get_reg(struct task_struct *task, int regno) -{ - long *addr = (long *)task_pt_regs(task); - - if (regno == PT_TSR || regno == PT_CSR) - return 0; - - return addr[regno]; -} - -/* - * Write contents of register REGNO in task TASK. - */ -static inline int put_reg(struct task_struct *task, - int regno, - unsigned long data) -{ - unsigned long *addr = (unsigned long *)task_pt_regs(task); - - if (regno != PT_TSR && regno != PT_CSR) - addr[regno] = data; - - return 0; -} - -/* regset get/set implementations */ - -static int gpr_get(struct task_struct *target, - const struct user_regset *regset, - struct membuf to) -{ - return membuf_write(&to, task_pt_regs(target), sizeof(struct pt_regs)); -} - -enum c6x_regset { - REGSET_GPR, -}; - -static const struct user_regset c6x_regsets[] = { - [REGSET_GPR] = { - .core_note_type = NT_PRSTATUS, - .n = ELF_NGREG, - .size = sizeof(u32), - .align = sizeof(u32), - .regset_get = gpr_get, - }, -}; - -static const struct user_regset_view user_c6x_native_view = { - .name = "tic6x", - .e_machine = EM_TI_C6000, - .regsets = c6x_regsets, - .n = ARRAY_SIZE(c6x_regsets), -}; - -const struct user_regset_view *task_user_regset_view(struct task_struct *task) -{ - return &user_c6x_native_view; -} - -/* - * Perform ptrace request - */ -long arch_ptrace(struct task_struct *child, long request, - unsigned long addr, unsigned long data) -{ - int ret = 0; - - switch (request) { - /* - * write the word at location addr. - */ - case PTRACE_POKETEXT: - ret = generic_ptrace_pokedata(child, addr, data); - if (ret == 0 && request == PTRACE_POKETEXT) - flush_icache_range(addr, addr + 4); - break; - default: - ret = ptrace_request(child, request, addr, data); - break; - } - - return ret; -} - -/* - * handle tracing of system call entry - * - return the revised system call number or ULONG_MAX to cause ENOSYS - */ -asmlinkage unsigned long syscall_trace_entry(struct pt_regs *regs) -{ - if (tracehook_report_syscall_entry(regs)) - /* tracing decided this syscall should not happen, so - * We'll return a bogus call number to get an ENOSYS - * error, but leave the original number in - * regs->orig_a4 - */ - return ULONG_MAX; - - return regs->b0; -} - -/* - * handle tracing of system call exit - */ -asmlinkage void syscall_trace_exit(struct pt_regs *regs) -{ - tracehook_report_syscall_exit(regs, 0); -} diff --git a/arch/c6x/kernel/setup.c b/arch/c6x/kernel/setup.c deleted file mode 100644 index 9254c3b794a5..000000000000 --- a/arch/c6x/kernel/setup.c +++ /dev/null @@ -1,476 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2006, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -static const char *c6x_soc_name; - -struct screen_info screen_info; - -int c6x_num_cores; -EXPORT_SYMBOL_GPL(c6x_num_cores); - -unsigned int c6x_silicon_rev; -EXPORT_SYMBOL_GPL(c6x_silicon_rev); - -/* - * Device status register. This holds information - * about device configuration needed by some drivers. - */ -unsigned int c6x_devstat; -EXPORT_SYMBOL_GPL(c6x_devstat); - -/* - * Some SoCs have fuse registers holding a unique MAC - * address. This is parsed out of the device tree with - * the resulting MAC being held here. - */ -unsigned char c6x_fuse_mac[6]; - -unsigned long memory_start; -unsigned long memory_end; -EXPORT_SYMBOL(memory_end); - -unsigned long ram_start; -unsigned long ram_end; - -/* Uncached memory for DMA consistent use (memdma=) */ -static unsigned long dma_start __initdata; -static unsigned long dma_size __initdata; - -struct cpuinfo_c6x { - const char *cpu_name; - const char *cpu_voltage; - const char *mmu; - const char *fpu; - char *cpu_rev; - unsigned int core_id; - char __cpu_rev[5]; -}; - -static DEFINE_PER_CPU(struct cpuinfo_c6x, cpu_data); - -unsigned int ticks_per_ns_scaled; -EXPORT_SYMBOL(ticks_per_ns_scaled); - -unsigned int c6x_core_freq; - -static void __init get_cpuinfo(void) -{ - unsigned cpu_id, rev_id, csr; - struct clk *coreclk = clk_get_sys(NULL, "core"); - unsigned long core_khz; - u64 tmp; - struct cpuinfo_c6x *p; - struct device_node *node; - - p = &per_cpu(cpu_data, smp_processor_id()); - - if (!IS_ERR(coreclk)) - c6x_core_freq = clk_get_rate(coreclk); - else { - printk(KERN_WARNING - "Cannot find core clock frequency. Using 700MHz\n"); - c6x_core_freq = 700000000; - } - - core_khz = c6x_core_freq / 1000; - - tmp = (uint64_t)core_khz << C6X_NDELAY_SCALE; - do_div(tmp, 1000000); - ticks_per_ns_scaled = tmp; - - csr = get_creg(CSR); - cpu_id = csr >> 24; - rev_id = (csr >> 16) & 0xff; - - p->mmu = "none"; - p->fpu = "none"; - p->cpu_voltage = "unknown"; - - switch (cpu_id) { - case 0: - p->cpu_name = "C67x"; - p->fpu = "yes"; - break; - case 2: - p->cpu_name = "C62x"; - break; - case 8: - p->cpu_name = "C64x"; - break; - case 12: - p->cpu_name = "C64x"; - break; - case 16: - p->cpu_name = "C64x+"; - p->cpu_voltage = "1.2"; - break; - case 21: - p->cpu_name = "C66X"; - p->cpu_voltage = "1.2"; - break; - default: - p->cpu_name = "unknown"; - break; - } - - if (cpu_id < 16) { - switch (rev_id) { - case 0x1: - if (cpu_id > 8) { - p->cpu_rev = "DM640/DM641/DM642/DM643"; - p->cpu_voltage = "1.2 - 1.4"; - } else { - p->cpu_rev = "C6201"; - p->cpu_voltage = "2.5"; - } - break; - case 0x2: - p->cpu_rev = "C6201B/C6202/C6211"; - p->cpu_voltage = "1.8"; - break; - case 0x3: - p->cpu_rev = "C6202B/C6203/C6204/C6205"; - p->cpu_voltage = "1.5"; - break; - case 0x201: - p->cpu_rev = "C6701 revision 0 (early CPU)"; - p->cpu_voltage = "1.8"; - break; - case 0x202: - p->cpu_rev = "C6701/C6711/C6712"; - p->cpu_voltage = "1.8"; - break; - case 0x801: - p->cpu_rev = "C64x"; - p->cpu_voltage = "1.5"; - break; - default: - p->cpu_rev = "unknown"; - } - } else { - p->cpu_rev = p->__cpu_rev; - snprintf(p->__cpu_rev, sizeof(p->__cpu_rev), "0x%x", cpu_id); - } - - p->core_id = get_coreid(); - - for_each_of_cpu_node(node) - ++c6x_num_cores; - - node = of_find_node_by_name(NULL, "soc"); - if (node) { - if (of_property_read_string(node, "model", &c6x_soc_name)) - c6x_soc_name = "unknown"; - of_node_put(node); - } else - c6x_soc_name = "unknown"; - - printk(KERN_INFO "CPU%d: %s rev %s, %s volts, %uMHz\n", - p->core_id, p->cpu_name, p->cpu_rev, - p->cpu_voltage, c6x_core_freq / 1000000); -} - -/* - * Early parsing of the command line - */ -static u32 mem_size __initdata; - -/* "mem=" parsing. */ -static int __init early_mem(char *p) -{ - if (!p) - return -EINVAL; - - mem_size = memparse(p, &p); - /* don't remove all of memory when handling "mem={invalid}" */ - if (mem_size == 0) - return -EINVAL; - - return 0; -} -early_param("mem", early_mem); - -/* "memdma=[@
]" parsing. */ -static int __init early_memdma(char *p) -{ - if (!p) - return -EINVAL; - - dma_size = memparse(p, &p); - if (*p == '@') - dma_start = memparse(p, &p); - - return 0; -} -early_param("memdma", early_memdma); - -int __init c6x_add_memory(phys_addr_t start, unsigned long size) -{ - static int ram_found __initdata; - - /* We only handle one bank (the one with PAGE_OFFSET) for now */ - if (ram_found) - return -EINVAL; - - if (start > PAGE_OFFSET || PAGE_OFFSET >= (start + size)) - return 0; - - ram_start = start; - ram_end = start + size; - - ram_found = 1; - return 0; -} - -/* - * Do early machine setup and device tree parsing. This is called very - * early on the boot process. - */ -notrace void __init machine_init(unsigned long dt_ptr) -{ - void *dtb = __va(dt_ptr); - void *fdt = __dtb_start; - - /* interrupts must be masked */ - set_creg(IER, 2); - - /* - * Set the Interrupt Service Table (IST) to the beginning of the - * vector table. - */ - set_ist(_vectors_start); - - /* - * dtb is passed in from bootloader. - * fdt is linked in blob. - */ - if (dtb && dtb != fdt) - fdt = dtb; - - /* Do some early initialization based on the flat device tree */ - early_init_dt_scan(fdt); - - parse_early_param(); -} - -void __init setup_arch(char **cmdline_p) -{ - phys_addr_t start, end; - u64 i; - - printk(KERN_INFO "Initializing kernel\n"); - - /* Initialize command line */ - *cmdline_p = boot_command_line; - - memory_end = ram_end; - memory_end &= ~(PAGE_SIZE - 1); - - if (mem_size && (PAGE_OFFSET + PAGE_ALIGN(mem_size)) < memory_end) - memory_end = PAGE_OFFSET + PAGE_ALIGN(mem_size); - - /* add block that this kernel can use */ - memblock_add(PAGE_OFFSET, memory_end - PAGE_OFFSET); - - /* reserve kernel text/data/bss */ - memblock_reserve(PAGE_OFFSET, - PAGE_ALIGN((unsigned long)&_end - PAGE_OFFSET)); - - if (dma_size) { - /* align to cacheability granularity */ - dma_size = CACHE_REGION_END(dma_size); - - if (!dma_start) - dma_start = memory_end - dma_size; - - /* align to cacheability granularity */ - dma_start = CACHE_REGION_START(dma_start); - - /* reserve DMA memory taken from kernel memory */ - if (memblock_is_region_memory(dma_start, dma_size)) - memblock_reserve(dma_start, dma_size); - } - - memory_start = PAGE_ALIGN((unsigned int) &_end); - - printk(KERN_INFO "Memory Start=%08lx, Memory End=%08lx\n", - memory_start, memory_end); - -#ifdef CONFIG_BLK_DEV_INITRD - /* - * Reserve initrd memory if in kernel memory. - */ - if (initrd_start < initrd_end) - if (memblock_is_region_memory(initrd_start, - initrd_end - initrd_start)) - memblock_reserve(initrd_start, - initrd_end - initrd_start); -#endif - - init_mm.start_code = (unsigned long) &_stext; - init_mm.end_code = (unsigned long) &_etext; - init_mm.end_data = memory_start; - init_mm.brk = memory_start; - - unflatten_and_copy_device_tree(); - - c6x_cache_init(); - - /* Set the whole external memory as non-cacheable */ - disable_caching(ram_start, ram_end - 1); - - /* Set caching of external RAM used by Linux */ - for_each_mem_range(i, &start, &end) - enable_caching(CACHE_REGION_START(start), - CACHE_REGION_START(end - 1)); - -#ifdef CONFIG_BLK_DEV_INITRD - /* - * Enable caching for initrd which falls outside kernel memory. - */ - if (initrd_start < initrd_end) { - if (!memblock_is_region_memory(initrd_start, - initrd_end - initrd_start)) - enable_caching(CACHE_REGION_START(initrd_start), - CACHE_REGION_START(initrd_end - 1)); - } -#endif - - /* - * Disable caching for dma coherent memory taken from kernel memory. - */ - if (dma_size && memblock_is_region_memory(dma_start, dma_size)) - disable_caching(dma_start, - CACHE_REGION_START(dma_start + dma_size - 1)); - - /* Initialize the coherent memory allocator */ - coherent_mem_init(dma_start, dma_size); - - max_low_pfn = PFN_DOWN(memory_end); - min_low_pfn = PFN_UP(memory_start); - max_pfn = max_low_pfn; - max_mapnr = max_low_pfn - min_low_pfn; - - /* Get kmalloc into gear */ - paging_init(); - - /* - * Probe for Device State Configuration Registers. - * We have to do this early in case timer needs to be enabled - * through DSCR. - */ - dscr_probe(); - - /* We do this early for timer and core clock frequency */ - c64x_setup_clocks(); - - /* Get CPU info */ - get_cpuinfo(); - -#if defined(CONFIG_VT) && defined(CONFIG_DUMMY_CONSOLE) - conswitchp = &dummy_con; -#endif -} - -#define cpu_to_ptr(n) ((void *)((long)(n)+1)) -#define ptr_to_cpu(p) ((long)(p) - 1) - -static int show_cpuinfo(struct seq_file *m, void *v) -{ - int n = ptr_to_cpu(v); - struct cpuinfo_c6x *p = &per_cpu(cpu_data, n); - - if (n == 0) { - seq_printf(m, - "soc\t\t: %s\n" - "soc revision\t: 0x%x\n" - "soc cores\t: %d\n", - c6x_soc_name, c6x_silicon_rev, c6x_num_cores); - } - - seq_printf(m, - "\n" - "processor\t: %d\n" - "cpu\t\t: %s\n" - "core revision\t: %s\n" - "core voltage\t: %s\n" - "core id\t\t: %d\n" - "mmu\t\t: %s\n" - "fpu\t\t: %s\n" - "cpu MHz\t\t: %u\n" - "bogomips\t: %lu.%02lu\n\n", - n, - p->cpu_name, p->cpu_rev, p->cpu_voltage, - p->core_id, p->mmu, p->fpu, - (c6x_core_freq + 500000) / 1000000, - (loops_per_jiffy/(500000/HZ)), - (loops_per_jiffy/(5000/HZ))%100); - - return 0; -} - -static void *c_start(struct seq_file *m, loff_t *pos) -{ - return *pos < nr_cpu_ids ? cpu_to_ptr(*pos) : NULL; -} -static void *c_next(struct seq_file *m, void *v, loff_t *pos) -{ - ++*pos; - return NULL; -} -static void c_stop(struct seq_file *m, void *v) -{ -} - -const struct seq_operations cpuinfo_op = { - c_start, - c_stop, - c_next, - show_cpuinfo -}; - -static struct cpu cpu_devices[NR_CPUS]; - -static int __init topology_init(void) -{ - int i; - - for_each_present_cpu(i) - register_cpu(&cpu_devices[i], i); - - return 0; -} - -subsys_initcall(topology_init); diff --git a/arch/c6x/kernel/signal.c b/arch/c6x/kernel/signal.c deleted file mode 100644 index 862460c3b183..000000000000 --- a/arch/c6x/kernel/signal.c +++ /dev/null @@ -1,322 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2006, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - * - * Updated for 2.6.34: Mark Salter - */ - -#include -#include -#include -#include - -#include -#include -#include - - -/* - * Do a signal return, undo the signal stack. - */ - -#define RETCODE_SIZE (9 << 2) /* 9 instructions = 36 bytes */ - -struct rt_sigframe { - struct siginfo __user *pinfo; - void __user *puc; - struct siginfo info; - struct ucontext uc; - unsigned long retcode[RETCODE_SIZE >> 2]; -}; - -static int restore_sigcontext(struct pt_regs *regs, - struct sigcontext __user *sc) -{ - int err = 0; - - /* The access_ok check was done by caller, so use __get_user here */ -#define COPY(x) (err |= __get_user(regs->x, &sc->sc_##x)) - - COPY(sp); COPY(a4); COPY(b4); COPY(a6); COPY(b6); COPY(a8); COPY(b8); - COPY(a0); COPY(a1); COPY(a2); COPY(a3); COPY(a5); COPY(a7); COPY(a9); - COPY(b0); COPY(b1); COPY(b2); COPY(b3); COPY(b5); COPY(b7); COPY(b9); - - COPY(a16); COPY(a17); COPY(a18); COPY(a19); - COPY(a20); COPY(a21); COPY(a22); COPY(a23); - COPY(a24); COPY(a25); COPY(a26); COPY(a27); - COPY(a28); COPY(a29); COPY(a30); COPY(a31); - COPY(b16); COPY(b17); COPY(b18); COPY(b19); - COPY(b20); COPY(b21); COPY(b22); COPY(b23); - COPY(b24); COPY(b25); COPY(b26); COPY(b27); - COPY(b28); COPY(b29); COPY(b30); COPY(b31); - - COPY(csr); COPY(pc); - -#undef COPY - - return err; -} - -asmlinkage int do_rt_sigreturn(struct pt_regs *regs) -{ - struct rt_sigframe __user *frame; - sigset_t set; - - /* Always make any pending restarted system calls return -EINTR */ - current->restart_block.fn = do_no_restart_syscall; - - /* - * Since we stacked the signal on a dword boundary, - * 'sp' should be dword aligned here. If it's - * not, then the user is trying to mess with us. - */ - if (regs->sp & 7) - goto badframe; - - frame = (struct rt_sigframe __user *) ((unsigned long) regs->sp + 8); - - if (!access_ok(frame, sizeof(*frame))) - goto badframe; - if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) - goto badframe; - - set_current_blocked(&set); - - if (restore_sigcontext(regs, &frame->uc.uc_mcontext)) - goto badframe; - - return regs->a4; - -badframe: - force_sig(SIGSEGV); - return 0; -} - -static int setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, - unsigned long mask) -{ - int err = 0; - - err |= __put_user(mask, &sc->sc_mask); - - /* The access_ok check was done by caller, so use __put_user here */ -#define COPY(x) (err |= __put_user(regs->x, &sc->sc_##x)) - - COPY(sp); COPY(a4); COPY(b4); COPY(a6); COPY(b6); COPY(a8); COPY(b8); - COPY(a0); COPY(a1); COPY(a2); COPY(a3); COPY(a5); COPY(a7); COPY(a9); - COPY(b0); COPY(b1); COPY(b2); COPY(b3); COPY(b5); COPY(b7); COPY(b9); - - COPY(a16); COPY(a17); COPY(a18); COPY(a19); - COPY(a20); COPY(a21); COPY(a22); COPY(a23); - COPY(a24); COPY(a25); COPY(a26); COPY(a27); - COPY(a28); COPY(a29); COPY(a30); COPY(a31); - COPY(b16); COPY(b17); COPY(b18); COPY(b19); - COPY(b20); COPY(b21); COPY(b22); COPY(b23); - COPY(b24); COPY(b25); COPY(b26); COPY(b27); - COPY(b28); COPY(b29); COPY(b30); COPY(b31); - - COPY(csr); COPY(pc); - -#undef COPY - - return err; -} - -static inline void __user *get_sigframe(struct ksignal *ksig, - struct pt_regs *regs, - unsigned long framesize) -{ - unsigned long sp = sigsp(regs->sp, ksig); - - /* - * No matter what happens, 'sp' must be dword - * aligned. Otherwise, nasty things will happen - */ - return (void __user *)((sp - framesize) & ~7); -} - -static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, - struct pt_regs *regs) -{ - struct rt_sigframe __user *frame; - unsigned long __user *retcode; - int err = 0; - - frame = get_sigframe(ksig, regs, sizeof(*frame)); - - if (!access_ok(frame, sizeof(*frame))) - return -EFAULT; - - err |= __put_user(&frame->info, &frame->pinfo); - err |= __put_user(&frame->uc, &frame->puc); - err |= copy_siginfo_to_user(&frame->info, &ksig->info); - - /* Clear all the bits of the ucontext we don't use. */ - err |= __clear_user(&frame->uc, offsetof(struct ucontext, uc_mcontext)); - - err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, set->sig[0]); - err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); - - /* Set up to return from userspace */ - retcode = (unsigned long __user *) &frame->retcode; - - /* The access_ok check was done above, so use __put_user here */ -#define COPY(x) (err |= __put_user(x, retcode++)) - - COPY(0x0000002AUL | (__NR_rt_sigreturn << 7)); - /* MVK __NR_rt_sigreturn,B0 */ - COPY(0x10000000UL); /* SWE */ - COPY(0x00006000UL); /* NOP 4 */ - COPY(0x00006000UL); /* NOP 4 */ - COPY(0x00006000UL); /* NOP 4 */ - COPY(0x00006000UL); /* NOP 4 */ - COPY(0x00006000UL); /* NOP 4 */ - COPY(0x00006000UL); /* NOP 4 */ - COPY(0x00006000UL); /* NOP 4 */ - -#undef COPY - - if (err) - return -EFAULT; - - flush_icache_range((unsigned long) &frame->retcode, - (unsigned long) &frame->retcode + RETCODE_SIZE); - - retcode = (unsigned long __user *) &frame->retcode; - - /* Change user context to branch to signal handler */ - regs->sp = (unsigned long) frame - 8; - regs->b3 = (unsigned long) retcode; - regs->pc = (unsigned long) ksig->ka.sa.sa_handler; - - /* Give the signal number to the handler */ - regs->a4 = ksig->sig; - - /* - * For realtime signals we must also set the second and third - * arguments for the signal handler. - * -- Peter Maydell 2000-12-06 - */ - regs->b4 = (unsigned long)&frame->info; - regs->a6 = (unsigned long)&frame->uc; - - return 0; -} - -static inline void -handle_restart(struct pt_regs *regs, struct k_sigaction *ka, int has_handler) -{ - switch (regs->a4) { - case -ERESTARTNOHAND: - if (!has_handler) - goto do_restart; - regs->a4 = -EINTR; - break; - - case -ERESTARTSYS: - if (has_handler && !(ka->sa.sa_flags & SA_RESTART)) { - regs->a4 = -EINTR; - break; - } - fallthrough; - case -ERESTARTNOINTR: -do_restart: - regs->a4 = regs->orig_a4; - regs->pc -= 4; - break; - } -} - -/* - * handle the actual delivery of a signal to userspace - */ -static void handle_signal(struct ksignal *ksig, struct pt_regs *regs, - int syscall) -{ - int ret; - - /* Are we from a system call? */ - if (syscall) { - /* If so, check system call restarting.. */ - switch (regs->a4) { - case -ERESTART_RESTARTBLOCK: - case -ERESTARTNOHAND: - regs->a4 = -EINTR; - break; - - case -ERESTARTSYS: - if (!(ksig->ka.sa.sa_flags & SA_RESTART)) { - regs->a4 = -EINTR; - break; - } - - fallthrough; - case -ERESTARTNOINTR: - regs->a4 = regs->orig_a4; - regs->pc -= 4; - } - } - - /* Set up the stack frame */ - ret = setup_rt_frame(ksig, sigmask_to_save(), regs); - signal_setup_done(ret, ksig, 0); -} - -/* - * handle a potential signal - */ -static void do_signal(struct pt_regs *regs, int syscall) -{ - struct ksignal ksig; - - /* we want the common case to go fast, which is why we may in certain - * cases get here from kernel mode */ - if (!user_mode(regs)) - return; - - if (get_signal(&ksig)) { - handle_signal(&ksig, regs, syscall); - return; - } - - /* did we come from a system call? */ - if (syscall) { - /* restart the system call - no handlers present */ - switch (regs->a4) { - case -ERESTARTNOHAND: - case -ERESTARTSYS: - case -ERESTARTNOINTR: - regs->a4 = regs->orig_a4; - regs->pc -= 4; - break; - - case -ERESTART_RESTARTBLOCK: - regs->a4 = regs->orig_a4; - regs->b0 = __NR_restart_syscall; - regs->pc -= 4; - break; - } - } - - /* if there's no signal to deliver, we just put the saved sigmask - * back */ - restore_saved_sigmask(); -} - -/* - * notification of userspace execution resumption - * - triggered by current->work.notify_resume - */ -asmlinkage void do_notify_resume(struct pt_regs *regs, u32 thread_info_flags, - int syscall) -{ - /* deal with pending signal delivery */ - if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) - do_signal(regs, syscall); - - if (thread_info_flags & (1 << TIF_NOTIFY_RESUME)) - tracehook_notify_resume(regs); -} diff --git a/arch/c6x/kernel/soc.c b/arch/c6x/kernel/soc.c deleted file mode 100644 index 8362f9390e03..000000000000 --- a/arch/c6x/kernel/soc.c +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Miscellaneous SoC-specific hooks. - * - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - */ -#include -#include -#include -#include -#include - -struct soc_ops soc_ops; - -int soc_get_exception(void) -{ - if (!soc_ops.get_exception) - return -1; - return soc_ops.get_exception(); -} - -void soc_assert_event(unsigned int evt) -{ - if (soc_ops.assert_event) - soc_ops.assert_event(evt); -} - -static u8 cmdline_mac[6]; - -static int __init get_mac_addr_from_cmdline(char *str) -{ - int count, i, val; - - for (count = 0; count < 6 && *str; count++, str += 3) { - if (!isxdigit(str[0]) || !isxdigit(str[1])) - return 0; - if (str[2] != ((count < 5) ? ':' : '\0')) - return 0; - - for (i = 0, val = 0; i < 2; i++) { - val = val << 4; - val |= isdigit(str[i]) ? - str[i] - '0' : toupper(str[i]) - 'A' + 10; - } - cmdline_mac[count] = val; - } - return 1; -} -__setup("emac_addr=", get_mac_addr_from_cmdline); - -/* - * Setup the MAC address for SoC ethernet devices. - * - * Before calling this function, the ethernet driver will have - * initialized the addr with local-mac-address from the device - * tree (if found). Allow command line to override, but not - * the fused address. - */ -int soc_mac_addr(unsigned int index, u8 *addr) -{ - int i, have_dt_mac = 0, have_cmdline_mac = 0, have_fuse_mac = 0; - - for (i = 0; i < 6; i++) { - if (cmdline_mac[i]) - have_cmdline_mac = 1; - if (c6x_fuse_mac[i]) - have_fuse_mac = 1; - if (addr[i]) - have_dt_mac = 1; - } - - /* cmdline overrides all */ - if (have_cmdline_mac) - memcpy(addr, cmdline_mac, 6); - else if (!have_dt_mac) { - if (have_fuse_mac) - memcpy(addr, c6x_fuse_mac, 6); - else - eth_random_addr(addr); - } - - /* adjust for specific EMAC device */ - addr[5] += index * c6x_num_cores; - return 1; -} -EXPORT_SYMBOL_GPL(soc_mac_addr); diff --git a/arch/c6x/kernel/switch_to.S b/arch/c6x/kernel/switch_to.S deleted file mode 100644 index b7f9f607042e..000000000000 --- a/arch/c6x/kernel/switch_to.S +++ /dev/null @@ -1,71 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter (msalter@redhat.com) - */ - -#include -#include - -#define SP B15 - - /* - * void __switch_to(struct thread_info *prev, - * struct thread_info *next, - * struct task_struct *tsk) ; - */ -ENTRY(__switch_to) - LDDW .D2T2 *+B4(THREAD_B15_14),B7:B6 - || MV .L2X A4,B5 ; prev - || MV .L1X B4,A5 ; next - || MVC .S2 RILC,B1 - - STW .D2T2 B3,*+B5(THREAD_PC) - || STDW .D1T1 A13:A12,*+A4(THREAD_A13_12) - || MVC .S2 ILC,B0 - - LDW .D2T2 *+B4(THREAD_PC),B3 - || LDDW .D1T1 *+A5(THREAD_A13_12),A13:A12 - - STDW .D1T1 A11:A10,*+A4(THREAD_A11_10) - || STDW .D2T2 B1:B0,*+B5(THREAD_RICL_ICL) -#ifndef __DSBT__ - || MVKL .S2 current_ksp,B1 -#endif - - STDW .D2T2 B15:B14,*+B5(THREAD_B15_14) - || STDW .D1T1 A15:A14,*+A4(THREAD_A15_14) -#ifndef __DSBT__ - || MVKH .S2 current_ksp,B1 -#endif - - ;; Switch to next SP - MV .S2 B7,SP -#ifdef __DSBT__ - || STW .D2T2 B7,*+B14(current_ksp) -#else - || STW .D2T2 B7,*B1 - || MV .L2 B6,B14 -#endif - || LDDW .D1T1 *+A5(THREAD_RICL_ICL),A1:A0 - - STDW .D2T2 B11:B10,*+B5(THREAD_B11_10) - || LDDW .D1T1 *+A5(THREAD_A15_14),A15:A14 - - STDW .D2T2 B13:B12,*+B5(THREAD_B13_12) - || LDDW .D1T1 *+A5(THREAD_A11_10),A11:A10 - - B .S2 B3 ; return in next E1 - || LDDW .D2T2 *+B4(THREAD_B13_12),B13:B12 - - LDDW .D2T2 *+B4(THREAD_B11_10),B11:B10 - NOP - - MV .L2X A0,B0 - || MV .S1 A6,A4 - - MVC .S2 B0,ILC - || MV .L2X A1,B1 - - MVC .S2 B1,RILC -ENDPROC(__switch_to) diff --git a/arch/c6x/kernel/sys_c6x.c b/arch/c6x/kernel/sys_c6x.c deleted file mode 100644 index 600277f057cf..000000000000 --- a/arch/c6x/kernel/sys_c6x.c +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#include -#include -#include - -#include - -#ifdef CONFIG_ACCESS_CHECK -int _access_ok(unsigned long addr, unsigned long size) -{ - if (!size) - return 1; - - if (!addr || addr > (0xffffffffUL - (size - 1))) - goto _bad_access; - - if (uaccess_kernel()) - return 1; - - if (memory_start <= addr && (addr + size - 1) < memory_end) - return 1; - -_bad_access: - pr_debug("Bad access attempt: pid[%d] addr[%08lx] size[0x%lx]\n", - current->pid, addr, size); - return 0; -} -EXPORT_SYMBOL(_access_ok); -#endif - -/* sys_cache_sync -- sync caches over given range */ -asmlinkage int sys_cache_sync(unsigned long s, unsigned long e) -{ - L1D_cache_block_writeback_invalidate(s, e); - L1P_cache_block_invalidate(s, e); - - return 0; -} - -/* Provide the actual syscall number to call mapping. */ -#undef __SYSCALL -#define __SYSCALL(nr, call) [nr] = (call), - -/* - * Use trampolines - */ -#define sys_pread64 sys_pread_c6x -#define sys_pwrite64 sys_pwrite_c6x -#define sys_truncate64 sys_truncate64_c6x -#define sys_ftruncate64 sys_ftruncate64_c6x -#define sys_fadvise64 sys_fadvise64_c6x -#define sys_fadvise64_64 sys_fadvise64_64_c6x -#define sys_fallocate sys_fallocate_c6x - -/* Use sys_mmap_pgoff directly */ -#define sys_mmap2 sys_mmap_pgoff - -/* - * Note that we can't include here since the header - * guard will defeat us; checks for __SYSCALL as well. - */ -void *sys_call_table[__NR_syscalls] = { - [0 ... __NR_syscalls-1] = sys_ni_syscall, -#include -}; diff --git a/arch/c6x/kernel/time.c b/arch/c6x/kernel/time.c deleted file mode 100644 index f3ec91a87f4f..000000000000 --- a/arch/c6x/kernel/time.c +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -static u32 sched_clock_multiplier; -#define SCHED_CLOCK_SHIFT 16 - -static u64 tsc_read(struct clocksource *cs) -{ - return get_cycles(); -} - -static struct clocksource clocksource_tsc = { - .name = "timestamp", - .rating = 300, - .read = tsc_read, - .mask = CLOCKSOURCE_MASK(64), - .flags = CLOCK_SOURCE_IS_CONTINUOUS, -}; - -/* - * scheduler clock - returns current time in nanoseconds. - */ -u64 sched_clock(void) -{ - u64 tsc = get_cycles(); - - return (tsc * sched_clock_multiplier) >> SCHED_CLOCK_SHIFT; -} - -void __init time_init(void) -{ - u64 tmp = (u64)NSEC_PER_SEC << SCHED_CLOCK_SHIFT; - - do_div(tmp, c6x_core_freq); - sched_clock_multiplier = tmp; - - clocksource_register_hz(&clocksource_tsc, c6x_core_freq); - - /* write anything into TSCL to enable counting */ - set_creg(TSCL, 0); - - /* probe for timer64 event timer */ - timer64_init(); -} diff --git a/arch/c6x/kernel/traps.c b/arch/c6x/kernel/traps.c deleted file mode 100644 index 2b9121c755be..000000000000 --- a/arch/c6x/kernel/traps.c +++ /dev/null @@ -1,409 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2006, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#include -#include -#include -#include - -#include -#include -#include - -int (*c6x_nmi_handler)(struct pt_regs *regs); - -void __init trap_init(void) -{ - ack_exception(EXCEPT_TYPE_NXF); - ack_exception(EXCEPT_TYPE_EXC); - ack_exception(EXCEPT_TYPE_IXF); - ack_exception(EXCEPT_TYPE_SXF); - enable_exception(); -} - -void show_regs(struct pt_regs *regs) -{ - pr_err("\n"); - show_regs_print_info(KERN_ERR); - pr_err("PC: %08lx SP: %08lx\n", regs->pc, regs->sp); - pr_err("Status: %08lx ORIG_A4: %08lx\n", regs->csr, regs->orig_a4); - pr_err("A0: %08lx B0: %08lx\n", regs->a0, regs->b0); - pr_err("A1: %08lx B1: %08lx\n", regs->a1, regs->b1); - pr_err("A2: %08lx B2: %08lx\n", regs->a2, regs->b2); - pr_err("A3: %08lx B3: %08lx\n", regs->a3, regs->b3); - pr_err("A4: %08lx B4: %08lx\n", regs->a4, regs->b4); - pr_err("A5: %08lx B5: %08lx\n", regs->a5, regs->b5); - pr_err("A6: %08lx B6: %08lx\n", regs->a6, regs->b6); - pr_err("A7: %08lx B7: %08lx\n", regs->a7, regs->b7); - pr_err("A8: %08lx B8: %08lx\n", regs->a8, regs->b8); - pr_err("A9: %08lx B9: %08lx\n", regs->a9, regs->b9); - pr_err("A10: %08lx B10: %08lx\n", regs->a10, regs->b10); - pr_err("A11: %08lx B11: %08lx\n", regs->a11, regs->b11); - pr_err("A12: %08lx B12: %08lx\n", regs->a12, regs->b12); - pr_err("A13: %08lx B13: %08lx\n", regs->a13, regs->b13); - pr_err("A14: %08lx B14: %08lx\n", regs->a14, regs->dp); - pr_err("A15: %08lx B15: %08lx\n", regs->a15, regs->sp); - pr_err("A16: %08lx B16: %08lx\n", regs->a16, regs->b16); - pr_err("A17: %08lx B17: %08lx\n", regs->a17, regs->b17); - pr_err("A18: %08lx B18: %08lx\n", regs->a18, regs->b18); - pr_err("A19: %08lx B19: %08lx\n", regs->a19, regs->b19); - pr_err("A20: %08lx B20: %08lx\n", regs->a20, regs->b20); - pr_err("A21: %08lx B21: %08lx\n", regs->a21, regs->b21); - pr_err("A22: %08lx B22: %08lx\n", regs->a22, regs->b22); - pr_err("A23: %08lx B23: %08lx\n", regs->a23, regs->b23); - pr_err("A24: %08lx B24: %08lx\n", regs->a24, regs->b24); - pr_err("A25: %08lx B25: %08lx\n", regs->a25, regs->b25); - pr_err("A26: %08lx B26: %08lx\n", regs->a26, regs->b26); - pr_err("A27: %08lx B27: %08lx\n", regs->a27, regs->b27); - pr_err("A28: %08lx B28: %08lx\n", regs->a28, regs->b28); - pr_err("A29: %08lx B29: %08lx\n", regs->a29, regs->b29); - pr_err("A30: %08lx B30: %08lx\n", regs->a30, regs->b30); - pr_err("A31: %08lx B31: %08lx\n", regs->a31, regs->b31); -} - -void die(char *str, struct pt_regs *fp, int nr) -{ - console_verbose(); - pr_err("%s: %08x\n", str, nr); - show_regs(fp); - - pr_err("Process %s (pid: %d, stackpage=%08lx)\n", - current->comm, current->pid, (PAGE_SIZE + - (unsigned long) current)); - - dump_stack(); - while (1) - ; -} - -static void die_if_kernel(char *str, struct pt_regs *fp, int nr) -{ - if (user_mode(fp)) - return; - - die(str, fp, nr); -} - - -/* Internal exceptions */ -static struct exception_info iexcept_table[10] = { - { "Oops - instruction fetch", SIGBUS, BUS_ADRERR }, - { "Oops - fetch packet", SIGBUS, BUS_ADRERR }, - { "Oops - execute packet", SIGILL, ILL_ILLOPC }, - { "Oops - undefined instruction", SIGILL, ILL_ILLOPC }, - { "Oops - resource conflict", SIGILL, ILL_ILLOPC }, - { "Oops - resource access", SIGILL, ILL_PRVREG }, - { "Oops - privilege", SIGILL, ILL_PRVOPC }, - { "Oops - loops buffer", SIGILL, ILL_ILLOPC }, - { "Oops - software exception", SIGILL, ILL_ILLTRP }, - { "Oops - unknown exception", SIGILL, ILL_ILLOPC } -}; - -/* External exceptions */ -static struct exception_info eexcept_table[128] = { - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - external exception", SIGBUS, BUS_ADRERR }, - { "Oops - CPU memory protection fault", SIGSEGV, SEGV_ACCERR }, - { "Oops - CPU memory protection fault in L1P", SIGSEGV, SEGV_ACCERR }, - { "Oops - DMA memory protection fault in L1P", SIGSEGV, SEGV_ACCERR }, - { "Oops - CPU memory protection fault in L1D", SIGSEGV, SEGV_ACCERR }, - { "Oops - DMA memory protection fault in L1D", SIGSEGV, SEGV_ACCERR }, - { "Oops - CPU memory protection fault in L2", SIGSEGV, SEGV_ACCERR }, - { "Oops - DMA memory protection fault in L2", SIGSEGV, SEGV_ACCERR }, - { "Oops - EMC CPU memory protection fault", SIGSEGV, SEGV_ACCERR }, - { "Oops - EMC bus error", SIGBUS, BUS_ADRERR } -}; - -static void do_trap(struct exception_info *except_info, struct pt_regs *regs) -{ - unsigned long addr = instruction_pointer(regs); - - if (except_info->code != TRAP_BRKPT) - pr_err("TRAP: %s PC[0x%lx] signo[%d] code[%d]\n", - except_info->kernel_str, regs->pc, - except_info->signo, except_info->code); - - die_if_kernel(except_info->kernel_str, regs, addr); - - force_sig_fault(except_info->signo, except_info->code, - (void __user *)addr); -} - -/* - * Process an internal exception (non maskable) - */ -static int process_iexcept(struct pt_regs *regs) -{ - unsigned int iexcept_report = get_iexcept(); - unsigned int iexcept_num; - - ack_exception(EXCEPT_TYPE_IXF); - - pr_err("IEXCEPT: PC[0x%lx]\n", regs->pc); - - while (iexcept_report) { - iexcept_num = __ffs(iexcept_report); - iexcept_report &= ~(1 << iexcept_num); - set_iexcept(iexcept_report); - if (*(unsigned int *)regs->pc == BKPT_OPCODE) { - /* This is a breakpoint */ - struct exception_info bkpt_exception = { - "Oops - undefined instruction", - SIGTRAP, TRAP_BRKPT - }; - do_trap(&bkpt_exception, regs); - iexcept_report &= ~(0xFF); - set_iexcept(iexcept_report); - continue; - } - - do_trap(&iexcept_table[iexcept_num], regs); - } - return 0; -} - -/* - * Process an external exception (maskable) - */ -static void process_eexcept(struct pt_regs *regs) -{ - int evt; - - pr_err("EEXCEPT: PC[0x%lx]\n", regs->pc); - - while ((evt = soc_get_exception()) >= 0) - do_trap(&eexcept_table[evt], regs); - - ack_exception(EXCEPT_TYPE_EXC); -} - -/* - * Main exception processing - */ -asmlinkage int process_exception(struct pt_regs *regs) -{ - unsigned int type; - unsigned int type_num; - unsigned int ie_num = 9; /* default is unknown exception */ - - while ((type = get_except_type()) != 0) { - type_num = fls(type) - 1; - - switch (type_num) { - case EXCEPT_TYPE_NXF: - ack_exception(EXCEPT_TYPE_NXF); - if (c6x_nmi_handler) - (c6x_nmi_handler)(regs); - else - pr_alert("NMI interrupt!\n"); - break; - - case EXCEPT_TYPE_IXF: - if (process_iexcept(regs)) - return 1; - break; - - case EXCEPT_TYPE_EXC: - process_eexcept(regs); - break; - - case EXCEPT_TYPE_SXF: - ie_num = 8; - default: - ack_exception(type_num); - do_trap(&iexcept_table[ie_num], regs); - break; - } - } - return 0; -} - -static int kstack_depth_to_print = 48; - -static void show_trace(unsigned long *stack, unsigned long *endstack, - const char *loglvl) -{ - unsigned long addr; - int i; - - printk("%sCall trace:", loglvl); - i = 0; - while (stack + 1 <= endstack) { - addr = *stack++; - /* - * If the address is either in the text segment of the - * kernel, or in the region which contains vmalloc'ed - * memory, it *may* be the address of a calling - * routine; if so, print it so that someone tracing - * down the cause of the crash will be able to figure - * out the call path that was taken. - */ - if (__kernel_text_address(addr)) { -#ifndef CONFIG_KALLSYMS - if (i % 5 == 0) - printk("%s\n ", loglvl); -#endif - printk("%s [<%08lx>] %pS\n", loglvl, addr, (void *)addr); - i++; - } - } - printk("%s\n", loglvl); -} - -void show_stack(struct task_struct *task, unsigned long *stack, - const char *loglvl) -{ - unsigned long *p, *endstack; - int i; - - if (!stack) { - if (task && task != current) - /* We know this is a kernel stack, - so this is the start/end */ - stack = (unsigned long *)thread_saved_ksp(task); - else - stack = (unsigned long *)&stack; - } - endstack = (unsigned long *)(((unsigned long)stack + THREAD_SIZE - 1) - & -THREAD_SIZE); - - pr_debug("Stack from %08lx:", (unsigned long)stack); - for (i = 0, p = stack; i < kstack_depth_to_print; i++) { - if (p + 1 > endstack) - break; - if (i % 8 == 0) - pr_cont("\n "); - pr_cont(" %08lx", *p++); - } - pr_cont("\n"); - show_trace(stack, endstack, loglvl); -} - -int is_valid_bugaddr(unsigned long addr) -{ - return __kernel_text_address(addr); -} diff --git a/arch/c6x/kernel/vectors.S b/arch/c6x/kernel/vectors.S deleted file mode 100644 index ad3dc006a6d3..000000000000 --- a/arch/c6x/kernel/vectors.S +++ /dev/null @@ -1,78 +0,0 @@ -; SPDX-License-Identifier: GPL-2.0-only -; -; Port on Texas Instruments TMS320C6x architecture -; -; Copyright (C) 2004, 2006, 2009, 2010, 2011 Texas Instruments Incorporated -; Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) -; -; This section handles all the interrupt vector routines. -; At RESET the processor sets up the DRAM timing parameters and -; branches to the label _c_int00 which handles initialization for the C code. -; - -#define ALIGNMENT 5 - - .macro IRQVEC name, handler - .align ALIGNMENT - .hidden \name - .global \name -\name: -#ifdef CONFIG_C6X_BIG_KERNEL - STW .D2T1 A0,*B15--[2] - || MVKL .S1 \handler,A0 - MVKH .S1 \handler,A0 - B .S2X A0 - LDW .D2T1 *++B15[2],A0 - NOP 4 - NOP - NOP - .endm -#else /* CONFIG_C6X_BIG_KERNEL */ - B .S2 \handler - NOP - NOP - NOP - NOP - NOP - NOP - NOP - .endm -#endif /* CONFIG_C6X_BIG_KERNEL */ - - .sect ".vectors","ax" - .align ALIGNMENT - .global RESET - .hidden RESET -RESET: -#ifdef CONFIG_C6X_BIG_KERNEL - MVKL .S1 _c_int00,A0 ; branch to _c_int00 - MVKH .S1 _c_int00,A0 - B .S2X A0 -#else - B .S2 _c_int00 - NOP - NOP -#endif - NOP - NOP - NOP - NOP - NOP - - - IRQVEC NMI,_nmi_handler ; NMI interrupt - IRQVEC AINT,_bad_interrupt ; reserved - IRQVEC MSGINT,_bad_interrupt ; reserved - - IRQVEC INT4,_int4_handler - IRQVEC INT5,_int5_handler - IRQVEC INT6,_int6_handler - IRQVEC INT7,_int7_handler - IRQVEC INT8,_int8_handler - IRQVEC INT9,_int9_handler - IRQVEC INT10,_int10_handler - IRQVEC INT11,_int11_handler - IRQVEC INT12,_int12_handler - IRQVEC INT13,_int13_handler - IRQVEC INT14,_int14_handler - IRQVEC INT15,_int15_handler diff --git a/arch/c6x/kernel/vmlinux.lds.S b/arch/c6x/kernel/vmlinux.lds.S deleted file mode 100644 index ac99ba0864bf..000000000000 --- a/arch/c6x/kernel/vmlinux.lds.S +++ /dev/null @@ -1,151 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * ld script for the c6x kernel - * - * Copyright (C) 2010, 2011 Texas Instruments Incorporated - * Mark Salter - */ - -#define RO_EXCEPTION_TABLE_ALIGN 16 - -#include -#include -#include - -ENTRY(_c_int00) - -#if defined(CONFIG_CPU_BIG_ENDIAN) -jiffies = jiffies_64 + 4; -#else -jiffies = jiffies_64; -#endif - -#define READONLY_SEGMENT_START \ - . = PAGE_OFFSET; -#define READWRITE_SEGMENT_START \ - . = ALIGN(128); \ - _data_lma = .; - -SECTIONS -{ - /* - * Start kernel read only segment - */ - READONLY_SEGMENT_START - - .vectors : - { - _vectors_start = .; - *(.vectors) - . = ALIGN(0x400); - _vectors_end = .; - } - - /* - * This section contains data which may be shared with other - * cores. It needs to be a fixed offset from PAGE_OFFSET - * regardless of kernel configuration. - */ - .virtio_ipc_dev : - { - *(.virtio_ipc_dev) - } - - . = ALIGN(PAGE_SIZE); - __init_begin = .; - .init : - { - _sinittext = .; - HEAD_TEXT - INIT_TEXT - _einittext = .; - } - - INIT_DATA_SECTION(16) - - PERCPU_SECTION(128) - - . = ALIGN(PAGE_SIZE); - __init_end = .; - - .text : - { - _text = .; - _stext = .; - TEXT_TEXT - SCHED_TEXT - CPUIDLE_TEXT - LOCK_TEXT - IRQENTRY_TEXT - SOFTIRQENTRY_TEXT - KPROBES_TEXT - *(.fixup) - *(.gnu.warning) - } - - RO_DATA(PAGE_SIZE) - .const : - { - *(.const .const.* .gnu.linkonce.r.*) - *(.switch) - } - - _etext = .; - - /* - * Start kernel read-write segment. - */ - READWRITE_SEGMENT_START - _sdata = .; - - .fardata : AT(ADDR(.fardata) - LOAD_OFFSET) - { - INIT_TASK_DATA(THREAD_SIZE) - NOSAVE_DATA - PAGE_ALIGNED_DATA(PAGE_SIZE) - CACHELINE_ALIGNED_DATA(128) - READ_MOSTLY_DATA(128) - DATA_DATA - CONSTRUCTORS - *(.data1) - *(.fardata .fardata.*) - *(.data.debug_bpt) - } - - .neardata ALIGN(8) : AT(ADDR(.neardata) - LOAD_OFFSET) - { - *(.neardata2 .neardata2.* .gnu.linkonce.s2.*) - *(.neardata .neardata.* .gnu.linkonce.s.*) - . = ALIGN(8); - } - - BUG_TABLE - - _edata = .; - - __bss_start = .; - SBSS(8) - BSS(8) - .far : - { - . = ALIGN(8); - *(.dynfar) - *(.far .far.* .gnu.linkonce.b.*) - . = ALIGN(8); - } - __bss_stop = .; - - _end = .; - - DWARF_DEBUG - - /DISCARD/ : - { - EXIT_TEXT - EXIT_DATA - EXIT_CALL - *(.discard) - *(.discard.*) - *(.interp) - } -} diff --git a/arch/c6x/lib/Makefile b/arch/c6x/lib/Makefile deleted file mode 100644 index e182004f82fe..000000000000 --- a/arch/c6x/lib/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for arch/c6x/lib/ -# - -lib-y := divu.o divi.o pop_rts.o push_rts.o remi.o remu.o strasgi.o llshru.o -lib-y += llshr.o llshl.o negll.o mpyll.o divremi.o divremu.o -lib-y += checksum.o csum_64plus.o memcpy_64plus.o strasgi_64plus.o diff --git a/arch/c6x/lib/checksum.c b/arch/c6x/lib/checksum.c deleted file mode 100644 index dff2e2ec6e64..000000000000 --- a/arch/c6x/lib/checksum.c +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - */ -#include -#include - -/* These are from csum_64plus.S */ -EXPORT_SYMBOL(csum_partial); -EXPORT_SYMBOL(csum_partial_copy_nocheck); -EXPORT_SYMBOL(ip_compute_csum); -EXPORT_SYMBOL(ip_fast_csum); diff --git a/arch/c6x/lib/csum_64plus.S b/arch/c6x/lib/csum_64plus.S deleted file mode 100644 index 57148866d8d3..000000000000 --- a/arch/c6x/lib/csum_64plus.S +++ /dev/null @@ -1,414 +0,0 @@ -; SPDX-License-Identifier: GPL-2.0-only -; -; linux/arch/c6x/lib/csum_64plus.s -; -; Port on Texas Instruments TMS320C6x architecture -; -; Copyright (C) 2006, 2009, 2010, 2011 Texas Instruments Incorporated -; Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) -; -#include - -; -;unsigned int csum_partial_copy_nocheck(const char *src, char * dst, -; int len, int sum) -; -; A4: src -; B4: dst -; A6: len -; B6: sum -; return csum in A4 -; - - .text -ENTRY(csum_partial_copy_nocheck) - MVC .S2 ILC,B30 - - ZERO .D1 A9 ; csum (a side) -|| ZERO .D2 B9 ; csum (b side) -|| SHRU .S2X A6,2,B5 ; len / 4 - - ;; Check alignment and size - AND .S1 3,A4,A1 -|| AND .S2 3,B4,B0 - OR .L2X B0,A1,B0 ; non aligned condition -|| MVC .S2 B5,ILC -|| MVK .D2 1,B2 -|| MV .D1X B5,A1 ; words condition - [!A1] B .S1 L8 - [B0] BNOP .S1 L6,5 - - SPLOOP 1 - - ;; Main loop for aligned words - LDW .D1T1 *A4++,A7 - NOP 4 - MV .S2X A7,B7 -|| EXTU .S1 A7,0,16,A16 - STW .D2T2 B7,*B4++ -|| MPYU .M2 B7,B2,B8 -|| ADD .L1 A16,A9,A9 - NOP - SPKERNEL 8,0 -|| ADD .L2 B8,B9,B9 - - ZERO .D1 A1 -|| ADD .L1X A9,B9,A9 ; add csum from a and b sides - -L6: - [!A1] BNOP .S1 L8,5 - - ;; Main loop for non-aligned words - SPLOOP 2 - || MVK .L1 1,A2 - - LDNW .D1T1 *A4++,A7 - NOP 3 - - NOP - MV .S2X A7,B7 - || EXTU .S1 A7,0,16,A16 - || MPYU .M1 A7,A2,A8 - - ADD .L1 A16,A9,A9 - SPKERNEL 6,0 - || STNW .D2T2 B7,*B4++ - || ADD .L1 A8,A9,A9 - -L8: AND .S2X 2,A6,B5 - CMPGT .L2 B5,0,B0 - [!B0] BNOP .S1 L82,4 - - ;; Manage half-word - ZERO .L1 A7 -|| ZERO .D1 A8 - -#ifdef CONFIG_CPU_BIG_ENDIAN - - LDBU .D1T1 *A4++,A7 - LDBU .D1T1 *A4++,A8 - NOP 3 - SHL .S1 A7,8,A0 - ADD .S1 A8,A9,A9 - STB .D2T1 A7,*B4++ -|| ADD .S1 A0,A9,A9 - STB .D2T1 A8,*B4++ - -#else - - LDBU .D1T1 *A4++,A7 - LDBU .D1T1 *A4++,A8 - NOP 3 - ADD .S1 A7,A9,A9 - SHL .S1 A8,8,A0 - - STB .D2T1 A7,*B4++ -|| ADD .S1 A0,A9,A9 - STB .D2T1 A8,*B4++ - -#endif - - ;; Manage eventually the last byte -L82: AND .S2X 1,A6,B0 - [!B0] BNOP .S1 L9,5 - -|| ZERO .L1 A7 - -L83: LDBU .D1T1 *A4++,A7 - NOP 4 - - MV .L2X A7,B7 - -#ifdef CONFIG_CPU_BIG_ENDIAN - - STB .D2T2 B7,*B4++ -|| SHL .S1 A7,8,A7 - ADD .S1 A7,A9,A9 - -#else - - STB .D2T2 B7,*B4++ -|| ADD .S1 A7,A9,A9 - -#endif - - ;; Fold the csum -L9: SHRU .S2X A9,16,B0 - [!B0] BNOP .S1 L10,5 - -L91: SHRU .S2X A9,16,B4 -|| EXTU .S1 A9,16,16,A3 - ADD .D1X A3,B4,A9 - - SHRU .S1 A9,16,A0 - [A0] BNOP .S1 L91,5 - -L10: MV .D1 A9,A4 - - BNOP .S2 B3,4 - MVC .S2 B30,ILC -ENDPROC(csum_partial_copy_nocheck) - -; -;unsigned short -;ip_fast_csum(unsigned char *iph, unsigned int ihl) -;{ -; unsigned int checksum = 0; -; unsigned short *tosum = (unsigned short *) iph; -; int len; -; -; len = ihl*4; -; -; if (len <= 0) -; return 0; -; -; while(len) { -; len -= 2; -; checksum += *tosum++; -; } -; if (len & 1) -; checksum += *(unsigned char*) tosum; -; -; while(checksum >> 16) -; checksum = (checksum & 0xffff) + (checksum >> 16); -; -; return ~checksum; -;} -; -; A4: iph -; B4: ihl -; return checksum in A4 -; - .text - -ENTRY(ip_fast_csum) - ZERO .D1 A5 - || MVC .S2 ILC,B30 - SHL .S2 B4,2,B0 - CMPGT .L2 B0,0,B1 - [!B1] BNOP .S1 L15,4 - [!B1] ZERO .D1 A3 - - [!B0] B .S1 L12 - SHRU .S2 B0,1,B0 - MVC .S2 B0,ILC - NOP 3 - - SPLOOP 1 - LDHU .D1T1 *A4++,A3 - NOP 3 - NOP - SPKERNEL 5,0 - || ADD .L1 A3,A5,A5 - -L12: SHRU .S1 A5,16,A0 - [!A0] BNOP .S1 L14,5 - -L13: SHRU .S2X A5,16,B4 - EXTU .S1 A5,16,16,A3 - ADD .D1X A3,B4,A5 - SHRU .S1 A5,16,A0 - [A0] BNOP .S1 L13,5 - -L14: NOT .D1 A5,A3 - EXTU .S1 A3,16,16,A3 - -L15: BNOP .S2 B3,3 - MVC .S2 B30,ILC - MV .D1 A3,A4 -ENDPROC(ip_fast_csum) - -; -;unsigned short -;do_csum(unsigned char *buff, unsigned int len) -;{ -; int odd, count; -; unsigned int result = 0; -; -; if (len <= 0) -; goto out; -; odd = 1 & (unsigned long) buff; -; if (odd) { -;#ifdef __LITTLE_ENDIAN -; result += (*buff << 8); -;#else -; result = *buff; -;#endif -; len--; -; buff++; -; } -; count = len >> 1; /* nr of 16-bit words.. */ -; if (count) { -; if (2 & (unsigned long) buff) { -; result += *(unsigned short *) buff; -; count--; -; len -= 2; -; buff += 2; -; } -; count >>= 1; /* nr of 32-bit words.. */ -; if (count) { -; unsigned int carry = 0; -; do { -; unsigned int w = *(unsigned int *) buff; -; count--; -; buff += 4; -; result += carry; -; result += w; -; carry = (w > result); -; } while (count); -; result += carry; -; result = (result & 0xffff) + (result >> 16); -; } -; if (len & 2) { -; result += *(unsigned short *) buff; -; buff += 2; -; } -; } -; if (len & 1) -;#ifdef __LITTLE_ENDIAN -; result += *buff; -;#else -; result += (*buff << 8); -;#endif -; result = (result & 0xffff) + (result >> 16); -; /* add up carry.. */ -; result = (result & 0xffff) + (result >> 16); -; if (odd) -; result = ((result >> 8) & 0xff) | ((result & 0xff) << 8); -;out: -; return result; -;} -; -; A4: buff -; B4: len -; return checksum in A4 -; - -ENTRY(do_csum) - CMPGT .L2 B4,0,B0 - [!B0] BNOP .S1 L26,3 - EXTU .S1 A4,31,31,A0 - - MV .L1 A0,A3 -|| MV .S1X B3,A5 -|| MV .L2 B4,B3 -|| ZERO .D1 A1 - -#ifdef CONFIG_CPU_BIG_ENDIAN - [A0] SUB .L2 B3,1,B3 -|| [A0] LDBU .D1T1 *A4++,A1 -#else - [!A0] BNOP .S1 L21,5 -|| [A0] LDBU .D1T1 *A4++,A0 - SUB .L2 B3,1,B3 -|| SHL .S1 A0,8,A1 -L21: -#endif - SHR .S2 B3,1,B0 - [!B0] BNOP .S1 L24,3 - MVK .L1 2,A0 - AND .L1 A4,A0,A0 - - [!A0] BNOP .S1 L22,5 -|| [A0] LDHU .D1T1 *A4++,A0 - SUB .L2 B0,1,B0 -|| SUB .S2 B3,2,B3 -|| ADD .L1 A0,A1,A1 -L22: - SHR .S2 B0,1,B0 -|| ZERO .L1 A0 - - [!B0] BNOP .S1 L23,5 -|| [B0] MVC .S2 B0,ILC - - SPLOOP 3 - SPMASK L1 -|| MV .L1 A1,A2 -|| LDW .D1T1 *A4++,A1 - - NOP 4 - ADD .L1 A0,A1,A0 - ADD .L1 A2,A0,A2 - - SPKERNEL 1,2 -|| CMPGTU .L1 A1,A2,A0 - - ADD .L1 A0,A2,A6 - EXTU .S1 A6,16,16,A7 - SHRU .S2X A6,16,B0 - NOP 1 - ADD .L1X A7,B0,A1 -L23: - MVK .L2 2,B0 - AND .L2 B3,B0,B0 - [B0] LDHU .D1T1 *A4++,A0 - NOP 4 - [B0] ADD .L1 A0,A1,A1 -L24: - EXTU .S2 B3,31,31,B0 -#ifdef CONFIG_CPU_BIG_ENDIAN - [!B0] BNOP .S1 L25,4 -|| [B0] LDBU .D1T1 *A4,A0 - SHL .S1 A0,8,A0 - ADD .L1 A0,A1,A1 -L25: -#else - [B0] LDBU .D1T1 *A4,A0 - NOP 4 - [B0] ADD .L1 A0,A1,A1 -#endif - EXTU .S1 A1,16,16,A0 - SHRU .S2X A1,16,B0 - NOP 1 - ADD .L1X A0,B0,A0 - SHRU .S1 A0,16,A1 - ADD .L1 A0,A1,A0 - EXTU .S1 A0,16,16,A1 - EXTU .S1 A1,16,24,A2 - - EXTU .S1 A1,24,16,A0 -|| MV .L2X A3,B0 - - [B0] OR .L1 A0,A2,A1 -L26: - NOP 1 - BNOP .S2X A5,4 - MV .L1 A1,A4 -ENDPROC(do_csum) - -;__wsum csum_partial(const void *buff, int len, __wsum wsum) -;{ -; unsigned int sum = (__force unsigned int)wsum; -; unsigned int result = do_csum(buff, len); -; -; /* add in old sum, and carry.. */ -; result += sum; -; if (sum > result) -; result += 1; -; return (__force __wsum)result; -;} -; -ENTRY(csum_partial) - MV .L1X B3,A9 -|| CALLP .S2 do_csum,B3 -|| MV .S1 A6,A8 - BNOP .S2X A9,2 - ADD .L1 A8,A4,A1 - CMPGTU .L1 A8,A1,A0 - ADD .L1 A1,A0,A4 -ENDPROC(csum_partial) - -;unsigned short -;ip_compute_csum(unsigned char *buff, unsigned int len) -; -; A4: buff -; B4: len -; return checksum in A4 - -ENTRY(ip_compute_csum) - MV .L1X B3,A9 -|| CALLP .S2 do_csum,B3 - BNOP .S2X A9,3 - NOT .S1 A4,A4 - CLR .S1 A4,16,31,A4 -ENDPROC(ip_compute_csum) diff --git a/arch/c6x/lib/divi.S b/arch/c6x/lib/divi.S deleted file mode 100644 index d1764ae0b519..000000000000 --- a/arch/c6x/lib/divi.S +++ /dev/null @@ -1,41 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright 2010 Free Software Foundation, Inc. -;; Contributed by Bernd Schmidt . -;; - -#include - - ;; ABI considerations for the divide functions - ;; The following registers are call-used: - ;; __c6xabi_divi A0,A1,A2,A4,A6,B0,B1,B2,B4,B5 - ;; __c6xabi_divu A0,A1,A2,A4,A6,B0,B1,B2,B4 - ;; __c6xabi_remi A1,A2,A4,A5,A6,B0,B1,B2,B4 - ;; __c6xabi_remu A1,A4,A5,A7,B0,B1,B2,B4 - ;; - ;; In our implementation, divu and remu are leaf functions, - ;; while both divi and remi call into divu. - ;; A0 is not clobbered by any of the functions. - ;; divu does not clobber B2 either, which is taken advantage of - ;; in remi. - ;; divi uses B5 to hold the original return address during - ;; the call to divu. - ;; remi uses B2 and A5 to hold the input values during the - ;; call to divu. It stores B3 in on the stack. - - .text -ENTRY(__c6xabi_divi) - call .s2 __c6xabi_divu -|| mv .d2 B3, B5 -|| cmpgt .l1 0, A4, A1 -|| cmpgt .l2 0, B4, B1 - - [A1] neg .l1 A4, A4 -|| [B1] neg .l2 B4, B4 -|| xor .s1x A1, B1, A1 - [A1] addkpc .s2 _divu_ret, B3, 4 -_divu_ret: - neg .l1 A4, A4 -|| mv .l2 B3,B5 -|| ret .s2 B5 - nop 5 -ENDPROC(__c6xabi_divi) diff --git a/arch/c6x/lib/divremi.S b/arch/c6x/lib/divremi.S deleted file mode 100644 index 575fc57a8a76..000000000000 --- a/arch/c6x/lib/divremi.S +++ /dev/null @@ -1,34 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright 2010 Free Software Foundation, Inc. -;; Contributed by Bernd Schmidt . -;; - -#include - - .text -ENTRY(__c6xabi_divremi) - stw .d2t2 B3, *B15--[2] -|| cmpgt .l1 0, A4, A1 -|| cmpgt .l2 0, B4, B2 -|| mv .s1 A4, A5 -|| call .s2 __c6xabi_divu - - [A1] neg .l1 A4, A4 -|| [B2] neg .l2 B4, B4 -|| xor .s2x B2, A1, B0 -|| mv .d2 B4, B2 - - [B0] addkpc .s2 _divu_ret_1, B3, 1 - [!B0] addkpc .s2 _divu_ret_2, B3, 1 - nop 2 -_divu_ret_1: - neg .l1 A4, A4 -_divu_ret_2: - ldw .d2t2 *++B15[2], B3 - - mpy32 .m1x A4, B2, A6 - nop 3 - ret .s2 B3 - sub .l1 A5, A6, A5 - nop 4 -ENDPROC(__c6xabi_divremi) diff --git a/arch/c6x/lib/divremu.S b/arch/c6x/lib/divremu.S deleted file mode 100644 index 5f6a6a2997ae..000000000000 --- a/arch/c6x/lib/divremu.S +++ /dev/null @@ -1,75 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright 2011 Free Software Foundation, Inc. -;; Contributed by Bernd Schmidt . -;; - -#include - - .text -ENTRY(__c6xabi_divremu) - ;; We use a series of up to 31 subc instructions. First, we find - ;; out how many leading zero bits there are in the divisor. This - ;; gives us both a shift count for aligning (shifting) the divisor - ;; to the, and the number of times we have to execute subc. - - ;; At the end, we have both the remainder and most of the quotient - ;; in A4. The top bit of the quotient is computed first and is - ;; placed in A2. - - ;; Return immediately if the dividend is zero. Setting B4 to 1 - ;; is a trick to allow us to leave the following insns in the jump - ;; delay slot without affecting the result. - mv .s2x A4, B1 - - [b1] lmbd .l2 1, B4, B1 -||[!b1] b .s2 B3 ; RETURN A -||[!b1] mvk .d2 1, B4 - -||[!b1] zero .s1 A5 - mv .l1x B1, A6 -|| shl .s2 B4, B1, B4 - - ;; The loop performs a maximum of 28 steps, so we do the - ;; first 3 here. - cmpltu .l1x A4, B4, A2 - [!A2] sub .l1x A4, B4, A4 -|| shru .s2 B4, 1, B4 -|| xor .s1 1, A2, A2 - - shl .s1 A2, 31, A2 -|| [b1] subc .l1x A4,B4,A4 -|| [b1] add .s2 -1, B1, B1 - [b1] subc .l1x A4,B4,A4 -|| [b1] add .s2 -1, B1, B1 - - ;; RETURN A may happen here (note: must happen before the next branch) -__divremu0: - cmpgt .l2 B1, 7, B0 -|| [b1] subc .l1x A4,B4,A4 -|| [b1] add .s2 -1, B1, B1 - [b1] subc .l1x A4,B4,A4 -|| [b1] add .s2 -1, B1, B1 -|| [b0] b .s1 __divremu0 - [b1] subc .l1x A4,B4,A4 -|| [b1] add .s2 -1, B1, B1 - [b1] subc .l1x A4,B4,A4 -|| [b1] add .s2 -1, B1, B1 - [b1] subc .l1x A4,B4,A4 -|| [b1] add .s2 -1, B1, B1 - [b1] subc .l1x A4,B4,A4 -|| [b1] add .s2 -1, B1, B1 - [b1] subc .l1x A4,B4,A4 -|| [b1] add .s2 -1, B1, B1 - ;; loop backwards branch happens here - - ret .s2 B3 -|| mvk .s1 32, A1 - sub .l1 A1, A6, A6 -|| extu .s1 A4, A6, A5 - shl .s1 A4, A6, A4 - shru .s1 A4, 1, A4 -|| sub .l1 A6, 1, A6 - or .l1 A2, A4, A4 - shru .s1 A4, A6, A4 - nop -ENDPROC(__c6xabi_divremu) diff --git a/arch/c6x/lib/divu.S b/arch/c6x/lib/divu.S deleted file mode 100644 index f0f6082944c2..000000000000 --- a/arch/c6x/lib/divu.S +++ /dev/null @@ -1,86 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright 2010 Free Software Foundation, Inc. -;; Contributed by Bernd Schmidt . -;; - -#include - - ;; ABI considerations for the divide functions - ;; The following registers are call-used: - ;; __c6xabi_divi A0,A1,A2,A4,A6,B0,B1,B2,B4,B5 - ;; __c6xabi_divu A0,A1,A2,A4,A6,B0,B1,B2,B4 - ;; __c6xabi_remi A1,A2,A4,A5,A6,B0,B1,B2,B4 - ;; __c6xabi_remu A1,A4,A5,A7,B0,B1,B2,B4 - ;; - ;; In our implementation, divu and remu are leaf functions, - ;; while both divi and remi call into divu. - ;; A0 is not clobbered by any of the functions. - ;; divu does not clobber B2 either, which is taken advantage of - ;; in remi. - ;; divi uses B5 to hold the original return address during - ;; the call to divu. - ;; remi uses B2 and A5 to hold the input values during the - ;; call to divu. It stores B3 in on the stack. - - .text -ENTRY(__c6xabi_divu) - ;; We use a series of up to 31 subc instructions. First, we find - ;; out how many leading zero bits there are in the divisor. This - ;; gives us both a shift count for aligning (shifting) the divisor - ;; to the, and the number of times we have to execute subc. - - ;; At the end, we have both the remainder and most of the quotient - ;; in A4. The top bit of the quotient is computed first and is - ;; placed in A2. - - ;; Return immediately if the dividend is zero. - mv .s2x A4, B1 - [B1] lmbd .l2 1, B4, B1 -|| [!B1] b .s2 B3 ; RETURN A -|| [!B1] mvk .d2 1, B4 - mv .l1x B1, A6 -|| shl .s2 B4, B1, B4 - - ;; The loop performs a maximum of 28 steps, so we do the - ;; first 3 here. - cmpltu .l1x A4, B4, A2 - [!A2] sub .l1x A4, B4, A4 -|| shru .s2 B4, 1, B4 -|| xor .s1 1, A2, A2 - - shl .s1 A2, 31, A2 -|| [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - - ;; RETURN A may happen here (note: must happen before the next branch) -_divu_loop: - cmpgt .l2 B1, 7, B0 -|| [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 -|| [B0] b .s1 _divu_loop - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - ;; loop backwards branch happens here - - ret .s2 B3 -|| mvk .s1 32, A1 - sub .l1 A1, A6, A6 - shl .s1 A4, A6, A4 - shru .s1 A4, 1, A4 -|| sub .l1 A6, 1, A6 - or .l1 A2, A4, A4 - shru .s1 A4, A6, A4 - nop -ENDPROC(__c6xabi_divu) diff --git a/arch/c6x/lib/llshl.S b/arch/c6x/lib/llshl.S deleted file mode 100644 index 3272499618e0..000000000000 --- a/arch/c6x/lib/llshl.S +++ /dev/null @@ -1,25 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright (C) 2010 Texas Instruments Incorporated -;; Contributed by Mark Salter . -;; - -;; uint64_t __c6xabi_llshl(uint64_t val, uint shift) - -#include - - .text -ENTRY(__c6xabi_llshl) - mv .l1x B4,A1 - [!A1] b .s2 B3 ; just return if zero shift - mvk .s1 32,A0 - sub .d1 A0,A1,A0 - cmplt .l1 0,A0,A2 - [A2] shru .s1 A4,A0,A0 - [!A2] neg .l1 A0,A5 -|| [A2] shl .s1 A5,A1,A5 - [!A2] shl .s1 A4,A5,A5 -|| [A2] or .d1 A5,A0,A5 -|| [!A2] mvk .l1 0,A4 - [A2] shl .s1 A4,A1,A4 - bnop .s2 B3,5 -ENDPROC(__c6xabi_llshl) diff --git a/arch/c6x/lib/llshr.S b/arch/c6x/lib/llshr.S deleted file mode 100644 index 6bfaacd15e73..000000000000 --- a/arch/c6x/lib/llshr.S +++ /dev/null @@ -1,26 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright (C) 2010 Texas Instruments Incorporated -;; Contributed by Mark Salter . -;; - -;; uint64_t __c6xabi_llshr(uint64_t val, uint shift) - -#include - - .text -ENTRY(__c6xabi_llshr) - mv .l1x B4,A1 - [!A1] b .s2 B3 ; return if zero shift count - mvk .s1 32,A0 - sub .d1 A0,A1,A0 - cmplt .l1 0,A0,A2 - [A2] shl .s1 A5,A0,A0 - nop - [!A2] neg .l1 A0,A4 -|| [A2] shru .s1 A4,A1,A4 - [!A2] shr .s1 A5,A4,A4 -|| [A2] or .d1 A4,A0,A4 - [!A2] shr .s1 A5,0x1f,A5 - [A2] shr .s1 A5,A1,A5 - bnop .s2 B3,5 -ENDPROC(__c6xabi_llshr) diff --git a/arch/c6x/lib/llshru.S b/arch/c6x/lib/llshru.S deleted file mode 100644 index 103128f50770..000000000000 --- a/arch/c6x/lib/llshru.S +++ /dev/null @@ -1,26 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright (C) 2010 Texas Instruments Incorporated -;; Contributed by Mark Salter . -;; - -;; uint64_t __c6xabi_llshru(uint64_t val, uint shift) - -#include - - .text -ENTRY(__c6xabi_llshru) - mv .l1x B4,A1 - [!A1] b .s2 B3 ; return if zero shift count - mvk .s1 32,A0 - sub .d1 A0,A1,A0 - cmplt .l1 0,A0,A2 - [A2] shl .s1 A5,A0,A0 - nop - [!A2] neg .l1 A0,A4 -|| [A2] shru .s1 A4,A1,A4 - [!A2] shru .s1 A5,A4,A4 -|| [A2] or .d1 A4,A0,A4 -|| [!A2] mvk .l1 0,A5 - [A2] shru .s1 A5,A1,A5 - bnop .s2 B3,5 -ENDPROC(__c6xabi_llshru) diff --git a/arch/c6x/lib/memcpy_64plus.S b/arch/c6x/lib/memcpy_64plus.S deleted file mode 100644 index 157a30486bfd..000000000000 --- a/arch/c6x/lib/memcpy_64plus.S +++ /dev/null @@ -1,43 +0,0 @@ -; SPDX-License-Identifier: GPL-2.0-only -; Port on Texas Instruments TMS320C6x architecture -; -; Copyright (C) 2006, 2009, 2010 Texas Instruments Incorporated -; Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) -; - -#include - - .text - -ENTRY(memcpy) - AND .L1 0x1,A6,A0 - || AND .S1 0x2,A6,A1 - || AND .L2X 0x4,A6,B0 - || MV .D1 A4,A3 - || MVC .S2 ILC,B2 - - [A0] LDB .D2T1 *B4++,A5 - [A1] LDB .D2T1 *B4++,A7 - [A1] LDB .D2T1 *B4++,A8 - [B0] LDNW .D2T1 *B4++,A9 - || SHRU .S2X A6,0x3,B1 - [!B1] BNOP .S2 B3,1 - - [A0] STB .D1T1 A5,*A3++ - ||[B1] MVC .S2 B1,ILC - [A1] STB .D1T1 A7,*A3++ - [A1] STB .D1T1 A8,*A3++ - [B0] STNW .D1T1 A9,*A3++ ; return when len < 8 - - SPLOOP 2 - - LDNDW .D2T1 *B4++,A9:A8 - NOP 3 - - NOP - SPKERNEL 0,0 - || STNDW .D1T1 A9:A8,*A3++ - - BNOP .S2 B3,4 - MVC .S2 B2,ILC -ENDPROC(memcpy) diff --git a/arch/c6x/lib/mpyll.S b/arch/c6x/lib/mpyll.S deleted file mode 100644 index d07c13ec4fd4..000000000000 --- a/arch/c6x/lib/mpyll.S +++ /dev/null @@ -1,37 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright (C) 2010 Texas Instruments Incorporated -;; Contributed by Mark Salter . -;; - -#include - - ;; uint64_t __c6xabi_mpyll(uint64_t x, uint64_t y) - ;; - ;; 64x64 multiply - ;; First compute partial results using 32-bit parts of x and y: - ;; - ;; b63 b32 b31 b0 - ;; ----------------------------- - ;; | 1 | 0 | - ;; ----------------------------- - ;; - ;; P0 = X0*Y0 - ;; P1 = X0*Y1 + X1*Y0 - ;; P2 = X1*Y1 - ;; - ;; result = (P2 << 64) + (P1 << 32) + P0 - ;; - ;; Since the result is also 64-bit, we can skip the P2 term. - - .text -ENTRY(__c6xabi_mpyll) - mpy32u .m1x A4,B4,A1:A0 ; X0*Y0 - b .s2 B3 - || mpy32u .m2x B5,A4,B1:B0 ; X0*Y1 (don't need upper 32-bits) - || mpy32u .m1x A5,B4,A3:A2 ; X1*Y0 (don't need upper 32-bits) - nop - nop - mv .s1 A0,A4 - add .l1x A2,B0,A5 - add .s1 A1,A5,A5 -ENDPROC(__c6xabi_mpyll) diff --git a/arch/c6x/lib/negll.S b/arch/c6x/lib/negll.S deleted file mode 100644 index 9ba434db5366..000000000000 --- a/arch/c6x/lib/negll.S +++ /dev/null @@ -1,19 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright (C) 2010 Texas Instruments Incorporated -;; Contributed by Mark Salter . -;; - -;; int64_t __c6xabi_negll(int64_t val) - -#include - - .text -ENTRY(__c6xabi_negll) - b .s2 B3 - mvk .l1 0,A0 - subu .l1 A0,A4,A3:A2 - sub .l1 A0,A5,A0 -|| ext .s1 A3,24,24,A5 - add .l1 A5,A0,A5 - mv .s1 A2,A4 -ENDPROC(__c6xabi_negll) diff --git a/arch/c6x/lib/pop_rts.S b/arch/c6x/lib/pop_rts.S deleted file mode 100644 index f129e32943c5..000000000000 --- a/arch/c6x/lib/pop_rts.S +++ /dev/null @@ -1,20 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright 2010 Free Software Foundation, Inc. -;; Contributed by Bernd Schmidt . -;; - -#include - - .text - -ENTRY(__c6xabi_pop_rts) - lddw .d2t2 *++B15, B3:B2 - lddw .d2t1 *++B15, A11:A10 - lddw .d2t2 *++B15, B11:B10 - lddw .d2t1 *++B15, A13:A12 - lddw .d2t2 *++B15, B13:B12 - lddw .d2t1 *++B15, A15:A14 -|| b .s2 B3 - ldw .d2t2 *++B15[2], B14 - nop 4 -ENDPROC(__c6xabi_pop_rts) diff --git a/arch/c6x/lib/push_rts.S b/arch/c6x/lib/push_rts.S deleted file mode 100644 index 40b0a4fe937c..000000000000 --- a/arch/c6x/lib/push_rts.S +++ /dev/null @@ -1,19 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright 2010 Free Software Foundation, Inc. -;; Contributed by Bernd Schmidt . -;; - -#include - - .text - -ENTRY(__c6xabi_push_rts) - stw .d2t2 B14, *B15--[2] - stdw .d2t1 A15:A14, *B15-- -|| b .s2x A3 - stdw .d2t2 B13:B12, *B15-- - stdw .d2t1 A13:A12, *B15-- - stdw .d2t2 B11:B10, *B15-- - stdw .d2t1 A11:A10, *B15-- - stdw .d2t2 B3:B2, *B15-- -ENDPROC(__c6xabi_push_rts) diff --git a/arch/c6x/lib/remi.S b/arch/c6x/lib/remi.S deleted file mode 100644 index 96a1335eac20..000000000000 --- a/arch/c6x/lib/remi.S +++ /dev/null @@ -1,52 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright 2010 Free Software Foundation, Inc. -;; Contributed by Bernd Schmidt . -;; - -#include - - ;; ABI considerations for the divide functions - ;; The following registers are call-used: - ;; __c6xabi_divi A0,A1,A2,A4,A6,B0,B1,B2,B4,B5 - ;; __c6xabi_divu A0,A1,A2,A4,A6,B0,B1,B2,B4 - ;; __c6xabi_remi A1,A2,A4,A5,A6,B0,B1,B2,B4 - ;; __c6xabi_remu A1,A4,A5,A7,B0,B1,B2,B4 - ;; - ;; In our implementation, divu and remu are leaf functions, - ;; while both divi and remi call into divu. - ;; A0 is not clobbered by any of the functions. - ;; divu does not clobber B2 either, which is taken advantage of - ;; in remi. - ;; divi uses B5 to hold the original return address during - ;; the call to divu. - ;; remi uses B2 and A5 to hold the input values during the - ;; call to divu. It stores B3 in on the stack. - - .text - -ENTRY(__c6xabi_remi) - stw .d2t2 B3, *B15--[2] -|| cmpgt .l1 0, A4, A1 -|| cmpgt .l2 0, B4, B2 -|| mv .s1 A4, A5 -|| call .s2 __c6xabi_divu - - [A1] neg .l1 A4, A4 -|| [B2] neg .l2 B4, B4 -|| xor .s2x B2, A1, B0 -|| mv .d2 B4, B2 - - [B0] addkpc .s2 _divu_ret_1, B3, 1 - [!B0] addkpc .s2 _divu_ret_2, B3, 1 - nop 2 -_divu_ret_1: - neg .l1 A4, A4 -_divu_ret_2: - ldw .d2t2 *++B15[2], B3 - - mpy32 .m1x A4, B2, A6 - nop 3 - ret .s2 B3 - sub .l1 A5, A6, A4 - nop 4 -ENDPROC(__c6xabi_remi) diff --git a/arch/c6x/lib/remu.S b/arch/c6x/lib/remu.S deleted file mode 100644 index 428feb9c06c0..000000000000 --- a/arch/c6x/lib/remu.S +++ /dev/null @@ -1,70 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright 2010 Free Software Foundation, Inc. -;; Contributed by Bernd Schmidt . -;; - -#include - - ;; ABI considerations for the divide functions - ;; The following registers are call-used: - ;; __c6xabi_divi A0,A1,A2,A4,A6,B0,B1,B2,B4,B5 - ;; __c6xabi_divu A0,A1,A2,A4,A6,B0,B1,B2,B4 - ;; __c6xabi_remi A1,A2,A4,A5,A6,B0,B1,B2,B4 - ;; __c6xabi_remu A1,A4,A5,A7,B0,B1,B2,B4 - ;; - ;; In our implementation, divu and remu are leaf functions, - ;; while both divi and remi call into divu. - ;; A0 is not clobbered by any of the functions. - ;; divu does not clobber B2 either, which is taken advantage of - ;; in remi. - ;; divi uses B5 to hold the original return address during - ;; the call to divu. - ;; remi uses B2 and A5 to hold the input values during the - ;; call to divu. It stores B3 in on the stack. - - - .text - -ENTRY(__c6xabi_remu) - ;; The ABI seems designed to prevent these functions calling each other, - ;; so we duplicate most of the divsi3 code here. - mv .s2x A4, B1 - lmbd .l2 1, B4, B1 -|| [!B1] b .s2 B3 ; RETURN A -|| [!B1] mvk .d2 1, B4 - - mv .l1x B1, A7 -|| shl .s2 B4, B1, B4 - - cmpltu .l1x A4, B4, A1 - [!A1] sub .l1x A4, B4, A4 - shru .s2 B4, 1, B4 - -_remu_loop: - cmpgt .l2 B1, 7, B0 -|| [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - ;; RETURN A may happen here (note: must happen before the next branch) - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 -|| [B0] b .s1 _remu_loop - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - ;; loop backwards branch happens here - - ret .s2 B3 - [B1] subc .l1x A4,B4,A4 -|| [B1] add .s2 -1, B1, B1 - [B1] subc .l1x A4,B4,A4 - - extu .s1 A4, A7, A4 - nop 2 -ENDPROC(__c6xabi_remu) diff --git a/arch/c6x/lib/strasgi.S b/arch/c6x/lib/strasgi.S deleted file mode 100644 index 715aeb200792..000000000000 --- a/arch/c6x/lib/strasgi.S +++ /dev/null @@ -1,77 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright 2010 Free Software Foundation, Inc. -;; Contributed by Bernd Schmidt . -;; - -#include - - .text - -ENTRY(__c6xabi_strasgi) - ;; This is essentially memcpy, with alignment known to be at least - ;; 4, and the size a multiple of 4 greater than or equal to 28. - ldw .d2t1 *B4++, A0 -|| mvk .s2 16, B1 - ldw .d2t1 *B4++, A1 -|| mvk .s2 20, B2 -|| sub .d1 A6, 24, A6 - ldw .d2t1 *B4++, A5 - ldw .d2t1 *B4++, A7 -|| mv .l2x A6, B7 - ldw .d2t1 *B4++, A8 - ldw .d2t1 *B4++, A9 -|| mv .s2x A0, B5 -|| cmpltu .l2 B2, B7, B0 - -_strasgi_loop: - stw .d1t2 B5, *A4++ -|| [B0] ldw .d2t1 *B4++, A0 -|| mv .s2x A1, B5 -|| mv .l2 B7, B6 - - [B0] sub .d2 B6, 24, B7 -|| [B0] b .s2 _strasgi_loop -|| cmpltu .l2 B1, B6, B0 - - [B0] ldw .d2t1 *B4++, A1 -|| stw .d1t2 B5, *A4++ -|| mv .s2x A5, B5 -|| cmpltu .l2 12, B6, B0 - - [B0] ldw .d2t1 *B4++, A5 -|| stw .d1t2 B5, *A4++ -|| mv .s2x A7, B5 -|| cmpltu .l2 8, B6, B0 - - [B0] ldw .d2t1 *B4++, A7 -|| stw .d1t2 B5, *A4++ -|| mv .s2x A8, B5 -|| cmpltu .l2 4, B6, B0 - - [B0] ldw .d2t1 *B4++, A8 -|| stw .d1t2 B5, *A4++ -|| mv .s2x A9, B5 -|| cmpltu .l2 0, B6, B0 - - [B0] ldw .d2t1 *B4++, A9 -|| stw .d1t2 B5, *A4++ -|| mv .s2x A0, B5 -|| cmpltu .l2 B2, B7, B0 - - ;; loop back branch happens here - - cmpltu .l2 B1, B6, B0 -|| ret .s2 b3 - - [B0] stw .d1t1 A1, *A4++ -|| cmpltu .l2 12, B6, B0 - [B0] stw .d1t1 A5, *A4++ -|| cmpltu .l2 8, B6, B0 - [B0] stw .d1t1 A7, *A4++ -|| cmpltu .l2 4, B6, B0 - [B0] stw .d1t1 A8, *A4++ -|| cmpltu .l2 0, B6, B0 - [B0] stw .d1t1 A9, *A4++ - - ;; return happens here -ENDPROC(__c6xabi_strasgi) diff --git a/arch/c6x/lib/strasgi_64plus.S b/arch/c6x/lib/strasgi_64plus.S deleted file mode 100644 index d10aa2dc3249..000000000000 --- a/arch/c6x/lib/strasgi_64plus.S +++ /dev/null @@ -1,27 +0,0 @@ -;; SPDX-License-Identifier: GPL-2.0-or-later -;; Copyright 2010 Free Software Foundation, Inc. -;; Contributed by Bernd Schmidt . -;; - -#include - - .text - -ENTRY(__c6xabi_strasgi_64plus) - shru .s2x a6, 2, b31 -|| mv .s1 a4, a30 -|| mv .d2 b4, b30 - - add .s2 -4, b31, b31 - - sploopd 1 -|| mvc .s2 b31, ilc - ldw .d2t2 *b30++, b31 - nop 4 - mv .s1x b31,a31 - spkernel 6, 0 -|| stw .d1t1 a31, *a30++ - - ret .s2 b3 - nop 5 -ENDPROC(__c6xabi_strasgi_64plus) diff --git a/arch/c6x/mm/Makefile b/arch/c6x/mm/Makefile deleted file mode 100644 index 19d05e972dd1..000000000000 --- a/arch/c6x/mm/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for the linux c6x-specific parts of the memory manager. -# - -obj-y := init.o dma-coherent.o diff --git a/arch/c6x/mm/dma-coherent.c b/arch/c6x/mm/dma-coherent.c deleted file mode 100644 index 03df07a831fc..000000000000 --- a/arch/c6x/mm/dma-coherent.c +++ /dev/null @@ -1,173 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot - * - * DMA uncached mapping support. - * - * Using code pulled from ARM - * Copyright (C) 2000-2004 Russell King - */ -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -/* - * DMA coherent memory management, can be redefined using the memdma= - * kernel command line - */ - -/* none by default */ -static phys_addr_t dma_base; -static u32 dma_size; -static u32 dma_pages; - -static unsigned long *dma_bitmap; - -/* bitmap lock */ -static DEFINE_SPINLOCK(dma_lock); - -/* - * Return a DMA coherent and contiguous memory chunk from the DMA memory - */ -static inline u32 __alloc_dma_pages(int order) -{ - unsigned long flags; - u32 pos; - - spin_lock_irqsave(&dma_lock, flags); - pos = bitmap_find_free_region(dma_bitmap, dma_pages, order); - spin_unlock_irqrestore(&dma_lock, flags); - - return dma_base + (pos << PAGE_SHIFT); -} - -static void __free_dma_pages(u32 addr, int order) -{ - unsigned long flags; - u32 pos = (addr - dma_base) >> PAGE_SHIFT; - - if (addr < dma_base || (pos + (1 << order)) >= dma_pages) { - printk(KERN_ERR "%s: freeing outside range.\n", __func__); - BUG(); - } - - spin_lock_irqsave(&dma_lock, flags); - bitmap_release_region(dma_bitmap, pos, order); - spin_unlock_irqrestore(&dma_lock, flags); -} - -/* - * Allocate DMA coherent memory space and return both the kernel - * virtual and DMA address for that space. - */ -void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t gfp, unsigned long attrs) -{ - void *ret; - u32 paddr; - int order; - - if (!dma_size || !size) - return NULL; - - order = get_count_order(((size - 1) >> PAGE_SHIFT) + 1); - - paddr = __alloc_dma_pages(order); - - if (handle) - *handle = paddr; - - if (!paddr) - return NULL; - - ret = phys_to_virt(paddr); - memset(ret, 0, 1 << order); - return ret; -} - -/* - * Free DMA coherent memory as defined by the above mapping. - */ -void arch_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, unsigned long attrs) -{ - int order; - - if (!dma_size || !size) - return; - - order = get_count_order(((size - 1) >> PAGE_SHIFT) + 1); - - __free_dma_pages(virt_to_phys(vaddr), order); -} - -/* - * Initialise the coherent DMA memory allocator using the given uncached region. - */ -void __init coherent_mem_init(phys_addr_t start, u32 size) -{ - if (!size) - return; - - printk(KERN_INFO - "Coherent memory (DMA) region start=0x%x size=0x%x\n", - start, size); - - dma_base = start; - dma_size = size; - - /* allocate bitmap */ - dma_pages = dma_size >> PAGE_SHIFT; - if (dma_size & (PAGE_SIZE - 1)) - ++dma_pages; - - dma_bitmap = memblock_alloc(BITS_TO_LONGS(dma_pages) * sizeof(long), - sizeof(long)); - if (!dma_bitmap) - panic("%s: Failed to allocate %zu bytes align=0x%zx\n", - __func__, BITS_TO_LONGS(dma_pages) * sizeof(long), - sizeof(long)); -} - -static void c6x_dma_sync(phys_addr_t paddr, size_t size, - enum dma_data_direction dir) -{ - BUG_ON(!valid_dma_direction(dir)); - - switch (dir) { - case DMA_FROM_DEVICE: - L2_cache_block_invalidate(paddr, paddr + size); - break; - case DMA_TO_DEVICE: - L2_cache_block_writeback(paddr, paddr + size); - break; - case DMA_BIDIRECTIONAL: - L2_cache_block_writeback_invalidate(paddr, paddr + size); - break; - default: - break; - } -} - -void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, - enum dma_data_direction dir) -{ - return c6x_dma_sync(paddr, size, dir); -} - -void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, - enum dma_data_direction dir) -{ - return c6x_dma_sync(paddr, size, dir); -} diff --git a/arch/c6x/mm/init.c b/arch/c6x/mm/init.c deleted file mode 100644 index a97e51a3e26d..000000000000 --- a/arch/c6x/mm/init.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2004, 2009, 2010, 2011 Texas Instruments Incorporated - * Author: Aurelien Jacquiot (aurelien.jacquiot@jaluna.com) - */ -#include -#include -#include -#include -#ifdef CONFIG_BLK_DEV_RAM -#include -#endif -#include - -#include -#include - -/* - * ZERO_PAGE is a special page that is used for zero-initialized - * data and COW. - */ -unsigned long empty_zero_page; -EXPORT_SYMBOL(empty_zero_page); - -/* - * paging_init() continues the virtual memory environment setup which - * was begun by the code in arch/head.S. - * The parameters are pointers to where to stick the starting and ending - * addresses of available kernel virtual memory. - */ -void __init paging_init(void) -{ - struct pglist_data *pgdat = NODE_DATA(0); - unsigned long max_zone_pfn[MAX_NR_ZONES] = {0, }; - - empty_zero_page = (unsigned long) memblock_alloc(PAGE_SIZE, - PAGE_SIZE); - if (!empty_zero_page) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE); - - /* - * Set up user data space - */ - set_fs(KERNEL_DS); - - /* - * Define zones - */ - max_zone_pfn[ZONE_NORMAL] = memory_end >> PAGE_SHIFT; - - free_area_init(max_zone_pfn); -} - -void __init mem_init(void) -{ - high_memory = (void *)(memory_end & PAGE_MASK); - - /* this will put all memory onto the freelists */ - memblock_free_all(); - - mem_init_print_info(NULL); -} diff --git a/arch/c6x/platforms/Kconfig b/arch/c6x/platforms/Kconfig deleted file mode 100644 index f3a9ae6e0e82..000000000000 --- a/arch/c6x/platforms/Kconfig +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -config SOC_TMS320C6455 - bool "TMS320C6455" - default n - -config SOC_TMS320C6457 - bool "TMS320C6457" - default n - -config SOC_TMS320C6472 - bool "TMS320C6472" - default n - -config SOC_TMS320C6474 - bool "TMS320C6474" - default n - -config SOC_TMS320C6678 - bool "TMS320C6678" - default n diff --git a/arch/c6x/platforms/Makefile b/arch/c6x/platforms/Makefile deleted file mode 100644 index b320f1c68884..000000000000 --- a/arch/c6x/platforms/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for arch/c6x/platforms -# -# Copyright 2010, 2011 Texas Instruments Incorporated -# - -obj-y = cache.o megamod-pic.o pll.o plldata.o timer64.o -obj-y += dscr.o - -# SoC objects -obj-$(CONFIG_SOC_TMS320C6455) += emif.o -obj-$(CONFIG_SOC_TMS320C6457) += emif.o diff --git a/arch/c6x/platforms/cache.c b/arch/c6x/platforms/cache.c deleted file mode 100644 index fff027b72513..000000000000 --- a/arch/c6x/platforms/cache.c +++ /dev/null @@ -1,444 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - */ -#include -#include -#include - -#include -#include - -/* - * Internal Memory Control Registers for caches - */ -#define IMCR_CCFG 0x0000 -#define IMCR_L1PCFG 0x0020 -#define IMCR_L1PCC 0x0024 -#define IMCR_L1DCFG 0x0040 -#define IMCR_L1DCC 0x0044 -#define IMCR_L2ALLOC0 0x2000 -#define IMCR_L2ALLOC1 0x2004 -#define IMCR_L2ALLOC2 0x2008 -#define IMCR_L2ALLOC3 0x200c -#define IMCR_L2WBAR 0x4000 -#define IMCR_L2WWC 0x4004 -#define IMCR_L2WIBAR 0x4010 -#define IMCR_L2WIWC 0x4014 -#define IMCR_L2IBAR 0x4018 -#define IMCR_L2IWC 0x401c -#define IMCR_L1PIBAR 0x4020 -#define IMCR_L1PIWC 0x4024 -#define IMCR_L1DWIBAR 0x4030 -#define IMCR_L1DWIWC 0x4034 -#define IMCR_L1DWBAR 0x4040 -#define IMCR_L1DWWC 0x4044 -#define IMCR_L1DIBAR 0x4048 -#define IMCR_L1DIWC 0x404c -#define IMCR_L2WB 0x5000 -#define IMCR_L2WBINV 0x5004 -#define IMCR_L2INV 0x5008 -#define IMCR_L1PINV 0x5028 -#define IMCR_L1DWB 0x5040 -#define IMCR_L1DWBINV 0x5044 -#define IMCR_L1DINV 0x5048 -#define IMCR_MAR_BASE 0x8000 -#define IMCR_MAR96_111 0x8180 -#define IMCR_MAR128_191 0x8200 -#define IMCR_MAR224_239 0x8380 -#define IMCR_L2MPFAR 0xa000 -#define IMCR_L2MPFSR 0xa004 -#define IMCR_L2MPFCR 0xa008 -#define IMCR_L2MPLK0 0xa100 -#define IMCR_L2MPLK1 0xa104 -#define IMCR_L2MPLK2 0xa108 -#define IMCR_L2MPLK3 0xa10c -#define IMCR_L2MPLKCMD 0xa110 -#define IMCR_L2MPLKSTAT 0xa114 -#define IMCR_L2MPPA_BASE 0xa200 -#define IMCR_L1PMPFAR 0xa400 -#define IMCR_L1PMPFSR 0xa404 -#define IMCR_L1PMPFCR 0xa408 -#define IMCR_L1PMPLK0 0xa500 -#define IMCR_L1PMPLK1 0xa504 -#define IMCR_L1PMPLK2 0xa508 -#define IMCR_L1PMPLK3 0xa50c -#define IMCR_L1PMPLKCMD 0xa510 -#define IMCR_L1PMPLKSTAT 0xa514 -#define IMCR_L1PMPPA_BASE 0xa600 -#define IMCR_L1DMPFAR 0xac00 -#define IMCR_L1DMPFSR 0xac04 -#define IMCR_L1DMPFCR 0xac08 -#define IMCR_L1DMPLK0 0xad00 -#define IMCR_L1DMPLK1 0xad04 -#define IMCR_L1DMPLK2 0xad08 -#define IMCR_L1DMPLK3 0xad0c -#define IMCR_L1DMPLKCMD 0xad10 -#define IMCR_L1DMPLKSTAT 0xad14 -#define IMCR_L1DMPPA_BASE 0xae00 -#define IMCR_L2PDWAKE0 0xc040 -#define IMCR_L2PDWAKE1 0xc044 -#define IMCR_L2PDSLEEP0 0xc050 -#define IMCR_L2PDSLEEP1 0xc054 -#define IMCR_L2PDSTAT0 0xc060 -#define IMCR_L2PDSTAT1 0xc064 - -/* - * CCFG register values and bits - */ -#define L2MODE_0K_CACHE 0x0 -#define L2MODE_32K_CACHE 0x1 -#define L2MODE_64K_CACHE 0x2 -#define L2MODE_128K_CACHE 0x3 -#define L2MODE_256K_CACHE 0x7 - -#define L2PRIO_URGENT 0x0 -#define L2PRIO_HIGH 0x1 -#define L2PRIO_MEDIUM 0x2 -#define L2PRIO_LOW 0x3 - -#define CCFG_ID 0x100 /* Invalidate L1P bit */ -#define CCFG_IP 0x200 /* Invalidate L1D bit */ - -static void __iomem *cache_base; - -/* - * L1 & L2 caches generic functions - */ -#define imcr_get(reg) soc_readl(cache_base + (reg)) -#define imcr_set(reg, value) \ -do { \ - soc_writel((value), cache_base + (reg)); \ - soc_readl(cache_base + (reg)); \ -} while (0) - -static void cache_block_operation_wait(unsigned int wc_reg) -{ - /* Wait for completion */ - while (imcr_get(wc_reg)) - cpu_relax(); -} - -static DEFINE_SPINLOCK(cache_lock); - -/* - * Generic function to perform a block cache operation as - * invalidate or writeback/invalidate - */ -static void cache_block_operation(unsigned int *start, - unsigned int *end, - unsigned int bar_reg, - unsigned int wc_reg) -{ - unsigned long flags; - unsigned int wcnt = - (L2_CACHE_ALIGN_CNT((unsigned int) end) - - L2_CACHE_ALIGN_LOW((unsigned int) start)) >> 2; - unsigned int wc = 0; - - for (; wcnt; wcnt -= wc, start += wc) { -loop: - spin_lock_irqsave(&cache_lock, flags); - - /* - * If another cache operation is occurring - */ - if (unlikely(imcr_get(wc_reg))) { - spin_unlock_irqrestore(&cache_lock, flags); - - /* Wait for previous operation completion */ - cache_block_operation_wait(wc_reg); - - /* Try again */ - goto loop; - } - - imcr_set(bar_reg, L2_CACHE_ALIGN_LOW((unsigned int) start)); - - if (wcnt > 0xffff) - wc = 0xffff; - else - wc = wcnt; - - /* Set word count value in the WC register */ - imcr_set(wc_reg, wc & 0xffff); - - spin_unlock_irqrestore(&cache_lock, flags); - - /* Wait for completion */ - cache_block_operation_wait(wc_reg); - } -} - -static void cache_block_operation_nowait(unsigned int *start, - unsigned int *end, - unsigned int bar_reg, - unsigned int wc_reg) -{ - unsigned long flags; - unsigned int wcnt = - (L2_CACHE_ALIGN_CNT((unsigned int) end) - - L2_CACHE_ALIGN_LOW((unsigned int) start)) >> 2; - unsigned int wc = 0; - - for (; wcnt; wcnt -= wc, start += wc) { - - spin_lock_irqsave(&cache_lock, flags); - - imcr_set(bar_reg, L2_CACHE_ALIGN_LOW((unsigned int) start)); - - if (wcnt > 0xffff) - wc = 0xffff; - else - wc = wcnt; - - /* Set word count value in the WC register */ - imcr_set(wc_reg, wc & 0xffff); - - spin_unlock_irqrestore(&cache_lock, flags); - - /* Don't wait for completion on last cache operation */ - if (wcnt > 0xffff) - cache_block_operation_wait(wc_reg); - } -} - -/* - * L1 caches management - */ - -/* - * Disable L1 caches - */ -void L1_cache_off(void) -{ - unsigned int dummy; - - imcr_set(IMCR_L1PCFG, 0); - dummy = imcr_get(IMCR_L1PCFG); - - imcr_set(IMCR_L1DCFG, 0); - dummy = imcr_get(IMCR_L1DCFG); -} - -/* - * Enable L1 caches - */ -void L1_cache_on(void) -{ - unsigned int dummy; - - imcr_set(IMCR_L1PCFG, 7); - dummy = imcr_get(IMCR_L1PCFG); - - imcr_set(IMCR_L1DCFG, 7); - dummy = imcr_get(IMCR_L1DCFG); -} - -/* - * L1P global-invalidate all - */ -void L1P_cache_global_invalidate(void) -{ - unsigned int set = 1; - imcr_set(IMCR_L1PINV, set); - while (imcr_get(IMCR_L1PINV) & 1) - cpu_relax(); -} - -/* - * L1D global-invalidate all - * - * Warning: this operation causes all updated data in L1D to - * be discarded rather than written back to the lower levels of - * memory - */ -void L1D_cache_global_invalidate(void) -{ - unsigned int set = 1; - imcr_set(IMCR_L1DINV, set); - while (imcr_get(IMCR_L1DINV) & 1) - cpu_relax(); -} - -void L1D_cache_global_writeback(void) -{ - unsigned int set = 1; - imcr_set(IMCR_L1DWB, set); - while (imcr_get(IMCR_L1DWB) & 1) - cpu_relax(); -} - -void L1D_cache_global_writeback_invalidate(void) -{ - unsigned int set = 1; - imcr_set(IMCR_L1DWBINV, set); - while (imcr_get(IMCR_L1DWBINV) & 1) - cpu_relax(); -} - -/* - * L2 caches management - */ - -/* - * Set L2 operation mode - */ -void L2_cache_set_mode(unsigned int mode) -{ - unsigned int ccfg = imcr_get(IMCR_CCFG); - - /* Clear and set the L2MODE bits in CCFG */ - ccfg &= ~7; - ccfg |= (mode & 7); - imcr_set(IMCR_CCFG, ccfg); - ccfg = imcr_get(IMCR_CCFG); -} - -/* - * L2 global-writeback and global-invalidate all - */ -void L2_cache_global_writeback_invalidate(void) -{ - imcr_set(IMCR_L2WBINV, 1); - while (imcr_get(IMCR_L2WBINV)) - cpu_relax(); -} - -/* - * L2 global-writeback all - */ -void L2_cache_global_writeback(void) -{ - imcr_set(IMCR_L2WB, 1); - while (imcr_get(IMCR_L2WB)) - cpu_relax(); -} - -/* - * Cacheability controls - */ -void enable_caching(unsigned long start, unsigned long end) -{ - unsigned int mar = IMCR_MAR_BASE + ((start >> 24) << 2); - unsigned int mar_e = IMCR_MAR_BASE + ((end >> 24) << 2); - - for (; mar <= mar_e; mar += 4) - imcr_set(mar, imcr_get(mar) | 1); -} - -void disable_caching(unsigned long start, unsigned long end) -{ - unsigned int mar = IMCR_MAR_BASE + ((start >> 24) << 2); - unsigned int mar_e = IMCR_MAR_BASE + ((end >> 24) << 2); - - for (; mar <= mar_e; mar += 4) - imcr_set(mar, imcr_get(mar) & ~1); -} - - -/* - * L1 block operations - */ -void L1P_cache_block_invalidate(unsigned int start, unsigned int end) -{ - cache_block_operation((unsigned int *) start, - (unsigned int *) end, - IMCR_L1PIBAR, IMCR_L1PIWC); -} -EXPORT_SYMBOL(L1P_cache_block_invalidate); - -void L1D_cache_block_invalidate(unsigned int start, unsigned int end) -{ - cache_block_operation((unsigned int *) start, - (unsigned int *) end, - IMCR_L1DIBAR, IMCR_L1DIWC); -} - -void L1D_cache_block_writeback_invalidate(unsigned int start, unsigned int end) -{ - cache_block_operation((unsigned int *) start, - (unsigned int *) end, - IMCR_L1DWIBAR, IMCR_L1DWIWC); -} - -void L1D_cache_block_writeback(unsigned int start, unsigned int end) -{ - cache_block_operation((unsigned int *) start, - (unsigned int *) end, - IMCR_L1DWBAR, IMCR_L1DWWC); -} -EXPORT_SYMBOL(L1D_cache_block_writeback); - -/* - * L2 block operations - */ -void L2_cache_block_invalidate(unsigned int start, unsigned int end) -{ - cache_block_operation((unsigned int *) start, - (unsigned int *) end, - IMCR_L2IBAR, IMCR_L2IWC); -} - -void L2_cache_block_writeback(unsigned int start, unsigned int end) -{ - cache_block_operation((unsigned int *) start, - (unsigned int *) end, - IMCR_L2WBAR, IMCR_L2WWC); -} - -void L2_cache_block_writeback_invalidate(unsigned int start, unsigned int end) -{ - cache_block_operation((unsigned int *) start, - (unsigned int *) end, - IMCR_L2WIBAR, IMCR_L2WIWC); -} - -void L2_cache_block_invalidate_nowait(unsigned int start, unsigned int end) -{ - cache_block_operation_nowait((unsigned int *) start, - (unsigned int *) end, - IMCR_L2IBAR, IMCR_L2IWC); -} - -void L2_cache_block_writeback_nowait(unsigned int start, unsigned int end) -{ - cache_block_operation_nowait((unsigned int *) start, - (unsigned int *) end, - IMCR_L2WBAR, IMCR_L2WWC); -} - -void L2_cache_block_writeback_invalidate_nowait(unsigned int start, - unsigned int end) -{ - cache_block_operation_nowait((unsigned int *) start, - (unsigned int *) end, - IMCR_L2WIBAR, IMCR_L2WIWC); -} - - -/* - * L1 and L2 caches configuration - */ -void __init c6x_cache_init(void) -{ - struct device_node *node; - - node = of_find_compatible_node(NULL, NULL, "ti,c64x+cache"); - if (!node) - return; - - cache_base = of_iomap(node, 0); - - of_node_put(node); - - if (!cache_base) - return; - - /* Set L2 caches on the the whole L2 SRAM memory */ - L2_cache_set_mode(L2MODE_SIZE); - - /* Enable L1 */ - L1_cache_on(); -} diff --git a/arch/c6x/platforms/dscr.c b/arch/c6x/platforms/dscr.c deleted file mode 100644 index 4571615b589f..000000000000 --- a/arch/c6x/platforms/dscr.c +++ /dev/null @@ -1,595 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Device State Control Registers driver - * - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - */ - -/* - * The Device State Control Registers (DSCR) provide SoC level control over - * a number of peripherals. Details vary considerably among the various SoC - * parts. In general, the DSCR block will provide one or more configuration - * registers often protected by a lock register. One or more key values must - * be written to a lock register in order to unlock the configuration register. - * The configuration register may be used to enable (and disable in some - * cases) SoC pin drivers, peripheral clock sources (internal or pin), etc. - * In some cases, a configuration register is write once or the individual - * bits are write once. That is, you may be able to enable a device, but - * will not be able to disable it. - * - * In addition to device configuration, the DSCR block may provide registers - * which are used to reset SoC peripherals, provide device ID information, - * provide MAC addresses, and other miscellaneous functions. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#define MAX_DEVSTATE_IDS 32 -#define MAX_DEVCTL_REGS 8 -#define MAX_DEVSTAT_REGS 8 -#define MAX_LOCKED_REGS 4 -#define MAX_SOC_EMACS 2 - -struct rmii_reset_reg { - u32 reg; - u32 mask; -}; - -/* - * Some registerd may be locked. In order to write to these - * registers, the key value must first be written to the lockreg. - */ -struct locked_reg { - u32 reg; /* offset from base */ - u32 lockreg; /* offset from base */ - u32 key; /* unlock key */ -}; - -/* - * This describes a contiguous area of like control bits used to enable/disable - * SoC devices. Each controllable device is given an ID which is used by the - * individual device drivers to control the device state. These IDs start at - * zero and are assigned sequentially to the control bitfield ranges described - * by this structure. - */ -struct devstate_ctl_reg { - u32 reg; /* register holding the control bits */ - u8 start_id; /* start id of this range */ - u8 num_ids; /* number of devices in this range */ - u8 enable_only; /* bits are write-once to enable only */ - u8 enable; /* value used to enable device */ - u8 disable; /* value used to disable device */ - u8 shift; /* starting (rightmost) bit in range */ - u8 nbits; /* number of bits per device */ -}; - - -/* - * This describes a region of status bits indicating the state of - * various devices. This is used internally to wait for status - * change completion when enabling/disabling a device. Status is - * optional and not all device controls will have a corresponding - * status. - */ -struct devstate_stat_reg { - u32 reg; /* register holding the status bits */ - u8 start_id; /* start id of this range */ - u8 num_ids; /* number of devices in this range */ - u8 enable; /* value indicating enabled state */ - u8 disable; /* value indicating disabled state */ - u8 shift; /* starting (rightmost) bit in range */ - u8 nbits; /* number of bits per device */ -}; - -struct devstate_info { - struct devstate_ctl_reg *ctl; - struct devstate_stat_reg *stat; -}; - -/* These are callbacks to SOC-specific code. */ -struct dscr_ops { - void (*init)(struct device_node *node); -}; - -struct dscr_regs { - spinlock_t lock; - void __iomem *base; - u32 kick_reg[2]; - u32 kick_key[2]; - struct locked_reg locked[MAX_LOCKED_REGS]; - struct devstate_info devstate_info[MAX_DEVSTATE_IDS]; - struct rmii_reset_reg rmii_resets[MAX_SOC_EMACS]; - struct devstate_ctl_reg devctl[MAX_DEVCTL_REGS]; - struct devstate_stat_reg devstat[MAX_DEVSTAT_REGS]; -}; - -static struct dscr_regs dscr; - -static struct locked_reg *find_locked_reg(u32 reg) -{ - int i; - - for (i = 0; i < MAX_LOCKED_REGS; i++) - if (dscr.locked[i].key && reg == dscr.locked[i].reg) - return &dscr.locked[i]; - return NULL; -} - -/* - * Write to a register with one lock - */ -static void dscr_write_locked1(u32 reg, u32 val, - u32 lock, u32 key) -{ - void __iomem *reg_addr = dscr.base + reg; - void __iomem *lock_addr = dscr.base + lock; - - /* - * For some registers, the lock is relocked after a short number - * of cycles. We have to put the lock write and register write in - * the same fetch packet to meet this timing. The .align ensures - * the two stw instructions are in the same fetch packet. - */ - asm volatile ("b .s2 0f\n" - "nop 5\n" - " .align 5\n" - "0:\n" - "stw .D1T2 %3,*%2\n" - "stw .D1T2 %1,*%0\n" - : - : "a"(reg_addr), "b"(val), "a"(lock_addr), "b"(key) - ); - - /* in case the hw doesn't reset the lock */ - soc_writel(0, lock_addr); -} - -/* - * Write to a register protected by two lock registers - */ -static void dscr_write_locked2(u32 reg, u32 val, - u32 lock0, u32 key0, - u32 lock1, u32 key1) -{ - soc_writel(key0, dscr.base + lock0); - soc_writel(key1, dscr.base + lock1); - soc_writel(val, dscr.base + reg); - soc_writel(0, dscr.base + lock0); - soc_writel(0, dscr.base + lock1); -} - -static void dscr_write(u32 reg, u32 val) -{ - struct locked_reg *lock; - - lock = find_locked_reg(reg); - if (lock) - dscr_write_locked1(reg, val, lock->lockreg, lock->key); - else if (dscr.kick_key[0]) - dscr_write_locked2(reg, val, dscr.kick_reg[0], dscr.kick_key[0], - dscr.kick_reg[1], dscr.kick_key[1]); - else - soc_writel(val, dscr.base + reg); -} - - -/* - * Drivers can use this interface to enable/disable SoC IP blocks. - */ -void dscr_set_devstate(int id, enum dscr_devstate_t state) -{ - struct devstate_ctl_reg *ctl; - struct devstate_stat_reg *stat; - struct devstate_info *info; - u32 ctl_val, val; - int ctl_shift, ctl_mask; - unsigned long flags; - - if (!dscr.base) - return; - - if (id < 0 || id >= MAX_DEVSTATE_IDS) - return; - - info = &dscr.devstate_info[id]; - ctl = info->ctl; - stat = info->stat; - - if (ctl == NULL) - return; - - ctl_shift = ctl->shift + ctl->nbits * (id - ctl->start_id); - ctl_mask = ((1 << ctl->nbits) - 1) << ctl_shift; - - switch (state) { - case DSCR_DEVSTATE_ENABLED: - ctl_val = ctl->enable << ctl_shift; - break; - case DSCR_DEVSTATE_DISABLED: - if (ctl->enable_only) - return; - ctl_val = ctl->disable << ctl_shift; - break; - default: - return; - } - - spin_lock_irqsave(&dscr.lock, flags); - - val = soc_readl(dscr.base + ctl->reg); - val &= ~ctl_mask; - val |= ctl_val; - - dscr_write(ctl->reg, val); - - spin_unlock_irqrestore(&dscr.lock, flags); - - if (!stat) - return; - - ctl_shift = stat->shift + stat->nbits * (id - stat->start_id); - - if (state == DSCR_DEVSTATE_ENABLED) - ctl_val = stat->enable; - else - ctl_val = stat->disable; - - do { - val = soc_readl(dscr.base + stat->reg); - val >>= ctl_shift; - val &= ((1 << stat->nbits) - 1); - } while (val != ctl_val); -} -EXPORT_SYMBOL(dscr_set_devstate); - -/* - * Drivers can use this to reset RMII module. - */ -void dscr_rmii_reset(int id, int assert) -{ - struct rmii_reset_reg *r; - unsigned long flags; - u32 val; - - if (id < 0 || id >= MAX_SOC_EMACS) - return; - - r = &dscr.rmii_resets[id]; - if (r->mask == 0) - return; - - spin_lock_irqsave(&dscr.lock, flags); - - val = soc_readl(dscr.base + r->reg); - if (assert) - dscr_write(r->reg, val | r->mask); - else - dscr_write(r->reg, val & ~(r->mask)); - - spin_unlock_irqrestore(&dscr.lock, flags); -} -EXPORT_SYMBOL(dscr_rmii_reset); - -static void __init dscr_parse_devstat(struct device_node *node, - void __iomem *base) -{ - u32 val; - int err; - - err = of_property_read_u32_array(node, "ti,dscr-devstat", &val, 1); - if (!err) - c6x_devstat = soc_readl(base + val); - printk(KERN_INFO "DEVSTAT: %08x\n", c6x_devstat); -} - -static void __init dscr_parse_silicon_rev(struct device_node *node, - void __iomem *base) -{ - u32 vals[3]; - int err; - - err = of_property_read_u32_array(node, "ti,dscr-silicon-rev", vals, 3); - if (!err) { - c6x_silicon_rev = soc_readl(base + vals[0]); - c6x_silicon_rev >>= vals[1]; - c6x_silicon_rev &= vals[2]; - } -} - -/* - * Some SoCs will have a pair of fuse registers which hold - * an ethernet MAC address. The "ti,dscr-mac-fuse-regs" - * property is a mapping from fuse register bytes to MAC - * address bytes. The expected format is: - * - * ti,dscr-mac-fuse-regs = - * - * reg0 and reg1 are the offsets of the two fuse registers. - * b3-b0 positionally represent bytes within the fuse register. - * b3 is the most significant byte and b0 is the least. - * Allowable values for b3-b0 are: - * - * 0 = fuse register byte not used in MAC address - * 1-6 = index+1 into c6x_fuse_mac[] - */ -static void __init dscr_parse_mac_fuse(struct device_node *node, - void __iomem *base) -{ - u32 vals[10], fuse; - int f, i, j, err; - - err = of_property_read_u32_array(node, "ti,dscr-mac-fuse-regs", - vals, 10); - if (err) - return; - - for (f = 0; f < 2; f++) { - fuse = soc_readl(base + vals[f * 5]); - for (j = (f * 5) + 1, i = 24; i >= 0; i -= 8, j++) - if (vals[j] && vals[j] <= 6) - c6x_fuse_mac[vals[j] - 1] = fuse >> i; - } -} - -static void __init dscr_parse_rmii_resets(struct device_node *node, - void __iomem *base) -{ - const __be32 *p; - int i, size; - - /* look for RMII reset registers */ - p = of_get_property(node, "ti,dscr-rmii-resets", &size); - if (p) { - /* parse all the reg/mask pairs we can handle */ - size /= (sizeof(*p) * 2); - if (size > MAX_SOC_EMACS) - size = MAX_SOC_EMACS; - - for (i = 0; i < size; i++) { - dscr.rmii_resets[i].reg = be32_to_cpup(p++); - dscr.rmii_resets[i].mask = be32_to_cpup(p++); - } - } -} - - -static void __init dscr_parse_privperm(struct device_node *node, - void __iomem *base) -{ - u32 vals[2]; - int err; - - err = of_property_read_u32_array(node, "ti,dscr-privperm", vals, 2); - if (err) - return; - dscr_write(vals[0], vals[1]); -} - -/* - * SoCs may have "locked" DSCR registers which can only be written - * to only after writing a key value to a lock registers. These - * regisers can be described with the "ti,dscr-locked-regs" property. - * This property provides a list of register descriptions with each - * description consisting of three values. - * - * ti,dscr-locked-regs = ; - * - * reg is the offset of the locked register - * lockreg is the offset of the lock register - * key is the unlock key written to lockreg - * - */ -static void __init dscr_parse_locked_regs(struct device_node *node, - void __iomem *base) -{ - struct locked_reg *r; - const __be32 *p; - int i, size; - - p = of_get_property(node, "ti,dscr-locked-regs", &size); - if (p) { - /* parse all the register descriptions we can handle */ - size /= (sizeof(*p) * 3); - if (size > MAX_LOCKED_REGS) - size = MAX_LOCKED_REGS; - - for (i = 0; i < size; i++) { - r = &dscr.locked[i]; - - r->reg = be32_to_cpup(p++); - r->lockreg = be32_to_cpup(p++); - r->key = be32_to_cpup(p++); - } - } -} - -/* - * SoCs may have DSCR registers which are only write enabled after - * writing specific key values to two registers. The two key registers - * and the key values can be parsed from a "ti,dscr-kick-regs" - * propety with the following layout: - * - * ti,dscr-kick-regs = - * - * kickreg is the offset of the "kick" register - * key is the value which unlocks writing for protected regs - */ -static void __init dscr_parse_kick_regs(struct device_node *node, - void __iomem *base) -{ - u32 vals[4]; - int err; - - err = of_property_read_u32_array(node, "ti,dscr-kick-regs", vals, 4); - if (!err) { - dscr.kick_reg[0] = vals[0]; - dscr.kick_key[0] = vals[1]; - dscr.kick_reg[1] = vals[2]; - dscr.kick_key[1] = vals[3]; - } -} - - -/* - * SoCs may provide controls to enable/disable individual IP blocks. These - * controls in the DSCR usually control pin drivers but also may control - * clocking and or resets. The device tree is used to describe the bitfields - * in registers used to control device state. The number of bits and their - * values may vary even within the same register. - * - * The layout of these bitfields is described by the ti,dscr-devstate-ctl-regs - * property. This property is a list where each element describes a contiguous - * range of control fields with like properties. Each element of the list - * consists of 7 cells with the following values: - * - * start_id num_ids reg enable disable start_bit nbits - * - * start_id is device id for the first device control in the range - * num_ids is the number of device controls in the range - * reg is the offset of the register holding the control bits - * enable is the value to enable a device - * disable is the value to disable a device (0xffffffff if cannot disable) - * start_bit is the bit number of the first bit in the range - * nbits is the number of bits per device control - */ -static void __init dscr_parse_devstate_ctl_regs(struct device_node *node, - void __iomem *base) -{ - struct devstate_ctl_reg *r; - const __be32 *p; - int i, j, size; - - p = of_get_property(node, "ti,dscr-devstate-ctl-regs", &size); - if (p) { - /* parse all the ranges we can handle */ - size /= (sizeof(*p) * 7); - if (size > MAX_DEVCTL_REGS) - size = MAX_DEVCTL_REGS; - - for (i = 0; i < size; i++) { - r = &dscr.devctl[i]; - - r->start_id = be32_to_cpup(p++); - r->num_ids = be32_to_cpup(p++); - r->reg = be32_to_cpup(p++); - r->enable = be32_to_cpup(p++); - r->disable = be32_to_cpup(p++); - if (r->disable == 0xffffffff) - r->enable_only = 1; - r->shift = be32_to_cpup(p++); - r->nbits = be32_to_cpup(p++); - - for (j = r->start_id; - j < (r->start_id + r->num_ids); - j++) - dscr.devstate_info[j].ctl = r; - } - } -} - -/* - * SoCs may provide status registers indicating the state (enabled/disabled) of - * devices on the SoC. The device tree is used to describe the bitfields in - * registers used to provide device status. The number of bits and their - * values used to provide status may vary even within the same register. - * - * The layout of these bitfields is described by the ti,dscr-devstate-stat-regs - * property. This property is a list where each element describes a contiguous - * range of status fields with like properties. Each element of the list - * consists of 7 cells with the following values: - * - * start_id num_ids reg enable disable start_bit nbits - * - * start_id is device id for the first device status in the range - * num_ids is the number of devices covered by the range - * reg is the offset of the register holding the status bits - * enable is the value indicating device is enabled - * disable is the value indicating device is disabled - * start_bit is the bit number of the first bit in the range - * nbits is the number of bits per device status - */ -static void __init dscr_parse_devstate_stat_regs(struct device_node *node, - void __iomem *base) -{ - struct devstate_stat_reg *r; - const __be32 *p; - int i, j, size; - - p = of_get_property(node, "ti,dscr-devstate-stat-regs", &size); - if (p) { - /* parse all the ranges we can handle */ - size /= (sizeof(*p) * 7); - if (size > MAX_DEVSTAT_REGS) - size = MAX_DEVSTAT_REGS; - - for (i = 0; i < size; i++) { - r = &dscr.devstat[i]; - - r->start_id = be32_to_cpup(p++); - r->num_ids = be32_to_cpup(p++); - r->reg = be32_to_cpup(p++); - r->enable = be32_to_cpup(p++); - r->disable = be32_to_cpup(p++); - r->shift = be32_to_cpup(p++); - r->nbits = be32_to_cpup(p++); - - for (j = r->start_id; - j < (r->start_id + r->num_ids); - j++) - dscr.devstate_info[j].stat = r; - } - } -} - -static struct of_device_id dscr_ids[] __initdata = { - { .compatible = "ti,c64x+dscr" }, - {} -}; - -/* - * Probe for DSCR area. - * - * This has to be done early on in case timer or interrupt controller - * needs something. e.g. On C6455 SoC, timer must be enabled through - * DSCR before it is functional. - */ -void __init dscr_probe(void) -{ - struct device_node *node; - void __iomem *base; - - spin_lock_init(&dscr.lock); - - node = of_find_matching_node(NULL, dscr_ids); - if (!node) - return; - - base = of_iomap(node, 0); - if (!base) { - of_node_put(node); - return; - } - - dscr.base = base; - - dscr_parse_devstat(node, base); - dscr_parse_silicon_rev(node, base); - dscr_parse_mac_fuse(node, base); - dscr_parse_rmii_resets(node, base); - dscr_parse_locked_regs(node, base); - dscr_parse_kick_regs(node, base); - dscr_parse_devstate_ctl_regs(node, base); - dscr_parse_devstate_stat_regs(node, base); - dscr_parse_privperm(node, base); -} diff --git a/arch/c6x/platforms/emif.c b/arch/c6x/platforms/emif.c deleted file mode 100644 index 6142ecc2cd88..000000000000 --- a/arch/c6x/platforms/emif.c +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * External Memory Interface - * - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - */ -#include -#include -#include -#include -#include - -#define NUM_EMIFA_CHIP_ENABLES 4 - -struct emifa_regs { - u32 midr; - u32 stat; - u32 reserved1[6]; - u32 bprio; - u32 reserved2[23]; - u32 cecfg[NUM_EMIFA_CHIP_ENABLES]; - u32 reserved3[4]; - u32 awcc; - u32 reserved4[7]; - u32 intraw; - u32 intmsk; - u32 intmskset; - u32 intmskclr; -}; - -static struct of_device_id emifa_match[] __initdata = { - { .compatible = "ti,c64x+emifa" }, - {} -}; - -/* - * Parse device tree for existence of an EMIF (External Memory Interface) - * and initialize it if found. - */ -static int __init c6x_emifa_init(void) -{ - struct emifa_regs __iomem *regs; - struct device_node *node; - const __be32 *p; - u32 val; - int i, len, err; - - node = of_find_matching_node(NULL, emifa_match); - if (!node) - return 0; - - regs = of_iomap(node, 0); - if (!regs) - return 0; - - /* look for a dscr-based enable for emifa pin buffers */ - err = of_property_read_u32_array(node, "ti,dscr-dev-enable", &val, 1); - if (!err) - dscr_set_devstate(val, DSCR_DEVSTATE_ENABLED); - - /* set up the chip enables */ - p = of_get_property(node, "ti,emifa-ce-config", &len); - if (p) { - len /= sizeof(u32); - if (len > NUM_EMIFA_CHIP_ENABLES) - len = NUM_EMIFA_CHIP_ENABLES; - for (i = 0; i <= len; i++) - soc_writel(be32_to_cpup(&p[i]), ®s->cecfg[i]); - } - - err = of_property_read_u32_array(node, "ti,emifa-burst-priority", &val, 1); - if (!err) - soc_writel(val, ®s->bprio); - - err = of_property_read_u32_array(node, "ti,emifa-async-wait-control", &val, 1); - if (!err) - soc_writel(val, ®s->awcc); - - iounmap(regs); - of_node_put(node); - return 0; -} -pure_initcall(c6x_emifa_init); diff --git a/arch/c6x/platforms/megamod-pic.c b/arch/c6x/platforms/megamod-pic.c deleted file mode 100644 index 56189e50728c..000000000000 --- a/arch/c6x/platforms/megamod-pic.c +++ /dev/null @@ -1,344 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Support for C64x+ Megamodule Interrupt Controller - * - * Copyright (C) 2010, 2011 Texas Instruments Incorporated - * Contributed by: Mark Salter - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define NR_COMBINERS 4 -#define NR_MUX_OUTPUTS 12 - -#define IRQ_UNMAPPED 0xffff - -/* - * Megamodule Interrupt Controller register layout - */ -struct megamod_regs { - u32 evtflag[8]; - u32 evtset[8]; - u32 evtclr[8]; - u32 reserved0[8]; - u32 evtmask[8]; - u32 mevtflag[8]; - u32 expmask[8]; - u32 mexpflag[8]; - u32 intmux_unused; - u32 intmux[7]; - u32 reserved1[8]; - u32 aegmux[2]; - u32 reserved2[14]; - u32 intxstat; - u32 intxclr; - u32 intdmask; - u32 reserved3[13]; - u32 evtasrt; -}; - -struct megamod_pic { - struct irq_domain *irqhost; - struct megamod_regs __iomem *regs; - raw_spinlock_t lock; - - /* hw mux mapping */ - unsigned int output_to_irq[NR_MUX_OUTPUTS]; -}; - -static struct megamod_pic *mm_pic; - -struct megamod_cascade_data { - struct megamod_pic *pic; - int index; -}; - -static struct megamod_cascade_data cascade_data[NR_COMBINERS]; - -static void mask_megamod(struct irq_data *data) -{ - struct megamod_pic *pic = irq_data_get_irq_chip_data(data); - irq_hw_number_t src = irqd_to_hwirq(data); - u32 __iomem *evtmask = &pic->regs->evtmask[src / 32]; - - raw_spin_lock(&pic->lock); - soc_writel(soc_readl(evtmask) | (1 << (src & 31)), evtmask); - raw_spin_unlock(&pic->lock); -} - -static void unmask_megamod(struct irq_data *data) -{ - struct megamod_pic *pic = irq_data_get_irq_chip_data(data); - irq_hw_number_t src = irqd_to_hwirq(data); - u32 __iomem *evtmask = &pic->regs->evtmask[src / 32]; - - raw_spin_lock(&pic->lock); - soc_writel(soc_readl(evtmask) & ~(1 << (src & 31)), evtmask); - raw_spin_unlock(&pic->lock); -} - -static struct irq_chip megamod_chip = { - .name = "megamod", - .irq_mask = mask_megamod, - .irq_unmask = unmask_megamod, -}; - -static void megamod_irq_cascade(struct irq_desc *desc) -{ - struct megamod_cascade_data *cascade; - struct megamod_pic *pic; - unsigned int irq; - u32 events; - int n, idx; - - cascade = irq_desc_get_handler_data(desc); - - pic = cascade->pic; - idx = cascade->index; - - while ((events = soc_readl(&pic->regs->mevtflag[idx])) != 0) { - n = __ffs(events); - - irq = irq_linear_revmap(pic->irqhost, idx * 32 + n); - - soc_writel(1 << n, &pic->regs->evtclr[idx]); - - generic_handle_irq(irq); - } -} - -static int megamod_map(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hw) -{ - struct megamod_pic *pic = h->host_data; - int i; - - /* We shouldn't see a hwirq which is muxed to core controller */ - for (i = 0; i < NR_MUX_OUTPUTS; i++) - if (pic->output_to_irq[i] == hw) - return -1; - - irq_set_chip_data(virq, pic); - irq_set_chip_and_handler(virq, &megamod_chip, handle_level_irq); - - /* Set default irq type */ - irq_set_irq_type(virq, IRQ_TYPE_NONE); - - return 0; -} - -static const struct irq_domain_ops megamod_domain_ops = { - .map = megamod_map, - .xlate = irq_domain_xlate_onecell, -}; - -static void __init set_megamod_mux(struct megamod_pic *pic, int src, int output) -{ - int index, offset; - u32 val; - - if (src < 0 || src >= (NR_COMBINERS * 32)) { - pic->output_to_irq[output] = IRQ_UNMAPPED; - return; - } - - /* four mappings per mux register */ - index = output / 4; - offset = (output & 3) * 8; - - val = soc_readl(&pic->regs->intmux[index]); - val &= ~(0xff << offset); - val |= src << offset; - soc_writel(val, &pic->regs->intmux[index]); -} - -/* - * Parse the MUX mapping, if one exists. - * - * The MUX map is an array of up to 12 cells; one for each usable core priority - * interrupt. The value of a given cell is the megamodule interrupt source - * which is to me MUXed to the output corresponding to the cell position - * withing the array. The first cell in the array corresponds to priority - * 4 and the last (12th) cell corresponds to priority 15. The allowed - * values are 4 - ((NR_COMBINERS * 32) - 1). Note that the combined interrupt - * sources (0 - 3) are not allowed to be mapped through this property. They - * are handled through the "interrupts" property. This allows us to use a - * value of zero as a "do not map" placeholder. - */ -static void __init parse_priority_map(struct megamod_pic *pic, - int *mapping, int size) -{ - struct device_node *np = irq_domain_get_of_node(pic->irqhost); - const __be32 *map; - int i, maplen; - u32 val; - - map = of_get_property(np, "ti,c64x+megamod-pic-mux", &maplen); - if (map) { - maplen /= 4; - if (maplen > size) - maplen = size; - - for (i = 0; i < maplen; i++) { - val = be32_to_cpup(map); - if (val && val >= 4) - mapping[i] = val; - ++map; - } - } -} - -static struct megamod_pic * __init init_megamod_pic(struct device_node *np) -{ - struct megamod_pic *pic; - int i, irq; - int mapping[NR_MUX_OUTPUTS]; - - pr_info("Initializing C64x+ Megamodule PIC\n"); - - pic = kzalloc(sizeof(struct megamod_pic), GFP_KERNEL); - if (!pic) { - pr_err("%pOF: Could not alloc PIC structure.\n", np); - return NULL; - } - - pic->irqhost = irq_domain_add_linear(np, NR_COMBINERS * 32, - &megamod_domain_ops, pic); - if (!pic->irqhost) { - pr_err("%pOF: Could not alloc host.\n", np); - goto error_free; - } - - pic->irqhost->host_data = pic; - - raw_spin_lock_init(&pic->lock); - - pic->regs = of_iomap(np, 0); - if (!pic->regs) { - pr_err("%pOF: Could not map registers.\n", np); - goto error_free; - } - - /* Initialize MUX map */ - for (i = 0; i < ARRAY_SIZE(mapping); i++) - mapping[i] = IRQ_UNMAPPED; - - parse_priority_map(pic, mapping, ARRAY_SIZE(mapping)); - - /* - * We can have up to 12 interrupts cascading to the core controller. - * These cascades can be from the combined interrupt sources or for - * individual interrupt sources. The "interrupts" property only - * deals with the cascaded combined interrupts. The individual - * interrupts muxed to the core controller use the core controller - * as their interrupt parent. - */ - for (i = 0; i < NR_COMBINERS; i++) { - struct irq_data *irq_data; - irq_hw_number_t hwirq; - - irq = irq_of_parse_and_map(np, i); - if (irq == NO_IRQ) - continue; - - irq_data = irq_get_irq_data(irq); - if (!irq_data) { - pr_err("%pOF: combiner-%d no irq_data for virq %d!\n", - np, i, irq); - continue; - } - - hwirq = irq_data->hwirq; - - /* - * Check that device tree provided something in the range - * of the core priority interrupts (4 - 15). - */ - if (hwirq < 4 || hwirq >= NR_PRIORITY_IRQS) { - pr_err("%pOF: combiner-%d core irq %ld out of range!\n", - np, i, hwirq); - continue; - } - - /* record the mapping */ - mapping[hwirq - 4] = i; - - pr_debug("%pOF: combiner-%d cascading to hwirq %ld\n", - np, i, hwirq); - - cascade_data[i].pic = pic; - cascade_data[i].index = i; - - /* mask and clear all events in combiner */ - soc_writel(~0, &pic->regs->evtmask[i]); - soc_writel(~0, &pic->regs->evtclr[i]); - - irq_set_chained_handler_and_data(irq, megamod_irq_cascade, - &cascade_data[i]); - } - - /* Finally, set up the MUX registers */ - for (i = 0; i < NR_MUX_OUTPUTS; i++) { - if (mapping[i] != IRQ_UNMAPPED) { - pr_debug("%pOF: setting mux %d to priority %d\n", - np, mapping[i], i + 4); - set_megamod_mux(pic, mapping[i], i); - } - } - - return pic; - -error_free: - kfree(pic); - - return NULL; -} - -/* - * Return next active event after ACK'ing it. - * Return -1 if no events active. - */ -static int get_exception(void) -{ - int i, bit; - u32 mask; - - for (i = 0; i < NR_COMBINERS; i++) { - mask = soc_readl(&mm_pic->regs->mexpflag[i]); - if (mask) { - bit = __ffs(mask); - soc_writel(1 << bit, &mm_pic->regs->evtclr[i]); - return (i * 32) + bit; - } - } - return -1; -} - -static void assert_event(unsigned int val) -{ - soc_writel(val, &mm_pic->regs->evtasrt); -} - -void __init megamod_pic_init(void) -{ - struct device_node *np; - - np = of_find_compatible_node(NULL, NULL, "ti,c64x+megamod-pic"); - if (!np) - return; - - mm_pic = init_megamod_pic(np); - of_node_put(np); - - soc_ops.get_exception = get_exception; - soc_ops.assert_event = assert_event; - - return; -} diff --git a/arch/c6x/platforms/pll.c b/arch/c6x/platforms/pll.c deleted file mode 100644 index 6fdf20d64dc7..000000000000 --- a/arch/c6x/platforms/pll.c +++ /dev/null @@ -1,440 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Clock and PLL control for C64x+ devices - * - * Copyright (C) 2010, 2011 Texas Instruments. - * Contributed by: Mark Salter - * - * Copied heavily from arm/mach-davinci/clock.c, so: - * - * Copyright (C) 2006-2007 Texas Instruments. - * Copyright (C) 2008-2009 Deep Root Systems, LLC - */ - -#include -#include -#include -#include -#include - -#include -#include - -static LIST_HEAD(clocks); -static DEFINE_MUTEX(clocks_mutex); -static DEFINE_SPINLOCK(clockfw_lock); - -static void __clk_enable(struct clk *clk) -{ - if (clk->parent) - __clk_enable(clk->parent); - clk->usecount++; -} - -static void __clk_disable(struct clk *clk) -{ - if (WARN_ON(clk->usecount == 0)) - return; - --clk->usecount; - - if (clk->parent) - __clk_disable(clk->parent); -} - -int clk_enable(struct clk *clk) -{ - unsigned long flags; - - if (clk == NULL || IS_ERR(clk)) - return -EINVAL; - - spin_lock_irqsave(&clockfw_lock, flags); - __clk_enable(clk); - spin_unlock_irqrestore(&clockfw_lock, flags); - - return 0; -} -EXPORT_SYMBOL(clk_enable); - -void clk_disable(struct clk *clk) -{ - unsigned long flags; - - if (clk == NULL || IS_ERR(clk)) - return; - - spin_lock_irqsave(&clockfw_lock, flags); - __clk_disable(clk); - spin_unlock_irqrestore(&clockfw_lock, flags); -} -EXPORT_SYMBOL(clk_disable); - -unsigned long clk_get_rate(struct clk *clk) -{ - if (clk == NULL || IS_ERR(clk)) - return -EINVAL; - - return clk->rate; -} -EXPORT_SYMBOL(clk_get_rate); - -long clk_round_rate(struct clk *clk, unsigned long rate) -{ - if (clk == NULL || IS_ERR(clk)) - return -EINVAL; - - if (clk->round_rate) - return clk->round_rate(clk, rate); - - return clk->rate; -} -EXPORT_SYMBOL(clk_round_rate); - -/* Propagate rate to children */ -static void propagate_rate(struct clk *root) -{ - struct clk *clk; - - list_for_each_entry(clk, &root->children, childnode) { - if (clk->recalc) - clk->rate = clk->recalc(clk); - propagate_rate(clk); - } -} - -int clk_set_rate(struct clk *clk, unsigned long rate) -{ - unsigned long flags; - int ret = -EINVAL; - - if (clk == NULL || IS_ERR(clk)) - return ret; - - if (clk->set_rate) - ret = clk->set_rate(clk, rate); - - spin_lock_irqsave(&clockfw_lock, flags); - if (ret == 0) { - if (clk->recalc) - clk->rate = clk->recalc(clk); - propagate_rate(clk); - } - spin_unlock_irqrestore(&clockfw_lock, flags); - - return ret; -} -EXPORT_SYMBOL(clk_set_rate); - -int clk_set_parent(struct clk *clk, struct clk *parent) -{ - unsigned long flags; - - if (clk == NULL || IS_ERR(clk)) - return -EINVAL; - - /* Cannot change parent on enabled clock */ - if (WARN_ON(clk->usecount)) - return -EINVAL; - - mutex_lock(&clocks_mutex); - clk->parent = parent; - list_del_init(&clk->childnode); - list_add(&clk->childnode, &clk->parent->children); - mutex_unlock(&clocks_mutex); - - spin_lock_irqsave(&clockfw_lock, flags); - if (clk->recalc) - clk->rate = clk->recalc(clk); - propagate_rate(clk); - spin_unlock_irqrestore(&clockfw_lock, flags); - - return 0; -} -EXPORT_SYMBOL(clk_set_parent); - -int clk_register(struct clk *clk) -{ - if (clk == NULL || IS_ERR(clk)) - return -EINVAL; - - if (WARN(clk->parent && !clk->parent->rate, - "CLK: %s parent %s has no rate!\n", - clk->name, clk->parent->name)) - return -EINVAL; - - mutex_lock(&clocks_mutex); - list_add_tail(&clk->node, &clocks); - if (clk->parent) - list_add_tail(&clk->childnode, &clk->parent->children); - mutex_unlock(&clocks_mutex); - - /* If rate is already set, use it */ - if (clk->rate) - return 0; - - /* Else, see if there is a way to calculate it */ - if (clk->recalc) - clk->rate = clk->recalc(clk); - - /* Otherwise, default to parent rate */ - else if (clk->parent) - clk->rate = clk->parent->rate; - - return 0; -} -EXPORT_SYMBOL(clk_register); - -void clk_unregister(struct clk *clk) -{ - if (clk == NULL || IS_ERR(clk)) - return; - - mutex_lock(&clocks_mutex); - list_del(&clk->node); - list_del(&clk->childnode); - mutex_unlock(&clocks_mutex); -} -EXPORT_SYMBOL(clk_unregister); - - -static u32 pll_read(struct pll_data *pll, int reg) -{ - return soc_readl(pll->base + reg); -} - -static unsigned long clk_sysclk_recalc(struct clk *clk) -{ - u32 v, plldiv = 0; - struct pll_data *pll; - unsigned long rate = clk->rate; - - if (WARN_ON(!clk->parent)) - return rate; - - rate = clk->parent->rate; - - /* the parent must be a PLL */ - if (WARN_ON(!clk->parent->pll_data)) - return rate; - - pll = clk->parent->pll_data; - - /* If pre-PLL, source clock is before the multiplier and divider(s) */ - if (clk->flags & PRE_PLL) - rate = pll->input_rate; - - if (!clk->div) { - pr_debug("%s: (no divider) rate = %lu KHz\n", - clk->name, rate / 1000); - return rate; - } - - if (clk->flags & FIXED_DIV_PLL) { - rate /= clk->div; - pr_debug("%s: (fixed divide by %d) rate = %lu KHz\n", - clk->name, clk->div, rate / 1000); - return rate; - } - - v = pll_read(pll, clk->div); - if (v & PLLDIV_EN) - plldiv = (v & PLLDIV_RATIO_MASK) + 1; - - if (plldiv == 0) - plldiv = 1; - - rate /= plldiv; - - pr_debug("%s: (divide by %d) rate = %lu KHz\n", - clk->name, plldiv, rate / 1000); - - return rate; -} - -static unsigned long clk_leafclk_recalc(struct clk *clk) -{ - if (WARN_ON(!clk->parent)) - return clk->rate; - - pr_debug("%s: (parent %s) rate = %lu KHz\n", - clk->name, clk->parent->name, clk->parent->rate / 1000); - - return clk->parent->rate; -} - -static unsigned long clk_pllclk_recalc(struct clk *clk) -{ - u32 ctrl, mult = 0, prediv = 0, postdiv = 0; - u8 bypass; - struct pll_data *pll = clk->pll_data; - unsigned long rate = clk->rate; - - if (clk->flags & FIXED_RATE_PLL) - return rate; - - ctrl = pll_read(pll, PLLCTL); - rate = pll->input_rate = clk->parent->rate; - - if (ctrl & PLLCTL_PLLEN) - bypass = 0; - else - bypass = 1; - - if (pll->flags & PLL_HAS_MUL) { - mult = pll_read(pll, PLLM); - mult = (mult & PLLM_PLLM_MASK) + 1; - } - if (pll->flags & PLL_HAS_PRE) { - prediv = pll_read(pll, PLLPRE); - if (prediv & PLLDIV_EN) - prediv = (prediv & PLLDIV_RATIO_MASK) + 1; - else - prediv = 0; - } - if (pll->flags & PLL_HAS_POST) { - postdiv = pll_read(pll, PLLPOST); - if (postdiv & PLLDIV_EN) - postdiv = (postdiv & PLLDIV_RATIO_MASK) + 1; - else - postdiv = 1; - } - - if (!bypass) { - if (prediv) - rate /= prediv; - if (mult) - rate *= mult; - if (postdiv) - rate /= postdiv; - - pr_debug("PLL%d: input = %luMHz, pre[%d] mul[%d] post[%d] " - "--> %luMHz output.\n", - pll->num, clk->parent->rate / 1000000, - prediv, mult, postdiv, rate / 1000000); - } else - pr_debug("PLL%d: input = %luMHz, bypass mode.\n", - pll->num, clk->parent->rate / 1000000); - - return rate; -} - - -static void __init __init_clk(struct clk *clk) -{ - INIT_LIST_HEAD(&clk->node); - INIT_LIST_HEAD(&clk->children); - INIT_LIST_HEAD(&clk->childnode); - - if (!clk->recalc) { - - /* Check if clock is a PLL */ - if (clk->pll_data) - clk->recalc = clk_pllclk_recalc; - - /* Else, if it is a PLL-derived clock */ - else if (clk->flags & CLK_PLL) - clk->recalc = clk_sysclk_recalc; - - /* Otherwise, it is a leaf clock (PSC clock) */ - else if (clk->parent) - clk->recalc = clk_leafclk_recalc; - } -} - -void __init c6x_clks_init(struct clk_lookup *clocks) -{ - struct clk_lookup *c; - struct clk *clk; - size_t num_clocks = 0; - - for (c = clocks; c->clk; c++) { - clk = c->clk; - - __init_clk(clk); - clk_register(clk); - num_clocks++; - - /* Turn on clocks that Linux doesn't otherwise manage */ - if (clk->flags & ALWAYS_ENABLED) - clk_enable(clk); - } - - clkdev_add_table(clocks, num_clocks); -} - -#ifdef CONFIG_DEBUG_FS - -#include -#include - -#define CLKNAME_MAX 10 /* longest clock name */ -#define NEST_DELTA 2 -#define NEST_MAX 4 - -static void -dump_clock(struct seq_file *s, unsigned nest, struct clk *parent) -{ - char *state; - char buf[CLKNAME_MAX + NEST_DELTA * NEST_MAX]; - struct clk *clk; - unsigned i; - - if (parent->flags & CLK_PLL) - state = "pll"; - else - state = ""; - - /* name */ - memset(buf, ' ', sizeof(buf) - 1); - buf[sizeof(buf) - 1] = 0; - i = strlen(parent->name); - memcpy(buf + nest, parent->name, - min(i, (unsigned)(sizeof(buf) - 1 - nest))); - - seq_printf(s, "%s users=%2d %-3s %9ld Hz\n", - buf, parent->usecount, state, clk_get_rate(parent)); - /* REVISIT show device associations too */ - - /* cost is now small, but not linear... */ - list_for_each_entry(clk, &parent->children, childnode) { - dump_clock(s, nest + NEST_DELTA, clk); - } -} - -static int c6x_ck_show(struct seq_file *m, void *v) -{ - struct clk *clk; - - /* - * Show clock tree; We trust nonzero usecounts equate to PSC enables... - */ - mutex_lock(&clocks_mutex); - list_for_each_entry(clk, &clocks, node) - if (!clk->parent) - dump_clock(m, 0, clk); - mutex_unlock(&clocks_mutex); - - return 0; -} - -static int c6x_ck_open(struct inode *inode, struct file *file) -{ - return single_open(file, c6x_ck_show, NULL); -} - -static const struct file_operations c6x_ck_operations = { - .open = c6x_ck_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -static int __init c6x_clk_debugfs_init(void) -{ - debugfs_create_file("c6x_clocks", S_IFREG | S_IRUGO, NULL, NULL, - &c6x_ck_operations); - - return 0; -} -device_initcall(c6x_clk_debugfs_init); -#endif /* CONFIG_DEBUG_FS */ diff --git a/arch/c6x/platforms/plldata.c b/arch/c6x/platforms/plldata.c deleted file mode 100644 index a799e04edefe..000000000000 --- a/arch/c6x/platforms/plldata.c +++ /dev/null @@ -1,467 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Port on Texas Instruments TMS320C6x architecture - * - * Copyright (C) 2011 Texas Instruments Incorporated - * Author: Mark Salter - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -/* - * Common SoC clock support. - */ - -/* Default input for PLL1 */ -struct clk clkin1 = { - .name = "clkin1", - .node = LIST_HEAD_INIT(clkin1.node), - .children = LIST_HEAD_INIT(clkin1.children), - .childnode = LIST_HEAD_INIT(clkin1.childnode), -}; - -struct pll_data c6x_soc_pll1 = { - .num = 1, - .sysclks = { - { - .name = "pll1", - .parent = &clkin1, - .pll_data = &c6x_soc_pll1, - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk1", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk2", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk3", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk4", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk5", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk6", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk7", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk8", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk9", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk10", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk11", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk12", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk13", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk14", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk15", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - { - .name = "pll1_sysclk16", - .parent = &c6x_soc_pll1.sysclks[0], - .flags = CLK_PLL, - }, - }, -}; - -/* CPU core clock */ -struct clk c6x_core_clk = { - .name = "core", -}; - -/* miscellaneous IO clocks */ -struct clk c6x_i2c_clk = { - .name = "i2c", -}; - -struct clk c6x_watchdog_clk = { - .name = "watchdog", -}; - -struct clk c6x_mcbsp1_clk = { - .name = "mcbsp1", -}; - -struct clk c6x_mcbsp2_clk = { - .name = "mcbsp2", -}; - -struct clk c6x_mdio_clk = { - .name = "mdio", -}; - - -#ifdef CONFIG_SOC_TMS320C6455 -static struct clk_lookup c6455_clks[] = { - CLK(NULL, "pll1", &c6x_soc_pll1.sysclks[0]), - CLK(NULL, "pll1_sysclk2", &c6x_soc_pll1.sysclks[2]), - CLK(NULL, "pll1_sysclk3", &c6x_soc_pll1.sysclks[3]), - CLK(NULL, "pll1_sysclk4", &c6x_soc_pll1.sysclks[4]), - CLK(NULL, "pll1_sysclk5", &c6x_soc_pll1.sysclks[5]), - CLK(NULL, "core", &c6x_core_clk), - CLK("i2c_davinci.1", NULL, &c6x_i2c_clk), - CLK("watchdog", NULL, &c6x_watchdog_clk), - CLK("2c81800.mdio", NULL, &c6x_mdio_clk), - CLK("", NULL, NULL) -}; - - -static void __init c6455_setup_clocks(struct device_node *node) -{ - struct pll_data *pll = &c6x_soc_pll1; - struct clk *sysclks = pll->sysclks; - - pll->flags = PLL_HAS_PRE | PLL_HAS_MUL; - - sysclks[2].flags |= FIXED_DIV_PLL; - sysclks[2].div = 3; - sysclks[3].flags |= FIXED_DIV_PLL; - sysclks[3].div = 6; - sysclks[4].div = PLLDIV4; - sysclks[5].div = PLLDIV5; - - c6x_core_clk.parent = &sysclks[0]; - c6x_i2c_clk.parent = &sysclks[3]; - c6x_watchdog_clk.parent = &sysclks[3]; - c6x_mdio_clk.parent = &sysclks[3]; - - c6x_clks_init(c6455_clks); -} -#endif /* CONFIG_SOC_TMS320C6455 */ - -#ifdef CONFIG_SOC_TMS320C6457 -static struct clk_lookup c6457_clks[] = { - CLK(NULL, "pll1", &c6x_soc_pll1.sysclks[0]), - CLK(NULL, "pll1_sysclk1", &c6x_soc_pll1.sysclks[1]), - CLK(NULL, "pll1_sysclk2", &c6x_soc_pll1.sysclks[2]), - CLK(NULL, "pll1_sysclk3", &c6x_soc_pll1.sysclks[3]), - CLK(NULL, "pll1_sysclk4", &c6x_soc_pll1.sysclks[4]), - CLK(NULL, "pll1_sysclk5", &c6x_soc_pll1.sysclks[5]), - CLK(NULL, "core", &c6x_core_clk), - CLK("i2c_davinci.1", NULL, &c6x_i2c_clk), - CLK("watchdog", NULL, &c6x_watchdog_clk), - CLK("2c81800.mdio", NULL, &c6x_mdio_clk), - CLK("", NULL, NULL) -}; - -static void __init c6457_setup_clocks(struct device_node *node) -{ - struct pll_data *pll = &c6x_soc_pll1; - struct clk *sysclks = pll->sysclks; - - pll->flags = PLL_HAS_MUL | PLL_HAS_POST; - - sysclks[1].flags |= FIXED_DIV_PLL; - sysclks[1].div = 1; - sysclks[2].flags |= FIXED_DIV_PLL; - sysclks[2].div = 3; - sysclks[3].flags |= FIXED_DIV_PLL; - sysclks[3].div = 6; - sysclks[4].div = PLLDIV4; - sysclks[5].div = PLLDIV5; - - c6x_core_clk.parent = &sysclks[1]; - c6x_i2c_clk.parent = &sysclks[3]; - c6x_watchdog_clk.parent = &sysclks[5]; - c6x_mdio_clk.parent = &sysclks[5]; - - c6x_clks_init(c6457_clks); -} -#endif /* CONFIG_SOC_TMS320C6455 */ - -#ifdef CONFIG_SOC_TMS320C6472 -static struct clk_lookup c6472_clks[] = { - CLK(NULL, "pll1", &c6x_soc_pll1.sysclks[0]), - CLK(NULL, "pll1_sysclk1", &c6x_soc_pll1.sysclks[1]), - CLK(NULL, "pll1_sysclk2", &c6x_soc_pll1.sysclks[2]), - CLK(NULL, "pll1_sysclk3", &c6x_soc_pll1.sysclks[3]), - CLK(NULL, "pll1_sysclk4", &c6x_soc_pll1.sysclks[4]), - CLK(NULL, "pll1_sysclk5", &c6x_soc_pll1.sysclks[5]), - CLK(NULL, "pll1_sysclk6", &c6x_soc_pll1.sysclks[6]), - CLK(NULL, "pll1_sysclk7", &c6x_soc_pll1.sysclks[7]), - CLK(NULL, "pll1_sysclk8", &c6x_soc_pll1.sysclks[8]), - CLK(NULL, "pll1_sysclk9", &c6x_soc_pll1.sysclks[9]), - CLK(NULL, "pll1_sysclk10", &c6x_soc_pll1.sysclks[10]), - CLK(NULL, "core", &c6x_core_clk), - CLK("i2c_davinci.1", NULL, &c6x_i2c_clk), - CLK("watchdog", NULL, &c6x_watchdog_clk), - CLK("2c81800.mdio", NULL, &c6x_mdio_clk), - CLK("", NULL, NULL) -}; - -/* assumptions used for delay loop calculations */ -#define MIN_CLKIN1_KHz 15625 -#define MAX_CORE_KHz 700000 -#define MIN_PLLOUT_KHz MIN_CLKIN1_KHz - -static void __init c6472_setup_clocks(struct device_node *node) -{ - struct pll_data *pll = &c6x_soc_pll1; - struct clk *sysclks = pll->sysclks; - int i; - - pll->flags = PLL_HAS_MUL; - - for (i = 1; i <= 6; i++) { - sysclks[i].flags |= FIXED_DIV_PLL; - sysclks[i].div = 1; - } - - sysclks[7].flags |= FIXED_DIV_PLL; - sysclks[7].div = 3; - sysclks[8].flags |= FIXED_DIV_PLL; - sysclks[8].div = 6; - sysclks[9].flags |= FIXED_DIV_PLL; - sysclks[9].div = 2; - sysclks[10].div = PLLDIV10; - - c6x_core_clk.parent = &sysclks[get_coreid() + 1]; - c6x_i2c_clk.parent = &sysclks[8]; - c6x_watchdog_clk.parent = &sysclks[8]; - c6x_mdio_clk.parent = &sysclks[5]; - - c6x_clks_init(c6472_clks); -} -#endif /* CONFIG_SOC_TMS320C6472 */ - - -#ifdef CONFIG_SOC_TMS320C6474 -static struct clk_lookup c6474_clks[] = { - CLK(NULL, "pll1", &c6x_soc_pll1.sysclks[0]), - CLK(NULL, "pll1_sysclk7", &c6x_soc_pll1.sysclks[7]), - CLK(NULL, "pll1_sysclk9", &c6x_soc_pll1.sysclks[9]), - CLK(NULL, "pll1_sysclk10", &c6x_soc_pll1.sysclks[10]), - CLK(NULL, "pll1_sysclk11", &c6x_soc_pll1.sysclks[11]), - CLK(NULL, "pll1_sysclk12", &c6x_soc_pll1.sysclks[12]), - CLK(NULL, "pll1_sysclk13", &c6x_soc_pll1.sysclks[13]), - CLK(NULL, "core", &c6x_core_clk), - CLK("i2c_davinci.1", NULL, &c6x_i2c_clk), - CLK("mcbsp.1", NULL, &c6x_mcbsp1_clk), - CLK("mcbsp.2", NULL, &c6x_mcbsp2_clk), - CLK("watchdog", NULL, &c6x_watchdog_clk), - CLK("2c81800.mdio", NULL, &c6x_mdio_clk), - CLK("", NULL, NULL) -}; - -static void __init c6474_setup_clocks(struct device_node *node) -{ - struct pll_data *pll = &c6x_soc_pll1; - struct clk *sysclks = pll->sysclks; - - pll->flags = PLL_HAS_MUL; - - sysclks[7].flags |= FIXED_DIV_PLL; - sysclks[7].div = 1; - sysclks[9].flags |= FIXED_DIV_PLL; - sysclks[9].div = 3; - sysclks[10].flags |= FIXED_DIV_PLL; - sysclks[10].div = 6; - - sysclks[11].div = PLLDIV11; - - sysclks[12].flags |= FIXED_DIV_PLL; - sysclks[12].div = 2; - - sysclks[13].div = PLLDIV13; - - c6x_core_clk.parent = &sysclks[7]; - c6x_i2c_clk.parent = &sysclks[10]; - c6x_watchdog_clk.parent = &sysclks[10]; - c6x_mcbsp1_clk.parent = &sysclks[10]; - c6x_mcbsp2_clk.parent = &sysclks[10]; - - c6x_clks_init(c6474_clks); -} -#endif /* CONFIG_SOC_TMS320C6474 */ - -#ifdef CONFIG_SOC_TMS320C6678 -static struct clk_lookup c6678_clks[] = { - CLK(NULL, "pll1", &c6x_soc_pll1.sysclks[0]), - CLK(NULL, "pll1_refclk", &c6x_soc_pll1.sysclks[1]), - CLK(NULL, "pll1_sysclk2", &c6x_soc_pll1.sysclks[2]), - CLK(NULL, "pll1_sysclk3", &c6x_soc_pll1.sysclks[3]), - CLK(NULL, "pll1_sysclk4", &c6x_soc_pll1.sysclks[4]), - CLK(NULL, "pll1_sysclk5", &c6x_soc_pll1.sysclks[5]), - CLK(NULL, "pll1_sysclk6", &c6x_soc_pll1.sysclks[6]), - CLK(NULL, "pll1_sysclk7", &c6x_soc_pll1.sysclks[7]), - CLK(NULL, "pll1_sysclk8", &c6x_soc_pll1.sysclks[8]), - CLK(NULL, "pll1_sysclk9", &c6x_soc_pll1.sysclks[9]), - CLK(NULL, "pll1_sysclk10", &c6x_soc_pll1.sysclks[10]), - CLK(NULL, "pll1_sysclk11", &c6x_soc_pll1.sysclks[11]), - CLK(NULL, "core", &c6x_core_clk), - CLK("", NULL, NULL) -}; - -static void __init c6678_setup_clocks(struct device_node *node) -{ - struct pll_data *pll = &c6x_soc_pll1; - struct clk *sysclks = pll->sysclks; - - pll->flags = PLL_HAS_MUL; - - sysclks[1].flags |= FIXED_DIV_PLL; - sysclks[1].div = 1; - - sysclks[2].div = PLLDIV2; - - sysclks[3].flags |= FIXED_DIV_PLL; - sysclks[3].div = 2; - - sysclks[4].flags |= FIXED_DIV_PLL; - sysclks[4].div = 3; - - sysclks[5].div = PLLDIV5; - - sysclks[6].flags |= FIXED_DIV_PLL; - sysclks[6].div = 64; - - sysclks[7].flags |= FIXED_DIV_PLL; - sysclks[7].div = 6; - - sysclks[8].div = PLLDIV8; - - sysclks[9].flags |= FIXED_DIV_PLL; - sysclks[9].div = 12; - - sysclks[10].flags |= FIXED_DIV_PLL; - sysclks[10].div = 3; - - sysclks[11].flags |= FIXED_DIV_PLL; - sysclks[11].div = 6; - - c6x_core_clk.parent = &sysclks[0]; - c6x_i2c_clk.parent = &sysclks[7]; - - c6x_clks_init(c6678_clks); -} -#endif /* CONFIG_SOC_TMS320C6678 */ - -static struct of_device_id c6x_clkc_match[] __initdata = { -#ifdef CONFIG_SOC_TMS320C6455 - { .compatible = "ti,c6455-pll", .data = c6455_setup_clocks }, -#endif -#ifdef CONFIG_SOC_TMS320C6457 - { .compatible = "ti,c6457-pll", .data = c6457_setup_clocks }, -#endif -#ifdef CONFIG_SOC_TMS320C6472 - { .compatible = "ti,c6472-pll", .data = c6472_setup_clocks }, -#endif -#ifdef CONFIG_SOC_TMS320C6474 - { .compatible = "ti,c6474-pll", .data = c6474_setup_clocks }, -#endif -#ifdef CONFIG_SOC_TMS320C6678 - { .compatible = "ti,c6678-pll", .data = c6678_setup_clocks }, -#endif - { .compatible = "ti,c64x+pll" }, - {} -}; - -void __init c64x_setup_clocks(void) -{ - void (*__setup_clocks)(struct device_node *np); - struct pll_data *pll = &c6x_soc_pll1; - struct device_node *node; - const struct of_device_id *id; - int err; - u32 val; - - node = of_find_matching_node(NULL, c6x_clkc_match); - if (!node) - return; - - pll->base = of_iomap(node, 0); - if (!pll->base) - goto out; - - err = of_property_read_u32(node, "clock-frequency", &val); - if (err || val == 0) { - pr_err("%pOF: no clock-frequency found! Using %dMHz\n", - node, (int)val / 1000000); - val = 25000000; - } - clkin1.rate = val; - - err = of_property_read_u32(node, "ti,c64x+pll-bypass-delay", &val); - if (err) - val = 5000; - pll->bypass_delay = val; - - err = of_property_read_u32(node, "ti,c64x+pll-reset-delay", &val); - if (err) - val = 30000; - pll->reset_delay = val; - - err = of_property_read_u32(node, "ti,c64x+pll-lock-delay", &val); - if (err) - val = 30000; - pll->lock_delay = val; - - /* id->data is a pointer to SoC-specific setup */ - id = of_match_node(c6x_clkc_match, node); - if (id && id->data) { - __setup_clocks = id->data; - __setup_clocks(node); - } - -out: - of_node_put(node); -} diff --git a/arch/c6x/platforms/timer64.c b/arch/c6x/platforms/timer64.c deleted file mode 100644 index 661f4c7c6ef6..000000000000 --- a/arch/c6x/platforms/timer64.c +++ /dev/null @@ -1,241 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2010, 2011 Texas Instruments Incorporated - * Contributed by: Mark Salter (msalter@redhat.com) - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct timer_regs { - u32 reserved0; - u32 emumgt; - u32 reserved1; - u32 reserved2; - u32 cntlo; - u32 cnthi; - u32 prdlo; - u32 prdhi; - u32 tcr; - u32 tgcr; - u32 wdtcr; -}; - -static struct timer_regs __iomem *timer; - -#define TCR_TSTATLO 0x001 -#define TCR_INVOUTPLO 0x002 -#define TCR_INVINPLO 0x004 -#define TCR_CPLO 0x008 -#define TCR_ENAMODELO_ONCE 0x040 -#define TCR_ENAMODELO_CONT 0x080 -#define TCR_ENAMODELO_MASK 0x0c0 -#define TCR_PWIDLO_MASK 0x030 -#define TCR_CLKSRCLO 0x100 -#define TCR_TIENLO 0x200 -#define TCR_TSTATHI (0x001 << 16) -#define TCR_INVOUTPHI (0x002 << 16) -#define TCR_CPHI (0x008 << 16) -#define TCR_PWIDHI_MASK (0x030 << 16) -#define TCR_ENAMODEHI_ONCE (0x040 << 16) -#define TCR_ENAMODEHI_CONT (0x080 << 16) -#define TCR_ENAMODEHI_MASK (0x0c0 << 16) - -#define TGCR_TIMLORS 0x001 -#define TGCR_TIMHIRS 0x002 -#define TGCR_TIMMODE_UD32 0x004 -#define TGCR_TIMMODE_WDT64 0x008 -#define TGCR_TIMMODE_CD32 0x00c -#define TGCR_TIMMODE_MASK 0x00c -#define TGCR_PSCHI_MASK (0x00f << 8) -#define TGCR_TDDRHI_MASK (0x00f << 12) - -/* - * Timer clocks are divided down from the CPU clock - * The divisor is in the EMUMGTCLKSPD register - */ -#define TIMER_DIVISOR \ - ((soc_readl(&timer->emumgt) & (0xf << 16)) >> 16) - -#define TIMER64_RATE (c6x_core_freq / TIMER_DIVISOR) - -#define TIMER64_MODE_DISABLED 0 -#define TIMER64_MODE_ONE_SHOT TCR_ENAMODELO_ONCE -#define TIMER64_MODE_PERIODIC TCR_ENAMODELO_CONT - -static int timer64_mode; -static int timer64_devstate_id = -1; - -static void timer64_config(unsigned long period) -{ - u32 tcr = soc_readl(&timer->tcr) & ~TCR_ENAMODELO_MASK; - - soc_writel(tcr, &timer->tcr); - soc_writel(period - 1, &timer->prdlo); - soc_writel(0, &timer->cntlo); - tcr |= timer64_mode; - soc_writel(tcr, &timer->tcr); -} - -static void timer64_enable(void) -{ - u32 val; - - if (timer64_devstate_id >= 0) - dscr_set_devstate(timer64_devstate_id, DSCR_DEVSTATE_ENABLED); - - /* disable timer, reset count */ - soc_writel(soc_readl(&timer->tcr) & ~TCR_ENAMODELO_MASK, &timer->tcr); - soc_writel(0, &timer->prdlo); - - /* use internal clock and 1 cycle pulse width */ - val = soc_readl(&timer->tcr); - soc_writel(val & ~(TCR_CLKSRCLO | TCR_PWIDLO_MASK), &timer->tcr); - - /* dual 32-bit unchained mode */ - val = soc_readl(&timer->tgcr) & ~TGCR_TIMMODE_MASK; - soc_writel(val, &timer->tgcr); - soc_writel(val | (TGCR_TIMLORS | TGCR_TIMMODE_UD32), &timer->tgcr); -} - -static void timer64_disable(void) -{ - /* disable timer, reset count */ - soc_writel(soc_readl(&timer->tcr) & ~TCR_ENAMODELO_MASK, &timer->tcr); - soc_writel(0, &timer->prdlo); - - if (timer64_devstate_id >= 0) - dscr_set_devstate(timer64_devstate_id, DSCR_DEVSTATE_DISABLED); -} - -static int next_event(unsigned long delta, - struct clock_event_device *evt) -{ - timer64_config(delta); - return 0; -} - -static int set_periodic(struct clock_event_device *evt) -{ - timer64_enable(); - timer64_mode = TIMER64_MODE_PERIODIC; - timer64_config(TIMER64_RATE / HZ); - return 0; -} - -static int set_oneshot(struct clock_event_device *evt) -{ - timer64_enable(); - timer64_mode = TIMER64_MODE_ONE_SHOT; - return 0; -} - -static int shutdown(struct clock_event_device *evt) -{ - timer64_mode = TIMER64_MODE_DISABLED; - timer64_disable(); - return 0; -} - -static struct clock_event_device t64_clockevent_device = { - .name = "TIMER64_EVT32_TIMER", - .features = CLOCK_EVT_FEAT_ONESHOT | - CLOCK_EVT_FEAT_PERIODIC, - .rating = 200, - .set_state_shutdown = shutdown, - .set_state_periodic = set_periodic, - .set_state_oneshot = set_oneshot, - .set_next_event = next_event, -}; - -static irqreturn_t timer_interrupt(int irq, void *dev_id) -{ - struct clock_event_device *cd = &t64_clockevent_device; - - cd->event_handler(cd); - - return IRQ_HANDLED; -} - -void __init timer64_init(void) -{ - struct clock_event_device *cd = &t64_clockevent_device; - struct device_node *np, *first = NULL; - u32 val; - int err, found = 0; - - for_each_compatible_node(np, NULL, "ti,c64x+timer64") { - err = of_property_read_u32(np, "ti,core-mask", &val); - if (!err) { - if (val & (1 << get_coreid())) { - found = 1; - break; - } - } else if (!first) - first = np; - } - if (!found) { - /* try first one with no core-mask */ - if (first) - np = of_node_get(first); - else { - pr_debug("Cannot find ti,c64x+timer64 timer.\n"); - return; - } - } - - timer = of_iomap(np, 0); - if (!timer) { - pr_debug("%pOF: Cannot map timer registers.\n", np); - goto out; - } - pr_debug("%pOF: Timer registers=%p.\n", np, timer); - - cd->irq = irq_of_parse_and_map(np, 0); - if (cd->irq == NO_IRQ) { - pr_debug("%pOF: Cannot find interrupt.\n", np); - iounmap(timer); - goto out; - } - - /* If there is a device state control, save the ID. */ - err = of_property_read_u32(np, "ti,dscr-dev-enable", &val); - if (!err) { - timer64_devstate_id = val; - - /* - * It is necessary to enable the timer block here because - * the TIMER_DIVISOR macro needs to read a timer register - * to get the divisor. - */ - dscr_set_devstate(timer64_devstate_id, DSCR_DEVSTATE_ENABLED); - } - - pr_debug("%pOF: Timer irq=%d.\n", np, cd->irq); - - clockevents_calc_mult_shift(cd, c6x_core_freq / TIMER_DIVISOR, 5); - - cd->max_delta_ns = clockevent_delta2ns(0x7fffffff, cd); - cd->max_delta_ticks = 0x7fffffff; - cd->min_delta_ns = clockevent_delta2ns(250, cd); - cd->min_delta_ticks = 250; - - cd->cpumask = cpumask_of(smp_processor_id()); - - clockevents_register_device(cd); - if (request_irq(cd->irq, timer_interrupt, IRQF_TIMER, "timer", - &t64_clockevent_device)) - pr_err("Failed to request irq %d (timer)\n", cd->irq); - -out: - of_node_put(np); - return; -} diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig index 0c262c2aeaf2..e7f7eee6ee9a 100644 --- a/drivers/bus/Kconfig +++ b/drivers/bus/Kconfig @@ -80,7 +80,7 @@ config MOXTET config HISILICON_LPC bool "Support for ISA I/O space on HiSilicon Hip06/7" - depends on (ARM64 && ARCH_HISI) || (COMPILE_TEST && !ALPHA && !HEXAGON && !PARISC && !C6X) + depends on (ARM64 && ARCH_HISI) || (COMPILE_TEST && !ALPHA && !HEXAGON && !PARISC) depends on HAS_IOMEM select INDIRECT_PIO if ARM64 help diff --git a/fs/Kconfig.binfmt b/fs/Kconfig.binfmt index 885da6d983b4..647439c2c05a 100644 --- a/fs/Kconfig.binfmt +++ b/fs/Kconfig.binfmt @@ -45,7 +45,7 @@ config ARCH_USE_GNU_PROPERTY config BINFMT_ELF_FDPIC bool "Kernel support for FDPIC ELF binaries" default y if !BINFMT_ELF - depends on (ARM || (SUPERH && !MMU) || C6X) + depends on (ARM || (SUPERH && !MMU)) select ELFCORE help ELF FDPIC binaries are based on ELF, but allow the individual load diff --git a/include/asm-generic/page.h b/include/asm-generic/page.h index fe801f01625e..6fc47561814c 100644 --- a/include/asm-generic/page.h +++ b/include/asm-generic/page.h @@ -63,11 +63,7 @@ extern unsigned long memory_end; #endif /* !__ASSEMBLY__ */ -#ifdef CONFIG_KERNEL_RAM_BASE_ADDRESS -#define PAGE_OFFSET (CONFIG_KERNEL_RAM_BASE_ADDRESS) -#else #define PAGE_OFFSET (0) -#endif #ifndef ARCH_PFN_OFFSET #define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) -- cgit v1.2.3 From f9ce0be71d1fbb038ada15ced83474b0e63f264d Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Sat, 19 Dec 2020 15:19:23 +0300 Subject: mm: Cleanup faultaround and finish_fault() codepaths alloc_set_pte() has two users with different requirements: in the faultaround code, it called from an atomic context and PTE page table has to be preallocated. finish_fault() can sleep and allocate page table as needed. PTL locking rules are also strange, hard to follow and overkill for finish_fault(). Let's untangle the mess. alloc_set_pte() has gone now. All locking is explicit. The price is some code duplication to handle huge pages in faultaround path, but it should be fine, having overall improvement in readability. Link: https://lore.kernel.org/r/20201229132819.najtavneutnf7ajp@box Signed-off-by: Kirill A. Shutemov [will: s/from from/from/ in comment; spotted by willy] Signed-off-by: Will Deacon --- fs/xfs/xfs_file.c | 6 +- include/linux/mm.h | 12 +-- include/linux/pgtable.h | 11 +++ mm/filemap.c | 177 +++++++++++++++++++++++++++++++----------- mm/memory.c | 199 ++++++++++++++---------------------------------- 5 files changed, 213 insertions(+), 192 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 5b0f93f73837..111fe73bb8a7 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -1319,17 +1319,19 @@ xfs_filemap_pfn_mkwrite( return __xfs_filemap_fault(vmf, PE_SIZE_PTE, true); } -static void +static vm_fault_t xfs_filemap_map_pages( struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff) { struct inode *inode = file_inode(vmf->vma->vm_file); + vm_fault_t ret; xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED); - filemap_map_pages(vmf, start_pgoff, end_pgoff); + ret = filemap_map_pages(vmf, start_pgoff, end_pgoff); xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED); + return ret; } static const struct vm_operations_struct xfs_file_vm_ops = { diff --git a/include/linux/mm.h b/include/linux/mm.h index ecdf8a8cd6ae..4572a9bc5862 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -542,8 +542,8 @@ struct vm_fault { * is not NULL, otherwise pmd. */ pgtable_t prealloc_pte; /* Pre-allocated pte page table. - * vm_ops->map_pages() calls - * alloc_set_pte() from atomic context. + * vm_ops->map_pages() sets up a page + * table from atomic context. * do_fault_around() pre-allocates * page table to avoid allocation from * atomic context. @@ -578,7 +578,7 @@ struct vm_operations_struct { vm_fault_t (*fault)(struct vm_fault *vmf); vm_fault_t (*huge_fault)(struct vm_fault *vmf, enum page_entry_size pe_size); - void (*map_pages)(struct vm_fault *vmf, + vm_fault_t (*map_pages)(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff); unsigned long (*pagesize)(struct vm_area_struct * area); @@ -988,7 +988,9 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) return pte; } -vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page); +vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page); +void do_set_pte(struct vm_fault *vmf, struct page *page); + vm_fault_t finish_fault(struct vm_fault *vmf); vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf); #endif @@ -2622,7 +2624,7 @@ extern void truncate_inode_pages_final(struct address_space *); /* generic vm_area_ops exported for stackable file systems */ extern vm_fault_t filemap_fault(struct vm_fault *vmf); -extern void filemap_map_pages(struct vm_fault *vmf, +extern vm_fault_t filemap_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff); extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf); diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 8fcdfa52eb4b..36eb748f3c97 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1314,6 +1314,17 @@ static inline int pmd_trans_unstable(pmd_t *pmd) #endif } +/* + * the ordering of these checks is important for pmds with _page_devmap set. + * if we check pmd_trans_unstable() first we will trip the bad_pmd() check + * inside of pmd_none_or_trans_huge_or_clear_bad(). this will end up correctly + * returning 1 but not before it spams dmesg with the pmd_clear_bad() output. + */ +static inline int pmd_devmap_trans_unstable(pmd_t *pmd) +{ + return pmd_devmap(*pmd) || pmd_trans_unstable(pmd); +} + #ifndef CONFIG_NUMA_BALANCING /* * Technically a PTE can be PROTNONE even when not doing NUMA balancing but diff --git a/mm/filemap.c b/mm/filemap.c index 5c9d564317a5..c1f2dc89b8a7 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -42,6 +42,7 @@ #include #include #include +#include #include "internal.h" #define CREATE_TRACE_POINTS @@ -2911,74 +2912,164 @@ out_retry: } EXPORT_SYMBOL(filemap_fault); -void filemap_map_pages(struct vm_fault *vmf, - pgoff_t start_pgoff, pgoff_t end_pgoff) +static bool filemap_map_pmd(struct vm_fault *vmf, struct page *page) { - struct file *file = vmf->vma->vm_file; + struct mm_struct *mm = vmf->vma->vm_mm; + + /* Huge page is mapped? No need to proceed. */ + if (pmd_trans_huge(*vmf->pmd)) { + unlock_page(page); + put_page(page); + return true; + } + + if (pmd_none(*vmf->pmd) && PageTransHuge(page)) { + vm_fault_t ret = do_set_pmd(vmf, page); + if (!ret) { + /* The page is mapped successfully, reference consumed. */ + unlock_page(page); + return true; + } + } + + if (pmd_none(*vmf->pmd)) { + vmf->ptl = pmd_lock(mm, vmf->pmd); + if (likely(pmd_none(*vmf->pmd))) { + mm_inc_nr_ptes(mm); + pmd_populate(mm, vmf->pmd, vmf->prealloc_pte); + vmf->prealloc_pte = NULL; + } + spin_unlock(vmf->ptl); + } + + /* See comment in handle_pte_fault() */ + if (pmd_devmap_trans_unstable(vmf->pmd)) { + unlock_page(page); + put_page(page); + return true; + } + + return false; +} + +static struct page *next_uptodate_page(struct page *page, + struct address_space *mapping, + struct xa_state *xas, pgoff_t end_pgoff) +{ + unsigned long max_idx; + + do { + if (!page) + return NULL; + if (xas_retry(xas, page)) + continue; + if (xa_is_value(page)) + continue; + if (PageLocked(page)) + continue; + if (!page_cache_get_speculative(page)) + continue; + /* Has the page moved or been split? */ + if (unlikely(page != xas_reload(xas))) + goto skip; + if (!PageUptodate(page) || PageReadahead(page)) + goto skip; + if (PageHWPoison(page)) + goto skip; + if (!trylock_page(page)) + goto skip; + if (page->mapping != mapping) + goto unlock; + if (!PageUptodate(page)) + goto unlock; + max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); + if (xas->xa_index >= max_idx) + goto unlock; + return page; +unlock: + unlock_page(page); +skip: + put_page(page); + } while ((page = xas_next_entry(xas, end_pgoff)) != NULL); + + return NULL; +} + +static inline struct page *first_map_page(struct address_space *mapping, + struct xa_state *xas, + pgoff_t end_pgoff) +{ + return next_uptodate_page(xas_find(xas, end_pgoff), + mapping, xas, end_pgoff); +} + +static inline struct page *next_map_page(struct address_space *mapping, + struct xa_state *xas, + pgoff_t end_pgoff) +{ + return next_uptodate_page(xas_next_entry(xas, end_pgoff), + mapping, xas, end_pgoff); +} + +vm_fault_t filemap_map_pages(struct vm_fault *vmf, + pgoff_t start_pgoff, pgoff_t end_pgoff) +{ + struct vm_area_struct *vma = vmf->vma; + struct file *file = vma->vm_file; struct address_space *mapping = file->f_mapping; pgoff_t last_pgoff = start_pgoff; - unsigned long max_idx; + unsigned long address = vmf->address; XA_STATE(xas, &mapping->i_pages, start_pgoff); struct page *head, *page; unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); + vm_fault_t ret = 0; rcu_read_lock(); - xas_for_each(&xas, head, end_pgoff) { - if (xas_retry(&xas, head)) - continue; - if (xa_is_value(head)) - goto next; + head = first_map_page(mapping, &xas, end_pgoff); + if (!head) + goto out; - /* - * Check for a locked page first, as a speculative - * reference may adversely influence page migration. - */ - if (PageLocked(head)) - goto next; - if (!page_cache_get_speculative(head)) - goto next; + if (filemap_map_pmd(vmf, head)) { + ret = VM_FAULT_NOPAGE; + goto out; + } - /* Has the page moved or been split? */ - if (unlikely(head != xas_reload(&xas))) - goto skip; + vmf->address = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); + do { page = find_subpage(head, xas.xa_index); - - if (!PageUptodate(head) || - PageReadahead(page) || - PageHWPoison(page)) - goto skip; - if (!trylock_page(head)) - goto skip; - - if (head->mapping != mapping || !PageUptodate(head)) - goto unlock; - - max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); - if (xas.xa_index >= max_idx) + if (PageHWPoison(page)) goto unlock; if (mmap_miss > 0) mmap_miss--; vmf->address += (xas.xa_index - last_pgoff) << PAGE_SHIFT; - if (vmf->pte) - vmf->pte += xas.xa_index - last_pgoff; + vmf->pte += xas.xa_index - last_pgoff; last_pgoff = xas.xa_index; - if (alloc_set_pte(vmf, page)) + + if (!pte_none(*vmf->pte)) goto unlock; + + do_set_pte(vmf, page); + /* no need to invalidate: a not-present page won't be cached */ + update_mmu_cache(vma, vmf->address, vmf->pte); unlock_page(head); - goto next; + + /* The fault is handled */ + if (vmf->address == address) + ret = VM_FAULT_NOPAGE; + continue; unlock: unlock_page(head); -skip: put_page(head); -next: - /* Huge page is mapped? No need to proceed. */ - if (pmd_trans_huge(*vmf->pmd)) - break; - } + } while ((head = next_map_page(mapping, &xas, end_pgoff)) != NULL); + pte_unmap_unlock(vmf->pte, vmf->ptl); +out: rcu_read_unlock(); + vmf->address = address; WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss); + return ret; } EXPORT_SYMBOL(filemap_map_pages); diff --git a/mm/memory.c b/mm/memory.c index feff48e1465a..3e2fc2950ad7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3503,7 +3503,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (pte_alloc(vma->vm_mm, vmf->pmd)) return VM_FAULT_OOM; - /* See the comment in pte_alloc_one_map() */ + /* See comment in handle_pte_fault() */ if (unlikely(pmd_trans_unstable(vmf->pmd))) return 0; @@ -3643,66 +3643,6 @@ static vm_fault_t __do_fault(struct vm_fault *vmf) return ret; } -/* - * The ordering of these checks is important for pmds with _PAGE_DEVMAP set. - * If we check pmd_trans_unstable() first we will trip the bad_pmd() check - * inside of pmd_none_or_trans_huge_or_clear_bad(). This will end up correctly - * returning 1 but not before it spams dmesg with the pmd_clear_bad() output. - */ -static int pmd_devmap_trans_unstable(pmd_t *pmd) -{ - return pmd_devmap(*pmd) || pmd_trans_unstable(pmd); -} - -static vm_fault_t pte_alloc_one_map(struct vm_fault *vmf) -{ - struct vm_area_struct *vma = vmf->vma; - - if (!pmd_none(*vmf->pmd)) - goto map_pte; - if (vmf->prealloc_pte) { - vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); - if (unlikely(!pmd_none(*vmf->pmd))) { - spin_unlock(vmf->ptl); - goto map_pte; - } - - mm_inc_nr_ptes(vma->vm_mm); - pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); - spin_unlock(vmf->ptl); - vmf->prealloc_pte = NULL; - } else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) { - return VM_FAULT_OOM; - } -map_pte: - /* - * If a huge pmd materialized under us just retry later. Use - * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead of - * pmd_trans_huge() to ensure the pmd didn't become pmd_trans_huge - * under us and then back to pmd_none, as a result of MADV_DONTNEED - * running immediately after a huge pmd fault in a different thread of - * this mm, in turn leading to a misleading pmd_trans_huge() retval. - * All we have to ensure is that it is a regular pmd that we can walk - * with pte_offset_map() and we can do that through an atomic read in - * C, which is what pmd_trans_unstable() provides. - */ - if (pmd_devmap_trans_unstable(vmf->pmd)) - return VM_FAULT_NOPAGE; - - /* - * At this point we know that our vmf->pmd points to a page of ptes - * and it cannot become pmd_none(), pmd_devmap() or pmd_trans_huge() - * for the duration of the fault. If a racing MADV_DONTNEED runs and - * we zap the ptes pointed to by our vmf->pmd, the vmf->ptl will still - * be valid and we will re-check to make sure the vmf->pte isn't - * pte_none() under vmf->ptl protection when we return to - * alloc_set_pte(). - */ - vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, - &vmf->ptl); - return 0; -} - #ifdef CONFIG_TRANSPARENT_HUGEPAGE static void deposit_prealloc_pte(struct vm_fault *vmf) { @@ -3717,7 +3657,7 @@ static void deposit_prealloc_pte(struct vm_fault *vmf) vmf->prealloc_pte = NULL; } -static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) +vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) { struct vm_area_struct *vma = vmf->vma; bool write = vmf->flags & FAULT_FLAG_WRITE; @@ -3775,52 +3715,17 @@ out: return ret; } #else -static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) +vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) { - BUILD_BUG(); - return 0; + return VM_FAULT_FALLBACK; } #endif -/** - * alloc_set_pte - setup new PTE entry for given page and add reverse page - * mapping. If needed, the function allocates page table or use pre-allocated. - * - * @vmf: fault environment - * @page: page to map - * - * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on - * return. - * - * Target users are page handler itself and implementations of - * vm_ops->map_pages. - * - * Return: %0 on success, %VM_FAULT_ code in case of error. - */ -vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page) +void do_set_pte(struct vm_fault *vmf, struct page *page) { struct vm_area_struct *vma = vmf->vma; bool write = vmf->flags & FAULT_FLAG_WRITE; pte_t entry; - vm_fault_t ret; - - if (pmd_none(*vmf->pmd) && PageTransCompound(page)) { - ret = do_set_pmd(vmf, page); - if (ret != VM_FAULT_FALLBACK) - return ret; - } - - if (!vmf->pte) { - ret = pte_alloc_one_map(vmf); - if (ret) - return ret; - } - - /* Re-check under ptl */ - if (unlikely(!pte_none(*vmf->pte))) { - update_mmu_tlb(vma, vmf->address, vmf->pte); - return VM_FAULT_NOPAGE; - } flush_icache_page(vma, page); entry = mk_pte(page, vma->vm_page_prot); @@ -3837,14 +3742,8 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page) page_add_file_rmap(page, false); } set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry); - - /* no need to invalidate: a not-present page won't be cached */ - update_mmu_cache(vma, vmf->address, vmf->pte); - - return 0; } - /** * finish_fault - finish page fault once we have prepared the page to fault * @@ -3862,12 +3761,12 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page) */ vm_fault_t finish_fault(struct vm_fault *vmf) { + struct vm_area_struct *vma = vmf->vma; struct page *page; - vm_fault_t ret = 0; + vm_fault_t ret; /* Did we COW the page? */ - if ((vmf->flags & FAULT_FLAG_WRITE) && - !(vmf->vma->vm_flags & VM_SHARED)) + if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) page = vmf->cow_page; else page = vmf->page; @@ -3876,12 +3775,38 @@ vm_fault_t finish_fault(struct vm_fault *vmf) * check even for read faults because we might have lost our CoWed * page */ - if (!(vmf->vma->vm_flags & VM_SHARED)) - ret = check_stable_address_space(vmf->vma->vm_mm); - if (!ret) - ret = alloc_set_pte(vmf, page); - if (vmf->pte) - pte_unmap_unlock(vmf->pte, vmf->ptl); + if (!(vma->vm_flags & VM_SHARED)) { + ret = check_stable_address_space(vma->vm_mm); + if (ret) + return ret; + } + + if (pmd_none(*vmf->pmd)) { + if (PageTransCompound(page)) { + ret = do_set_pmd(vmf, page); + if (ret != VM_FAULT_FALLBACK) + return ret; + } + + if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) + return VM_FAULT_OOM; + } + + /* See comment in handle_pte_fault() */ + if (pmd_devmap_trans_unstable(vmf->pmd)) + return 0; + + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, + vmf->address, &vmf->ptl); + ret = 0; + /* Re-check under ptl */ + if (likely(pte_none(*vmf->pte))) + do_set_pte(vmf, page); + else + ret = VM_FAULT_NOPAGE; + + update_mmu_tlb(vma, vmf->address, vmf->pte); + pte_unmap_unlock(vmf->pte, vmf->ptl); return ret; } @@ -3951,13 +3876,12 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) pgoff_t start_pgoff = vmf->pgoff; pgoff_t end_pgoff; int off; - vm_fault_t ret = 0; nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT; mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK; - vmf->address = max(address & mask, vmf->vma->vm_start); - off = ((address - vmf->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); + address = max(address & mask, vmf->vma->vm_start); + off = ((vmf->address - address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); start_pgoff -= off; /* @@ -3965,7 +3889,7 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) * the vma or nr_pages from start_pgoff, depending what is nearest. */ end_pgoff = start_pgoff - - ((vmf->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) + + ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) + PTRS_PER_PTE - 1; end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1, start_pgoff + nr_pages - 1); @@ -3973,31 +3897,11 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) if (pmd_none(*vmf->pmd)) { vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); if (!vmf->prealloc_pte) - goto out; + return VM_FAULT_OOM; smp_wmb(); /* See comment in __pte_alloc() */ } - vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff); - - /* Huge page is mapped? Page fault is solved */ - if (pmd_trans_huge(*vmf->pmd)) { - ret = VM_FAULT_NOPAGE; - goto out; - } - - /* ->map_pages() haven't done anything useful. Cold page cache? */ - if (!vmf->pte) - goto out; - - /* check if the page fault is solved */ - vmf->pte -= (vmf->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT); - if (!pte_none(*vmf->pte)) - ret = VM_FAULT_NOPAGE; - pte_unmap_unlock(vmf->pte, vmf->ptl); -out: - vmf->address = address; - vmf->pte = NULL; - return ret; + return vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff); } static vm_fault_t do_read_fault(struct vm_fault *vmf) @@ -4353,7 +4257,18 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) */ vmf->pte = NULL; } else { - /* See comment in pte_alloc_one_map() */ + /* + * If a huge pmd materialized under us just retry later. Use + * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead + * of pmd_trans_huge() to ensure the pmd didn't become + * pmd_trans_huge under us and then back to pmd_none, as a + * result of MADV_DONTNEED running immediately after a huge pmd + * fault in a different thread of this mm, in turn leading to a + * misleading pmd_trans_huge() retval. All we have to ensure is + * that it is a regular pmd that we can walk with + * pte_offset_map() and we can do that through an atomic read + * in C, which is what pmd_trans_unstable() provides. + */ if (pmd_devmap_trans_unstable(vmf->pmd)) return 0; /* -- cgit v1.2.3 From 961f3c898e86de9b530648b6b0319f9240508f16 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 14 Jan 2021 09:04:39 +0100 Subject: fs: fix kernel-doc markups Two markups are at the wrong place. Kernel-doc only support having the comment just before the identifier. Also, some identifiers have different names between their prototypes and the kernel-doc markup. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/96b1e1b388600ab092331f6c4e88ff8e8779ce6c.1610610937.git.mchehab+huawei@kernel.org Signed-off-by: Jonathan Corbet --- fs/dcache.c | 63 ++++++++++++++++++++++++++++++----------------------------- fs/inode.c | 4 ++-- fs/seq_file.c | 5 +++-- fs/super.c | 12 ++++++------ 4 files changed, 43 insertions(+), 41 deletions(-) (limited to 'fs') diff --git a/fs/dcache.c b/fs/dcache.c index 97e81a844a96..c17fd15b01d4 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -456,23 +456,6 @@ static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry, list_lru_isolate_move(lru, &dentry->d_lru, list); } -/** - * d_drop - drop a dentry - * @dentry: dentry to drop - * - * d_drop() unhashes the entry from the parent dentry hashes, so that it won't - * be found through a VFS lookup any more. Note that this is different from - * deleting the dentry - d_delete will try to mark the dentry negative if - * possible, giving a successful _negative_ lookup, while d_drop will - * just make the cache lookup fail. - * - * d_drop() is used mainly for stuff that wants to invalidate a dentry for some - * reason (NFS timeouts or autofs deletes). - * - * __d_drop requires dentry->d_lock - * ___d_drop doesn't mark dentry as "unhashed" - * (dentry->d_hash.pprev will be LIST_POISON2, not NULL). - */ static void ___d_drop(struct dentry *dentry) { struct hlist_bl_head *b; @@ -501,6 +484,24 @@ void __d_drop(struct dentry *dentry) } EXPORT_SYMBOL(__d_drop); +/** + * d_drop - drop a dentry + * @dentry: dentry to drop + * + * d_drop() unhashes the entry from the parent dentry hashes, so that it won't + * be found through a VFS lookup any more. Note that this is different from + * deleting the dentry - d_delete will try to mark the dentry negative if + * possible, giving a successful _negative_ lookup, while d_drop will + * just make the cache lookup fail. + * + * d_drop() is used mainly for stuff that wants to invalidate a dentry for some + * reason (NFS timeouts or autofs deletes). + * + * __d_drop requires dentry->d_lock + * + * ___d_drop doesn't mark dentry as "unhashed" + * (dentry->d_hash.pprev will be LIST_POISON2, not NULL). + */ void d_drop(struct dentry *dentry) { spin_lock(&dentry->d_lock); @@ -996,20 +997,6 @@ struct dentry *d_find_any_alias(struct inode *inode) } EXPORT_SYMBOL(d_find_any_alias); -/** - * d_find_alias - grab a hashed alias of inode - * @inode: inode in question - * - * If inode has a hashed alias, or is a directory and has any alias, - * acquire the reference to alias and return it. Otherwise return NULL. - * Notice that if inode is a directory there can be only one alias and - * it can be unhashed only if it has no children, or if it is the root - * of a filesystem, or if the directory was renamed and d_revalidate - * was the first vfs operation to notice. - * - * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer - * any other hashed alias over that one. - */ static struct dentry *__d_find_alias(struct inode *inode) { struct dentry *alias; @@ -1029,6 +1016,20 @@ static struct dentry *__d_find_alias(struct inode *inode) return NULL; } +/** + * d_find_alias - grab a hashed alias of inode + * @inode: inode in question + * + * If inode has a hashed alias, or is a directory and has any alias, + * acquire the reference to alias and return it. Otherwise return NULL. + * Notice that if inode is a directory there can be only one alias and + * it can be unhashed only if it has no children, or if it is the root + * of a filesystem, or if the directory was renamed and d_revalidate + * was the first vfs operation to notice. + * + * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer + * any other hashed alias over that one. + */ struct dentry *d_find_alias(struct inode *inode) { struct dentry *de = NULL; diff --git a/fs/inode.c b/fs/inode.c index 6442d97d9a4a..1dc9e032f659 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -1493,7 +1493,7 @@ struct inode *find_inode_rcu(struct super_block *sb, unsigned long hashval, EXPORT_SYMBOL(find_inode_rcu); /** - * find_inode_by_rcu - Find an inode in the inode cache + * find_inode_by_ino_rcu - Find an inode in the inode cache * @sb: Super block of file system to search * @ino: The inode number to match * @@ -1777,7 +1777,7 @@ static int update_time(struct inode *inode, struct timespec64 *time, int flags) } /** - * touch_atime - update the access time + * atime_needs_update - update the access time * @path: the &struct path to update * @inode: inode to update * diff --git a/fs/seq_file.c b/fs/seq_file.c index 03a369ccd28c..cb11a34fb871 100644 --- a/fs/seq_file.c +++ b/fs/seq_file.c @@ -669,7 +669,8 @@ void seq_puts(struct seq_file *m, const char *s) EXPORT_SYMBOL(seq_puts); /** - * A helper routine for putting decimal numbers without rich format of printf(). + * seq_put_decimal_ull_width - A helper routine for putting decimal numbers + * without rich format of printf(). * only 'unsigned long long' is supported. * @m: seq_file identifying the buffer to which data should be written * @delimiter: a string which is printed before the number @@ -1044,7 +1045,7 @@ struct hlist_node *seq_hlist_next_rcu(void *v, EXPORT_SYMBOL(seq_hlist_next_rcu); /** - * seq_hlist_start_precpu - start an iteration of a percpu hlist array + * seq_hlist_start_percpu - start an iteration of a percpu hlist array * @head: pointer to percpu array of struct hlist_heads * @cpu: pointer to cpu "cursor" * @pos: start position of sequence diff --git a/fs/super.c b/fs/super.c index 2c6cdea2ab2d..18a3de0b93f5 100644 --- a/fs/super.c +++ b/fs/super.c @@ -1718,12 +1718,6 @@ int freeze_super(struct super_block *sb) } EXPORT_SYMBOL(freeze_super); -/** - * thaw_super -- unlock filesystem - * @sb: the super to thaw - * - * Unlocks the filesystem and marks it writeable again after freeze_super(). - */ static int thaw_super_locked(struct super_block *sb) { int error; @@ -1759,6 +1753,12 @@ out: return 0; } +/** + * thaw_super -- unlock filesystem + * @sb: the super to thaw + * + * Unlocks the filesystem and marks it writeable again after freeze_super(). + */ int thaw_super(struct super_block *sb) { down_write(&sb->s_umount); -- cgit v1.2.3 From 4fd15eaf429a1e7de9d653f0ff2e8a50733495b8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 14 Jan 2021 09:04:40 +0100 Subject: pstore/zone: fix a kernel-doc markup The documented struct is psz_head and not psz_buffer. Reviewed-by: Kees Cook Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/a67ca4d12c3ef277dadb9e0d0df8450158e637cc.1610610937.git.mchehab+huawei@kernel.org Signed-off-by: Jonathan Corbet --- fs/pstore/zone.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/pstore/zone.c b/fs/pstore/zone.c index 5266ccbec007..7c8f8feac6c3 100644 --- a/fs/pstore/zone.c +++ b/fs/pstore/zone.c @@ -23,7 +23,7 @@ #include "internal.h" /** - * struct psz_head - header of zone to flush to storage + * struct psz_buffer - header of zone to flush to storage * * @sig: signature to indicate header (PSZ_SIG xor PSZONE-type value) * @datalen: length of data in @data -- cgit v1.2.3 From 00e8e9bc3c46c95bfa191294a60beaf6d2ead01e Mon Sep 17 00:00:00 2001 From: Zhaoyang Huang Date: Tue, 5 Jan 2021 07:48:00 -0500 Subject: gfs2: amend SLAB_RECLAIM_ACCOUNT on gfs2 related slab cache As gfs2_quotad_cachep and gfs2_glock_cachep have registered shrinkers, amending SLAB_RECLAIM_ACCOUNT when creating them, which improves slab accounting. Signed-off-by: Zhaoyang Huang Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/main.c b/fs/gfs2/main.c index c7393ee9cf68..28d0eb23e18e 100644 --- a/fs/gfs2/main.c +++ b/fs/gfs2/main.c @@ -98,7 +98,7 @@ static int __init init_gfs2_fs(void) error = -ENOMEM; gfs2_glock_cachep = kmem_cache_create("gfs2_glock", sizeof(struct gfs2_glock), - 0, 0, + 0, SLAB_RECLAIM_ACCOUNT, gfs2_init_glock_once); if (!gfs2_glock_cachep) goto fail_cachep1; @@ -134,7 +134,7 @@ static int __init init_gfs2_fs(void) gfs2_quotad_cachep = kmem_cache_create("gfs2_quotad", sizeof(struct gfs2_quota_data), - 0, 0, NULL); + 0, SLAB_RECLAIM_ACCOUNT, NULL); if (!gfs2_quotad_cachep) goto fail_cachep6; -- cgit v1.2.3 From 6da1b4b1ab36d80a3994fd4811c8381de10af604 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:32 -0800 Subject: xfs: fix an ABBA deadlock in xfs_rename When overlayfs is running on top of xfs and the user unlinks a file in the overlay, overlayfs will create a whiteout inode and ask xfs to "rename" the whiteout file atop the one being unlinked. If the file being unlinked loses its one nlink, we then have to put the inode on the unlinked list. This requires us to grab the AGI buffer of the whiteout inode to take it off the unlinked list (which is where whiteouts are created) and to grab the AGI buffer of the file being deleted. If the whiteout was created in a higher numbered AG than the file being deleted, we'll lock the AGIs in the wrong order and deadlock. Therefore, grab all the AGI locks we think we'll need ahead of time, and in order of increasing AG number per the locking rules. Reported-by: wenli xie Fixes: 93597ae8dac0 ("xfs: Fix deadlock between AGI and AGF when target_ip exists in xfs_rename()") Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster --- fs/xfs/libxfs/xfs_dir2.h | 2 -- fs/xfs/libxfs/xfs_dir2_sf.c | 2 +- fs/xfs/xfs_inode.c | 42 +++++++++++++++++++++++++----------------- 3 files changed, 26 insertions(+), 20 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index e55378640b05..d03e6098ded9 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -47,8 +47,6 @@ extern int xfs_dir_lookup(struct xfs_trans *tp, struct xfs_inode *dp, extern int xfs_dir_removename(struct xfs_trans *tp, struct xfs_inode *dp, struct xfs_name *name, xfs_ino_t ino, xfs_extlen_t tot); -extern bool xfs_dir2_sf_replace_needblock(struct xfs_inode *dp, - xfs_ino_t inum); extern int xfs_dir_replace(struct xfs_trans *tp, struct xfs_inode *dp, struct xfs_name *name, xfs_ino_t inum, xfs_extlen_t tot); diff --git a/fs/xfs/libxfs/xfs_dir2_sf.c b/fs/xfs/libxfs/xfs_dir2_sf.c index 2463b5d73447..8c4f76bba88b 100644 --- a/fs/xfs/libxfs/xfs_dir2_sf.c +++ b/fs/xfs/libxfs/xfs_dir2_sf.c @@ -1018,7 +1018,7 @@ xfs_dir2_sf_removename( /* * Check whether the sf dir replace operation need more blocks. */ -bool +static bool xfs_dir2_sf_replace_needblock( struct xfs_inode *dp, xfs_ino_t inum) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index b7352bc4c815..e5dc41b10ebb 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3017,7 +3017,7 @@ xfs_rename( struct xfs_trans *tp; struct xfs_inode *wip = NULL; /* whiteout inode */ struct xfs_inode *inodes[__XFS_SORT_INODES]; - struct xfs_buf *agibp; + int i; int num_inodes = __XFS_SORT_INODES; bool new_parent = (src_dp != target_dp); bool src_is_directory = S_ISDIR(VFS_I(src_ip)->i_mode); @@ -3130,6 +3130,30 @@ xfs_rename( } } + /* + * Lock the AGI buffers we need to handle bumping the nlink of the + * whiteout inode off the unlinked list and to handle dropping the + * nlink of the target inode. Per locking order rules, do this in + * increasing AG order and before directory block allocation tries to + * grab AGFs because we grab AGIs before AGFs. + * + * The (vfs) caller must ensure that if src is a directory then + * target_ip is either null or an empty directory. + */ + for (i = 0; i < num_inodes && inodes[i] != NULL; i++) { + if (inodes[i] == wip || + (inodes[i] == target_ip && + (VFS_I(target_ip)->i_nlink == 1 || src_is_directory))) { + struct xfs_buf *bp; + xfs_agnumber_t agno; + + agno = XFS_INO_TO_AGNO(mp, inodes[i]->i_ino); + error = xfs_read_agi(mp, tp, agno, &bp); + if (error) + goto out_trans_cancel; + } + } + /* * Directory entry creation below may acquire the AGF. Remove * the whiteout from the unlinked list first to preserve correct @@ -3182,22 +3206,6 @@ xfs_rename( * In case there is already an entry with the same * name at the destination directory, remove it first. */ - - /* - * Check whether the replace operation will need to allocate - * blocks. This happens when the shortform directory lacks - * space and we have to convert it to a block format directory. - * When more blocks are necessary, we must lock the AGI first - * to preserve locking order (AGI -> AGF). - */ - if (xfs_dir2_sf_replace_needblock(target_dp, src_ip->i_ino)) { - error = xfs_read_agi(mp, tp, - XFS_INO_TO_AGNO(mp, target_ip->i_ino), - &agibp); - if (error) - goto out_trans_cancel; - } - error = xfs_dir_replace(tp, target_dp, target_name, src_ip->i_ino, spaceres); if (error) -- cgit v1.2.3 From b9b7e1dc56c5ca8d6fc37c410b054e9f26737d2e Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:10 -0800 Subject: xfs: Add helper for checking per-inode extent count overflow XFS does not check for possible overflow of per-inode extent counter fields when adding extents to either data or attr fork. For e.g. 1. Insert 5 million xattrs (each having a value size of 255 bytes) and then delete 50% of them in an alternating manner. 2. On a 4k block sized XFS filesystem instance, the above causes 98511 extents to be created in the attr fork of the inode. xfsaild/loop0 2008 [003] 1475.127209: probe:xfs_inode_to_disk: (ffffffffa43fb6b0) if_nextents=98511 i_ino=131 3. The incore inode fork extent counter is a signed 32-bit quantity. However the on-disk extent counter is an unsigned 16-bit quantity and hence cannot hold 98511 extents. 4. The following incorrect value is stored in the attr extent counter, # xfs_db -f -c 'inode 131' -c 'print core.naextents' /dev/loop0 core.naextents = -32561 This commit adds a new helper function (i.e. xfs_iext_count_may_overflow()) to check for overflow of the per-inode data and xattr extent counters. Future patches will use this function to make sure that an FS operation won't cause the extent counter to overflow. Suggested-by: Darrick J. Wong Reviewed-by: Allison Henderson Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_fork.c | 23 +++++++++++++++++++++++ fs/xfs/libxfs/xfs_inode_fork.h | 2 ++ 2 files changed, 25 insertions(+) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 7575de5cecb1..8d48716547e5 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -23,6 +23,7 @@ #include "xfs_da_btree.h" #include "xfs_dir2_priv.h" #include "xfs_attr_leaf.h" +#include "xfs_types.h" kmem_zone_t *xfs_ifork_zone; @@ -728,3 +729,25 @@ xfs_ifork_verify_local_attr( return 0; } + +int +xfs_iext_count_may_overflow( + struct xfs_inode *ip, + int whichfork, + int nr_to_add) +{ + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); + uint64_t max_exts; + uint64_t nr_exts; + + if (whichfork == XFS_COW_FORK) + return 0; + + max_exts = (whichfork == XFS_ATTR_FORK) ? MAXAEXTNUM : MAXEXTNUM; + + nr_exts = ifp->if_nextents + nr_to_add; + if (nr_exts < ifp->if_nextents || nr_exts > max_exts) + return -EFBIG; + + return 0; +} diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index a4953e95c4f3..0beb8e2a00be 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -172,5 +172,7 @@ extern void xfs_ifork_init_cow(struct xfs_inode *ip); int xfs_ifork_verify_local_data(struct xfs_inode *ip); int xfs_ifork_verify_local_attr(struct xfs_inode *ip); +int xfs_iext_count_may_overflow(struct xfs_inode *ip, int whichfork, + int nr_to_add); #endif /* __XFS_INODE_FORK_H__ */ -- cgit v1.2.3 From 727e1acd297cae15449607d6e2ee39c71216cf1a Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:11 -0800 Subject: xfs: Check for extent overflow when trivally adding a new extent When adding a new data extent (without modifying an inode's existing extents) the extent count increases only by 1. This commit checks for extent count overflow in such cases. Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Henderson Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_bmap.c | 6 ++++++ fs/xfs/libxfs/xfs_inode_fork.h | 6 ++++++ fs/xfs/xfs_bmap_item.c | 7 +++++++ fs/xfs/xfs_bmap_util.c | 5 +++++ fs/xfs/xfs_dquot.c | 8 +++++++- fs/xfs/xfs_iomap.c | 5 +++++ fs/xfs/xfs_rtalloc.c | 5 +++++ 7 files changed, 41 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index bc446418e227..32aeacf6f055 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4527,6 +4527,12 @@ xfs_bmapi_convert_delalloc( return error; xfs_ilock(ip, XFS_ILOCK_EXCL); + + error = xfs_iext_count_may_overflow(ip, whichfork, + XFS_IEXT_ADD_NOSPLIT_CNT); + if (error) + goto out_trans_cancel; + xfs_trans_ijoin(tp, ip, 0); if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &bma.icur, &bma.got) || diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index 0beb8e2a00be..7fc2b129a2e7 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -34,6 +34,12 @@ struct xfs_ifork { #define XFS_IFEXTENTS 0x02 /* All extent pointers are read in */ #define XFS_IFBROOT 0x04 /* i_broot points to the bmap b-tree root */ +/* + * Worst-case increase in the fork extent count when we're adding a single + * extent to a fork and there's no possibility of splitting an existing mapping. + */ +#define XFS_IEXT_ADD_NOSPLIT_CNT (1) + /* * Fork handling. */ diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 93e4d8ae6e92..0534304ed0a7 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -508,6 +508,13 @@ xfs_bui_item_recover( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, 0); + if (bui_type == XFS_BMAP_MAP) { + error = xfs_iext_count_may_overflow(ip, whichfork, + XFS_IEXT_ADD_NOSPLIT_CNT); + if (error) + goto err_cancel; + } + count = bmap->me_len; error = xfs_trans_log_finish_bmap_update(tp, budp, bui_type, ip, whichfork, bmap->me_startoff, bmap->me_startblock, diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 7371a7f7c652..db44bfaabe88 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -822,6 +822,11 @@ xfs_alloc_file_space( if (error) goto error1; + error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + XFS_IEXT_ADD_NOSPLIT_CNT); + if (error) + goto error0; + xfs_trans_ijoin(tp, ip, 0); error = xfs_bmapi_write(tp, ip, startoffset_fsb, diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 1d95ed387d66..175f544f7c45 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -314,8 +314,14 @@ xfs_dquot_disk_alloc( return -ESRCH; } - /* Create the block mapping. */ xfs_trans_ijoin(tp, quotip, XFS_ILOCK_EXCL); + + error = xfs_iext_count_may_overflow(quotip, XFS_DATA_FORK, + XFS_IEXT_ADD_NOSPLIT_CNT); + if (error) + return error; + + /* Create the block mapping. */ error = xfs_bmapi_write(tp, quotip, dqp->q_fileoffset, XFS_DQUOT_CLUSTER_SIZE_FSB, XFS_BMAPI_METADATA, 0, &map, &nmaps); diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 7b9ff824e82d..32246d90b9cb 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -250,6 +250,11 @@ xfs_iomap_write_direct( if (error) goto out_trans_cancel; + error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + XFS_IEXT_ADD_NOSPLIT_CNT); + if (error) + goto out_res_cancel; + xfs_trans_ijoin(tp, ip, 0); /* diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c index b4999fb01ff7..161b0e8992ba 100644 --- a/fs/xfs/xfs_rtalloc.c +++ b/fs/xfs/xfs_rtalloc.c @@ -804,6 +804,11 @@ xfs_growfs_rt_alloc( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + XFS_IEXT_ADD_NOSPLIT_CNT); + if (error) + goto out_trans_cancel; + /* * Allocate blocks to the bitmap file. */ -- cgit v1.2.3 From 85ef08b5a667615bc7be5058259753dc42a7adcd Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:11 -0800 Subject: xfs: Check for extent overflow when punching a hole The extent mapping the file offset at which a hole has to be inserted will be split into two extents causing extent count to increase by 1. Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Henderson Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_fork.h | 7 +++++++ fs/xfs/xfs_bmap_item.c | 15 +++++++++------ fs/xfs/xfs_bmap_util.c | 10 ++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index 7fc2b129a2e7..bcac769a7df6 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -40,6 +40,13 @@ struct xfs_ifork { */ #define XFS_IEXT_ADD_NOSPLIT_CNT (1) +/* + * Punching out an extent from the middle of an existing extent can cause the + * extent count to increase by 1. + * i.e. | Old extent | Hole | Old extent | + */ +#define XFS_IEXT_PUNCH_HOLE_CNT (1) + /* * Fork handling. */ diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 0534304ed0a7..2344757ede63 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -471,6 +471,7 @@ xfs_bui_item_recover( xfs_exntst_t state; unsigned int bui_type; int whichfork; + int iext_delta; int error = 0; if (!xfs_bui_validate(mp, buip)) { @@ -508,12 +509,14 @@ xfs_bui_item_recover( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, 0); - if (bui_type == XFS_BMAP_MAP) { - error = xfs_iext_count_may_overflow(ip, whichfork, - XFS_IEXT_ADD_NOSPLIT_CNT); - if (error) - goto err_cancel; - } + if (bui_type == XFS_BMAP_MAP) + iext_delta = XFS_IEXT_ADD_NOSPLIT_CNT; + else + iext_delta = XFS_IEXT_PUNCH_HOLE_CNT; + + error = xfs_iext_count_may_overflow(ip, whichfork, iext_delta); + if (error) + goto err_cancel; count = bmap->me_len; error = xfs_trans_log_finish_bmap_update(tp, budp, bui_type, ip, diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index db44bfaabe88..6ac7a6ac2658 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -891,6 +891,11 @@ xfs_unmap_extent( xfs_trans_ijoin(tp, ip, 0); + error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + XFS_IEXT_PUNCH_HOLE_CNT); + if (error) + goto out_trans_cancel; + error = xfs_bunmapi(tp, ip, startoffset_fsb, len_fsb, 0, 2, done); if (error) goto out_trans_cancel; @@ -1168,6 +1173,11 @@ xfs_insert_file_space( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, 0); + error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + XFS_IEXT_PUNCH_HOLE_CNT); + if (error) + goto out_trans_cancel; + /* * The extent shifting code works on extent granularity. So, if stop_fsb * is not the starting block of extent, we need to split the extent at -- cgit v1.2.3 From f5d92749191402c50e32ac83dd9da3b910f5680f Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:12 -0800 Subject: xfs: Check for extent overflow when adding dir entries Directory entry addition can cause the following, 1. Data block can be added/removed. A new extent can cause extent count to increase by 1. 2. Free disk block can be added/removed. Same behaviour as described above for Data block. 3. Dabtree blocks. XFS_DA_NODE_MAXDEPTH blocks can be added. Each of these can be new extents. Hence extent count can increase by XFS_DA_NODE_MAXDEPTH. Signed-off-by: Chandan Babu R Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_fork.h | 13 +++++++++++++ fs/xfs/xfs_inode.c | 10 ++++++++++ fs/xfs/xfs_symlink.c | 5 +++++ 3 files changed, 28 insertions(+) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index bcac769a7df6..ea1a9dd8a763 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -47,6 +47,19 @@ struct xfs_ifork { */ #define XFS_IEXT_PUNCH_HOLE_CNT (1) +/* + * Directory entry addition can cause the following, + * 1. Data block can be added/removed. + * A new extent can cause extent count to increase by 1. + * 2. Free disk block can be added/removed. + * Same behaviour as described above for Data block. + * 3. Dabtree blocks. + * XFS_DA_NODE_MAXDEPTH blocks can be added. Each of these can be new + * extents. Hence extent count can increase by XFS_DA_NODE_MAXDEPTH. + */ +#define XFS_IEXT_DIR_MANIP_CNT(mp) \ + ((XFS_DA_NODE_MAXDEPTH + 1 + 1) * (mp)->m_dir_geo->fsbcount) + /* * Fork handling. */ diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index e5dc41b10ebb..3cb41b5d5a26 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -1042,6 +1042,11 @@ xfs_create( if (error) goto out_trans_cancel; + error = xfs_iext_count_may_overflow(dp, XFS_DATA_FORK, + XFS_IEXT_DIR_MANIP_CNT(mp)); + if (error) + goto out_trans_cancel; + /* * A newly created regular or special file just has one directory * entry pointing to them, but a directory also the "." entry @@ -1258,6 +1263,11 @@ xfs_link( xfs_trans_ijoin(tp, sip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, tdp, XFS_ILOCK_EXCL); + error = xfs_iext_count_may_overflow(tdp, XFS_DATA_FORK, + XFS_IEXT_DIR_MANIP_CNT(mp)); + if (error) + goto error_return; + /* * If we are using project inheritance, we only allow hard link * creation in our tree when the project IDs are the same; else diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 1f43fd7f3209..0b8136a32484 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -220,6 +220,11 @@ xfs_symlink( if (error) goto out_trans_cancel; + error = xfs_iext_count_may_overflow(dp, XFS_DATA_FORK, + XFS_IEXT_DIR_MANIP_CNT(mp)); + if (error) + goto out_trans_cancel; + /* * Allocate an inode for the symlink. */ -- cgit v1.2.3 From 0dbc5cb1a91cc8c44b1c75429f5b9351837114fd Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:12 -0800 Subject: xfs: Check for extent overflow when removing dir entries Directory entry removal must always succeed; Hence XFS does the following during low disk space scenario: 1. Data/Free blocks linger until a future remove operation. 2. Dabtree blocks would be swapped with the last block in the leaf space and then the new last block will be unmapped. This facility is reused during low inode extent count scenario i.e. this commit causes xfs_bmap_del_extent_real() to return -ENOSPC error code so that the above mentioned behaviour is exercised causing no change to the directory's extent count. Signed-off-by: Chandan Babu R Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_bmap.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 32aeacf6f055..6c8f17a0e247 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -5151,6 +5151,24 @@ xfs_bmap_del_extent_real( /* * Deleting the middle of the extent. */ + + /* + * For directories, -ENOSPC is returned since a directory entry + * remove operation must not fail due to low extent count + * availability. -ENOSPC will be handled by higher layers of XFS + * by letting the corresponding empty Data/Free blocks to linger + * until a future remove operation. Dabtree blocks would be + * swapped with the last block in the leaf space and then the + * new last block will be unmapped. + */ + error = xfs_iext_count_may_overflow(ip, whichfork, 1); + if (error) { + ASSERT(S_ISDIR(VFS_I(ip)->i_mode) && + whichfork == XFS_DATA_FORK); + error = -ENOSPC; + goto done; + } + old = got; got.br_blockcount = del->br_startoff - got.br_startoff; -- cgit v1.2.3 From 02092a2f034fdeabab524ae39c2de86ba9ffa15a Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:13 -0800 Subject: xfs: Check for extent overflow when renaming dir entries A rename operation is essentially a directory entry remove operation from the perspective of parent directory (i.e. src_dp) of rename's source. Hence the only place where we check for extent count overflow for src_dp is in xfs_bmap_del_extent_real(). xfs_bmap_del_extent_real() returns -ENOSPC when it detects a possible extent count overflow and in response, the higher layers of directory handling code do the following: 1. Data/Free blocks: XFS lets these blocks linger until a future remove operation removes them. 2. Dabtree blocks: XFS swaps the blocks with the last block in the Leaf space and unmaps the last block. For target_dp, there are two cases depending on whether the destination directory entry exists or not. When destination directory entry does not exist (i.e. target_ip == NULL), extent count overflow check is performed only when transaction has a non-zero sized space reservation associated with it. With a zero-sized space reservation, XFS allows a rename operation to continue only when the directory has sufficient free space in its data/leaf/free space blocks to hold the new entry. When destination directory entry exists (i.e. target_ip != NULL), all we need to do is change the inode number associated with the already existing entry. Hence there is no need to perform an extent count overflow check. Signed-off-by: Chandan Babu R Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_bmap.c | 3 +++ fs/xfs/xfs_inode.c | 44 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 6c8f17a0e247..8ebe5f13279c 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -5160,6 +5160,9 @@ xfs_bmap_del_extent_real( * until a future remove operation. Dabtree blocks would be * swapped with the last block in the leaf space and then the * new last block will be unmapped. + * + * The above logic also applies to the source directory entry of + * a rename operation. */ error = xfs_iext_count_may_overflow(ip, whichfork, 1); if (error) { diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 3cb41b5d5a26..8ebd9c64aa48 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3116,6 +3116,35 @@ xfs_rename( /* * Check for expected errors before we dirty the transaction * so we can return an error without a transaction abort. + * + * Extent count overflow check: + * + * From the perspective of src_dp, a rename operation is essentially a + * directory entry remove operation. Hence the only place where we check + * for extent count overflow for src_dp is in + * xfs_bmap_del_extent_real(). xfs_bmap_del_extent_real() returns + * -ENOSPC when it detects a possible extent count overflow and in + * response, the higher layers of directory handling code do the + * following: + * 1. Data/Free blocks: XFS lets these blocks linger until a + * future remove operation removes them. + * 2. Dabtree blocks: XFS swaps the blocks with the last block in the + * Leaf space and unmaps the last block. + * + * For target_dp, there are two cases depending on whether the + * destination directory entry exists or not. + * + * When destination directory entry does not exist (i.e. target_ip == + * NULL), extent count overflow check is performed only when transaction + * has a non-zero sized space reservation associated with it. With a + * zero-sized space reservation, XFS allows a rename operation to + * continue only when the directory has sufficient free space in its + * data/leaf/free space blocks to hold the new entry. + * + * When destination directory entry exists (i.e. target_ip != NULL), all + * we need to do is change the inode number associated with the already + * existing entry. Hence there is no need to perform an extent count + * overflow check. */ if (target_ip == NULL) { /* @@ -3126,6 +3155,12 @@ xfs_rename( error = xfs_dir_canenter(tp, target_dp, target_name); if (error) goto out_trans_cancel; + } else { + error = xfs_iext_count_may_overflow(target_dp, + XFS_DATA_FORK, + XFS_IEXT_DIR_MANIP_CNT(mp)); + if (error) + goto out_trans_cancel; } } else { /* @@ -3291,9 +3326,16 @@ xfs_rename( if (wip) { error = xfs_dir_replace(tp, src_dp, src_name, wip->i_ino, spaceres); - } else + } else { + /* + * NOTE: We don't need to check for extent count overflow here + * because the dir remove name code will leave the dir block in + * place if the extent count would overflow. + */ error = xfs_dir_removename(tp, src_dp, src_name, src_ip->i_ino, spaceres); + } + if (error) goto out_trans_cancel; -- cgit v1.2.3 From 3a19bb147c72d2e9b77137bf5130b9cfb50a5eef Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:13 -0800 Subject: xfs: Check for extent overflow when adding/removing xattrs Adding/removing an xattr can cause XFS_DA_NODE_MAXDEPTH extents to be added. One extra extent for dabtree in case a local attr is large enough to cause a double split. It can also cause extent count to increase proportional to the size of a remote xattr's value. Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Henderson Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_attr.c | 13 +++++++++++++ fs/xfs/libxfs/xfs_inode_fork.h | 10 ++++++++++ 2 files changed, 23 insertions(+) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index fd8e6418a0d3..be51e7068dcd 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -396,6 +396,7 @@ xfs_attr_set( struct xfs_trans_res tres; bool rsvd = (args->attr_filter & XFS_ATTR_ROOT); int error, local; + int rmt_blks = 0; unsigned int total; if (XFS_FORCED_SHUTDOWN(dp->i_mount)) @@ -442,11 +443,15 @@ xfs_attr_set( tres.tr_logcount = XFS_ATTRSET_LOG_COUNT; tres.tr_logflags = XFS_TRANS_PERM_LOG_RES; total = args->total; + + if (!local) + rmt_blks = xfs_attr3_rmt_blocks(mp, args->valuelen); } else { XFS_STATS_INC(mp, xs_attr_remove); tres = M_RES(mp)->tr_attrrm; total = XFS_ATTRRM_SPACE_RES(mp); + rmt_blks = xfs_attr3_rmt_blocks(mp, XFS_XATTR_SIZE_MAX); } /* @@ -460,6 +465,14 @@ xfs_attr_set( xfs_ilock(dp, XFS_ILOCK_EXCL); xfs_trans_ijoin(args->trans, dp, 0); + + if (args->value || xfs_inode_hasattr(dp)) { + error = xfs_iext_count_may_overflow(dp, XFS_ATTR_FORK, + XFS_IEXT_ATTR_MANIP_CNT(rmt_blks)); + if (error) + goto out_trans_cancel; + } + if (args->value) { unsigned int quota_flags = XFS_QMOPT_RES_REGBLKS; diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index ea1a9dd8a763..8d89838e23f8 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -60,6 +60,16 @@ struct xfs_ifork { #define XFS_IEXT_DIR_MANIP_CNT(mp) \ ((XFS_DA_NODE_MAXDEPTH + 1 + 1) * (mp)->m_dir_geo->fsbcount) +/* + * Adding/removing an xattr can cause XFS_DA_NODE_MAXDEPTH extents to + * be added. One extra extent for dabtree in case a local attr is + * large enough to cause a double split. It can also cause extent + * count to increase proportional to the size of a remote xattr's + * value. + */ +#define XFS_IEXT_ATTR_MANIP_CNT(rmt_blks) \ + (XFS_DA_NODE_MAXDEPTH + max(1, rmt_blks)) + /* * Fork handling. */ -- cgit v1.2.3 From c442f3086d5a108b7ff086c8ade1923a8f389db5 Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:14 -0800 Subject: xfs: Check for extent overflow when writing to unwritten extent A write to a sub-interval of an existing unwritten extent causes the original extent to be split into 3 extents i.e. | Unwritten | Real | Unwritten | Hence extent count can increase by 2. Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Henderson Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_fork.h | 9 +++++++++ fs/xfs/xfs_iomap.c | 5 +++++ 2 files changed, 14 insertions(+) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index 8d89838e23f8..917e289ad962 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -70,6 +70,15 @@ struct xfs_ifork { #define XFS_IEXT_ATTR_MANIP_CNT(rmt_blks) \ (XFS_DA_NODE_MAXDEPTH + max(1, rmt_blks)) +/* + * A write to a sub-interval of an existing unwritten extent causes the original + * extent to be split into 3 extents + * i.e. | Unwritten | Real | Unwritten | + * Hence extent count can increase by 2. + */ +#define XFS_IEXT_WRITE_UNWRITTEN_CNT (2) + + /* * Fork handling. */ diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 32246d90b9cb..8f4b27cded20 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -566,6 +566,11 @@ xfs_iomap_write_unwritten( if (error) goto error_on_bmapi_transaction; + error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + XFS_IEXT_WRITE_UNWRITTEN_CNT); + if (error) + goto error_on_bmapi_transaction; + /* * Modify the unwritten extent state of the buffer. */ -- cgit v1.2.3 From 5f1d5bbfb2e674052a9fe542f53678978af20770 Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:14 -0800 Subject: xfs: Check for extent overflow when moving extent from cow to data fork Moving an extent to data fork can cause a sub-interval of an existing extent to be unmapped. This will increase extent count by 1. Mapping in the new extent can increase the extent count by 1 again i.e. | Old extent | New extent | Old extent | Hence number of extents increases by 2. Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Henderson Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_fork.h | 9 +++++++++ fs/xfs/xfs_reflink.c | 5 +++++ 2 files changed, 14 insertions(+) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index 917e289ad962..c8f279edc5c1 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -79,6 +79,15 @@ struct xfs_ifork { #define XFS_IEXT_WRITE_UNWRITTEN_CNT (2) +/* + * Moving an extent to data fork can cause a sub-interval of an existing extent + * to be unmapped. This will increase extent count by 1. Mapping in the new + * extent can increase the extent count by 1 again i.e. + * | Old extent | New extent | Old extent | + * Hence number of extents increases by 2. + */ +#define XFS_IEXT_REFLINK_END_COW_CNT (2) + /* * Fork handling. */ diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 6fa05fb78189..ca0ac1426d74 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -628,6 +628,11 @@ xfs_reflink_end_cow_extent( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, 0); + error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + XFS_IEXT_REFLINK_END_COW_CNT); + if (error) + goto out_cancel; + /* * In case of racing, overlapping AIO writes no COW extents might be * left by the time I/O completes for the loser of the race. In that -- cgit v1.2.3 From ee898d78c3540b44270a5fdffe208d7bbb219d93 Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:14 -0800 Subject: xfs: Check for extent overflow when remapping an extent Remapping an extent involves unmapping the existing extent and mapping in the new extent. When unmapping, an extent containing the entire unmap range can be split into two extents, i.e. | Old extent | hole | Old extent | Hence extent count increases by 1. Mapping in the new extent into the destination file can increase the extent count by 1. Reviewed-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_reflink.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'fs') diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index ca0ac1426d74..e1c98dbf79e4 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -1006,6 +1006,7 @@ xfs_reflink_remap_extent( unsigned int resblks; bool smap_real; bool dmap_written = xfs_bmap_is_written_extent(dmap); + int iext_delta = 0; int nimaps; int error; @@ -1099,6 +1100,16 @@ xfs_reflink_remap_extent( goto out_cancel; } + if (smap_real) + ++iext_delta; + + if (dmap_written) + ++iext_delta; + + error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, iext_delta); + if (error) + goto out_cancel; + if (smap_real) { /* * If the extent we're unmapping is backed by storage (written -- cgit v1.2.3 From bcc561f21f115437a010307420fc43d91be91c66 Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:15 -0800 Subject: xfs: Check for extent overflow when swapping extents Removing an initial range of source/donor file's extent and adding a new extent (from donor/source file) in its place will cause extent count to increase by 1. Reviewed-by: Darrick J. Wong Reviewed-by: Allison Henderson Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_fork.h | 7 +++++++ fs/xfs/xfs_bmap_util.c | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index c8f279edc5c1..9e2137cd7372 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -88,6 +88,13 @@ struct xfs_ifork { */ #define XFS_IEXT_REFLINK_END_COW_CNT (2) +/* + * Removing an initial range of source/donor file's extent and adding a new + * extent (from donor/source file) in its place will cause extent count to + * increase by 1. + */ +#define XFS_IEXT_SWAP_RMAP_CNT (1) + /* * Fork handling. */ diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 6ac7a6ac2658..f3f8c48ff5bf 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -1399,6 +1399,22 @@ xfs_swap_extent_rmap( irec.br_blockcount); trace_xfs_swap_extent_rmap_remap_piece(tip, &uirec); + if (xfs_bmap_is_real_extent(&uirec)) { + error = xfs_iext_count_may_overflow(ip, + XFS_DATA_FORK, + XFS_IEXT_SWAP_RMAP_CNT); + if (error) + goto out; + } + + if (xfs_bmap_is_real_extent(&irec)) { + error = xfs_iext_count_may_overflow(tip, + XFS_DATA_FORK, + XFS_IEXT_SWAP_RMAP_CNT); + if (error) + goto out; + } + /* Remove the mapping from the donor file. */ xfs_bmap_unmap_extent(tp, tip, &uirec); -- cgit v1.2.3 From f9fa87169d2bc1bf55ab42bb6085114378c53b86 Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:15 -0800 Subject: xfs: Introduce error injection to reduce maximum inode fork extent count This commit adds XFS_ERRTAG_REDUCE_MAX_IEXTENTS error tag which enables userspace programs to test "Inode fork extent count overflow detection" by reducing maximum possible inode fork extent count to 10. Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Henderson Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_errortag.h | 4 +++- fs/xfs/libxfs/xfs_inode_fork.c | 4 ++++ fs/xfs/xfs_error.c | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_errortag.h b/fs/xfs/libxfs/xfs_errortag.h index 53b305dea381..1c56fcceeea6 100644 --- a/fs/xfs/libxfs/xfs_errortag.h +++ b/fs/xfs/libxfs/xfs_errortag.h @@ -56,7 +56,8 @@ #define XFS_ERRTAG_FORCE_SUMMARY_RECALC 33 #define XFS_ERRTAG_IUNLINK_FALLBACK 34 #define XFS_ERRTAG_BUF_IOERROR 35 -#define XFS_ERRTAG_MAX 36 +#define XFS_ERRTAG_REDUCE_MAX_IEXTENTS 36 +#define XFS_ERRTAG_MAX 37 /* * Random factors for above tags, 1 means always, 2 means 1/2 time, etc. @@ -97,5 +98,6 @@ #define XFS_RANDOM_FORCE_SUMMARY_RECALC 1 #define XFS_RANDOM_IUNLINK_FALLBACK (XFS_RANDOM_DEFAULT/10) #define XFS_RANDOM_BUF_IOERROR XFS_RANDOM_DEFAULT +#define XFS_RANDOM_REDUCE_MAX_IEXTENTS 1 #endif /* __XFS_ERRORTAG_H_ */ diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 8d48716547e5..e080d7e07643 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -24,6 +24,7 @@ #include "xfs_dir2_priv.h" #include "xfs_attr_leaf.h" #include "xfs_types.h" +#include "xfs_errortag.h" kmem_zone_t *xfs_ifork_zone; @@ -745,6 +746,9 @@ xfs_iext_count_may_overflow( max_exts = (whichfork == XFS_ATTR_FORK) ? MAXAEXTNUM : MAXEXTNUM; + if (XFS_TEST_ERROR(false, ip->i_mount, XFS_ERRTAG_REDUCE_MAX_IEXTENTS)) + max_exts = 10; + nr_exts = ifp->if_nextents + nr_to_add; if (nr_exts < ifp->if_nextents || nr_exts > max_exts) return -EFBIG; diff --git a/fs/xfs/xfs_error.c b/fs/xfs/xfs_error.c index 7f6e20899473..3780b118cc47 100644 --- a/fs/xfs/xfs_error.c +++ b/fs/xfs/xfs_error.c @@ -54,6 +54,7 @@ static unsigned int xfs_errortag_random_default[] = { XFS_RANDOM_FORCE_SUMMARY_RECALC, XFS_RANDOM_IUNLINK_FALLBACK, XFS_RANDOM_BUF_IOERROR, + XFS_RANDOM_REDUCE_MAX_IEXTENTS, }; struct xfs_errortag_attr { @@ -164,6 +165,7 @@ XFS_ERRORTAG_ATTR_RW(force_repair, XFS_ERRTAG_FORCE_SCRUB_REPAIR); XFS_ERRORTAG_ATTR_RW(bad_summary, XFS_ERRTAG_FORCE_SUMMARY_RECALC); XFS_ERRORTAG_ATTR_RW(iunlink_fallback, XFS_ERRTAG_IUNLINK_FALLBACK); XFS_ERRORTAG_ATTR_RW(buf_ioerror, XFS_ERRTAG_BUF_IOERROR); +XFS_ERRORTAG_ATTR_RW(reduce_max_iextents, XFS_ERRTAG_REDUCE_MAX_IEXTENTS); static struct attribute *xfs_errortag_attrs[] = { XFS_ERRORTAG_ATTR_LIST(noerror), @@ -202,6 +204,7 @@ static struct attribute *xfs_errortag_attrs[] = { XFS_ERRORTAG_ATTR_LIST(bad_summary), XFS_ERRORTAG_ATTR_LIST(iunlink_fallback), XFS_ERRORTAG_ATTR_LIST(buf_ioerror), + XFS_ERRORTAG_ATTR_LIST(reduce_max_iextents), NULL, }; -- cgit v1.2.3 From aff4db57d510082f11194ca915d8101463c92d46 Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:16 -0800 Subject: xfs: Remove duplicate assert statement in xfs_bmap_btalloc() The check for verifying if the allocated extent is from an AG whose index is greater than or equal to that of tp->t_firstblock is already done a couple of statements earlier in the same function. Hence this commit removes the redundant assert statement. Reviewed-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_bmap.c | 1 - 1 file changed, 1 deletion(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 8ebe5f13279c..0b15b1ff4bdd 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -3699,7 +3699,6 @@ xfs_bmap_btalloc( ap->blkno = args.fsbno; if (ap->tp->t_firstblock == NULLFSBLOCK) ap->tp->t_firstblock = args.fsbno; - ASSERT(nullfb || fb_agno <= args.agno); ap->length = args.len; /* * If the extent size hint is active, we tried to round the -- cgit v1.2.3 From 0961fddfdd3f8ccd6302af2e7718abbaf18c9fff Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:16 -0800 Subject: xfs: Compute bmap extent alignments in a separate function This commit moves over the code which computes stripe alignment and extent size hint alignment into a separate function. Apart from xfs_bmap_btalloc(), the new function will be used by another function introduced in a future commit. Reviewed-by: Darrick J. Wong Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_bmap.c | 89 ++++++++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 37 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 0b15b1ff4bdd..8955a0a938d5 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -3463,13 +3463,59 @@ xfs_bmap_btalloc_accounting( args->len); } +static int +xfs_bmap_compute_alignments( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args) +{ + struct xfs_mount *mp = args->mp; + xfs_extlen_t align = 0; /* minimum allocation alignment */ + int stripe_align = 0; + int error; + + /* stripe alignment for allocation is determined by mount parameters */ + if (mp->m_swidth && (mp->m_flags & XFS_MOUNT_SWALLOC)) + stripe_align = mp->m_swidth; + else if (mp->m_dalign) + stripe_align = mp->m_dalign; + + if (ap->flags & XFS_BMAPI_COWFORK) + align = xfs_get_cowextsz_hint(ap->ip); + else if (ap->datatype & XFS_ALLOC_USERDATA) + align = xfs_get_extsz_hint(ap->ip); + if (align) { + error = xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, + align, 0, ap->eof, 0, ap->conv, + &ap->offset, &ap->length); + ASSERT(!error); + ASSERT(ap->length); + } + + /* apply extent size hints if obtained earlier */ + if (align) { + args->prod = align; + div_u64_rem(ap->offset, args->prod, &args->mod); + if (args->mod) + args->mod = args->prod - args->mod; + } else if (mp->m_sb.sb_blocksize >= PAGE_SIZE) { + args->prod = 1; + args->mod = 0; + } else { + args->prod = PAGE_SIZE >> mp->m_sb.sb_blocklog; + div_u64_rem(ap->offset, args->prod, &args->mod); + if (args->mod) + args->mod = args->prod - args->mod; + } + + return stripe_align; +} + STATIC int xfs_bmap_btalloc( struct xfs_bmalloca *ap) /* bmap alloc argument struct */ { xfs_mount_t *mp; /* mount point structure */ xfs_alloctype_t atype = 0; /* type for allocation routines */ - xfs_extlen_t align = 0; /* minimum allocation alignment */ xfs_agnumber_t fb_agno; /* ag number of ap->firstblock */ xfs_agnumber_t ag; xfs_alloc_arg_t args; @@ -3489,25 +3535,11 @@ xfs_bmap_btalloc( mp = ap->ip->i_mount; - /* stripe alignment for allocation is determined by mount parameters */ - stripe_align = 0; - if (mp->m_swidth && (mp->m_flags & XFS_MOUNT_SWALLOC)) - stripe_align = mp->m_swidth; - else if (mp->m_dalign) - stripe_align = mp->m_dalign; - - if (ap->flags & XFS_BMAPI_COWFORK) - align = xfs_get_cowextsz_hint(ap->ip); - else if (ap->datatype & XFS_ALLOC_USERDATA) - align = xfs_get_extsz_hint(ap->ip); - if (align) { - error = xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, - align, 0, ap->eof, 0, ap->conv, - &ap->offset, &ap->length); - ASSERT(!error); - ASSERT(ap->length); - } + memset(&args, 0, sizeof(args)); + args.tp = ap->tp; + args.mp = mp; + stripe_align = xfs_bmap_compute_alignments(ap, &args); nullfb = ap->tp->t_firstblock == NULLFSBLOCK; fb_agno = nullfb ? NULLAGNUMBER : XFS_FSB_TO_AGNO(mp, @@ -3538,9 +3570,6 @@ xfs_bmap_btalloc( * Normal allocation, done through xfs_alloc_vextent. */ tryagain = isaligned = 0; - memset(&args, 0, sizeof(args)); - args.tp = ap->tp; - args.mp = mp; args.fsbno = ap->blkno; args.oinfo = XFS_RMAP_OINFO_SKIP_UPDATE; @@ -3571,21 +3600,7 @@ xfs_bmap_btalloc( args.total = ap->total; args.minlen = ap->minlen; } - /* apply extent size hints if obtained earlier */ - if (align) { - args.prod = align; - div_u64_rem(ap->offset, args.prod, &args.mod); - if (args.mod) - args.mod = args.prod - args.mod; - } else if (mp->m_sb.sb_blocksize >= PAGE_SIZE) { - args.prod = 1; - args.mod = 0; - } else { - args.prod = PAGE_SIZE >> mp->m_sb.sb_blocklog; - div_u64_rem(ap->offset, args.prod, &args.mod); - if (args.mod) - args.mod = args.prod - args.mod; - } + /* * If we are not low on available data blocks, and the underlying * logical volume manager is a stripe, and the file offset is zero then -- cgit v1.2.3 From 07c72e556299a7fea448912b1330b9ebfd418662 Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:17 -0800 Subject: xfs: Process allocated extent in a separate function This commit moves over the code in xfs_bmap_btalloc() which is responsible for processing an allocated extent to a new function. Apart from xfs_bmap_btalloc(), the new function will be invoked by another function introduced in a future commit. Reviewed-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_bmap.c | 74 +++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 29 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 8955a0a938d5..bf53a0b1eff3 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -3510,6 +3510,48 @@ xfs_bmap_compute_alignments( return stripe_align; } +static void +xfs_bmap_process_allocated_extent( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, + xfs_fileoff_t orig_offset, + xfs_extlen_t orig_length) +{ + int nullfb; + + nullfb = ap->tp->t_firstblock == NULLFSBLOCK; + + /* + * check the allocation happened at the same or higher AG than + * the first block that was allocated. + */ + ASSERT(nullfb || + XFS_FSB_TO_AGNO(args->mp, ap->tp->t_firstblock) <= + XFS_FSB_TO_AGNO(args->mp, args->fsbno)); + + ap->blkno = args->fsbno; + if (nullfb) + ap->tp->t_firstblock = args->fsbno; + ap->length = args->len; + /* + * If the extent size hint is active, we tried to round the + * caller's allocation request offset down to extsz and the + * length up to another extsz boundary. If we found a free + * extent we mapped it in starting at this new offset. If the + * newly mapped space isn't long enough to cover any of the + * range of offsets that was originally requested, move the + * mapping up so that we can fill as much of the caller's + * original request as possible. Free space is apparently + * very fragmented so we're unlikely to be able to satisfy the + * hints anyway. + */ + if (ap->length <= orig_length) + ap->offset = orig_offset; + else if (ap->offset + ap->length < orig_offset + orig_length) + ap->offset = orig_offset + orig_length - ap->length; + xfs_bmap_btalloc_accounting(ap, args); +} + STATIC int xfs_bmap_btalloc( struct xfs_bmalloca *ap) /* bmap alloc argument struct */ @@ -3702,36 +3744,10 @@ xfs_bmap_btalloc( return error; ap->tp->t_flags |= XFS_TRANS_LOWMODE; } + if (args.fsbno != NULLFSBLOCK) { - /* - * check the allocation happened at the same or higher AG than - * the first block that was allocated. - */ - ASSERT(ap->tp->t_firstblock == NULLFSBLOCK || - XFS_FSB_TO_AGNO(mp, ap->tp->t_firstblock) <= - XFS_FSB_TO_AGNO(mp, args.fsbno)); - - ap->blkno = args.fsbno; - if (ap->tp->t_firstblock == NULLFSBLOCK) - ap->tp->t_firstblock = args.fsbno; - ap->length = args.len; - /* - * If the extent size hint is active, we tried to round the - * caller's allocation request offset down to extsz and the - * length up to another extsz boundary. If we found a free - * extent we mapped it in starting at this new offset. If the - * newly mapped space isn't long enough to cover any of the - * range of offsets that was originally requested, move the - * mapping up so that we can fill as much of the caller's - * original request as possible. Free space is apparently - * very fragmented so we're unlikely to be able to satisfy the - * hints anyway. - */ - if (ap->length <= orig_length) - ap->offset = orig_offset; - else if (ap->offset + ap->length < orig_offset + orig_length) - ap->offset = orig_offset + orig_length - ap->length; - xfs_bmap_btalloc_accounting(ap, &args); + xfs_bmap_process_allocated_extent(ap, &args, orig_offset, + orig_length); } else { ap->blkno = NULLFSBLOCK; ap->length = 0; -- cgit v1.2.3 From 301519674699aa9b80a15b2b2165e08532b176e6 Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Fri, 22 Jan 2021 16:48:17 -0800 Subject: xfs: Introduce error injection to allocate only minlen size extents for files This commit adds XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT error tag which helps userspace test programs to get xfs_bmap_btalloc() to always allocate minlen sized extents. This is required for test programs which need a guarantee that minlen extents allocated for a file do not get merged with their existing neighbours in the inode's BMBT. "Inode fork extent overflow check" for Directories, Xattrs and extension of realtime inodes need this since the file offset at which the extents are being allocated cannot be explicitly controlled from userspace. One way to use this error tag is to, 1. Consume all of the free space by sequentially writing to a file. 2. Punch alternate blocks of the file. This causes CNTBT to contain sufficient number of one block sized extent records. 3. Inject XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT error tag. After step 3, xfs_bmap_btalloc() will issue space allocation requests for minlen sized extents only. ENOSPC error code is returned to userspace when there aren't any "one block sized" extents left in any of the AGs. Reviewed-by: Darrick J. Wong Signed-off-by: Chandan Babu R Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_alloc.c | 50 +++++++++++++++++ fs/xfs/libxfs/xfs_alloc.h | 3 ++ fs/xfs/libxfs/xfs_bmap.c | 124 ++++++++++++++++++++++++++++++++++--------- fs/xfs/libxfs/xfs_errortag.h | 4 +- fs/xfs/xfs_error.c | 3 ++ 5 files changed, 159 insertions(+), 25 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c index 7cb9f064ac64..0c623d3c1036 100644 --- a/fs/xfs/libxfs/xfs_alloc.c +++ b/fs/xfs/libxfs/xfs_alloc.c @@ -2474,6 +2474,47 @@ xfs_defer_agfl_block( xfs_defer_add(tp, XFS_DEFER_OPS_TYPE_AGFL_FREE, &new->xefi_list); } +#ifdef DEBUG +/* + * Check if an AGF has a free extent record whose length is equal to + * args->minlen. + */ +STATIC int +xfs_exact_minlen_extent_available( + struct xfs_alloc_arg *args, + struct xfs_buf *agbp, + int *stat) +{ + struct xfs_btree_cur *cnt_cur; + xfs_agblock_t fbno; + xfs_extlen_t flen; + int error = 0; + + cnt_cur = xfs_allocbt_init_cursor(args->mp, args->tp, agbp, + args->agno, XFS_BTNUM_CNT); + error = xfs_alloc_lookup_ge(cnt_cur, 0, args->minlen, stat); + if (error) + goto out; + + if (*stat == 0) { + error = -EFSCORRUPTED; + goto out; + } + + error = xfs_alloc_get_rec(cnt_cur, &fbno, &flen, stat); + if (error) + goto out; + + if (*stat == 1 && flen != args->minlen) + *stat = 0; + +out: + xfs_btree_del_cursor(cnt_cur, error); + + return error; +} +#endif + /* * Decide whether to use this allocation group for this allocation. * If so, fix up the btree freelist's size. @@ -2545,6 +2586,15 @@ xfs_alloc_fix_freelist( if (!xfs_alloc_space_available(args, need, flags)) goto out_agbp_relse; +#ifdef DEBUG + if (args->alloc_minlen_only) { + int stat; + + error = xfs_exact_minlen_extent_available(args, agbp, &stat); + if (error || !stat) + goto out_agbp_relse; + } +#endif /* * Make the freelist shorter if it's too long. * diff --git a/fs/xfs/libxfs/xfs_alloc.h b/fs/xfs/libxfs/xfs_alloc.h index 6c22b12176b8..a4427c5775c2 100644 --- a/fs/xfs/libxfs/xfs_alloc.h +++ b/fs/xfs/libxfs/xfs_alloc.h @@ -75,6 +75,9 @@ typedef struct xfs_alloc_arg { char wasfromfl; /* set if allocation is from freelist */ struct xfs_owner_info oinfo; /* owner of blocks being allocated */ enum xfs_ag_resv_type resv; /* block reservation to use */ +#ifdef DEBUG + bool alloc_minlen_only; /* allocate exact minlen extent */ +#endif } xfs_alloc_arg_t; /* diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index bf53a0b1eff3..2cd24bb06040 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -3552,34 +3552,101 @@ xfs_bmap_process_allocated_extent( xfs_bmap_btalloc_accounting(ap, args); } -STATIC int -xfs_bmap_btalloc( - struct xfs_bmalloca *ap) /* bmap alloc argument struct */ +#ifdef DEBUG +static int +xfs_bmap_exact_minlen_extent_alloc( + struct xfs_bmalloca *ap) { - xfs_mount_t *mp; /* mount point structure */ - xfs_alloctype_t atype = 0; /* type for allocation routines */ - xfs_agnumber_t fb_agno; /* ag number of ap->firstblock */ - xfs_agnumber_t ag; - xfs_alloc_arg_t args; - xfs_fileoff_t orig_offset; - xfs_extlen_t orig_length; - xfs_extlen_t blen; - xfs_extlen_t nextminlen = 0; - int nullfb; /* true if ap->firstblock isn't set */ - int isaligned; - int tryagain; - int error; - int stripe_align; + struct xfs_mount *mp = ap->ip->i_mount; + struct xfs_alloc_arg args = { .tp = ap->tp, .mp = mp }; + xfs_fileoff_t orig_offset; + xfs_extlen_t orig_length; + int error; ASSERT(ap->length); + + if (ap->minlen != 1) { + ap->blkno = NULLFSBLOCK; + ap->length = 0; + return 0; + } + orig_offset = ap->offset; orig_length = ap->length; - mp = ap->ip->i_mount; + args.alloc_minlen_only = 1; - memset(&args, 0, sizeof(args)); - args.tp = ap->tp; - args.mp = mp; + xfs_bmap_compute_alignments(ap, &args); + + if (ap->tp->t_firstblock == NULLFSBLOCK) { + /* + * Unlike the longest extent available in an AG, we don't track + * the length of an AG's shortest extent. + * XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT is a debug only knob and + * hence we can afford to start traversing from the 0th AG since + * we need not be concerned about a drop in performance in + * "debug only" code paths. + */ + ap->blkno = XFS_AGB_TO_FSB(mp, 0, 0); + } else { + ap->blkno = ap->tp->t_firstblock; + } + + args.fsbno = ap->blkno; + args.oinfo = XFS_RMAP_OINFO_SKIP_UPDATE; + args.type = XFS_ALLOCTYPE_FIRST_AG; + args.total = args.minlen = args.maxlen = ap->minlen; + + args.alignment = 1; + args.minalignslop = 0; + + args.minleft = ap->minleft; + args.wasdel = ap->wasdel; + args.resv = XFS_AG_RESV_NONE; + args.datatype = ap->datatype; + + error = xfs_alloc_vextent(&args); + if (error) + return error; + + if (args.fsbno != NULLFSBLOCK) { + xfs_bmap_process_allocated_extent(ap, &args, orig_offset, + orig_length); + } else { + ap->blkno = NULLFSBLOCK; + ap->length = 0; + } + + return 0; +} +#else + +#define xfs_bmap_exact_minlen_extent_alloc(bma) (-EFSCORRUPTED) + +#endif + +STATIC int +xfs_bmap_btalloc( + struct xfs_bmalloca *ap) +{ + struct xfs_mount *mp = ap->ip->i_mount; + struct xfs_alloc_arg args = { .tp = ap->tp, .mp = mp }; + xfs_alloctype_t atype = 0; + xfs_agnumber_t fb_agno; /* ag number of ap->firstblock */ + xfs_agnumber_t ag; + xfs_fileoff_t orig_offset; + xfs_extlen_t orig_length; + xfs_extlen_t blen; + xfs_extlen_t nextminlen = 0; + int nullfb; /* true if ap->firstblock isn't set */ + int isaligned; + int tryagain; + int error; + int stripe_align; + + ASSERT(ap->length); + orig_offset = ap->offset; + orig_length = ap->length; stripe_align = xfs_bmap_compute_alignments(ap, &args); @@ -4113,6 +4180,10 @@ xfs_bmap_alloc_userdata( return xfs_bmap_rtalloc(bma); } + if (unlikely(XFS_TEST_ERROR(false, mp, + XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT))) + return xfs_bmap_exact_minlen_extent_alloc(bma); + return xfs_bmap_btalloc(bma); } @@ -4149,10 +4220,15 @@ xfs_bmapi_allocate( else bma->minlen = 1; - if (bma->flags & XFS_BMAPI_METADATA) - error = xfs_bmap_btalloc(bma); - else + if (bma->flags & XFS_BMAPI_METADATA) { + if (unlikely(XFS_TEST_ERROR(false, mp, + XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT))) + error = xfs_bmap_exact_minlen_extent_alloc(bma); + else + error = xfs_bmap_btalloc(bma); + } else { error = xfs_bmap_alloc_userdata(bma); + } if (error || bma->blkno == NULLFSBLOCK) return error; diff --git a/fs/xfs/libxfs/xfs_errortag.h b/fs/xfs/libxfs/xfs_errortag.h index 1c56fcceeea6..6ca9084b6934 100644 --- a/fs/xfs/libxfs/xfs_errortag.h +++ b/fs/xfs/libxfs/xfs_errortag.h @@ -57,7 +57,8 @@ #define XFS_ERRTAG_IUNLINK_FALLBACK 34 #define XFS_ERRTAG_BUF_IOERROR 35 #define XFS_ERRTAG_REDUCE_MAX_IEXTENTS 36 -#define XFS_ERRTAG_MAX 37 +#define XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT 37 +#define XFS_ERRTAG_MAX 38 /* * Random factors for above tags, 1 means always, 2 means 1/2 time, etc. @@ -99,5 +100,6 @@ #define XFS_RANDOM_IUNLINK_FALLBACK (XFS_RANDOM_DEFAULT/10) #define XFS_RANDOM_BUF_IOERROR XFS_RANDOM_DEFAULT #define XFS_RANDOM_REDUCE_MAX_IEXTENTS 1 +#define XFS_RANDOM_BMAP_ALLOC_MINLEN_EXTENT 1 #endif /* __XFS_ERRORTAG_H_ */ diff --git a/fs/xfs/xfs_error.c b/fs/xfs/xfs_error.c index 3780b118cc47..185b4915b7bf 100644 --- a/fs/xfs/xfs_error.c +++ b/fs/xfs/xfs_error.c @@ -55,6 +55,7 @@ static unsigned int xfs_errortag_random_default[] = { XFS_RANDOM_IUNLINK_FALLBACK, XFS_RANDOM_BUF_IOERROR, XFS_RANDOM_REDUCE_MAX_IEXTENTS, + XFS_RANDOM_BMAP_ALLOC_MINLEN_EXTENT, }; struct xfs_errortag_attr { @@ -166,6 +167,7 @@ XFS_ERRORTAG_ATTR_RW(bad_summary, XFS_ERRTAG_FORCE_SUMMARY_RECALC); XFS_ERRORTAG_ATTR_RW(iunlink_fallback, XFS_ERRTAG_IUNLINK_FALLBACK); XFS_ERRORTAG_ATTR_RW(buf_ioerror, XFS_ERRTAG_BUF_IOERROR); XFS_ERRORTAG_ATTR_RW(reduce_max_iextents, XFS_ERRTAG_REDUCE_MAX_IEXTENTS); +XFS_ERRORTAG_ATTR_RW(bmap_alloc_minlen_extent, XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT); static struct attribute *xfs_errortag_attrs[] = { XFS_ERRORTAG_ATTR_LIST(noerror), @@ -205,6 +207,7 @@ static struct attribute *xfs_errortag_attrs[] = { XFS_ERRORTAG_ATTR_LIST(iunlink_fallback), XFS_ERRORTAG_ATTR_LIST(buf_ioerror), XFS_ERRORTAG_ATTR_LIST(reduce_max_iextents), + XFS_ERRORTAG_ATTR_LIST(bmap_alloc_minlen_extent), NULL, }; -- cgit v1.2.3 From eaf92540a9189851672d33215a34f22ea8d30446 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 22 Jan 2021 16:48:18 -0800 Subject: xfs: remove a stale comment from xfs_file_aio_write_checks() The comment in xfs_file_aio_write_checks() about calling file_modified() after dropping the ilock doesn't make sense, because the code that unconditionally acquires and drops the ilock was removed by commit 467f78992a07 ("xfs: reduce ilock hold times in xfs_file_aio_write_checks"). Remove this outdated comment. Reviewed-by: Christoph Hellwig Signed-off-by: Eric Biggers Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_file.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 5b0f93f73837..4927c6653f15 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -389,12 +389,6 @@ restart: } else spin_unlock(&ip->i_flags_lock); - /* - * Updating the timestamps will grab the ilock again from - * xfs_fs_dirty_inode, so we have to call it after dropping the - * lock above. Eventually we should look into a way to avoid - * the pointless lock roundtrip. - */ return file_modified(file); } -- cgit v1.2.3 From 01ea173e103edd5ec41acec65b9261b87e123fc2 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 22 Jan 2021 16:48:18 -0800 Subject: xfs: fix up non-directory creation in SGID directories XFS always inherits the SGID bit if it is set on the parent inode, while the generic inode_init_owner does not do this in a few cases where it can create a possible security problem, see commit 0fa3ecd87848 ("Fix up non-directory creation in SGID directories") for details. Switch XFS to use the generic helper for the normal path to fix this, just keeping the simple field inheritance open coded for the case of the non-sgid case with the bsdgrpid mount option. Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Reported-by: Christian Brauner Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_inode.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 8ebd9c64aa48..e2a1db4cee43 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -775,6 +775,7 @@ xfs_init_new_inode( prid_t prid, struct xfs_inode **ipp) { + struct inode *dir = pip ? VFS_I(pip) : NULL; struct xfs_mount *mp = tp->t_mountp; struct xfs_inode *ip; unsigned int flags; @@ -804,18 +805,17 @@ xfs_init_new_inode( ASSERT(ip != NULL); inode = VFS_I(ip); - inode->i_mode = mode; set_nlink(inode, nlink); - inode->i_uid = current_fsuid(); inode->i_rdev = rdev; ip->i_d.di_projid = prid; - if (pip && XFS_INHERIT_GID(pip)) { - inode->i_gid = VFS_I(pip)->i_gid; - if ((VFS_I(pip)->i_mode & S_ISGID) && S_ISDIR(mode)) - inode->i_mode |= S_ISGID; + if (dir && !(dir->i_mode & S_ISGID) && + (mp->m_flags & XFS_MOUNT_GRPID)) { + inode->i_uid = current_fsuid(); + inode->i_gid = dir->i_gid; + inode->i_mode = mode; } else { - inode->i_gid = current_fsgid(); + inode_init_owner(inode, dir, mode); } /* -- cgit v1.2.3 From 88a9e03beef22cc5fabea344f54b9a0dfe63de08 Mon Sep 17 00:00:00 2001 From: Yumei Huang Date: Fri, 22 Jan 2021 16:48:19 -0800 Subject: xfs: Fix assert failure in xfs_setattr_size() An assert failure is triggered by syzkaller test due to ATTR_KILL_PRIV is not cleared before xfs_setattr_size. As ATTR_KILL_PRIV is not checked/used by xfs_setattr_size, just remove it from the assert. Signed-off-by: Yumei Huang Reviewed-by: Brian Foster Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_iops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 67c8dc9de8aa..f1e21b6cfa48 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -846,7 +846,7 @@ xfs_setattr_size( ASSERT(xfs_isilocked(ip, XFS_MMAPLOCK_EXCL)); ASSERT(S_ISREG(inode->i_mode)); ASSERT((iattr->ia_valid & (ATTR_UID|ATTR_GID|ATTR_ATIME|ATTR_ATIME_SET| - ATTR_MTIME_SET|ATTR_KILL_PRIV|ATTR_TIMES_SET)) == 0); + ATTR_MTIME_SET|ATTR_TIMES_SET)) == 0); oldsize = inode->i_size; newsize = iattr->ia_size; -- cgit v1.2.3 From 10fb9ac1251fd0daa645c9e6a22270bfc72bd5e8 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:19 -0800 Subject: xfs: rename xfs_wait_buftarg() to xfs_buftarg_drain() xfs_wait_buftarg() is vaguely named and somewhat overloaded. Its primary purpose is to reclaim all buffers from the provided buffer target LRU. In preparation to refactor xfs_wait_buftarg() into serialization and LRU draining components, rename the function and associated helpers to something more descriptive. This patch has no functional changes with the minor exception of renaming a tracepoint. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf.c | 12 ++++++------ fs/xfs/xfs_buf.h | 10 +++++----- fs/xfs/xfs_log.c | 6 +++--- fs/xfs/xfs_mount.c | 4 ++-- fs/xfs/xfs_trace.h | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index f8400bbd6473..99ee8b0f499c 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -43,7 +43,7 @@ static kmem_zone_t *xfs_buf_zone; * pag_buf_lock * lru_lock * - * xfs_buftarg_wait_rele + * xfs_buftarg_drain_rele * lru_lock * b_lock (trylock due to inversion) * @@ -88,7 +88,7 @@ xfs_buf_vmap_len( * because the corresponding decrement is deferred to buffer release. Buffers * can undergo I/O multiple times in a hold-release cycle and per buffer I/O * tracking adds unnecessary overhead. This is used for sychronization purposes - * with unmount (see xfs_wait_buftarg()), so all we really need is a count of + * with unmount (see xfs_buftarg_drain()), so all we really need is a count of * in-flight buffers. * * Buffers that are never released (e.g., superblock, iclog buffers) must set @@ -1786,7 +1786,7 @@ __xfs_buf_mark_corrupt( * while freeing all the buffers only held by the LRU. */ static enum lru_status -xfs_buftarg_wait_rele( +xfs_buftarg_drain_rele( struct list_head *item, struct list_lru_one *lru, spinlock_t *lru_lock, @@ -1798,7 +1798,7 @@ xfs_buftarg_wait_rele( if (atomic_read(&bp->b_hold) > 1) { /* need to wait, so skip it this pass */ - trace_xfs_buf_wait_buftarg(bp, _RET_IP_); + trace_xfs_buf_drain_buftarg(bp, _RET_IP_); return LRU_SKIP; } if (!spin_trylock(&bp->b_lock)) @@ -1816,7 +1816,7 @@ xfs_buftarg_wait_rele( } void -xfs_wait_buftarg( +xfs_buftarg_drain( struct xfs_buftarg *btp) { LIST_HEAD(dispose); @@ -1841,7 +1841,7 @@ xfs_wait_buftarg( /* loop until there is nothing left on the lru list. */ while (list_lru_count(&btp->bt_lru)) { - list_lru_walk(&btp->bt_lru, xfs_buftarg_wait_rele, + list_lru_walk(&btp->bt_lru, xfs_buftarg_drain_rele, &dispose, LONG_MAX); while (!list_empty(&dispose)) { diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index 5d91a31298a4..d5e31ba205e0 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -152,7 +152,7 @@ struct xfs_buf { struct list_head b_list; struct xfs_perag *b_pag; /* contains rbtree root */ struct xfs_mount *b_mount; - xfs_buftarg_t *b_target; /* buffer target (device) */ + struct xfs_buftarg *b_target; /* buffer target (device) */ void *b_addr; /* virtual address of buffer */ struct work_struct b_ioend_work; struct completion b_iowait; /* queue for I/O waiters */ @@ -344,11 +344,11 @@ xfs_buf_update_cksum(struct xfs_buf *bp, unsigned long cksum_offset) /* * Handling of buftargs. */ -extern xfs_buftarg_t *xfs_alloc_buftarg(struct xfs_mount *, - struct block_device *, struct dax_device *); +extern struct xfs_buftarg *xfs_alloc_buftarg(struct xfs_mount *, + struct block_device *, struct dax_device *); extern void xfs_free_buftarg(struct xfs_buftarg *); -extern void xfs_wait_buftarg(xfs_buftarg_t *); -extern int xfs_setsize_buftarg(xfs_buftarg_t *, unsigned int); +extern void xfs_buftarg_drain(struct xfs_buftarg *); +extern int xfs_setsize_buftarg(struct xfs_buftarg *, unsigned int); #define xfs_getsize_buftarg(buftarg) block_size((buftarg)->bt_bdev) #define xfs_readonly_buftarg(buftarg) bdev_read_only((buftarg)->bt_bdev) diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index fa2d05e65ff1..5ad4d5e78019 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -741,7 +741,7 @@ xfs_log_mount_finish( xfs_log_force(mp, XFS_LOG_SYNC); xfs_ail_push_all_sync(mp->m_ail); } - xfs_wait_buftarg(mp->m_ddev_targp); + xfs_buftarg_drain(mp->m_ddev_targp); if (readonly) mp->m_flags |= XFS_MOUNT_RDONLY; @@ -936,13 +936,13 @@ xfs_log_quiesce( /* * The superblock buffer is uncached and while xfs_ail_push_all_sync() - * will push it, xfs_wait_buftarg() will not wait for it. Further, + * will push it, xfs_buftarg_drain() will not wait for it. Further, * xfs_buf_iowait() cannot be used because it was pushed with the * XBF_ASYNC flag set, so we need to use a lock/unlock pair to wait for * the IO to complete. */ xfs_ail_push_all_sync(mp->m_ail); - xfs_wait_buftarg(mp->m_ddev_targp); + xfs_buftarg_drain(mp->m_ddev_targp); xfs_buf_lock(mp->m_sb_bp); xfs_buf_unlock(mp->m_sb_bp); diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index 7110507a2b6b..29a553f0877d 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1023,8 +1023,8 @@ xfs_mountfs( xfs_log_mount_cancel(mp); out_fail_wait: if (mp->m_logdev_targp && mp->m_logdev_targp != mp->m_ddev_targp) - xfs_wait_buftarg(mp->m_logdev_targp); - xfs_wait_buftarg(mp->m_ddev_targp); + xfs_buftarg_drain(mp->m_logdev_targp); + xfs_buftarg_drain(mp->m_ddev_targp); out_free_perag: xfs_free_perag(mp); out_free_dir: diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 5a263ae3d4f0..e2e0092c331d 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -358,7 +358,7 @@ DEFINE_BUF_EVENT(xfs_buf_get_uncached); DEFINE_BUF_EVENT(xfs_buf_item_relse); DEFINE_BUF_EVENT(xfs_buf_iodone_async); DEFINE_BUF_EVENT(xfs_buf_error_relse); -DEFINE_BUF_EVENT(xfs_buf_wait_buftarg); +DEFINE_BUF_EVENT(xfs_buf_drain_buftarg); DEFINE_BUF_EVENT(xfs_trans_read_buf_shut); /* not really buffer traces, but the buf provides useful information */ -- cgit v1.2.3 From 8321ddb2fa2964bffbc61400894a47dc3462323f Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:20 -0800 Subject: xfs: don't drain buffer lru on freeze and read-only remount xfs_buftarg_drain() is called from xfs_log_quiesce() to ensure the buffer cache is reclaimed during unmount. xfs_log_quiesce() is also called from xfs_quiesce_attr(), however, which means that cache state is completely drained for filesystem freeze and read-only remount. While technically harmless, this is unnecessarily heavyweight. Both freeze and read-only mounts allow reads and thus allow population of the buffer cache. Therefore, the transitional sequence in either case really only needs to quiesce outstanding writes to return the filesystem in a generally read-only state. Additionally, some users have reported that attempts to freeze a filesystem concurrent with a read-heavy workload causes the freeze process to stall for a significant amount of time. This occurs because, as mentioned above, the read workload repopulates the buffer LRU while the freeze task attempts to drain it. To improve this situation, replace the drain in xfs_log_quiesce() with a buffer I/O quiesce and lift the drain into the unmount path. This removes buffer LRU reclaim from freeze and read-only [re]mount, but ensures the LRU is still drained before the filesystem unmounts. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf.c | 20 +++++++++++++++----- fs/xfs/xfs_buf.h | 1 + fs/xfs/xfs_log.c | 6 ++++-- 3 files changed, 20 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 99ee8b0f499c..f6e5235df7c9 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1815,14 +1815,13 @@ xfs_buftarg_drain_rele( return LRU_REMOVED; } +/* + * Wait for outstanding I/O on the buftarg to complete. + */ void -xfs_buftarg_drain( +xfs_buftarg_wait( struct xfs_buftarg *btp) { - LIST_HEAD(dispose); - int loop = 0; - bool write_fail = false; - /* * First wait on the buftarg I/O count for all in-flight buffers to be * released. This is critical as new buffers do not make the LRU until @@ -1838,6 +1837,17 @@ xfs_buftarg_drain( while (percpu_counter_sum(&btp->bt_io_count)) delay(100); flush_workqueue(btp->bt_mount->m_buf_workqueue); +} + +void +xfs_buftarg_drain( + struct xfs_buftarg *btp) +{ + LIST_HEAD(dispose); + int loop = 0; + bool write_fail = false; + + xfs_buftarg_wait(btp); /* loop until there is nothing left on the lru list. */ while (list_lru_count(&btp->bt_lru)) { diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index d5e31ba205e0..459ca34f26f5 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -347,6 +347,7 @@ xfs_buf_update_cksum(struct xfs_buf *bp, unsigned long cksum_offset) extern struct xfs_buftarg *xfs_alloc_buftarg(struct xfs_mount *, struct block_device *, struct dax_device *); extern void xfs_free_buftarg(struct xfs_buftarg *); +extern void xfs_buftarg_wait(struct xfs_buftarg *); extern void xfs_buftarg_drain(struct xfs_buftarg *); extern int xfs_setsize_buftarg(struct xfs_buftarg *, unsigned int); diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 5ad4d5e78019..46ea4017fcec 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -936,13 +936,13 @@ xfs_log_quiesce( /* * The superblock buffer is uncached and while xfs_ail_push_all_sync() - * will push it, xfs_buftarg_drain() will not wait for it. Further, + * will push it, xfs_buftarg_wait() will not wait for it. Further, * xfs_buf_iowait() cannot be used because it was pushed with the * XBF_ASYNC flag set, so we need to use a lock/unlock pair to wait for * the IO to complete. */ xfs_ail_push_all_sync(mp->m_ail); - xfs_buftarg_drain(mp->m_ddev_targp); + xfs_buftarg_wait(mp->m_ddev_targp); xfs_buf_lock(mp->m_sb_bp); xfs_buf_unlock(mp->m_sb_bp); @@ -962,6 +962,8 @@ xfs_log_unmount( { xfs_log_quiesce(mp); + xfs_buftarg_drain(mp->m_ddev_targp); + xfs_trans_ail_destroy(mp); xfs_sysfs_del(&mp->m_log->l_kobj); -- cgit v1.2.3 From 8aa921a95335d0a8c8e2be35a44467e7c91ec3e4 Mon Sep 17 00:00:00 2001 From: Jeffrey Mitchell Date: Fri, 22 Jan 2021 16:48:20 -0800 Subject: xfs: set inode size after creating symlink When XFS creates a new symlink, it writes its size to disk but not to the VFS inode. This causes i_size_read() to return 0 for that symlink until it is re-read from disk, for example when the system is rebooted. I found this inconsistency while protecting directories with eCryptFS. The command "stat path/to/symlink/in/ecryptfs" will report "Size: 0" if the symlink was created after the last reboot on an XFS root. Call i_size_write() in xfs_symlink() Signed-off-by: Jeffrey Mitchell Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_symlink.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 0b8136a32484..7f96649e918a 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -305,6 +305,7 @@ xfs_symlink( } ASSERT(pathlen == 0); } + i_size_write(VFS_I(ip), ip->i_d.di_size); /* * Create the directory entry for the symlink. -- cgit v1.2.3 From 50d25484bebe94320c49dd1347d3330c7063bbdb Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:20 -0800 Subject: xfs: sync lazy sb accounting on quiesce of read-only mounts xfs_log_sbcount() syncs the superblock specifically to accumulate the in-core percpu superblock counters and commit them to disk. This is required to maintain filesystem consistency across quiesce (freeze, read-only mount/remount) or unmount when lazy superblock accounting is enabled because individual transactions do not update the superblock directly. This mechanism works as expected for writable mounts, but xfs_log_sbcount() skips the update for read-only mounts. Read-only mounts otherwise still allow log recovery and write out an unmount record during log quiesce. If a read-only mount performs log recovery, it can modify the in-core superblock counters and write an unmount record when the filesystem unmounts without ever syncing the in-core counters. This leaves the filesystem with a clean log but in an inconsistent state with regard to lazy sb counters. Update xfs_log_sbcount() to use the same logic xfs_log_unmount_write() uses to determine when to write an unmount record. This ensures that lazy accounting is always synced before the log is cleaned. Refactor this logic into a new helper to distinguish between a writable filesystem and a writable log. Specifically, the log is writable unless the filesystem is mounted with the norecovery mount option, the underlying log device is read-only, or the filesystem is shutdown. Drop the freeze state check because the update is already allowed during the freezing process and no context calls this function on an already frozen fs. Also, retain the shutdown check in xfs_log_unmount_write() to catch the case where the preceding log force might have triggered a shutdown. Signed-off-by: Brian Foster Reviewed-by: Gao Xiang Reviewed-by: Allison Henderson Reviewed-by: Darrick J. Wong Reviewed-by: Bill O'Donnell Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_log.c | 28 ++++++++++++++++++++-------- fs/xfs/xfs_log.h | 1 + fs/xfs/xfs_mount.c | 3 +-- 3 files changed, 22 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 46ea4017fcec..62bcdaa07dc9 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -347,6 +347,25 @@ xlog_tic_add_region(xlog_ticket_t *tic, uint len, uint type) tic->t_res_num++; } +bool +xfs_log_writable( + struct xfs_mount *mp) +{ + /* + * Never write to the log on norecovery mounts, if the block device is + * read-only, or if the filesystem is shutdown. Read-only mounts still + * allow internal writes for log recovery and unmount purposes, so don't + * restrict that case here. + */ + if (mp->m_flags & XFS_MOUNT_NORECOVERY) + return false; + if (xfs_readonly_buftarg(mp->m_log->l_targ)) + return false; + if (XFS_FORCED_SHUTDOWN(mp)) + return false; + return true; +} + /* * Replenish the byte reservation required by moving the grant write head. */ @@ -886,15 +905,8 @@ xfs_log_unmount_write( { struct xlog *log = mp->m_log; - /* - * Don't write out unmount record on norecovery mounts or ro devices. - * Or, if we are doing a forced umount (typically because of IO errors). - */ - if (mp->m_flags & XFS_MOUNT_NORECOVERY || - xfs_readonly_buftarg(log->l_targ)) { - ASSERT(mp->m_flags & XFS_MOUNT_RDONLY); + if (!xfs_log_writable(mp)) return; - } xfs_log_force(mp, XFS_LOG_SYNC); diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h index 58c3fcbec94a..98c913da7587 100644 --- a/fs/xfs/xfs_log.h +++ b/fs/xfs/xfs_log.h @@ -127,6 +127,7 @@ int xfs_log_reserve(struct xfs_mount *mp, int xfs_log_regrant(struct xfs_mount *mp, struct xlog_ticket *tic); void xfs_log_unmount(struct xfs_mount *mp); int xfs_log_force_umount(struct xfs_mount *mp, int logerror); +bool xfs_log_writable(struct xfs_mount *mp); struct xlog_ticket *xfs_log_ticket_get(struct xlog_ticket *ticket); void xfs_log_ticket_put(struct xlog_ticket *ticket); diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index 29a553f0877d..8c24e1ee63ec 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1176,8 +1176,7 @@ xfs_fs_writable( int xfs_log_sbcount(xfs_mount_t *mp) { - /* allow this to proceed during the freeze sequence... */ - if (!xfs_fs_writable(mp, SB_FREEZE_COMPLETE)) + if (!xfs_log_writable(mp)) return 0; /* -- cgit v1.2.3 From 37444fc4cc398266fe0f71a9c0925620d44fb76a Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:21 -0800 Subject: xfs: lift writable fs check up into log worker task The log covering helper checks whether the filesystem is writable to determine whether to cover the log. The helper is currently only called from the background log worker. In preparation to reuse the helper from freezing contexts, lift the check into xfs_log_worker(). Signed-off-by: Brian Foster Reviewed-by: Allison Henderson Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_log.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 62bcdaa07dc9..3ede7ad431c0 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -1051,14 +1051,12 @@ xfs_log_space_wake( * there's no point in running a dummy transaction at this point because we * can't start trying to idle the log until both the CIL and AIL are empty. */ -static int -xfs_log_need_covered(xfs_mount_t *mp) +static bool +xfs_log_need_covered( + struct xfs_mount *mp) { - struct xlog *log = mp->m_log; - int needed = 0; - - if (!xfs_fs_writable(mp, SB_FREEZE_WRITE)) - return 0; + struct xlog *log = mp->m_log; + bool needed = false; if (!xlog_cil_empty(log)) return 0; @@ -1076,14 +1074,14 @@ xfs_log_need_covered(xfs_mount_t *mp) if (!xlog_iclogs_empty(log)) break; - needed = 1; + needed = true; if (log->l_covered_state == XLOG_STATE_COVER_NEED) log->l_covered_state = XLOG_STATE_COVER_DONE; else log->l_covered_state = XLOG_STATE_COVER_DONE2; break; default: - needed = 1; + needed = true; break; } spin_unlock(&log->l_icloglock); @@ -1273,7 +1271,7 @@ xfs_log_worker( struct xfs_mount *mp = log->l_mp; /* dgc: errors ignored - not fatal and nowhere to report them */ - if (xfs_log_need_covered(mp)) { + if (xfs_fs_writable(mp, SB_FREEZE_WRITE) && xfs_log_need_covered(mp)) { /* * Dump a transaction into the log that contains no real change. * This is needed to stamp the current tail LSN into the log -- cgit v1.2.3 From 9e54ee0fc9ef88ee255dc9770b291d047b38643c Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:21 -0800 Subject: xfs: separate log cleaning from log quiesce Log quiesce is currently associated with cleaning the log, which is accomplished by writing an unmount record as the last step of the quiesce sequence. The quiesce codepath is a bit convoluted in this regard due to how it is reused from various contexts. In preparation to create separate log cleaning and log covering interfaces, lift the write of the unmount record into a new cleaning helper and call that wherever xfs_log_quiesce() is currently invoked. No functional changes. Signed-off-by: Brian Foster Reviewed-by: Allison Henderson Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_log.c | 8 +++++++- fs/xfs/xfs_log.h | 1 + fs/xfs/xfs_super.c | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 3ede7ad431c0..d02e14054956 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -957,7 +957,13 @@ xfs_log_quiesce( xfs_buftarg_wait(mp->m_ddev_targp); xfs_buf_lock(mp->m_sb_bp); xfs_buf_unlock(mp->m_sb_bp); +} +void +xfs_log_clean( + struct xfs_mount *mp) +{ + xfs_log_quiesce(mp); xfs_log_unmount_write(mp); } @@ -972,7 +978,7 @@ void xfs_log_unmount( struct xfs_mount *mp) { - xfs_log_quiesce(mp); + xfs_log_clean(mp); xfs_buftarg_drain(mp->m_ddev_targp); diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h index 98c913da7587..b0400589f824 100644 --- a/fs/xfs/xfs_log.h +++ b/fs/xfs/xfs_log.h @@ -139,6 +139,7 @@ bool xfs_log_item_in_current_chkpt(struct xfs_log_item *lip); void xfs_log_work_queue(struct xfs_mount *mp); void xfs_log_quiesce(struct xfs_mount *mp); +void xfs_log_clean(struct xfs_mount *mp); bool xfs_log_check_lsn(struct xfs_mount *, xfs_lsn_t); bool xfs_log_in_recovery(struct xfs_mount *); diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 813be879a5e5..09d956e30fd8 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -897,7 +897,7 @@ xfs_quiesce_attr( if (error) xfs_warn(mp, "xfs_attr_quiesce: failed to log sb changes. " "Frozen image may not be consistent."); - xfs_log_quiesce(mp); + xfs_log_clean(mp); } /* -- cgit v1.2.3 From 303591a0a9473fc4842984080fdb619188426bad Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:22 -0800 Subject: xfs: cover the log during log quiesce The log quiesce mechanism historically terminates by marking the log clean with an unmount record. The primary objective is to indicate that log recovery is no longer required after the quiesce has flushed all in-core changes and written back filesystem metadata. While this is perfectly fine, it is somewhat hacky as currently used in certain contexts. For example, filesystem freeze quiesces (i.e. cleans) the log and immediately redirties it with a dummy superblock transaction to ensure that log recovery runs in the event of a crash. While this functions correctly, cleaning the log from freeze context is clearly superfluous given the current redirtying behavior. Instead, the desired behavior can be achieved by simply covering the log. This effectively retires all on-disk log items from the active range of the log by issuing two synchronous and sequential dummy superblock update transactions that serve to update the on-disk log head and tail. The subtle difference is that the log technically remains dirty due to the lack of an unmount record, though recovery is effectively a no-op due to the content of the checkpoints being clean (i.e. the unmodified on-disk superblock). Log covering currently runs in the background and only triggers once the filesystem and log has idled. The purpose of the background mechanism is to prevent log recovery from replaying the most recently logged items long after those items may have been written back. In the quiesce path, the log has been deliberately idled by forcing the log and pushing the AIL until empty in a context where no further mutable filesystem operations are allowed. Therefore, we can cover the log as the final step in the log quiesce codepath to reflect that all previously active items have been successfully written back. This facilitates selective log covering from certain contexts (i.e. freeze) that only seek to quiesce, but not necessarily clean the log. Note that as a side effect of this change, log covering now occurs when cleaning the log as well. This is harmless, facilitates subsequent cleanups, and is mostly temporary as various operations switch to use explicit log covering. Signed-off-by: Brian Foster Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson --- fs/xfs/xfs_log.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- fs/xfs/xfs_log.h | 2 +- 2 files changed, 47 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index d02e14054956..e941faea8e98 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -91,6 +91,9 @@ STATIC int xlog_iclogs_empty( struct xlog *log); +static int +xfs_log_cover(struct xfs_mount *); + static void xlog_grant_sub_space( struct xlog *log, @@ -936,10 +939,9 @@ xfs_log_unmount_write( * To do this, we first need to shut down the background log work so it is not * trying to cover the log as we clean up. We then need to unpin all objects in * the log so we can then flush them out. Once they have completed their IO and - * run the callbacks removing themselves from the AIL, we can write the unmount - * record. + * run the callbacks removing themselves from the AIL, we can cover the log. */ -void +int xfs_log_quiesce( struct xfs_mount *mp) { @@ -957,6 +959,8 @@ xfs_log_quiesce( xfs_buftarg_wait(mp->m_ddev_targp); xfs_buf_lock(mp->m_sb_bp); xfs_buf_unlock(mp->m_sb_bp); + + return xfs_log_cover(mp); } void @@ -1094,6 +1098,45 @@ xfs_log_need_covered( return needed; } +/* + * Explicitly cover the log. This is similar to background log covering but + * intended for usage in quiesce codepaths. The caller is responsible to ensure + * the log is idle and suitable for covering. The CIL, iclog buffers and AIL + * must all be empty. + */ +static int +xfs_log_cover( + struct xfs_mount *mp) +{ + struct xlog *log = mp->m_log; + int error = 0; + + ASSERT((xlog_cil_empty(log) && xlog_iclogs_empty(log) && + !xfs_ail_min_lsn(log->l_ailp)) || + XFS_FORCED_SHUTDOWN(mp)); + + if (!xfs_log_writable(mp)) + return 0; + + /* + * To cover the log, commit the superblock twice (at most) in + * independent checkpoints. The first serves as a reference for the + * tail pointer. The sync transaction and AIL push empties the AIL and + * updates the in-core tail to the LSN of the first checkpoint. The + * second commit updates the on-disk tail with the in-core LSN, + * covering the log. Push the AIL one more time to leave it empty, as + * we found it. + */ + while (xfs_log_need_covered(mp)) { + error = xfs_sync_sb(mp, true); + if (error) + break; + xfs_ail_push_all_sync(mp->m_ail); + } + + return error; +} + /* * We may be holding the log iclog lock upon entering this routine. */ diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h index b0400589f824..044e02cb8921 100644 --- a/fs/xfs/xfs_log.h +++ b/fs/xfs/xfs_log.h @@ -138,7 +138,7 @@ void xlog_cil_process_committed(struct list_head *list); bool xfs_log_item_in_current_chkpt(struct xfs_log_item *lip); void xfs_log_work_queue(struct xfs_mount *mp); -void xfs_log_quiesce(struct xfs_mount *mp); +int xfs_log_quiesce(struct xfs_mount *mp); void xfs_log_clean(struct xfs_mount *mp); bool xfs_log_check_lsn(struct xfs_mount *, xfs_lsn_t); bool xfs_log_in_recovery(struct xfs_mount *); -- cgit v1.2.3 From b0eb9e1182668b0e9cf81dbf38041cfb8c12887f Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:22 -0800 Subject: xfs: don't reset log idle state on covering checkpoints Now that log covering occurs on quiesce, we'd like to reuse the underlying superblock sync for final superblock updates. This includes things like lazy superblock counter updates, log feature incompat bits in the future, etc. One quirk to this approach is that once the log is in the IDLE (i.e. already covered) state, any subsequent log write resets the state back to NEED. This means that a final superblock sync to an already covered log requires two more sb syncs to return the log back to IDLE again. For example, if a lazy superblock enabled filesystem is mount cycled without any modifications, the unmount path syncs the superblock once and writes an unmount record. With the desired log quiesce covering behavior, we sync the superblock three times at unmount time: once for the lazy superblock counter update and twice more to cover the log. By contrast, if the log is active or only partially covered at unmount time, a final superblock sync would doubly serve as the one or two remaining syncs required to cover the log. This duplicate covering sequence is unnecessary because the filesystem remains consistent if a crash occurs at any point. The superblock will either be recovered in the event of a crash or written back before the log is quiesced and potentially cleaned with an unmount record. Update the log covering state machine to remain in the IDLE state if additional covering checkpoints pass through the log. This facilitates final superblock updates (such as lazy superblock counters) via a single sb sync without losing covered status. This provides some consistency with the active and partially covered cases and also avoids harmless, but spurious checkpoints when quiescing the log. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson --- fs/xfs/xfs_log.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index e941faea8e98..a613f008b95f 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -2599,12 +2599,15 @@ xlog_covered_state( int iclogs_changed) { /* - * We usually go to NEED. But we go to NEED2 if the changed indicates we - * are done writing the dummy record. If we are done with the second - * dummy recored (DONE2), then we go to IDLE. + * We go to NEED for any non-covering writes. We go to NEED2 if we just + * wrote the first covering record (DONE). We go to IDLE if we just + * wrote the second covering record (DONE2) and remain in IDLE until a + * non-covering write occurs. */ switch (prev_state) { case XLOG_STATE_COVER_IDLE: + if (iclogs_changed == 1) + return XLOG_STATE_COVER_IDLE; case XLOG_STATE_COVER_NEED: case XLOG_STATE_COVER_NEED2: break; -- cgit v1.2.3 From f46e5a174655fd0bdb73008f6a4967d9c706f691 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:23 -0800 Subject: xfs: fold sbcount quiesce logging into log covering xfs_log_sbcount() calls xfs_sync_sb() to sync superblock counters to disk when lazy superblock accounting is enabled. This occurs on unmount, freeze, and read-only (re)mount and ensures the final values are calculated and persisted to disk before each form of quiesce completes. Now that log covering occurs in all of these contexts and uses the same xfs_sync_sb() mechanism to update log state, there is no need to log the superblock separately for any reason. Update the log quiesce path to sync the superblock at least once for any mount where lazy superblock accounting is enabled. If the log is already covered, it will remain in the covered state. Otherwise, the next sync as part of the normal covering sequence will carry the associated superblock update with it. Remove xfs_log_sbcount() now that it is no longer needed. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson --- fs/xfs/xfs_log.c | 20 ++++++++++++++++++-- fs/xfs/xfs_mount.c | 31 ------------------------------- fs/xfs/xfs_mount.h | 1 - fs/xfs/xfs_super.c | 8 -------- 4 files changed, 18 insertions(+), 42 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index a613f008b95f..58699881c100 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -1110,6 +1110,7 @@ xfs_log_cover( { struct xlog *log = mp->m_log; int error = 0; + bool need_covered; ASSERT((xlog_cil_empty(log) && xlog_iclogs_empty(log) && !xfs_ail_min_lsn(log->l_ailp)) || @@ -1118,6 +1119,21 @@ xfs_log_cover( if (!xfs_log_writable(mp)) return 0; + /* + * xfs_log_need_covered() is not idempotent because it progresses the + * state machine if the log requires covering. Therefore, we must call + * this function once and use the result until we've issued an sb sync. + * Do so first to make that abundantly clear. + * + * Fall into the covering sequence if the log needs covering or the + * mount has lazy superblock accounting to sync to disk. The sb sync + * used for covering accumulates the in-core counters, so covering + * handles this for us. + */ + need_covered = xfs_log_need_covered(mp); + if (!need_covered && !xfs_sb_version_haslazysbcount(&mp->m_sb)) + return 0; + /* * To cover the log, commit the superblock twice (at most) in * independent checkpoints. The first serves as a reference for the @@ -1127,12 +1143,12 @@ xfs_log_cover( * covering the log. Push the AIL one more time to leave it empty, as * we found it. */ - while (xfs_log_need_covered(mp)) { + do { error = xfs_sync_sb(mp, true); if (error) break; xfs_ail_push_all_sync(mp->m_ail); - } + } while (xfs_log_need_covered(mp)); return error; } diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index 8c24e1ee63ec..b621b09899e5 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1124,12 +1124,6 @@ xfs_unmountfs( xfs_warn(mp, "Unable to free reserved block pool. " "Freespace may not be correct on next mount."); - error = xfs_log_sbcount(mp); - if (error) - xfs_warn(mp, "Unable to update superblock counters. " - "Freespace may not be correct on next mount."); - - xfs_log_unmount(mp); xfs_da_unmount(mp); xfs_uuid_unmount(mp); @@ -1164,31 +1158,6 @@ xfs_fs_writable( return true; } -/* - * xfs_log_sbcount - * - * Sync the superblock counters to disk. - * - * Note this code can be called during the process of freezing, so we use the - * transaction allocator that does not block when the transaction subsystem is - * in its frozen state. - */ -int -xfs_log_sbcount(xfs_mount_t *mp) -{ - if (!xfs_log_writable(mp)) - return 0; - - /* - * we don't need to do this if we are updating the superblock - * counters on every modification. - */ - if (!xfs_sb_version_haslazysbcount(&mp->m_sb)) - return 0; - - return xfs_sync_sb(mp, true); -} - /* * Deltas for the block count can vary from 1 to very large, but lock contention * only occurs on frequent small block count updates such as in the delayed diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index dfa429b77ee2..452ca7654dc5 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -399,7 +399,6 @@ int xfs_buf_hash_init(xfs_perag_t *pag); void xfs_buf_hash_destroy(xfs_perag_t *pag); extern void xfs_uuid_table_free(void); -extern int xfs_log_sbcount(xfs_mount_t *); extern uint64_t xfs_default_resblks(xfs_mount_t *mp); extern int xfs_mountfs(xfs_mount_t *mp); extern int xfs_initialize_perag(xfs_mount_t *mp, xfs_agnumber_t agcount, diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 09d956e30fd8..75ada867c665 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -884,19 +884,11 @@ void xfs_quiesce_attr( struct xfs_mount *mp) { - int error = 0; - cancel_delayed_work_sync(&mp->m_log->l_work); /* force the log to unpin objects from the now complete transactions */ xfs_log_force(mp, XFS_LOG_SYNC); - - /* Push the superblock and write an unmount record */ - error = xfs_log_sbcount(mp); - if (error) - xfs_warn(mp, "xfs_attr_quiesce: failed to log sb changes. " - "Frozen image may not be consistent."); xfs_log_clean(mp); } -- cgit v1.2.3 From 5232b9315034e45dba43b164aca3d5228948d05b Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:23 -0800 Subject: xfs: remove duplicate wq cancel and log force from attr quiesce These two calls are repeated at the beginning of xfs_log_quiesce(). Drop them from xfs_quiesce_attr(). Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson --- fs/xfs/xfs_super.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 75ada867c665..8fc9044131fc 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -884,11 +884,6 @@ void xfs_quiesce_attr( struct xfs_mount *mp) { - cancel_delayed_work_sync(&mp->m_log->l_work); - - /* force the log to unpin objects from the now complete transactions */ - xfs_log_force(mp, XFS_LOG_SYNC); - xfs_log_clean(mp); } -- cgit v1.2.3 From ea2064da4592723d7b96235ca9bba4091a7458e3 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:24 -0800 Subject: xfs: remove xfs_quiesce_attr() xfs_quiesce_attr() is now a wrapper for xfs_log_clean(). Remove it and call xfs_log_clean() directly. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson --- fs/xfs/xfs_mount.c | 2 +- fs/xfs/xfs_super.c | 24 ++---------------------- 2 files changed, 3 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index b621b09899e5..53b8ccab7235 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -946,7 +946,7 @@ xfs_mountfs( */ if ((mp->m_flags & (XFS_MOUNT_RDONLY|XFS_MOUNT_NORECOVERY)) == XFS_MOUNT_RDONLY) { - xfs_quiesce_attr(mp); + xfs_log_clean(mp); } /* diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 8fc9044131fc..aedf622d221b 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -867,26 +867,6 @@ xfs_restore_resvblks(struct xfs_mount *mp) xfs_reserve_blocks(mp, &resblks, NULL); } -/* - * Trigger writeback of all the dirty metadata in the file system. - * - * This ensures that the metadata is written to their location on disk rather - * than just existing in transactions in the log. This means after a quiesce - * there is no log replay required to write the inodes to disk - this is the - * primary difference between a sync and a quiesce. - * - * We cancel log work early here to ensure all transactions the log worker may - * run have finished before we clean up and log the superblock and write an - * unmount record. The unfreeze process is responsible for restarting the log - * worker correctly. - */ -void -xfs_quiesce_attr( - struct xfs_mount *mp) -{ - xfs_log_clean(mp); -} - /* * Second stage of a freeze. The data is already frozen so we only * need to take care of the metadata. Once that's done sync the superblock @@ -909,7 +889,7 @@ xfs_fs_freeze( flags = memalloc_nofs_save(); xfs_stop_block_reaping(mp); xfs_save_resvblks(mp); - xfs_quiesce_attr(mp); + xfs_log_clean(mp); ret = xfs_sync_sb(mp, true); memalloc_nofs_restore(flags); return ret; @@ -1752,7 +1732,7 @@ xfs_remount_ro( */ xfs_save_resvblks(mp); - xfs_quiesce_attr(mp); + xfs_log_clean(mp); mp->m_flags |= XFS_MOUNT_RDONLY; return 0; -- cgit v1.2.3 From 5b0ad7c2a52d4fdfec86a2c29096701783f46719 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 22 Jan 2021 16:48:24 -0800 Subject: xfs: cover the log on freeze instead of cleaning it Filesystem freeze cleans the log and immediately redirties it so log recovery runs if a crash occurs after the filesystem is frozen. Now that log quiesce covers the log, there is no need to clean the log and redirty it to trigger log recovery because covering has the same effect. Update xfs_fs_freeze() to quiesce (and thus cover) the log. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson --- fs/xfs/xfs_super.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index aedf622d221b..aed74a3fc787 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -889,8 +889,7 @@ xfs_fs_freeze( flags = memalloc_nofs_save(); xfs_stop_block_reaping(mp); xfs_save_resvblks(mp); - xfs_log_clean(mp); - ret = xfs_sync_sb(mp, true); + ret = xfs_log_quiesce(mp); memalloc_nofs_restore(flags); return ret; } -- cgit v1.2.3 From f22c7f87777361f94aa17f746fbadfa499248dc8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 22 Jan 2021 16:48:25 -0800 Subject: xfs: refactor xfs_file_fsync Factor out the log syncing logic into two helpers to make the code easier to read and more maintainable. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/xfs_file.c | 81 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 31 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 4927c6653f15..9441c257d86c 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -118,6 +118,54 @@ xfs_dir_fsync( return xfs_log_force_inode(ip); } +static xfs_lsn_t +xfs_fsync_lsn( + struct xfs_inode *ip, + bool datasync) +{ + if (!xfs_ipincount(ip)) + return 0; + if (datasync && !(ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP)) + return 0; + return ip->i_itemp->ili_last_lsn; +} + +/* + * All metadata updates are logged, which means that we just have to flush the + * log up to the latest LSN that touched the inode. + * + * If we have concurrent fsync/fdatasync() calls, we need them to all block on + * the log force before we clear the ili_fsync_fields field. This ensures that + * we don't get a racing sync operation that does not wait for the metadata to + * hit the journal before returning. If we race with clearing ili_fsync_fields, + * then all that will happen is the log force will do nothing as the lsn will + * already be on disk. We can't race with setting ili_fsync_fields because that + * is done under XFS_ILOCK_EXCL, and that can't happen because we hold the lock + * shared until after the ili_fsync_fields is cleared. + */ +static int +xfs_fsync_flush_log( + struct xfs_inode *ip, + bool datasync, + int *log_flushed) +{ + int error = 0; + xfs_lsn_t lsn; + + xfs_ilock(ip, XFS_ILOCK_SHARED); + lsn = xfs_fsync_lsn(ip, datasync); + if (lsn) { + error = xfs_log_force_lsn(ip->i_mount, lsn, XFS_LOG_SYNC, + log_flushed); + + spin_lock(&ip->i_itemp->ili_lock); + ip->i_itemp->ili_fsync_fields = 0; + spin_unlock(&ip->i_itemp->ili_lock); + } + xfs_iunlock(ip, XFS_ILOCK_SHARED); + return error; +} + STATIC int xfs_file_fsync( struct file *file, @@ -125,13 +173,10 @@ xfs_file_fsync( loff_t end, int datasync) { - struct inode *inode = file->f_mapping->host; - struct xfs_inode *ip = XFS_I(inode); - struct xfs_inode_log_item *iip = ip->i_itemp; + struct xfs_inode *ip = XFS_I(file->f_mapping->host); struct xfs_mount *mp = ip->i_mount; int error = 0; int log_flushed = 0; - xfs_lsn_t lsn = 0; trace_xfs_file_fsync(ip); @@ -155,33 +200,7 @@ xfs_file_fsync( else if (mp->m_logdev_targp != mp->m_ddev_targp) xfs_blkdev_issue_flush(mp->m_ddev_targp); - /* - * All metadata updates are logged, which means that we just have to - * flush the log up to the latest LSN that touched the inode. If we have - * concurrent fsync/fdatasync() calls, we need them to all block on the - * log force before we clear the ili_fsync_fields field. This ensures - * that we don't get a racing sync operation that does not wait for the - * metadata to hit the journal before returning. If we race with - * clearing the ili_fsync_fields, then all that will happen is the log - * force will do nothing as the lsn will already be on disk. We can't - * race with setting ili_fsync_fields because that is done under - * XFS_ILOCK_EXCL, and that can't happen because we hold the lock shared - * until after the ili_fsync_fields is cleared. - */ - xfs_ilock(ip, XFS_ILOCK_SHARED); - if (xfs_ipincount(ip)) { - if (!datasync || - (iip->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP)) - lsn = iip->ili_last_lsn; - } - - if (lsn) { - error = xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, &log_flushed); - spin_lock(&iip->ili_lock); - iip->ili_fsync_fields = 0; - spin_unlock(&iip->ili_lock); - } - xfs_iunlock(ip, XFS_ILOCK_SHARED); + error = xfs_fsync_flush_log(ip, datasync, &log_flushed); /* * If we only have a single device, and the log force about was -- cgit v1.2.3 From ae29e4220fd3047b5442e7e8db8027d7745093f5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 22 Jan 2021 16:48:25 -0800 Subject: xfs: reduce ilock acquisitions in xfs_file_fsync If the inode is not pinned by the time fsync is called we don't need the ilock to protect against concurrent clearing of ili_fsync_fields as the inode won't need a log flush or clearing of these fields. Not taking the iolock allows for full concurrency of fsync and thus O_DSYNC completions with io_uring/aio write submissions. Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/xfs_file.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 9441c257d86c..48f6f89a38a9 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -200,7 +200,14 @@ xfs_file_fsync( else if (mp->m_logdev_targp != mp->m_ddev_targp) xfs_blkdev_issue_flush(mp->m_ddev_targp); - error = xfs_fsync_flush_log(ip, datasync, &log_flushed); + /* + * Any inode that has dirty modifications in the log is pinned. The + * racy check here for a pinned inode while not catch modifications + * that happen concurrently to the fsync call, but fsync semantics + * only require to sync previously completed I/O. + */ + if (xfs_ipincount(ip)) + error = xfs_fsync_flush_log(ip, datasync, &log_flushed); /* * If we only have a single device, and the log force about was -- cgit v1.2.3 From 5724be5de88f5f6863d44c859f42f70d5cc667ed Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:09 -0800 Subject: iomap: rename the flags variable in __iomap_dio_rw Rename flags to iomap_flags to make the usage a little more clear. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/iomap/direct-io.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 933f234d5bec..604103ab76f9 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -427,7 +427,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, size_t count = iov_iter_count(iter); loff_t pos = iocb->ki_pos; loff_t end = iocb->ki_pos + count - 1, ret = 0; - unsigned int flags = IOMAP_DIRECT; + unsigned int iomap_flags = IOMAP_DIRECT; struct blk_plug plug; struct iomap_dio *dio; @@ -461,7 +461,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, if (iter_is_iovec(iter)) dio->flags |= IOMAP_DIO_DIRTY; } else { - flags |= IOMAP_WRITE; + iomap_flags |= IOMAP_WRITE; dio->flags |= IOMAP_DIO_WRITE; /* for data sync or sync, we need sync completion processing */ @@ -483,7 +483,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, ret = -EAGAIN; goto out_free_dio; } - flags |= IOMAP_NOWAIT; + iomap_flags |= IOMAP_NOWAIT; } ret = filemap_write_and_wait_range(mapping, pos, end); @@ -514,7 +514,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, blk_start_plug(&plug); do { - ret = iomap_apply(inode, pos, count, flags, ops, dio, + ret = iomap_apply(inode, pos, count, iomap_flags, ops, dio, iomap_dio_actor); if (ret <= 0) { /* magic error code to fall back to buffered I/O */ -- cgit v1.2.3 From 2f63296578cad1ae681152d5b2122a4595195f16 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:09 -0800 Subject: iomap: pass a flags argument to iomap_dio_rw Pass a set of flags to iomap_dio_rw instead of the boolean wait_for_completion argument. The IOMAP_DIO_FORCE_WAIT flag replaces the wait_for_completion, but only needs to be passed when the iocb isn't synchronous to start with to simplify the callers. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster [djwong: rework xfs_file.c so that we can push iomap changes separately] Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/btrfs/file.c | 7 +++---- fs/ext4/file.c | 5 ++--- fs/gfs2/file.c | 7 ++----- fs/iomap/direct-io.c | 11 +++++------ fs/xfs/xfs_file.c | 5 ++--- fs/zonefs/super.c | 4 ++-- include/linux/iomap.h | 10 ++++++++-- 7 files changed, 24 insertions(+), 25 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 0e41459b8de6..ddfd2e2adedf 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1949,8 +1949,8 @@ relock: goto buffered; } - dio = __iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, - &btrfs_dio_ops, is_sync_kiocb(iocb)); + dio = __iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, &btrfs_dio_ops, + 0); btrfs_inode_unlock(inode, ilock_flags); @@ -3622,8 +3622,7 @@ static ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to) return 0; btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED); - ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dio_ops, - is_sync_kiocb(iocb)); + ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dio_ops, 0); btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); return ret; } diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 349b27f0dda0..194f5d00fa32 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -74,8 +74,7 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) return generic_file_read_iter(iocb, to); } - ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, - is_sync_kiocb(iocb)); + ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0); inode_unlock_shared(inode); file_accessed(iocb->ki_filp); @@ -550,7 +549,7 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) if (ilock_shared) iomap_ops = &ext4_iomap_overwrite_ops; ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops, - is_sync_kiocb(iocb) || unaligned_io || extend); + (unaligned_io || extend) ? IOMAP_DIO_FORCE_WAIT : 0); if (ret == -ENOTBLK) ret = 0; diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index b39b339feddc..89609c299717 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -797,9 +797,7 @@ static ssize_t gfs2_file_direct_read(struct kiocb *iocb, struct iov_iter *to, if (ret) goto out_uninit; - ret = iomap_dio_rw(iocb, to, &gfs2_iomap_ops, NULL, - is_sync_kiocb(iocb)); - + ret = iomap_dio_rw(iocb, to, &gfs2_iomap_ops, NULL, 0); gfs2_glock_dq(gh); out_uninit: gfs2_holder_uninit(gh); @@ -833,8 +831,7 @@ static ssize_t gfs2_file_direct_write(struct kiocb *iocb, struct iov_iter *from, if (offset + len > i_size_read(&ip->i_inode)) goto out; - ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL, - is_sync_kiocb(iocb)); + ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL, 0); if (ret == -ENOTBLK) ret = 0; out: diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 604103ab76f9..947343730e2c 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -420,13 +420,15 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length, struct iomap_dio * __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, - bool wait_for_completion) + unsigned int dio_flags) { struct address_space *mapping = iocb->ki_filp->f_mapping; struct inode *inode = file_inode(iocb->ki_filp); size_t count = iov_iter_count(iter); loff_t pos = iocb->ki_pos; loff_t end = iocb->ki_pos + count - 1, ret = 0; + bool wait_for_completion = + is_sync_kiocb(iocb) || (dio_flags & IOMAP_DIO_FORCE_WAIT); unsigned int iomap_flags = IOMAP_DIRECT; struct blk_plug plug; struct iomap_dio *dio; @@ -434,9 +436,6 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, if (!count) return NULL; - if (WARN_ON(is_sync_kiocb(iocb) && !wait_for_completion)) - return ERR_PTR(-EIO); - dio = kmalloc(sizeof(*dio), GFP_KERNEL); if (!dio) return ERR_PTR(-ENOMEM); @@ -598,11 +597,11 @@ EXPORT_SYMBOL_GPL(__iomap_dio_rw); ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, - bool wait_for_completion) + unsigned int dio_flags) { struct iomap_dio *dio; - dio = __iomap_dio_rw(iocb, iter, ops, dops, wait_for_completion); + dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags); if (IS_ERR_OR_NULL(dio)) return PTR_ERR_OR_ZERO(dio); return iomap_dio_complete(dio); diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 5b0f93f73837..7f18dae84584 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -219,8 +219,7 @@ xfs_file_dio_aio_read( } else { xfs_ilock(ip, XFS_IOLOCK_SHARED); } - ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, - is_sync_kiocb(iocb)); + ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, 0); xfs_iunlock(ip, XFS_IOLOCK_SHARED); return ret; @@ -584,7 +583,7 @@ xfs_file_dio_aio_write( */ ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops, &xfs_dio_write_ops, - is_sync_kiocb(iocb) || unaligned_io); + unaligned_io ? IOMAP_DIO_FORCE_WAIT : 0); out: xfs_iunlock(ip, iolock); diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index bec47f2d074b..0e7ab0bc00ae 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -780,7 +780,7 @@ static ssize_t zonefs_file_dio_write(struct kiocb *iocb, struct iov_iter *from) ret = zonefs_file_dio_append(iocb, from); else ret = iomap_dio_rw(iocb, from, &zonefs_iomap_ops, - &zonefs_write_dio_ops, sync); + &zonefs_write_dio_ops, 0); if (zi->i_ztype == ZONEFS_ZTYPE_SEQ && (ret > 0 || ret == -EIOCBQUEUED)) { if (ret > 0) @@ -917,7 +917,7 @@ static ssize_t zonefs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) } file_accessed(iocb->ki_filp); ret = iomap_dio_rw(iocb, to, &zonefs_iomap_ops, - &zonefs_read_dio_ops, is_sync_kiocb(iocb)); + &zonefs_read_dio_ops, 0); } else { ret = generic_file_read_iter(iocb, to); if (ret == -EIO) diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 5bd3cac4df9c..be4e1e1e01e8 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -256,12 +256,18 @@ struct iomap_dio_ops { struct bio *bio, loff_t file_offset); }; +/* + * Wait for the I/O to complete in iomap_dio_rw even if the kiocb is not + * synchronous. + */ +#define IOMAP_DIO_FORCE_WAIT (1 << 0) + ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, - bool wait_for_completion); + unsigned int dio_flags); struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, - bool wait_for_completion); + unsigned int dio_flags); ssize_t iomap_dio_complete(struct iomap_dio *dio); int iomap_dio_iopoll(struct kiocb *kiocb, bool spin); -- cgit v1.2.3 From 213f627104daf8589aad8ee73fcaeb603ab0af15 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:10 -0800 Subject: iomap: add a IOMAP_DIO_OVERWRITE_ONLY flag Add a flag to signal that only pure overwrites are allowed. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/iomap/direct-io.c | 7 +++++++ include/linux/iomap.h | 8 ++++++++ 2 files changed, 15 insertions(+) (limited to 'fs') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 947343730e2c..65d32364345d 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -485,6 +485,13 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, iomap_flags |= IOMAP_NOWAIT; } + if (dio_flags & IOMAP_DIO_OVERWRITE_ONLY) { + ret = -EAGAIN; + if (pos >= dio->i_size || pos + count > dio->i_size) + goto out_free_dio; + iomap_flags |= IOMAP_OVERWRITE_ONLY; + } + ret = filemap_write_and_wait_range(mapping, pos, end); if (ret) goto out_free_dio; diff --git a/include/linux/iomap.h b/include/linux/iomap.h index be4e1e1e01e8..1a86f520de56 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -122,6 +122,7 @@ struct iomap_page_ops { #define IOMAP_FAULT (1 << 3) /* mapping for page fault */ #define IOMAP_DIRECT (1 << 4) /* direct I/O */ #define IOMAP_NOWAIT (1 << 5) /* do not block */ +#define IOMAP_OVERWRITE_ONLY (1 << 6) /* only pure overwrites allowed */ struct iomap_ops { /* @@ -262,6 +263,13 @@ struct iomap_dio_ops { */ #define IOMAP_DIO_FORCE_WAIT (1 << 0) +/* + * Do not allocate blocks or zero partial blocks, but instead fall back to + * the caller by returning -EAGAIN. Used to optimize direct I/O writes that + * are not aligned to the file system block size. + */ +#define IOMAP_DIO_OVERWRITE_ONLY (1 << 1) + ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, unsigned int dio_flags); -- cgit v1.2.3 From a6435940b62f81a1718bf2bd46a051379fc89b9d Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:20 +0100 Subject: mount: attach mappings to mounts In order to support per-mount idmappings vfsmounts are marked with user namespaces. The idmapping of the user namespace will be used to map the ids of vfs objects when they are accessed through that mount. By default all vfsmounts are marked with the initial user namespace. The initial user namespace is used to indicate that a mount is not idmapped. All operations behave as before. Based on prior discussions we want to attach the whole user namespace and not just a dedicated idmapping struct. This allows us to reuse all the helpers that already exist for dealing with idmappings instead of introducing a whole new range of helpers. In addition, if we decide in the future that we are confident enough to enable unprivileged users to setup idmapped mounts the permission checking can take into account whether the caller is privileged in the user namespace the mount is currently marked with. Later patches enforce that once a mount has been idmapped it can't be remapped. This keeps permission checking and life-cycle management simple. Users wanting to change the idmapped can always create a new detached mount with a different idmapping. Add a new mnt_userns member to vfsmount and two simple helpers to retrieve the mnt_userns from vfsmounts and files. The idea to attach user namespaces to vfsmounts has been floated around in various forms at Linux Plumbers in ~2018 with the original idea tracing back to a discussion in 2017 at a conference in St. Petersburg between Christoph, Tycho, and myself. Link: https://lore.kernel.org/r/20210121131959.646623-2-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/namespace.c | 9 +++++++++ include/linux/fs.h | 6 ++++++ include/linux/mount.h | 6 ++++++ 3 files changed, 21 insertions(+) (limited to 'fs') diff --git a/fs/namespace.c b/fs/namespace.c index 9d33909d0f9e..ecdc63ef881c 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -210,6 +210,7 @@ static struct mount *alloc_vfsmnt(const char *name) INIT_HLIST_NODE(&mnt->mnt_mp_list); INIT_LIST_HEAD(&mnt->mnt_umounting); INIT_HLIST_HEAD(&mnt->mnt_stuck_children); + mnt->mnt.mnt_userns = &init_user_ns; } return mnt; @@ -547,6 +548,11 @@ int sb_prepare_remount_readonly(struct super_block *sb) static void free_vfsmnt(struct mount *mnt) { + struct user_namespace *mnt_userns; + + mnt_userns = mnt_user_ns(&mnt->mnt); + if (mnt_userns != &init_user_ns) + put_user_ns(mnt_userns); kfree_const(mnt->mnt_devname); #ifdef CONFIG_SMP free_percpu(mnt->mnt_pcp); @@ -1055,6 +1061,9 @@ static struct mount *clone_mnt(struct mount *old, struct dentry *root, mnt->mnt.mnt_flags &= ~(MNT_WRITE_HOLD|MNT_MARKED|MNT_INTERNAL); atomic_inc(&sb->s_active); + mnt->mnt.mnt_userns = mnt_user_ns(&old->mnt); + if (mnt->mnt.mnt_userns != &init_user_ns) + mnt->mnt.mnt_userns = get_user_ns(mnt->mnt.mnt_userns); mnt->mnt.mnt_sb = sb; mnt->mnt.mnt_root = dget(root); mnt->mnt_mountpoint = mnt->mnt.mnt_root; diff --git a/include/linux/fs.h b/include/linux/fs.h index fd47deea7c17..fd0b80e6361d 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -2231,6 +2232,7 @@ struct file_system_type { #define FS_HAS_SUBTYPE 4 #define FS_USERNS_MOUNT 8 /* Can be mounted by userns root */ #define FS_DISALLOW_NOTIFY_PERM 16 /* Disable fanotify permission events */ +#define FS_ALLOW_IDMAP 32 /* FS has been updated to handle vfs idmappings. */ #define FS_THP_SUPPORT 8192 /* Remove once all fs converted */ #define FS_RENAME_DOES_D_MOVE 32768 /* FS will handle d_move() during rename() internally. */ int (*init_fs_context)(struct fs_context *); @@ -2517,6 +2519,10 @@ struct filename { }; static_assert(offsetof(struct filename, iname) % sizeof(long) == 0); +static inline struct user_namespace *file_mnt_user_ns(struct file *file) +{ + return mnt_user_ns(file->f_path.mnt); +} extern long vfs_truncate(const struct path *, loff_t); extern int do_truncate(struct dentry *, loff_t start, unsigned int time_attrs, struct file *filp); diff --git a/include/linux/mount.h b/include/linux/mount.h index aaf343b38671..52de25e08319 100644 --- a/include/linux/mount.h +++ b/include/linux/mount.h @@ -72,8 +72,14 @@ struct vfsmount { struct dentry *mnt_root; /* root of the mounted tree */ struct super_block *mnt_sb; /* pointer to superblock */ int mnt_flags; + struct user_namespace *mnt_userns; } __randomize_layout; +static inline struct user_namespace *mnt_user_ns(const struct vfsmount *mnt) +{ + return mnt->mnt_userns; +} + struct file; /* forward dec */ struct path; -- cgit v1.2.3 From 02f92b3868a1b34ab98464e76b0e4e060474ba10 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:22 +0100 Subject: fs: add file and path permissions helpers Add two simple helpers to check permissions on a file and path respectively and convert over some callers. It simplifies quite a few codepaths and also reduces the churn in later patches quite a bit. Christoph also correctly points out that this makes codepaths (e.g. ioctls) way easier to follow that would otherwise have to do more complex argument passing than necessary. Link: https://lore.kernel.org/r/20210121131959.646623-4-christian.brauner@ubuntu.com Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Signed-off-by: Christian Brauner --- fs/init.c | 6 +++--- fs/notify/fanotify/fanotify_user.c | 2 +- fs/notify/inotify/inotify_user.c | 2 +- fs/open.c | 6 +++--- fs/udf/file.c | 2 +- fs/verity/enable.c | 2 +- include/linux/fs.h | 8 ++++++++ kernel/bpf/inode.c | 2 +- kernel/sys.c | 2 +- mm/madvise.c | 2 +- mm/memcontrol.c | 2 +- mm/mincore.c | 2 +- net/unix/af_unix.c | 2 +- 13 files changed, 24 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/init.c b/fs/init.c index e9c320a48cf1..02723bea8499 100644 --- a/fs/init.c +++ b/fs/init.c @@ -49,7 +49,7 @@ int __init init_chdir(const char *filename) error = kern_path(filename, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); if (error) return error; - error = inode_permission(path.dentry->d_inode, MAY_EXEC | MAY_CHDIR); + error = path_permission(&path, MAY_EXEC | MAY_CHDIR); if (!error) set_fs_pwd(current->fs, &path); path_put(&path); @@ -64,7 +64,7 @@ int __init init_chroot(const char *filename) error = kern_path(filename, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); if (error) return error; - error = inode_permission(path.dentry->d_inode, MAY_EXEC | MAY_CHDIR); + error = path_permission(&path, MAY_EXEC | MAY_CHDIR); if (error) goto dput_and_out; error = -EPERM; @@ -118,7 +118,7 @@ int __init init_eaccess(const char *filename) error = kern_path(filename, LOOKUP_FOLLOW, &path); if (error) return error; - error = inode_permission(d_inode(path.dentry), MAY_ACCESS); + error = path_permission(&path, MAY_ACCESS); path_put(&path); return error; } diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index dcab112e1f00..64cfc1a3015d 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -702,7 +702,7 @@ static int fanotify_find_path(int dfd, const char __user *filename, } /* you can only watch an inode if you have read permissions on it */ - ret = inode_permission(path->dentry->d_inode, MAY_READ); + ret = path_permission(path, MAY_READ); if (ret) { path_put(path); goto out; diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index 59c177011a0f..e1155d32ef6f 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -352,7 +352,7 @@ static int inotify_find_inode(const char __user *dirname, struct path *path, if (error) return error; /* you can only watch an inode if you have read permissions on it */ - error = inode_permission(path->dentry->d_inode, MAY_READ); + error = path_permission(path, MAY_READ); if (error) { path_put(path); return error; diff --git a/fs/open.c b/fs/open.c index 1e06e443a565..cd1efd254cad 100644 --- a/fs/open.c +++ b/fs/open.c @@ -492,7 +492,7 @@ retry: if (error) goto out; - error = inode_permission(path.dentry->d_inode, MAY_EXEC | MAY_CHDIR); + error = path_permission(&path, MAY_EXEC | MAY_CHDIR); if (error) goto dput_and_out; @@ -521,7 +521,7 @@ SYSCALL_DEFINE1(fchdir, unsigned int, fd) if (!d_can_lookup(f.file->f_path.dentry)) goto out_putf; - error = inode_permission(file_inode(f.file), MAY_EXEC | MAY_CHDIR); + error = file_permission(f.file, MAY_EXEC | MAY_CHDIR); if (!error) set_fs_pwd(current->fs, &f.file->f_path); out_putf: @@ -540,7 +540,7 @@ retry: if (error) goto out; - error = inode_permission(path.dentry->d_inode, MAY_EXEC | MAY_CHDIR); + error = path_permission(&path, MAY_EXEC | MAY_CHDIR); if (error) goto dput_and_out; diff --git a/fs/udf/file.c b/fs/udf/file.c index ad8eefad27d7..3671a40ed3c3 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -183,7 +183,7 @@ long udf_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) long old_block, new_block; int result; - if (inode_permission(inode, MAY_READ) != 0) { + if (file_permission(filp, MAY_READ) != 0) { udf_debug("no permission to access inode %lu\n", inode->i_ino); return -EPERM; } diff --git a/fs/verity/enable.c b/fs/verity/enable.c index f7e997a01ad0..77e159a0346b 100644 --- a/fs/verity/enable.c +++ b/fs/verity/enable.c @@ -369,7 +369,7 @@ int fsverity_ioctl_enable(struct file *filp, const void __user *uarg) * has verity enabled, and to stabilize the data being hashed. */ - err = inode_permission(inode, MAY_WRITE); + err = file_permission(filp, MAY_WRITE); if (err) return err; diff --git a/include/linux/fs.h b/include/linux/fs.h index 3165998e2294..bcd17097d441 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2812,6 +2812,14 @@ static inline int bmap(struct inode *inode, sector_t *block) extern int notify_change(struct dentry *, struct iattr *, struct inode **); extern int inode_permission(struct inode *, int); extern int generic_permission(struct inode *, int); +static inline int file_permission(struct file *file, int mask) +{ + return inode_permission(file_inode(file), mask); +} +static inline int path_permission(const struct path *path, int mask) +{ + return inode_permission(d_inode(path->dentry), mask); +} extern int __check_sticky(struct inode *dir, struct inode *inode); static inline bool execute_ok(struct inode *inode) diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index dd4b7fd60ee7..8962f139521e 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -507,7 +507,7 @@ static void *bpf_obj_do_get(const char __user *pathname, return ERR_PTR(ret); inode = d_backing_inode(path.dentry); - ret = inode_permission(inode, ACC_MODE(flags)); + ret = path_permission(&path, ACC_MODE(flags)); if (ret) goto out; diff --git a/kernel/sys.c b/kernel/sys.c index 51f00fe20e4d..138fb253b344 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1848,7 +1848,7 @@ static int prctl_set_mm_exe_file(struct mm_struct *mm, unsigned int fd) if (!S_ISREG(inode->i_mode) || path_noexec(&exe.file->f_path)) goto exit; - err = inode_permission(inode, MAY_EXEC); + err = file_permission(exe.file, MAY_EXEC); if (err) goto exit; diff --git a/mm/madvise.c b/mm/madvise.c index 6a660858784b..175c5582d8a9 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -540,7 +540,7 @@ static inline bool can_do_pageout(struct vm_area_struct *vma) * opens a side channel. */ return inode_owner_or_capable(file_inode(vma->vm_file)) || - inode_permission(file_inode(vma->vm_file), MAY_WRITE) == 0; + file_permission(vma->vm_file, MAY_WRITE) == 0; } static long madvise_pageout(struct vm_area_struct *vma, diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 605f671203ef..cf9076f58582 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4899,7 +4899,7 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, /* the process need read permission on control file */ /* AV: shouldn't we check that it's been opened for read instead? */ - ret = inode_permission(file_inode(cfile.file), MAY_READ); + ret = file_permission(cfile.file, MAY_READ); if (ret < 0) goto out_put_cfile; diff --git a/mm/mincore.c b/mm/mincore.c index 02db1a834021..7bdb4673f776 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -167,7 +167,7 @@ static inline bool can_do_mincore(struct vm_area_struct *vma) * mappings, which opens a side channel. */ return inode_owner_or_capable(file_inode(vma->vm_file)) || - inode_permission(file_inode(vma->vm_file), MAY_WRITE) == 0; + file_permission(vma->vm_file, MAY_WRITE) == 0; } static const struct mm_walk_ops mincore_walk_ops = { diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 41c3303c3357..18453d15dddf 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -936,7 +936,7 @@ static struct sock *unix_find_other(struct net *net, if (err) goto fail; inode = d_backing_inode(path.dentry); - err = inode_permission(inode, MAY_WRITE); + err = path_permission(&path, MAY_WRITE); if (err) goto put_fail; -- cgit v1.2.3 From 0558c1bf5a0811bf5e3753eed911a15b9bd08271 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:23 +0100 Subject: capability: handle idmapped mounts In order to determine whether a caller holds privilege over a given inode the capability framework exposes the two helpers privileged_wrt_inode_uidgid() and capable_wrt_inode_uidgid(). The former verifies that the inode has a mapping in the caller's user namespace and the latter additionally verifies that the caller has the requested capability in their current user namespace. If the inode is accessed through an idmapped mount map it into the mount's user namespace. Afterwards the checks are identical to non-idmapped inodes. If the initial user namespace is passed all operations are a nop so non-idmapped mounts will not see a change in behavior. Link: https://lore.kernel.org/r/20210121131959.646623-5-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Acked-by: Serge Hallyn Signed-off-by: Christian Brauner --- fs/attr.c | 8 ++++---- fs/exec.c | 3 ++- fs/inode.c | 3 ++- fs/namei.c | 13 ++++++++----- fs/overlayfs/super.c | 2 +- fs/posix_acl.c | 2 +- fs/xfs/xfs_ioctl.c | 2 +- include/linux/capability.h | 7 +++++-- kernel/capability.c | 14 +++++++++----- security/commoncap.c | 5 +++-- 10 files changed, 36 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/attr.c b/fs/attr.c index b4bbdbd4c8ca..d270f640a192 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -23,7 +23,7 @@ static bool chown_ok(const struct inode *inode, kuid_t uid) if (uid_eq(current_fsuid(), inode->i_uid) && uid_eq(uid, inode->i_uid)) return true; - if (capable_wrt_inode_uidgid(inode, CAP_CHOWN)) + if (capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_CHOWN)) return true; if (uid_eq(inode->i_uid, INVALID_UID) && ns_capable(inode->i_sb->s_user_ns, CAP_CHOWN)) @@ -36,7 +36,7 @@ static bool chgrp_ok(const struct inode *inode, kgid_t gid) if (uid_eq(current_fsuid(), inode->i_uid) && (in_group_p(gid) || gid_eq(gid, inode->i_gid))) return true; - if (capable_wrt_inode_uidgid(inode, CAP_CHOWN)) + if (capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_CHOWN)) return true; if (gid_eq(inode->i_gid, INVALID_GID) && ns_capable(inode->i_sb->s_user_ns, CAP_CHOWN)) @@ -92,7 +92,7 @@ int setattr_prepare(struct dentry *dentry, struct iattr *attr) /* Also check the setgid bit! */ if (!in_group_p((ia_valid & ATTR_GID) ? attr->ia_gid : inode->i_gid) && - !capable_wrt_inode_uidgid(inode, CAP_FSETID)) + !capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_FSETID)) attr->ia_mode &= ~S_ISGID; } @@ -193,7 +193,7 @@ void setattr_copy(struct inode *inode, const struct iattr *attr) umode_t mode = attr->ia_mode; if (!in_group_p(inode->i_gid) && - !capable_wrt_inode_uidgid(inode, CAP_FSETID)) + !capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_FSETID)) mode &= ~S_ISGID; inode->i_mode = mode; } diff --git a/fs/exec.c b/fs/exec.c index 5d4d52039105..89d4780ff48f 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1411,7 +1411,8 @@ void would_dump(struct linux_binprm *bprm, struct file *file) /* Ensure mm->user_ns contains the executable */ user_ns = old = bprm->mm->user_ns; while ((user_ns != &init_user_ns) && - !privileged_wrt_inode_uidgid(user_ns, inode)) + !privileged_wrt_inode_uidgid(user_ns, &init_user_ns, + inode)) user_ns = user_ns->parent; if (old != user_ns) { diff --git a/fs/inode.c b/fs/inode.c index 6442d97d9a4a..cd40cbf87ce4 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -2146,7 +2146,8 @@ void inode_init_owner(struct inode *inode, const struct inode *dir, mode |= S_ISGID; else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) && !in_group_p(inode->i_gid) && - !capable_wrt_inode_uidgid(dir, CAP_FSETID)) + !capable_wrt_inode_uidgid(&init_user_ns, dir, + CAP_FSETID)) mode &= ~S_ISGID; } else inode->i_gid = current_fsgid(); diff --git a/fs/namei.c b/fs/namei.c index 78443a85480a..fd4724bce4f5 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -357,10 +357,11 @@ int generic_permission(struct inode *inode, int mask) if (S_ISDIR(inode->i_mode)) { /* DACs are overridable for directories */ if (!(mask & MAY_WRITE)) - if (capable_wrt_inode_uidgid(inode, + if (capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_DAC_READ_SEARCH)) return 0; - if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE)) + if (capable_wrt_inode_uidgid(&init_user_ns, inode, + CAP_DAC_OVERRIDE)) return 0; return -EACCES; } @@ -370,7 +371,8 @@ int generic_permission(struct inode *inode, int mask) */ mask &= MAY_READ | MAY_WRITE | MAY_EXEC; if (mask == MAY_READ) - if (capable_wrt_inode_uidgid(inode, CAP_DAC_READ_SEARCH)) + if (capable_wrt_inode_uidgid(&init_user_ns, inode, + CAP_DAC_READ_SEARCH)) return 0; /* * Read/write DACs are always overridable. @@ -378,7 +380,8 @@ int generic_permission(struct inode *inode, int mask) * at least one exec bit set. */ if (!(mask & MAY_EXEC) || (inode->i_mode & S_IXUGO)) - if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE)) + if (capable_wrt_inode_uidgid(&init_user_ns, inode, + CAP_DAC_OVERRIDE)) return 0; return -EACCES; @@ -2659,7 +2662,7 @@ int __check_sticky(struct inode *dir, struct inode *inode) return 0; if (uid_eq(dir->i_uid, fsuid)) return 0; - return !capable_wrt_inode_uidgid(inode, CAP_FOWNER); + return !capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_FOWNER); } EXPORT_SYMBOL(__check_sticky); diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 2bd570cbe8a4..88d877787770 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1017,7 +1017,7 @@ ovl_posix_acl_xattr_set(const struct xattr_handler *handler, if (unlikely(inode->i_mode & S_ISGID) && handler->flags == ACL_TYPE_ACCESS && !in_group_p(inode->i_gid) && - !capable_wrt_inode_uidgid(inode, CAP_FSETID)) { + !capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_FSETID)) { struct iattr iattr = { .ia_valid = ATTR_KILL_SGID }; err = ovl_setattr(dentry, &iattr); diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 95882b3f5f62..4ca6d53c6f0a 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -656,7 +656,7 @@ int posix_acl_update_mode(struct inode *inode, umode_t *mode_p, if (error == 0) *acl = NULL; if (!in_group_p(inode->i_gid) && - !capable_wrt_inode_uidgid(inode, CAP_FSETID)) + !capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_FSETID)) mode &= ~S_ISGID; *mode_p = mode; return 0; diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 3fbd98f61ea5..97bd29fc8c43 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1502,7 +1502,7 @@ xfs_ioctl_setattr( */ if ((VFS_I(ip)->i_mode & (S_ISUID|S_ISGID)) && - !capable_wrt_inode_uidgid(VFS_I(ip), CAP_FSETID)) + !capable_wrt_inode_uidgid(&init_user_ns, VFS_I(ip), CAP_FSETID)) VFS_I(ip)->i_mode &= ~(S_ISUID|S_ISGID); /* Change the ownerships and register project quota modifications */ diff --git a/include/linux/capability.h b/include/linux/capability.h index b2f698915c0f..62bfa3ed84d7 100644 --- a/include/linux/capability.h +++ b/include/linux/capability.h @@ -247,8 +247,11 @@ static inline bool ns_capable_setid(struct user_namespace *ns, int cap) return true; } #endif /* CONFIG_MULTIUSER */ -extern bool privileged_wrt_inode_uidgid(struct user_namespace *ns, const struct inode *inode); -extern bool capable_wrt_inode_uidgid(const struct inode *inode, int cap); +bool privileged_wrt_inode_uidgid(struct user_namespace *ns, + struct user_namespace *mnt_userns, + const struct inode *inode); +bool capable_wrt_inode_uidgid(struct user_namespace *mnt_userns, + const struct inode *inode, int cap); extern bool file_ns_capable(const struct file *file, struct user_namespace *ns, int cap); extern bool ptracer_capable(struct task_struct *tsk, struct user_namespace *ns); static inline bool perfmon_capable(void) diff --git a/kernel/capability.c b/kernel/capability.c index de7eac903a2a..46a361dde042 100644 --- a/kernel/capability.c +++ b/kernel/capability.c @@ -484,10 +484,12 @@ EXPORT_SYMBOL(file_ns_capable); * * Return true if the inode uid and gid are within the namespace. */ -bool privileged_wrt_inode_uidgid(struct user_namespace *ns, const struct inode *inode) +bool privileged_wrt_inode_uidgid(struct user_namespace *ns, + struct user_namespace *mnt_userns, + const struct inode *inode) { - return kuid_has_mapping(ns, inode->i_uid) && - kgid_has_mapping(ns, inode->i_gid); + return kuid_has_mapping(ns, i_uid_into_mnt(mnt_userns, inode)) && + kgid_has_mapping(ns, i_gid_into_mnt(mnt_userns, inode)); } /** @@ -499,11 +501,13 @@ bool privileged_wrt_inode_uidgid(struct user_namespace *ns, const struct inode * * its own user namespace and that the given inode's uid and gid are * mapped into the current user namespace. */ -bool capable_wrt_inode_uidgid(const struct inode *inode, int cap) +bool capable_wrt_inode_uidgid(struct user_namespace *mnt_userns, + const struct inode *inode, int cap) { struct user_namespace *ns = current_user_ns(); - return ns_capable(ns, cap) && privileged_wrt_inode_uidgid(ns, inode); + return ns_capable(ns, cap) && + privileged_wrt_inode_uidgid(ns, mnt_userns, inode); } EXPORT_SYMBOL(capable_wrt_inode_uidgid); diff --git a/security/commoncap.c b/security/commoncap.c index bacc1111d871..88ee345f7bfa 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -489,7 +489,7 @@ int cap_convert_nscap(struct dentry *dentry, const void **ivalue, size_t size) return -EINVAL; if (!validheader(size, cap)) return -EINVAL; - if (!capable_wrt_inode_uidgid(inode, CAP_SETFCAP)) + if (!capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_SETFCAP)) return -EPERM; if (size == XATTR_CAPS_SZ_2) if (ns_capable(inode->i_sb->s_user_ns, CAP_SETFCAP)) @@ -956,7 +956,8 @@ int cap_inode_removexattr(struct dentry *dentry, const char *name) struct inode *inode = d_backing_inode(dentry); if (!inode) return -EINVAL; - if (!capable_wrt_inode_uidgid(inode, CAP_SETFCAP)) + if (!capable_wrt_inode_uidgid(&init_user_ns, inode, + CAP_SETFCAP)) return -EPERM; return 0; } -- cgit v1.2.3 From 47291baa8ddfdae10663624ff0a15ab165952708 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:24 +0100 Subject: namei: make permission helpers idmapped mount aware The two helpers inode_permission() and generic_permission() are used by the vfs to perform basic permission checking by verifying that the caller is privileged over an inode. In order to handle idmapped mounts we extend the two helpers with an additional user namespace argument. On idmapped mounts the two helpers will make sure to map the inode according to the mount's user namespace and then peform identical permission checks to inode_permission() and generic_permission(). If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-6-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Acked-by: Serge Hallyn Signed-off-by: Christian Brauner --- fs/attr.c | 3 +- fs/btrfs/inode.c | 2 +- fs/btrfs/ioctl.c | 12 +++-- fs/ceph/inode.c | 2 +- fs/cifs/cifsfs.c | 2 +- fs/configfs/symlink.c | 3 +- fs/ecryptfs/inode.c | 3 +- fs/exec.c | 2 +- fs/fuse/dir.c | 5 +- fs/gfs2/inode.c | 2 +- fs/hostfs/hostfs_kern.c | 2 +- fs/kernfs/inode.c | 2 +- fs/libfs.c | 7 ++- fs/namei.c | 121 ++++++++++++++++++++++++++++++++-------------- fs/nfs/dir.c | 2 +- fs/nfsd/nfsfh.c | 3 +- fs/nfsd/vfs.c | 5 +- fs/nilfs2/inode.c | 2 +- fs/ocfs2/file.c | 2 +- fs/ocfs2/refcounttree.c | 4 +- fs/open.c | 4 +- fs/orangefs/inode.c | 2 +- fs/overlayfs/file.c | 2 +- fs/overlayfs/inode.c | 4 +- fs/overlayfs/util.c | 2 +- fs/posix_acl.c | 17 +++++-- fs/proc/base.c | 4 +- fs/proc/fd.c | 2 +- fs/reiserfs/xattr.c | 2 +- fs/remap_range.c | 2 +- fs/xattr.c | 2 +- include/linux/fs.h | 10 ++-- include/linux/posix_acl.h | 7 ++- ipc/mqueue.c | 2 +- kernel/bpf/inode.c | 2 +- kernel/cgroup/cgroup.c | 2 +- 36 files changed, 164 insertions(+), 88 deletions(-) (limited to 'fs') diff --git a/fs/attr.c b/fs/attr.c index d270f640a192..c9e29e589cec 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -244,7 +244,8 @@ int notify_change(struct dentry * dentry, struct iattr * attr, struct inode **de return -EPERM; if (!inode_owner_or_capable(inode)) { - error = inode_permission(inode, MAY_WRITE); + error = inode_permission(&init_user_ns, inode, + MAY_WRITE); if (error) return error; } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index a8e0a6b038d3..512ee2650bbb 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -9889,7 +9889,7 @@ static int btrfs_permission(struct inode *inode, int mask) if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) return -EACCES; } - return generic_permission(inode, mask); + return generic_permission(&init_user_ns, inode, mask); } static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index dde49a791f3e..8ced6dfefee4 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -922,7 +922,7 @@ static int btrfs_may_delete(struct inode *dir, struct dentry *victim, int isdir) BUG_ON(d_inode(victim->d_parent) != dir); audit_inode_child(dir, victim, AUDIT_TYPE_CHILD_DELETE); - error = inode_permission(dir, MAY_WRITE | MAY_EXEC); + error = inode_permission(&init_user_ns, dir, MAY_WRITE | MAY_EXEC); if (error) return error; if (IS_APPEND(dir)) @@ -951,7 +951,7 @@ static inline int btrfs_may_create(struct inode *dir, struct dentry *child) return -EEXIST; if (IS_DEADDIR(dir)) return -ENOENT; - return inode_permission(dir, MAY_WRITE | MAY_EXEC); + return inode_permission(&init_user_ns, dir, MAY_WRITE | MAY_EXEC); } /* @@ -2538,7 +2538,8 @@ static int btrfs_search_path_in_tree_user(struct inode *inode, ret = PTR_ERR(temp_inode); goto out_put; } - ret = inode_permission(temp_inode, MAY_READ | MAY_EXEC); + ret = inode_permission(&init_user_ns, temp_inode, + MAY_READ | MAY_EXEC); iput(temp_inode); if (ret) { ret = -EACCES; @@ -3068,7 +3069,8 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, if (root == dest) goto out_dput; - err = inode_permission(inode, MAY_WRITE | MAY_EXEC); + err = inode_permission(&init_user_ns, inode, + MAY_WRITE | MAY_EXEC); if (err) goto out_dput; } @@ -3139,7 +3141,7 @@ static int btrfs_ioctl_defrag(struct file *file, void __user *argp) * running and allows defrag on files open in read-only mode. */ if (!capable(CAP_SYS_ADMIN) && - inode_permission(inode, MAY_WRITE)) { + inode_permission(&init_user_ns, inode, MAY_WRITE)) { ret = -EPERM; goto out; } diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index adc8fc3c5d85..e8a15ee09bc1 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -2331,7 +2331,7 @@ int ceph_permission(struct inode *inode, int mask) err = ceph_do_getattr(inode, CEPH_CAP_AUTH_SHARED, false); if (!err) - err = generic_permission(inode, mask); + err = generic_permission(&init_user_ns, inode, mask); return err; } diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index ce0d0037fd0a..ce14e6f8adb6 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -320,7 +320,7 @@ static int cifs_permission(struct inode *inode, int mask) on the client (above and beyond ACL on servers) for servers which do not support setting and viewing mode bits, so allowing client to check permissions is useful */ - return generic_permission(inode, mask); + return generic_permission(&init_user_ns, inode, mask); } static struct kmem_cache *cifs_inode_cachep; diff --git a/fs/configfs/symlink.c b/fs/configfs/symlink.c index cb61467478ca..8ca36394fa30 100644 --- a/fs/configfs/symlink.c +++ b/fs/configfs/symlink.c @@ -197,7 +197,8 @@ int configfs_symlink(struct inode *dir, struct dentry *dentry, const char *symna if (dentry->d_inode || d_unhashed(dentry)) ret = -EEXIST; else - ret = inode_permission(dir, MAY_WRITE | MAY_EXEC); + ret = inode_permission(&init_user_ns, dir, + MAY_WRITE | MAY_EXEC); if (!ret) ret = type->ct_item_ops->allow_link(parent_item, target_item); if (!ret) { diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index e23752d9a79f..0b346baf110d 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -864,7 +864,8 @@ int ecryptfs_truncate(struct dentry *dentry, loff_t new_length) static int ecryptfs_permission(struct inode *inode, int mask) { - return inode_permission(ecryptfs_inode_to_lower(inode), mask); + return inode_permission(&init_user_ns, + ecryptfs_inode_to_lower(inode), mask); } /** diff --git a/fs/exec.c b/fs/exec.c index 89d4780ff48f..a8ec371cd3cd 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1404,7 +1404,7 @@ EXPORT_SYMBOL(begin_new_exec); void would_dump(struct linux_binprm *bprm, struct file *file) { struct inode *inode = file_inode(file); - if (inode_permission(inode, MAY_READ) < 0) { + if (inode_permission(&init_user_ns, inode, MAY_READ) < 0) { struct user_namespace *old, *user_ns; bprm->interp_flags |= BINPRM_FLAGS_ENFORCE_NONDUMP; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 78f9f209078c..7497009a5a45 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -1280,7 +1280,7 @@ static int fuse_permission(struct inode *inode, int mask) } if (fc->default_permissions) { - err = generic_permission(inode, mask); + err = generic_permission(&init_user_ns, inode, mask); /* If permission is denied, try to refresh file attributes. This is also needed, because the root @@ -1288,7 +1288,8 @@ static int fuse_permission(struct inode *inode, int mask) if (err == -EACCES && !refreshed) { err = fuse_perm_getattr(inode, mask); if (!err) - err = generic_permission(inode, mask); + err = generic_permission(&init_user_ns, + inode, mask); } /* Note: the opposite of the above test does not diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index c1b77e8d6b1c..5b2ff0c74b67 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -1852,7 +1852,7 @@ int gfs2_permission(struct inode *inode, int mask) if ((mask & MAY_WRITE) && IS_IMMUTABLE(inode)) error = -EPERM; else - error = generic_permission(inode, mask); + error = generic_permission(&init_user_ns, inode, mask); if (gfs2_holder_initialized(&i_gh)) gfs2_glock_dq_uninit(&i_gh); diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c index aea35459d390..b841a05a2b8c 100644 --- a/fs/hostfs/hostfs_kern.c +++ b/fs/hostfs/hostfs_kern.c @@ -779,7 +779,7 @@ static int hostfs_permission(struct inode *ino, int desired) err = access_file(name, r, w, x); __putname(name); if (!err) - err = generic_permission(ino, desired); + err = generic_permission(&init_user_ns, ino, desired); return err; } diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c index fc2469a20fed..ff5598cc1de0 100644 --- a/fs/kernfs/inode.c +++ b/fs/kernfs/inode.c @@ -285,7 +285,7 @@ int kernfs_iop_permission(struct inode *inode, int mask) kernfs_refresh_inode(kn, inode); mutex_unlock(&kernfs_mutex); - return generic_permission(inode, mask); + return generic_permission(&init_user_ns, inode, mask); } int kernfs_xattr_get(struct kernfs_node *kn, const char *name, diff --git a/fs/libfs.c b/fs/libfs.c index d1c3bade9f30..f8b3c02b4f0f 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -1318,9 +1318,14 @@ static ssize_t empty_dir_listxattr(struct dentry *dentry, char *list, size_t siz return -EOPNOTSUPP; } +static int empty_dir_permission(struct inode *inode, int mask) +{ + return generic_permission(&init_user_ns, inode, mask); +} + static const struct inode_operations empty_dir_inode_operations = { .lookup = empty_dir_lookup, - .permission = generic_permission, + .permission = empty_dir_permission, .setattr = empty_dir_setattr, .getattr = empty_dir_getattr, .listxattr = empty_dir_listxattr, diff --git a/fs/namei.c b/fs/namei.c index fd4724bce4f5..d78d74f5f5af 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -259,7 +259,24 @@ void putname(struct filename *name) __putname(name); } -static int check_acl(struct inode *inode, int mask) +/** + * check_acl - perform ACL permission checking + * @mnt_userns: user namespace of the mount the inode was found from + * @inode: inode to check permissions on + * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...) + * + * This function performs the ACL permission checking. Since this function + * retrieve POSIX acls it needs to know whether it is called from a blocking or + * non-blocking context and thus cares about the MAY_NOT_BLOCK bit. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. + */ +static int check_acl(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { #ifdef CONFIG_FS_POSIX_ACL struct posix_acl *acl; @@ -271,14 +288,14 @@ static int check_acl(struct inode *inode, int mask) /* no ->get_acl() calls in RCU mode... */ if (is_uncached_acl(acl)) return -ECHILD; - return posix_acl_permission(inode, acl, mask); + return posix_acl_permission(mnt_userns, inode, acl, mask); } acl = get_acl(inode, ACL_TYPE_ACCESS); if (IS_ERR(acl)) return PTR_ERR(acl); if (acl) { - int error = posix_acl_permission(inode, acl, mask); + int error = posix_acl_permission(mnt_userns, inode, acl, mask); posix_acl_release(acl); return error; } @@ -287,18 +304,31 @@ static int check_acl(struct inode *inode, int mask) return -EAGAIN; } -/* - * This does the basic UNIX permission checking. +/** + * acl_permission_check - perform basic UNIX permission checking + * @mnt_userns: user namespace of the mount the inode was found from + * @inode: inode to check permissions on + * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...) + * + * This function performs the basic UNIX permission checking. Since this + * function may retrieve POSIX acls it needs to know whether it is called from a + * blocking or non-blocking context and thus cares about the MAY_NOT_BLOCK bit. * - * Note that the POSIX ACL check cares about the MAY_NOT_BLOCK bit, - * for RCU walking. + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. */ -static int acl_permission_check(struct inode *inode, int mask) +static int acl_permission_check(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { unsigned int mode = inode->i_mode; + kuid_t i_uid; /* Are we the owner? If so, ACL's don't matter */ - if (likely(uid_eq(current_fsuid(), inode->i_uid))) { + i_uid = i_uid_into_mnt(mnt_userns, inode); + if (likely(uid_eq(current_fsuid(), i_uid))) { mask &= 7; mode >>= 6; return (mask & ~mode) ? -EACCES : 0; @@ -306,7 +336,7 @@ static int acl_permission_check(struct inode *inode, int mask) /* Do we have ACL's? */ if (IS_POSIXACL(inode) && (mode & S_IRWXG)) { - int error = check_acl(inode, mask); + int error = check_acl(mnt_userns, inode, mask); if (error != -EAGAIN) return error; } @@ -320,7 +350,8 @@ static int acl_permission_check(struct inode *inode, int mask) * about? Need to check group ownership if so. */ if (mask & (mode ^ (mode >> 3))) { - if (in_group_p(inode->i_gid)) + kgid_t kgid = i_gid_into_mnt(mnt_userns, inode); + if (in_group_p(kgid)) mode >>= 3; } @@ -330,6 +361,7 @@ static int acl_permission_check(struct inode *inode, int mask) /** * generic_permission - check for access rights on a Posix-like filesystem + * @mnt_userns: user namespace of the mount the inode was found from * @inode: inode to check access rights for * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC, * %MAY_NOT_BLOCK ...) @@ -342,25 +374,32 @@ static int acl_permission_check(struct inode *inode, int mask) * generic_permission is rcu-walk aware. It returns -ECHILD in case an rcu-walk * request cannot be satisfied (eg. requires blocking or too much complexity). * It would then be called again in ref-walk mode. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. */ -int generic_permission(struct inode *inode, int mask) +int generic_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask) { int ret; /* * Do the basic permission checks. */ - ret = acl_permission_check(inode, mask); + ret = acl_permission_check(mnt_userns, inode, mask); if (ret != -EACCES) return ret; if (S_ISDIR(inode->i_mode)) { /* DACs are overridable for directories */ if (!(mask & MAY_WRITE)) - if (capable_wrt_inode_uidgid(&init_user_ns, inode, + if (capable_wrt_inode_uidgid(mnt_userns, inode, CAP_DAC_READ_SEARCH)) return 0; - if (capable_wrt_inode_uidgid(&init_user_ns, inode, + if (capable_wrt_inode_uidgid(mnt_userns, inode, CAP_DAC_OVERRIDE)) return 0; return -EACCES; @@ -371,7 +410,7 @@ int generic_permission(struct inode *inode, int mask) */ mask &= MAY_READ | MAY_WRITE | MAY_EXEC; if (mask == MAY_READ) - if (capable_wrt_inode_uidgid(&init_user_ns, inode, + if (capable_wrt_inode_uidgid(mnt_userns, inode, CAP_DAC_READ_SEARCH)) return 0; /* @@ -380,7 +419,7 @@ int generic_permission(struct inode *inode, int mask) * at least one exec bit set. */ if (!(mask & MAY_EXEC) || (inode->i_mode & S_IXUGO)) - if (capable_wrt_inode_uidgid(&init_user_ns, inode, + if (capable_wrt_inode_uidgid(mnt_userns, inode, CAP_DAC_OVERRIDE)) return 0; @@ -388,13 +427,19 @@ int generic_permission(struct inode *inode, int mask) } EXPORT_SYMBOL(generic_permission); -/* +/** + * do_inode_permission - UNIX permission checking + * @mnt_userns: user namespace of the mount the inode was found from + * @inode: inode to check permissions on + * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...) + * * We _really_ want to just do "generic_permission()" without * even looking at the inode->i_op values. So we keep a cache * flag in inode->i_opflags, that says "this has not special * permission function, use the fast case". */ -static inline int do_inode_permission(struct inode *inode, int mask) +static inline int do_inode_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { if (unlikely(!(inode->i_opflags & IOP_FASTPERM))) { if (likely(inode->i_op->permission)) @@ -405,7 +450,7 @@ static inline int do_inode_permission(struct inode *inode, int mask) inode->i_opflags |= IOP_FASTPERM; spin_unlock(&inode->i_lock); } - return generic_permission(inode, mask); + return generic_permission(mnt_userns, inode, mask); } /** @@ -430,8 +475,9 @@ static int sb_permission(struct super_block *sb, struct inode *inode, int mask) /** * inode_permission - Check for access rights to a given inode - * @inode: Inode to check permission on - * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC) + * @mnt_userns: User namespace of the mount the inode was found from + * @inode: Inode to check permission on + * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC) * * Check for read/write/execute permissions on an inode. We use fs[ug]id for * this, letting us set arbitrary permissions for filesystem access without @@ -439,7 +485,8 @@ static int sb_permission(struct super_block *sb, struct inode *inode, int mask) * * When checking for MAY_APPEND, MAY_WRITE must also be set in @mask. */ -int inode_permission(struct inode *inode, int mask) +int inode_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { int retval; @@ -463,7 +510,7 @@ int inode_permission(struct inode *inode, int mask) return -EACCES; } - retval = do_inode_permission(inode, mask); + retval = do_inode_permission(mnt_userns, inode, mask); if (retval) return retval; @@ -1009,7 +1056,7 @@ static bool safe_hardlink_source(struct inode *inode) return false; /* Hardlinking to unreadable or unwritable sources is dangerous. */ - if (inode_permission(inode, MAY_READ | MAY_WRITE)) + if (inode_permission(&init_user_ns, inode, MAY_READ | MAY_WRITE)) return false; return true; @@ -1569,13 +1616,14 @@ static struct dentry *lookup_slow(const struct qstr *name, static inline int may_lookup(struct nameidata *nd) { if (nd->flags & LOOKUP_RCU) { - int err = inode_permission(nd->inode, MAY_EXEC|MAY_NOT_BLOCK); + int err = inode_permission(&init_user_ns, nd->inode, + MAY_EXEC | MAY_NOT_BLOCK); if (err != -ECHILD) return err; if (unlazy_walk(nd)) return -ECHILD; } - return inode_permission(nd->inode, MAY_EXEC); + return inode_permission(&init_user_ns, nd->inode, MAY_EXEC); } static int reserve_stack(struct nameidata *nd, struct path *link, unsigned seq) @@ -2509,7 +2557,7 @@ static int lookup_one_len_common(const char *name, struct dentry *base, return err; } - return inode_permission(base->d_inode, MAY_EXEC); + return inode_permission(&init_user_ns, base->d_inode, MAY_EXEC); } /** @@ -2703,7 +2751,7 @@ static int may_delete(struct inode *dir, struct dentry *victim, bool isdir) audit_inode_child(dir, victim, AUDIT_TYPE_CHILD_DELETE); - error = inode_permission(dir, MAY_WRITE | MAY_EXEC); + error = inode_permission(&init_user_ns, dir, MAY_WRITE | MAY_EXEC); if (error) return error; if (IS_APPEND(dir)) @@ -2747,7 +2795,7 @@ static inline int may_create(struct inode *dir, struct dentry *child) if (!kuid_has_mapping(s_user_ns, current_fsuid()) || !kgid_has_mapping(s_user_ns, current_fsgid())) return -EOVERFLOW; - return inode_permission(dir, MAY_WRITE | MAY_EXEC); + return inode_permission(&init_user_ns, dir, MAY_WRITE | MAY_EXEC); } /* @@ -2877,7 +2925,7 @@ static int may_open(const struct path *path, int acc_mode, int flag) break; } - error = inode_permission(inode, MAY_OPEN | acc_mode); + error = inode_permission(&init_user_ns, inode, MAY_OPEN | acc_mode); if (error) return error; @@ -2939,7 +2987,8 @@ static int may_o_create(const struct path *dir, struct dentry *dentry, umode_t m !kgid_has_mapping(s_user_ns, current_fsgid())) return -EOVERFLOW; - error = inode_permission(dir->dentry->d_inode, MAY_WRITE | MAY_EXEC); + error = inode_permission(&init_user_ns, dir->dentry->d_inode, + MAY_WRITE | MAY_EXEC); if (error) return error; @@ -3276,7 +3325,7 @@ struct dentry *vfs_tmpfile(struct dentry *dentry, umode_t mode, int open_flag) int error; /* we want directory to be writable */ - error = inode_permission(dir, MAY_WRITE | MAY_EXEC); + error = inode_permission(&init_user_ns, dir, MAY_WRITE | MAY_EXEC); if (error) goto out_err; error = -EOPNOTSUPP; @@ -4267,12 +4316,14 @@ int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, */ if (new_dir != old_dir) { if (is_dir) { - error = inode_permission(source, MAY_WRITE); + error = inode_permission(&init_user_ns, source, + MAY_WRITE); if (error) return error; } if ((flags & RENAME_EXCHANGE) && new_is_dir) { - error = inode_permission(target, MAY_WRITE); + error = inode_permission(&init_user_ns, target, + MAY_WRITE); if (error) return error; } diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index ef827ae193d2..727e01a84503 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2987,7 +2987,7 @@ out_notsup: res = nfs_revalidate_inode(NFS_SERVER(inode), inode); if (res == 0) - res = generic_permission(inode, mask); + res = generic_permission(&init_user_ns, inode, mask); goto out; } EXPORT_SYMBOL_GPL(nfs_permission); diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 66f2ef67792a..8d90796e236a 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -40,7 +40,8 @@ static int nfsd_acceptable(void *expv, struct dentry *dentry) /* make sure parents give x permission to user */ int err; parent = dget_parent(tdentry); - err = inode_permission(d_inode(parent), MAY_EXEC); + err = inode_permission(&init_user_ns, + d_inode(parent), MAY_EXEC); if (err < 0) { dput(parent); break; diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 04937e51de56..0edf11258aaa 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -2391,13 +2391,14 @@ nfsd_permission(struct svc_rqst *rqstp, struct svc_export *exp, return 0; /* This assumes NFSD_MAY_{READ,WRITE,EXEC} == MAY_{READ,WRITE,EXEC} */ - err = inode_permission(inode, acc & (MAY_READ|MAY_WRITE|MAY_EXEC)); + err = inode_permission(&init_user_ns, inode, + acc & (MAY_READ | MAY_WRITE | MAY_EXEC)); /* Allow read access to binaries even when mode 111 */ if (err == -EACCES && S_ISREG(inode->i_mode) && (acc == (NFSD_MAY_READ | NFSD_MAY_OWNER_OVERRIDE) || acc == (NFSD_MAY_READ | NFSD_MAY_READ_IF_EXEC))) - err = inode_permission(inode, MAY_EXEC); + err = inode_permission(&init_user_ns, inode, MAY_EXEC); return err? nfserrno(err) : 0; } diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index 745d371d6fea..b6517220cad5 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c @@ -851,7 +851,7 @@ int nilfs_permission(struct inode *inode, int mask) root->cno != NILFS_CPTREE_CURRENT_CNO) return -EROFS; /* snapshot is not writable */ - return generic_permission(inode, mask); + return generic_permission(&init_user_ns, inode, mask); } int nilfs_load_inode_block(struct inode *inode, struct buffer_head **pbh) diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 85979e2214b3..0c75619adf54 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -1355,7 +1355,7 @@ int ocfs2_permission(struct inode *inode, int mask) dump_stack(); } - ret = generic_permission(inode, mask); + ret = generic_permission(&init_user_ns, inode, mask); ocfs2_inode_unlock_tracker(inode, 0, &oh, had_lock); out: diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index 3b397fa9c9e8..c26937824be1 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -4346,7 +4346,7 @@ static inline int ocfs2_may_create(struct inode *dir, struct dentry *child) return -EEXIST; if (IS_DEADDIR(dir)) return -ENOENT; - return inode_permission(dir, MAY_WRITE | MAY_EXEC); + return inode_permission(&init_user_ns, dir, MAY_WRITE | MAY_EXEC); } /** @@ -4400,7 +4400,7 @@ static int ocfs2_vfs_reflink(struct dentry *old_dentry, struct inode *dir, * file. */ if (!preserve) { - error = inode_permission(inode, MAY_READ); + error = inode_permission(&init_user_ns, inode, MAY_READ); if (error) return error; } diff --git a/fs/open.c b/fs/open.c index cd1efd254cad..a6dac6d97988 100644 --- a/fs/open.c +++ b/fs/open.c @@ -83,7 +83,7 @@ long vfs_truncate(const struct path *path, loff_t length) if (error) goto out; - error = inode_permission(inode, MAY_WRITE); + error = inode_permission(&init_user_ns, inode, MAY_WRITE); if (error) goto mnt_drop_write_and_out; @@ -436,7 +436,7 @@ retry: goto out_path_release; } - res = inode_permission(inode, mode | MAY_ACCESS); + res = inode_permission(&init_user_ns, inode, mode | MAY_ACCESS); /* SuS v2 requires we report a read only fs too */ if (res || !(mode & S_IWOTH) || special_file(inode->i_mode)) goto out_path_release; diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c index 48f0547d4850..4c790cc8042d 100644 --- a/fs/orangefs/inode.c +++ b/fs/orangefs/inode.c @@ -933,7 +933,7 @@ int orangefs_permission(struct inode *inode, int mask) if (ret < 0) return ret; - return generic_permission(inode, mask); + return generic_permission(&init_user_ns, inode, mask); } int orangefs_update_time(struct inode *inode, struct timespec64 *time, int flags) diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c index bd9dd38347ae..b2948e7b3210 100644 --- a/fs/overlayfs/file.c +++ b/fs/overlayfs/file.c @@ -50,7 +50,7 @@ static struct file *ovl_open_realfile(const struct file *file, acc_mode |= MAY_APPEND; old_cred = ovl_override_creds(inode->i_sb); - err = inode_permission(realinode, MAY_OPEN | acc_mode); + err = inode_permission(&init_user_ns, realinode, MAY_OPEN | acc_mode); if (err) { realfile = ERR_PTR(err); } else { diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index d739e14c6814..c101ebbb7a77 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -294,7 +294,7 @@ int ovl_permission(struct inode *inode, int mask) * Check overlay inode with the creds of task and underlying inode * with creds of mounter */ - err = generic_permission(inode, mask); + err = generic_permission(&init_user_ns, inode, mask); if (err) return err; @@ -305,7 +305,7 @@ int ovl_permission(struct inode *inode, int mask) /* Make sure mounter can read file for copy up later */ mask |= MAY_READ; } - err = inode_permission(realinode, mask); + err = inode_permission(&init_user_ns, realinode, mask); revert_creds(old_cred); return err; diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 6569031af3cd..de5c2047a0e9 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -479,7 +479,7 @@ struct file *ovl_path_open(struct path *path, int flags) BUG(); } - err = inode_permission(inode, acc_mode | MAY_OPEN); + err = inode_permission(&init_user_ns, inode, acc_mode | MAY_OPEN); if (err) return ERR_PTR(err); diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 4ca6d53c6f0a..5d9fe2fb2953 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -345,10 +345,13 @@ EXPORT_SYMBOL(posix_acl_from_mode); * by the acl. Returns -E... otherwise. */ int -posix_acl_permission(struct inode *inode, const struct posix_acl *acl, int want) +posix_acl_permission(struct user_namespace *mnt_userns, struct inode *inode, + const struct posix_acl *acl, int want) { const struct posix_acl_entry *pa, *pe, *mask_obj; int found = 0; + kuid_t uid; + kgid_t gid; want &= MAY_READ | MAY_WRITE | MAY_EXEC; @@ -356,22 +359,26 @@ posix_acl_permission(struct inode *inode, const struct posix_acl *acl, int want) switch(pa->e_tag) { case ACL_USER_OBJ: /* (May have been checked already) */ - if (uid_eq(inode->i_uid, current_fsuid())) + uid = i_uid_into_mnt(mnt_userns, inode); + if (uid_eq(uid, current_fsuid())) goto check_perm; break; case ACL_USER: - if (uid_eq(pa->e_uid, current_fsuid())) + uid = kuid_into_mnt(mnt_userns, pa->e_uid); + if (uid_eq(uid, current_fsuid())) goto mask; break; case ACL_GROUP_OBJ: - if (in_group_p(inode->i_gid)) { + gid = i_gid_into_mnt(mnt_userns, inode); + if (in_group_p(gid)) { found = 1; if ((pa->e_perm & want) == want) goto mask; } break; case ACL_GROUP: - if (in_group_p(pa->e_gid)) { + gid = kgid_into_mnt(mnt_userns, pa->e_gid); + if (in_group_p(gid)) { found = 1; if ((pa->e_perm & want) == want) goto mask; diff --git a/fs/proc/base.c b/fs/proc/base.c index b3422cda2a91..b4ec9293625e 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -751,7 +751,7 @@ static int proc_pid_permission(struct inode *inode, int mask) return -EPERM; } - return generic_permission(inode, mask); + return generic_permission(&init_user_ns, inode, mask); } @@ -3492,7 +3492,7 @@ static int proc_tid_comm_permission(struct inode *inode, int mask) return 0; } - return generic_permission(inode, mask); + return generic_permission(&init_user_ns, inode, mask); } static const struct inode_operations proc_tid_comm_inode_operations = { diff --git a/fs/proc/fd.c b/fs/proc/fd.c index cb51763ed554..d6e76461e135 100644 --- a/fs/proc/fd.c +++ b/fs/proc/fd.c @@ -281,7 +281,7 @@ int proc_fd_permission(struct inode *inode, int mask) struct task_struct *p; int rv; - rv = generic_permission(inode, mask); + rv = generic_permission(&init_user_ns, inode, mask); if (rv == 0) return rv; diff --git a/fs/reiserfs/xattr.c b/fs/reiserfs/xattr.c index fe63a7c3e0da..ec440d1957a1 100644 --- a/fs/reiserfs/xattr.c +++ b/fs/reiserfs/xattr.c @@ -957,7 +957,7 @@ int reiserfs_permission(struct inode *inode, int mask) if (IS_PRIVATE(inode)) return 0; - return generic_permission(inode, mask); + return generic_permission(&init_user_ns, inode, mask); } static int xattr_hide_revalidate(struct dentry *dentry, unsigned int flags) diff --git a/fs/remap_range.c b/fs/remap_range.c index 77dba3a49e65..29a4a4dbfe12 100644 --- a/fs/remap_range.c +++ b/fs/remap_range.c @@ -438,7 +438,7 @@ static bool allow_file_dedupe(struct file *file) return true; if (uid_eq(current_fsuid(), file_inode(file)->i_uid)) return true; - if (!inode_permission(file_inode(file), MAY_WRITE)) + if (!inode_permission(&init_user_ns, file_inode(file), MAY_WRITE)) return true; return false; } diff --git a/fs/xattr.c b/fs/xattr.c index fd57153b1f61..56151bd9e642 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -131,7 +131,7 @@ xattr_permission(struct inode *inode, const char *name, int mask) return -EPERM; } - return inode_permission(inode, mask); + return inode_permission(&init_user_ns, inode, mask); } /* diff --git a/include/linux/fs.h b/include/linux/fs.h index bcd17097d441..a85dfe6962df 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2810,15 +2810,17 @@ static inline int bmap(struct inode *inode, sector_t *block) #endif extern int notify_change(struct dentry *, struct iattr *, struct inode **); -extern int inode_permission(struct inode *, int); -extern int generic_permission(struct inode *, int); +int inode_permission(struct user_namespace *, struct inode *, int); +int generic_permission(struct user_namespace *, struct inode *, int); static inline int file_permission(struct file *file, int mask) { - return inode_permission(file_inode(file), mask); + return inode_permission(file_mnt_user_ns(file), + file_inode(file), mask); } static inline int path_permission(const struct path *path, int mask) { - return inode_permission(d_inode(path->dentry), mask); + return inode_permission(mnt_user_ns(path->mnt), + d_inode(path->dentry), mask); } extern int __check_sticky(struct inode *dir, struct inode *inode); diff --git a/include/linux/posix_acl.h b/include/linux/posix_acl.h index 90797f1b421d..85fb4c0c650a 100644 --- a/include/linux/posix_acl.h +++ b/include/linux/posix_acl.h @@ -15,6 +15,8 @@ #include #include +struct user_namespace; + struct posix_acl_entry { short e_tag; unsigned short e_perm; @@ -61,8 +63,6 @@ posix_acl_release(struct posix_acl *acl) extern void posix_acl_init(struct posix_acl *, int); extern struct posix_acl *posix_acl_alloc(int, gfp_t); -extern int posix_acl_valid(struct user_namespace *, const struct posix_acl *); -extern int posix_acl_permission(struct inode *, const struct posix_acl *, int); extern struct posix_acl *posix_acl_from_mode(umode_t, gfp_t); extern int posix_acl_equiv_mode(const struct posix_acl *, umode_t *); extern int __posix_acl_create(struct posix_acl **, gfp_t, umode_t *); @@ -85,6 +85,9 @@ struct posix_acl *get_cached_acl_rcu(struct inode *inode, int type); void set_cached_acl(struct inode *inode, int type, struct posix_acl *acl); void forget_cached_acl(struct inode *inode, int type); void forget_all_cached_acls(struct inode *inode); +int posix_acl_valid(struct user_namespace *, const struct posix_acl *); +int posix_acl_permission(struct user_namespace *, struct inode *, + const struct posix_acl *, int); static inline void cache_no_acl(struct inode *inode) { diff --git a/ipc/mqueue.c b/ipc/mqueue.c index beff0cfcd1e8..693f01fe1216 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -873,7 +873,7 @@ static int prepare_open(struct dentry *dentry, int oflag, int ro, if ((oflag & O_ACCMODE) == (O_RDWR | O_WRONLY)) return -EINVAL; acc = oflag2acc[oflag & O_ACCMODE]; - return inode_permission(d_inode(dentry), acc); + return inode_permission(&init_user_ns, d_inode(dentry), acc); } static int do_mq_open(const char __user *u_name, int oflag, umode_t mode, diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 8962f139521e..e3226b65f5dc 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -558,7 +558,7 @@ int bpf_obj_get_user(const char __user *pathname, int flags) static struct bpf_prog *__get_prog_inode(struct inode *inode, enum bpf_prog_type type) { struct bpf_prog *prog; - int ret = inode_permission(inode, MAY_READ); + int ret = inode_permission(&init_user_ns, inode, MAY_READ); if (ret) return ERR_PTR(ret); diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 613845769103..091ffb5d2939 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -4670,7 +4670,7 @@ static int cgroup_may_write(const struct cgroup *cgrp, struct super_block *sb) if (!inode) return -ENOMEM; - ret = inode_permission(inode, MAY_WRITE); + ret = inode_permission(&init_user_ns, inode, MAY_WRITE); iput(inode); return ret; } -- cgit v1.2.3 From 21cb47be6fb9ece7e6ee63f6780986faa384a77c Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:25 +0100 Subject: inode: make init and permission helpers idmapped mount aware The inode_owner_or_capable() helper determines whether the caller is the owner of the inode or is capable with respect to that inode. Allow it to handle idmapped mounts. If the inode is accessed through an idmapped mount it according to the mount's user namespace. Afterwards the checks are identical to non-idmapped mounts. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Similarly, allow the inode_init_owner() helper to handle idmapped mounts. It initializes a new inode on idmapped mounts by mapping the fsuid and fsgid of the caller from the mount's user namespace. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-7-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Signed-off-by: Christian Brauner --- fs/9p/acl.c | 2 +- fs/9p/vfs_inode.c | 2 +- fs/attr.c | 6 +++--- fs/bfs/dir.c | 2 +- fs/btrfs/inode.c | 2 +- fs/btrfs/ioctl.c | 10 +++++----- fs/btrfs/tests/btrfs-tests.c | 2 +- fs/crypto/policy.c | 2 +- fs/efivarfs/file.c | 2 +- fs/ext2/ialloc.c | 2 +- fs/ext2/ioctl.c | 6 +++--- fs/ext4/ialloc.c | 2 +- fs/ext4/ioctl.c | 15 ++++++++------- fs/f2fs/file.c | 14 +++++++------- fs/f2fs/namei.c | 2 +- fs/f2fs/xattr.c | 2 +- fs/fcntl.c | 2 +- fs/gfs2/file.c | 2 +- fs/hfsplus/inode.c | 2 +- fs/hfsplus/ioctl.c | 2 +- fs/hugetlbfs/inode.c | 2 +- fs/inode.c | 36 ++++++++++++++++++++++++++---------- fs/jfs/ioctl.c | 2 +- fs/jfs/jfs_inode.c | 2 +- fs/minix/bitmap.c | 2 +- fs/namei.c | 5 +++-- fs/nilfs2/inode.c | 2 +- fs/nilfs2/ioctl.c | 2 +- fs/ocfs2/dlmfs/dlmfs.c | 4 ++-- fs/ocfs2/ioctl.c | 2 +- fs/ocfs2/namei.c | 2 +- fs/omfs/inode.c | 2 +- fs/overlayfs/dir.c | 2 +- fs/overlayfs/file.c | 4 ++-- fs/overlayfs/super.c | 2 +- fs/overlayfs/util.c | 2 +- fs/posix_acl.c | 2 +- fs/ramfs/inode.c | 2 +- fs/reiserfs/ioctl.c | 4 ++-- fs/reiserfs/namei.c | 2 +- fs/sysv/ialloc.c | 2 +- fs/ubifs/dir.c | 2 +- fs/ubifs/ioctl.c | 2 +- fs/udf/ialloc.c | 2 +- fs/ufs/ialloc.c | 2 +- fs/xattr.c | 3 ++- fs/xfs/xfs_ioctl.c | 2 +- fs/zonefs/super.c | 2 +- include/linux/fs.h | 8 ++++---- kernel/bpf/inode.c | 2 +- mm/madvise.c | 3 ++- mm/mincore.c | 3 ++- mm/shmem.c | 2 +- security/selinux/hooks.c | 4 ++-- 54 files changed, 112 insertions(+), 91 deletions(-) (limited to 'fs') diff --git a/fs/9p/acl.c b/fs/9p/acl.c index 6261719f6f2a..d77b28e8d57a 100644 --- a/fs/9p/acl.c +++ b/fs/9p/acl.c @@ -258,7 +258,7 @@ static int v9fs_xattr_set_acl(const struct xattr_handler *handler, if (S_ISLNK(inode->i_mode)) return -EOPNOTSUPP; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; if (value) { /* update the cached acl value */ diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index 4a937fac1acb..f66eb3c12c8a 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -251,7 +251,7 @@ int v9fs_init_inode(struct v9fs_session_info *v9ses, { int err = 0; - inode_init_owner(inode, NULL, mode); + inode_init_owner(&init_user_ns,inode, NULL, mode); inode->i_blocks = 0; inode->i_rdev = rdev; inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); diff --git a/fs/attr.c b/fs/attr.c index c9e29e589cec..00ae0b000146 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -87,7 +87,7 @@ int setattr_prepare(struct dentry *dentry, struct iattr *attr) /* Make sure a caller can chmod. */ if (ia_valid & ATTR_MODE) { - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; /* Also check the setgid bit! */ if (!in_group_p((ia_valid & ATTR_GID) ? attr->ia_gid : @@ -98,7 +98,7 @@ int setattr_prepare(struct dentry *dentry, struct iattr *attr) /* Check for setting the inode time. */ if (ia_valid & (ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET)) { - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; } @@ -243,7 +243,7 @@ int notify_change(struct dentry * dentry, struct iattr * attr, struct inode **de if (IS_IMMUTABLE(inode)) return -EPERM; - if (!inode_owner_or_capable(inode)) { + if (!inode_owner_or_capable(&init_user_ns, inode)) { error = inode_permission(&init_user_ns, inode, MAY_WRITE); if (error) diff --git a/fs/bfs/dir.c b/fs/bfs/dir.c index d8dfe3a0cb39..be1335a8d25b 100644 --- a/fs/bfs/dir.c +++ b/fs/bfs/dir.c @@ -96,7 +96,7 @@ static int bfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, } set_bit(ino, info->si_imap); info->si_freei--; - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); inode->i_blocks = 0; inode->i_op = &bfs_file_inops; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 512ee2650bbb..07fe8b2f3bab 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -6190,7 +6190,7 @@ static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans, if (ret != 0) goto fail_unlock; - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode_set_bytes(inode, 0); inode->i_mtime = current_time(inode); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 8ced6dfefee4..1f763c60415b 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -213,7 +213,7 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg) const char *comp = NULL; u32 binode_flags; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; if (btrfs_root_readonly(root)) @@ -429,7 +429,7 @@ static int btrfs_ioctl_fssetxattr(struct file *file, void __user *arg) unsigned old_i_flags; int ret = 0; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; if (btrfs_root_readonly(root)) @@ -1862,7 +1862,7 @@ static noinline int __btrfs_ioctl_snap_create(struct file *file, btrfs_info(BTRFS_I(file_inode(file))->root->fs_info, "Snapshot src from another FS"); ret = -EXDEV; - } else if (!inode_owner_or_capable(src_inode)) { + } else if (!inode_owner_or_capable(&init_user_ns, src_inode)) { /* * Subvolume creation is not restricted, but snapshots * are limited to own subvolumes only @@ -1982,7 +1982,7 @@ static noinline int btrfs_ioctl_subvol_setflags(struct file *file, u64 flags; int ret = 0; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; ret = mnt_want_write_file(file); @@ -4453,7 +4453,7 @@ static long _btrfs_ioctl_set_received_subvol(struct file *file, int ret = 0; int received_uuid_changed; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; ret = mnt_want_write_file(file); diff --git a/fs/btrfs/tests/btrfs-tests.c b/fs/btrfs/tests/btrfs-tests.c index 6bd97bd4cb37..3a4099a2bf05 100644 --- a/fs/btrfs/tests/btrfs-tests.c +++ b/fs/btrfs/tests/btrfs-tests.c @@ -62,7 +62,7 @@ struct inode *btrfs_new_test_inode(void) BTRFS_I(inode)->location.type = BTRFS_INODE_ITEM_KEY; BTRFS_I(inode)->location.objectid = BTRFS_FIRST_FREE_OBJECTID; BTRFS_I(inode)->location.offset = 0; - inode_init_owner(inode, NULL, S_IFREG); + inode_init_owner(&init_user_ns, inode, NULL, S_IFREG); return inode; } diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c index a51cef6bd27f..ed3d623724cd 100644 --- a/fs/crypto/policy.c +++ b/fs/crypto/policy.c @@ -465,7 +465,7 @@ int fscrypt_ioctl_set_policy(struct file *filp, const void __user *arg) return -EFAULT; policy.version = version; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; ret = mnt_want_write_file(filp); diff --git a/fs/efivarfs/file.c b/fs/efivarfs/file.c index feaa5e182b7b..e6bc0302643b 100644 --- a/fs/efivarfs/file.c +++ b/fs/efivarfs/file.c @@ -137,7 +137,7 @@ efivarfs_ioc_setxflags(struct file *file, void __user *arg) unsigned int oldflags = efivarfs_getflags(inode); int error; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; if (copy_from_user(&flags, arg, sizeof(flags))) diff --git a/fs/ext2/ialloc.c b/fs/ext2/ialloc.c index 432c3febea6d..df14e750e9fe 100644 --- a/fs/ext2/ialloc.c +++ b/fs/ext2/ialloc.c @@ -551,7 +551,7 @@ got: inode->i_uid = current_fsuid(); inode->i_gid = dir->i_gid; } else - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_ino = ino; inode->i_blocks = 0; diff --git a/fs/ext2/ioctl.c b/fs/ext2/ioctl.c index 32a8d10b579d..b399cbb7022d 100644 --- a/fs/ext2/ioctl.c +++ b/fs/ext2/ioctl.c @@ -39,7 +39,7 @@ long ext2_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (ret) return ret; - if (!inode_owner_or_capable(inode)) { + if (!inode_owner_or_capable(&init_user_ns, inode)) { ret = -EACCES; goto setflags_out; } @@ -84,7 +84,7 @@ setflags_out: case EXT2_IOC_SETVERSION: { __u32 generation; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; ret = mnt_want_write_file(filp); if (ret) @@ -117,7 +117,7 @@ setversion_out: if (!test_opt(inode->i_sb, RESERVATION) ||!S_ISREG(inode->i_mode)) return -ENOTTY; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; if (get_user(rsv_window_size, (int __user *)arg)) diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index b215c564bc31..00c1ec6eee16 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -972,7 +972,7 @@ struct inode *__ext4_new_inode(handle_t *handle, struct inode *dir, inode->i_uid = current_fsuid(); inode->i_gid = dir->i_gid; } else - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); if (ext4_has_feature_project(sb) && ext4_test_inode_flag(dir, EXT4_INODE_PROJINHERIT)) diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index d9665d2f82db..ab80e2493fdc 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -139,7 +139,8 @@ static long swap_inode_boot_loader(struct super_block *sb, } if (IS_RDONLY(inode) || IS_APPEND(inode) || IS_IMMUTABLE(inode) || - !inode_owner_or_capable(inode) || !capable(CAP_SYS_ADMIN)) { + !inode_owner_or_capable(&init_user_ns, inode) || + !capable(CAP_SYS_ADMIN)) { err = -EPERM; goto journal_err_out; } @@ -829,7 +830,7 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) case FS_IOC_SETFLAGS: { int err; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; if (get_user(flags, (int __user *) arg)) @@ -871,7 +872,7 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) __u32 generation; int err; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; if (ext4_has_metadata_csum(inode->i_sb)) { @@ -1010,7 +1011,7 @@ mext_out: case EXT4_IOC_MIGRATE: { int err; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; err = mnt_want_write_file(filp); @@ -1032,7 +1033,7 @@ mext_out: case EXT4_IOC_ALLOC_DA_BLKS: { int err; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; err = mnt_want_write_file(filp); @@ -1217,7 +1218,7 @@ resizefs_out: case EXT4_IOC_CLEAR_ES_CACHE: { - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; ext4_clear_inode_es(inode); return 0; @@ -1263,7 +1264,7 @@ resizefs_out: return -EFAULT; /* Make sure caller has proper permission */ - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; if (fa.fsx_xflags & ~EXT4_SUPPORTED_FS_XFLAGS) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index f585545277d7..5fc0ff28b5dd 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -1961,7 +1961,7 @@ static int f2fs_ioc_setflags(struct file *filp, unsigned long arg) u32 iflags; int ret; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; if (get_user(fsflags, (int __user *)arg)) @@ -2008,7 +2008,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp) struct f2fs_sb_info *sbi = F2FS_I_SB(inode); int ret; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; if (!S_ISREG(inode->i_mode)) @@ -2075,7 +2075,7 @@ static int f2fs_ioc_commit_atomic_write(struct file *filp) struct inode *inode = file_inode(filp); int ret; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; ret = mnt_want_write_file(filp); @@ -2117,7 +2117,7 @@ static int f2fs_ioc_start_volatile_write(struct file *filp) struct inode *inode = file_inode(filp); int ret; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; if (!S_ISREG(inode->i_mode)) @@ -2152,7 +2152,7 @@ static int f2fs_ioc_release_volatile_write(struct file *filp) struct inode *inode = file_inode(filp); int ret; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; ret = mnt_want_write_file(filp); @@ -2181,7 +2181,7 @@ static int f2fs_ioc_abort_volatile_write(struct file *filp) struct inode *inode = file_inode(filp); int ret; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; ret = mnt_want_write_file(filp); @@ -3158,7 +3158,7 @@ static int f2fs_ioc_fssetxattr(struct file *filp, unsigned long arg) return -EFAULT; /* Make sure caller has proper permission */ - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; if (fa.fsx_xflags & ~F2FS_SUPPORTED_XFLAGS) diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index 6edb1ab579a1..ad98926bacac 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -46,7 +46,7 @@ static struct inode *f2fs_new_inode(struct inode *dir, umode_t mode) nid_free = true; - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_ino = ino; inode->i_blocks = 0; diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index 65afcc3cc68a..d772bf13a814 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -114,7 +114,7 @@ static int f2fs_xattr_advise_set(const struct xattr_handler *handler, unsigned char old_advise = F2FS_I(inode)->i_advise; unsigned char new_advise; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; if (value == NULL) return -EINVAL; diff --git a/fs/fcntl.c b/fs/fcntl.c index 05b36b28f2e8..74d99731fd43 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -46,7 +46,7 @@ static int setfl(int fd, struct file * filp, unsigned long arg) /* O_NOATIME can only be set by the owner or superuser */ if ((arg & O_NOATIME) && !(filp->f_flags & O_NOATIME)) - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; /* required for strict SunOS emulation */ diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index b39b339feddc..1d994bdfffaa 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -238,7 +238,7 @@ static int do_gfs2_set_flags(struct file *filp, u32 reqflags, u32 mask, goto out; error = -EACCES; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) goto out; error = 0; diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index e3da9e96b835..21357046b039 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -376,7 +376,7 @@ struct inode *hfsplus_new_inode(struct super_block *sb, struct inode *dir, return NULL; inode->i_ino = sbi->next_cnid++; - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); set_nlink(inode, 1); inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); diff --git a/fs/hfsplus/ioctl.c b/fs/hfsplus/ioctl.c index ce15b9496b77..3edb1926d127 100644 --- a/fs/hfsplus/ioctl.c +++ b/fs/hfsplus/ioctl.c @@ -91,7 +91,7 @@ static int hfsplus_ioctl_setflags(struct file *file, int __user *user_flags) if (err) goto out; - if (!inode_owner_or_capable(inode)) { + if (!inode_owner_or_capable(&init_user_ns, inode)) { err = -EACCES; goto out_drop_write; } diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index b5c109703daa..6737929e443c 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -836,7 +836,7 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb, struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode); inode->i_ino = get_next_ino(); - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); lockdep_set_class(&inode->i_mapping->i_mmap_rwsem, &hugetlbfs_i_mmap_rwsem_key); inode->i_mapping->a_ops = &hugetlbfs_aops; diff --git a/fs/inode.c b/fs/inode.c index cd40cbf87ce4..a9ac97a27784 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -2130,14 +2130,21 @@ EXPORT_SYMBOL(init_special_inode); /** * inode_init_owner - Init uid,gid,mode for new inode according to posix standards + * @mnt_userns: User namespace of the mount the inode was created from * @inode: New inode * @dir: Directory inode * @mode: mode of the new inode + * + * If the inode has been created through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions + * and initializing i_uid and i_gid. On non-idmapped mounts or if permission + * checking is to be performed on the raw inode simply passs init_user_ns. */ -void inode_init_owner(struct inode *inode, const struct inode *dir, - umode_t mode) +void inode_init_owner(struct user_namespace *mnt_userns, struct inode *inode, + const struct inode *dir, umode_t mode) { - inode->i_uid = current_fsuid(); + inode->i_uid = fsuid_into_mnt(mnt_userns); if (dir && dir->i_mode & S_ISGID) { inode->i_gid = dir->i_gid; @@ -2145,32 +2152,41 @@ void inode_init_owner(struct inode *inode, const struct inode *dir, if (S_ISDIR(mode)) mode |= S_ISGID; else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) && - !in_group_p(inode->i_gid) && - !capable_wrt_inode_uidgid(&init_user_ns, dir, - CAP_FSETID)) + !in_group_p(i_gid_into_mnt(mnt_userns, dir)) && + !capable_wrt_inode_uidgid(mnt_userns, dir, CAP_FSETID)) mode &= ~S_ISGID; } else - inode->i_gid = current_fsgid(); + inode->i_gid = fsgid_into_mnt(mnt_userns); inode->i_mode = mode; } EXPORT_SYMBOL(inode_init_owner); /** * inode_owner_or_capable - check current task permissions to inode + * @mnt_userns: user namespace of the mount the inode was found from * @inode: inode being checked * * Return true if current either has CAP_FOWNER in a namespace with the * inode owner uid mapped, or owns the file. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. */ -bool inode_owner_or_capable(const struct inode *inode) +bool inode_owner_or_capable(struct user_namespace *mnt_userns, + const struct inode *inode) { + kuid_t i_uid; struct user_namespace *ns; - if (uid_eq(current_fsuid(), inode->i_uid)) + i_uid = i_uid_into_mnt(mnt_userns, inode); + if (uid_eq(current_fsuid(), i_uid)) return true; ns = current_user_ns(); - if (kuid_has_mapping(ns, inode->i_uid) && ns_capable(ns, CAP_FOWNER)) + if (kuid_has_mapping(ns, i_uid) && ns_capable(ns, CAP_FOWNER)) return true; return false; } diff --git a/fs/jfs/ioctl.c b/fs/jfs/ioctl.c index 10ee0ecca1a8..2581d4db58ff 100644 --- a/fs/jfs/ioctl.c +++ b/fs/jfs/ioctl.c @@ -76,7 +76,7 @@ long jfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (err) return err; - if (!inode_owner_or_capable(inode)) { + if (!inode_owner_or_capable(&init_user_ns, inode)) { err = -EACCES; goto setflags_out; } diff --git a/fs/jfs/jfs_inode.c b/fs/jfs/jfs_inode.c index 4cef170630db..59379089e939 100644 --- a/fs/jfs/jfs_inode.c +++ b/fs/jfs/jfs_inode.c @@ -64,7 +64,7 @@ struct inode *ialloc(struct inode *parent, umode_t mode) goto fail_put; } - inode_init_owner(inode, parent, mode); + inode_init_owner(&init_user_ns, inode, parent, mode); /* * New inodes need to save sane values on disk when * uid & gid mount options are used diff --git a/fs/minix/bitmap.c b/fs/minix/bitmap.c index f4e5e5181a14..9115948c624e 100644 --- a/fs/minix/bitmap.c +++ b/fs/minix/bitmap.c @@ -252,7 +252,7 @@ struct inode *minix_new_inode(const struct inode *dir, umode_t mode, int *error) iput(inode); return NULL; } - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_ino = j; inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); inode->i_blocks = 0; diff --git a/fs/namei.c b/fs/namei.c index d78d74f5f5af..04b001ddade3 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -1088,7 +1088,8 @@ int may_linkat(struct path *link) /* Source inode owner (or CAP_FOWNER) can hardlink all they like, * otherwise, it must be a safe source. */ - if (safe_hardlink_source(inode) || inode_owner_or_capable(inode)) + if (safe_hardlink_source(inode) || + inode_owner_or_capable(&init_user_ns, inode)) return 0; audit_log_path_denied(AUDIT_ANOM_LINK, "linkat"); @@ -2940,7 +2941,7 @@ static int may_open(const struct path *path, int acc_mode, int flag) } /* O_NOATIME can only be set by the owner or superuser */ - if (flag & O_NOATIME && !inode_owner_or_capable(inode)) + if (flag & O_NOATIME && !inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; return 0; diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index b6517220cad5..11225a659736 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c @@ -348,7 +348,7 @@ struct inode *nilfs_new_inode(struct inode *dir, umode_t mode) /* reference count of i_bh inherits from nilfs_mdt_read_block() */ atomic64_inc(&root->inodes_count); - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_ino = ino; inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); diff --git a/fs/nilfs2/ioctl.c b/fs/nilfs2/ioctl.c index 07d26f61f22a..b053b40315bf 100644 --- a/fs/nilfs2/ioctl.c +++ b/fs/nilfs2/ioctl.c @@ -132,7 +132,7 @@ static int nilfs_ioctl_setflags(struct inode *inode, struct file *filp, unsigned int flags, oldflags; int ret; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; if (get_user(flags, (int __user *)argp)) diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c index 583820ec63e2..37c7d03a6284 100644 --- a/fs/ocfs2/dlmfs/dlmfs.c +++ b/fs/ocfs2/dlmfs/dlmfs.c @@ -329,7 +329,7 @@ static struct inode *dlmfs_get_root_inode(struct super_block *sb) if (inode) { inode->i_ino = get_next_ino(); - inode_init_owner(inode, NULL, mode); + inode_init_owner(&init_user_ns, inode, NULL, mode); inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); inc_nlink(inode); @@ -352,7 +352,7 @@ static struct inode *dlmfs_get_inode(struct inode *parent, return NULL; inode->i_ino = get_next_ino(); - inode_init_owner(inode, parent, mode); + inode_init_owner(&init_user_ns, inode, parent, mode); inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); ip = DLMFS_I(inode); diff --git a/fs/ocfs2/ioctl.c b/fs/ocfs2/ioctl.c index 89984172fc4a..50c9b30ee9f6 100644 --- a/fs/ocfs2/ioctl.c +++ b/fs/ocfs2/ioctl.c @@ -96,7 +96,7 @@ static int ocfs2_set_inode_attr(struct inode *inode, unsigned flags, } status = -EACCES; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) goto bail_unlock; if (!S_ISDIR(inode->i_mode)) diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c index 2a237ab00453..908b79e1082b 100644 --- a/fs/ocfs2/namei.c +++ b/fs/ocfs2/namei.c @@ -198,7 +198,7 @@ static struct inode *ocfs2_get_init_inode(struct inode *dir, umode_t mode) * callers. */ if (S_ISDIR(mode)) set_nlink(inode, 2); - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); status = dquot_initialize(inode); if (status) return ERR_PTR(status); diff --git a/fs/omfs/inode.c b/fs/omfs/inode.c index ce93ccca8639..2a0e83236c01 100644 --- a/fs/omfs/inode.c +++ b/fs/omfs/inode.c @@ -48,7 +48,7 @@ struct inode *omfs_new_inode(struct inode *dir, umode_t mode) goto fail; inode->i_ino = new_block; - inode_init_owner(inode, NULL, mode); + inode_init_owner(&init_user_ns, inode, NULL, mode); inode->i_mapping->a_ops = &omfs_aops; inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index 28a075b5f5b2..98a23353b19a 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -636,7 +636,7 @@ static int ovl_create_object(struct dentry *dentry, int mode, dev_t rdev, inode->i_state |= I_CREATING; spin_unlock(&inode->i_lock); - inode_init_owner(inode, dentry->d_parent->d_inode, mode); + inode_init_owner(&init_user_ns, inode, dentry->d_parent->d_inode, mode); attr.mode = inode->i_mode; err = ovl_create_or_link(dentry, inode, &attr, false); diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c index b2948e7b3210..7d8b84c715b3 100644 --- a/fs/overlayfs/file.c +++ b/fs/overlayfs/file.c @@ -54,7 +54,7 @@ static struct file *ovl_open_realfile(const struct file *file, if (err) { realfile = ERR_PTR(err); } else { - if (!inode_owner_or_capable(realinode)) + if (!inode_owner_or_capable(&init_user_ns, realinode)) flags &= ~O_NOATIME; realfile = open_with_fake_path(&file->f_path, flags, realinode, @@ -520,7 +520,7 @@ static long ovl_ioctl_set_flags(struct file *file, unsigned int cmd, long ret; struct inode *inode = file_inode(file); - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; ret = mnt_want_write_file(file); diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 88d877787770..3e925deaa19a 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1005,7 +1005,7 @@ ovl_posix_acl_xattr_set(const struct xattr_handler *handler, goto out_acl_release; } err = -EPERM; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) goto out_acl_release; posix_acl_release(acl); diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index de5c2047a0e9..06013b7b1e87 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -484,7 +484,7 @@ struct file *ovl_path_open(struct path *path, int flags) return ERR_PTR(err); /* O_NOATIME is an optimization, don't fail if not permitted */ - if (inode_owner_or_capable(inode)) + if (inode_owner_or_capable(&init_user_ns, inode)) flags |= O_NOATIME; return dentry_open(path, flags, current_cred()); diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 5d9fe2fb2953..9ce8214bfdac 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -874,7 +874,7 @@ set_posix_acl(struct inode *inode, int type, struct posix_acl *acl) if (type == ACL_TYPE_DEFAULT && !S_ISDIR(inode->i_mode)) return acl ? -EACCES : 0; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; if (acl) { diff --git a/fs/ramfs/inode.c b/fs/ramfs/inode.c index ee179a81b3da..3fd4326f36b5 100644 --- a/fs/ramfs/inode.c +++ b/fs/ramfs/inode.c @@ -67,7 +67,7 @@ struct inode *ramfs_get_inode(struct super_block *sb, if (inode) { inode->i_ino = get_next_ino(); - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_mapping->a_ops = &ramfs_aops; mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); mapping_set_unevictable(inode->i_mapping); diff --git a/fs/reiserfs/ioctl.c b/fs/reiserfs/ioctl.c index adb21bea3d60..4f1cbd930179 100644 --- a/fs/reiserfs/ioctl.c +++ b/fs/reiserfs/ioctl.c @@ -59,7 +59,7 @@ long reiserfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (err) break; - if (!inode_owner_or_capable(inode)) { + if (!inode_owner_or_capable(&init_user_ns, inode)) { err = -EPERM; goto setflags_out; } @@ -101,7 +101,7 @@ setflags_out: err = put_user(inode->i_generation, (int __user *)arg); break; case REISERFS_IOC_SETVERSION: - if (!inode_owner_or_capable(inode)) { + if (!inode_owner_or_capable(&init_user_ns, inode)) { err = -EPERM; break; } diff --git a/fs/reiserfs/namei.c b/fs/reiserfs/namei.c index 1594687582f0..a67a7d371725 100644 --- a/fs/reiserfs/namei.c +++ b/fs/reiserfs/namei.c @@ -615,7 +615,7 @@ static int new_inode_init(struct inode *inode, struct inode *dir, umode_t mode) * the quota init calls have to know who to charge the quota to, so * we have to set uid and gid here */ - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); return dquot_initialize(inode); } diff --git a/fs/sysv/ialloc.c b/fs/sysv/ialloc.c index 6c9801986af6..50df794a3c1f 100644 --- a/fs/sysv/ialloc.c +++ b/fs/sysv/ialloc.c @@ -163,7 +163,7 @@ struct inode * sysv_new_inode(const struct inode * dir, umode_t mode) *sbi->s_sb_fic_count = cpu_to_fs16(sbi, count); fs16_add(sbi, sbi->s_sb_total_free_inodes, -1); dirty_sb(sb); - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_ino = fs16_to_cpu(sbi, ino); inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); inode->i_blocks = 0; diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c index 9a6b8660425a..694e7714545b 100644 --- a/fs/ubifs/dir.c +++ b/fs/ubifs/dir.c @@ -94,7 +94,7 @@ struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir, */ inode->i_flags |= S_NOCMTIME; - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); inode->i_mapping->nrpages = 0; diff --git a/fs/ubifs/ioctl.c b/fs/ubifs/ioctl.c index 4363d85a3fd4..2326d5122beb 100644 --- a/fs/ubifs/ioctl.c +++ b/fs/ubifs/ioctl.c @@ -155,7 +155,7 @@ long ubifs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) if (IS_RDONLY(inode)) return -EROFS; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EACCES; if (get_user(flags, (int __user *) arg)) diff --git a/fs/udf/ialloc.c b/fs/udf/ialloc.c index 84ed23edebfd..2ecf0e87660e 100644 --- a/fs/udf/ialloc.c +++ b/fs/udf/ialloc.c @@ -103,7 +103,7 @@ struct inode *udf_new_inode(struct inode *dir, umode_t mode) mutex_unlock(&sbi->s_alloc_mutex); } - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); if (UDF_QUERY_FLAG(sb, UDF_FLAG_UID_SET)) inode->i_uid = sbi->s_uid; if (UDF_QUERY_FLAG(sb, UDF_FLAG_GID_SET)) diff --git a/fs/ufs/ialloc.c b/fs/ufs/ialloc.c index 969fd60436d3..7e3e08c0166f 100644 --- a/fs/ufs/ialloc.c +++ b/fs/ufs/ialloc.c @@ -289,7 +289,7 @@ cg_found: ufs_mark_sb_dirty(sb); inode->i_ino = cg * uspi->s_ipg + bit; - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_blocks = 0; inode->i_generation = 0; inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); diff --git a/fs/xattr.c b/fs/xattr.c index 56151bd9e642..c669922e1bde 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -127,7 +127,8 @@ xattr_permission(struct inode *inode, const char *name, int mask) if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode)) return (mask & MAY_WRITE) ? -EPERM : -ENODATA; if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) && - (mask & MAY_WRITE) && !inode_owner_or_capable(inode)) + (mask & MAY_WRITE) && + !inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; } diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 97bd29fc8c43..218e80afc859 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1300,7 +1300,7 @@ xfs_ioctl_setattr_get_trans( * The user ID of the calling process must be equal to the file owner * ID, except in cases where the CAP_FSETID capability is applicable. */ - if (!inode_owner_or_capable(VFS_I(ip))) { + if (!inode_owner_or_capable(&init_user_ns, VFS_I(ip))) { error = -EPERM; goto out_cancel; } diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index bec47f2d074b..569525ee8f69 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -1223,7 +1223,7 @@ static void zonefs_init_dir_inode(struct inode *parent, struct inode *inode, struct super_block *sb = parent->i_sb; inode->i_ino = blkdev_nr_zones(sb->s_bdev->bd_disk) + type + 1; - inode_init_owner(inode, parent, S_IFDIR | 0555); + inode_init_owner(&init_user_ns, inode, parent, S_IFDIR | 0555); inode->i_op = &zonefs_dir_inode_operations; inode->i_fop = &simple_dir_operations; set_nlink(inode, 2); diff --git a/include/linux/fs.h b/include/linux/fs.h index a85dfe6962df..2a9d4af6a64d 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1762,8 +1762,8 @@ static inline bool sb_start_intwrite_trylock(struct super_block *sb) return __sb_start_write_trylock(sb, SB_FREEZE_FS); } - -extern bool inode_owner_or_capable(const struct inode *inode); +bool inode_owner_or_capable(struct user_namespace *mnt_userns, + const struct inode *inode); /* * VFS helper functions.. @@ -1805,8 +1805,8 @@ extern long compat_ptr_ioctl(struct file *file, unsigned int cmd, /* * VFS file helper functions. */ -extern void inode_init_owner(struct inode *inode, const struct inode *dir, - umode_t mode); +void inode_init_owner(struct user_namespace *mnt_userns, struct inode *inode, + const struct inode *dir, umode_t mode); extern bool may_open_dev(const struct path *path); /* diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index e3226b65f5dc..05b1f51d15e0 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -122,7 +122,7 @@ static struct inode *bpf_get_inode(struct super_block *sb, inode->i_mtime = inode->i_atime; inode->i_ctime = inode->i_atime; - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); return inode; } diff --git a/mm/madvise.c b/mm/madvise.c index 175c5582d8a9..d4f5eece9d56 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -539,7 +539,8 @@ static inline bool can_do_pageout(struct vm_area_struct *vma) * otherwise we'd be including shared non-exclusive mappings, which * opens a side channel. */ - return inode_owner_or_capable(file_inode(vma->vm_file)) || + return inode_owner_or_capable(&init_user_ns, + file_inode(vma->vm_file)) || file_permission(vma->vm_file, MAY_WRITE) == 0; } diff --git a/mm/mincore.c b/mm/mincore.c index 7bdb4673f776..9122676b54d6 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -166,7 +166,8 @@ static inline bool can_do_mincore(struct vm_area_struct *vma) * for writing; otherwise we'd be including shared non-exclusive * mappings, which opens a side channel. */ - return inode_owner_or_capable(file_inode(vma->vm_file)) || + return inode_owner_or_capable(&init_user_ns, + file_inode(vma->vm_file)) || file_permission(vma->vm_file, MAY_WRITE) == 0; } diff --git a/mm/shmem.c b/mm/shmem.c index 7c6b6d8f6c39..1c68c9edba5e 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2303,7 +2303,7 @@ static struct inode *shmem_get_inode(struct super_block *sb, const struct inode inode = new_inode(sb); if (inode) { inode->i_ino = ino; - inode_init_owner(inode, dir, mode); + inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_blocks = 0; inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); inode->i_generation = prandom_u32(); diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 644b17ec9e63..9d6d3da2caf2 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -3140,13 +3140,13 @@ static int selinux_inode_setxattr(struct dentry *dentry, const char *name, } if (!selinux_initialized(&selinux_state)) - return (inode_owner_or_capable(inode) ? 0 : -EPERM); + return (inode_owner_or_capable(&init_user_ns, inode) ? 0 : -EPERM); sbsec = inode->i_sb->s_security; if (!(sbsec->flags & SBLABEL_MNT)) return -EOPNOTSUPP; - if (!inode_owner_or_capable(inode)) + if (!inode_owner_or_capable(&init_user_ns, inode)) return -EPERM; ad.type = LSM_AUDIT_DATA_DENTRY; -- cgit v1.2.3 From 2f221d6f7b881d95de1f356a3097d755ab1e47d4 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:26 +0100 Subject: attr: handle idmapped mounts When file attributes are changed most filesystems rely on the setattr_prepare(), setattr_copy(), and notify_change() helpers for initialization and permission checking. Let them handle idmapped mounts. If the inode is accessed through an idmapped mount map it into the mount's user namespace. Afterwards the checks are identical to non-idmapped mounts. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Helpers that perform checks on the ia_uid and ia_gid fields in struct iattr assume that ia_uid and ia_gid are intended values and have already been mapped correctly at the userspace-kernelspace boundary as we already do today. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-8-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- arch/powerpc/platforms/cell/spufs/inode.c | 2 +- drivers/base/devtmpfs.c | 4 +- fs/9p/vfs_inode.c | 4 +- fs/9p/vfs_inode_dotl.c | 4 +- fs/adfs/inode.c | 2 +- fs/affs/inode.c | 4 +- fs/attr.c | 119 ++++++++++++++++++++++-------- fs/btrfs/inode.c | 4 +- fs/cachefiles/interface.c | 4 +- fs/ceph/inode.c | 2 +- fs/cifs/inode.c | 8 +- fs/ecryptfs/inode.c | 7 +- fs/exfat/file.c | 4 +- fs/ext2/inode.c | 4 +- fs/ext4/inode.c | 4 +- fs/f2fs/file.c | 10 ++- fs/fat/file.c | 4 +- fs/fuse/dir.c | 2 +- fs/gfs2/inode.c | 4 +- fs/hfs/inode.c | 4 +- fs/hfsplus/inode.c | 4 +- fs/hostfs/hostfs_kern.c | 4 +- fs/hpfs/inode.c | 4 +- fs/hugetlbfs/inode.c | 4 +- fs/inode.c | 2 +- fs/jffs2/fs.c | 2 +- fs/jfs/file.c | 4 +- fs/kernfs/inode.c | 4 +- fs/libfs.c | 4 +- fs/minix/file.c | 4 +- fs/nfsd/nfsproc.c | 2 +- fs/nfsd/vfs.c | 4 +- fs/nilfs2/inode.c | 4 +- fs/ntfs/inode.c | 2 +- fs/ocfs2/dlmfs/dlmfs.c | 4 +- fs/ocfs2/file.c | 4 +- fs/omfs/file.c | 4 +- fs/open.c | 8 +- fs/orangefs/inode.c | 4 +- fs/overlayfs/copy_up.c | 8 +- fs/overlayfs/dir.c | 2 +- fs/overlayfs/inode.c | 4 +- fs/overlayfs/super.c | 2 +- fs/proc/base.c | 4 +- fs/proc/generic.c | 4 +- fs/proc/proc_sysctl.c | 4 +- fs/ramfs/file-nommu.c | 4 +- fs/reiserfs/inode.c | 4 +- fs/sysv/file.c | 4 +- fs/ubifs/file.c | 2 +- fs/udf/file.c | 4 +- fs/ufs/inode.c | 4 +- fs/utimes.c | 3 +- fs/xfs/xfs_iops.c | 2 +- fs/zonefs/super.c | 4 +- include/linux/fs.h | 8 +- mm/shmem.c | 4 +- 57 files changed, 206 insertions(+), 137 deletions(-) (limited to 'fs') diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c index 25390569e24c..3de526eb2275 100644 --- a/arch/powerpc/platforms/cell/spufs/inode.c +++ b/arch/powerpc/platforms/cell/spufs/inode.c @@ -98,7 +98,7 @@ spufs_setattr(struct dentry *dentry, struct iattr *attr) if ((attr->ia_valid & ATTR_SIZE) && (attr->ia_size != inode->i_size)) return -EINVAL; - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c index eac184e6d657..2e0c3cdb4184 100644 --- a/drivers/base/devtmpfs.c +++ b/drivers/base/devtmpfs.c @@ -221,7 +221,7 @@ static int handle_create(const char *nodename, umode_t mode, kuid_t uid, newattrs.ia_gid = gid; newattrs.ia_valid = ATTR_MODE|ATTR_UID|ATTR_GID; inode_lock(d_inode(dentry)); - notify_change(dentry, &newattrs, NULL); + notify_change(&init_user_ns, dentry, &newattrs, NULL); inode_unlock(d_inode(dentry)); /* mark as kernel-created inode */ @@ -328,7 +328,7 @@ static int handle_remove(const char *nodename, struct device *dev) newattrs.ia_valid = ATTR_UID|ATTR_GID|ATTR_MODE; inode_lock(d_inode(dentry)); - notify_change(dentry, &newattrs, NULL); + notify_change(&init_user_ns, dentry, &newattrs, NULL); inode_unlock(d_inode(dentry)); err = vfs_unlink(d_inode(parent.dentry), dentry, NULL); if (!err || err == -ENOENT) diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index f66eb3c12c8a..9c3ff6e9ab82 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -1062,7 +1062,7 @@ static int v9fs_vfs_setattr(struct dentry *dentry, struct iattr *iattr) struct p9_wstat wstat; p9_debug(P9_DEBUG_VFS, "\n"); - retval = setattr_prepare(dentry, iattr); + retval = setattr_prepare(&init_user_ns, dentry, iattr); if (retval) return retval; @@ -1118,7 +1118,7 @@ static int v9fs_vfs_setattr(struct dentry *dentry, struct iattr *iattr) v9fs_invalidate_inode_attr(d_inode(dentry)); - setattr_copy(d_inode(dentry), iattr); + setattr_copy(&init_user_ns, d_inode(dentry), iattr); mark_inode_dirty(d_inode(dentry)); return 0; } diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c index 823c2eb5f1bf..302553101fcb 100644 --- a/fs/9p/vfs_inode_dotl.c +++ b/fs/9p/vfs_inode_dotl.c @@ -549,7 +549,7 @@ int v9fs_vfs_setattr_dotl(struct dentry *dentry, struct iattr *iattr) p9_debug(P9_DEBUG_VFS, "\n"); - retval = setattr_prepare(dentry, iattr); + retval = setattr_prepare(&init_user_ns, dentry, iattr); if (retval) return retval; @@ -590,7 +590,7 @@ int v9fs_vfs_setattr_dotl(struct dentry *dentry, struct iattr *iattr) truncate_setsize(inode, iattr->ia_size); v9fs_invalidate_inode_attr(inode); - setattr_copy(inode, iattr); + setattr_copy(&init_user_ns, inode, iattr); mark_inode_dirty(inode); if (iattr->ia_valid & ATTR_MODE) { /* We also want to update ACL when we update mode bits */ diff --git a/fs/adfs/inode.c b/fs/adfs/inode.c index 32620f4a7623..278dcee6ae22 100644 --- a/fs/adfs/inode.c +++ b/fs/adfs/inode.c @@ -299,7 +299,7 @@ adfs_notify_change(struct dentry *dentry, struct iattr *attr) unsigned int ia_valid = attr->ia_valid; int error; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); /* * we can't change the UID or GID of any file - diff --git a/fs/affs/inode.c b/fs/affs/inode.c index 044412110b52..767e5bdfb703 100644 --- a/fs/affs/inode.c +++ b/fs/affs/inode.c @@ -223,7 +223,7 @@ affs_notify_change(struct dentry *dentry, struct iattr *attr) pr_debug("notify_change(%lu,0x%x)\n", inode->i_ino, attr->ia_valid); - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) goto out; @@ -249,7 +249,7 @@ affs_notify_change(struct dentry *dentry, struct iattr *attr) affs_truncate(inode); } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); if (attr->ia_valid & ATTR_MODE) diff --git a/fs/attr.c b/fs/attr.c index 00ae0b000146..f4543d2abdfb 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -18,27 +18,55 @@ #include #include -static bool chown_ok(const struct inode *inode, kuid_t uid) +/** + * chown_ok - verify permissions to chown inode + * @mnt_userns: user namespace of the mount @inode was found from + * @inode: inode to check permissions on + * @uid: uid to chown @inode to + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. + */ +static bool chown_ok(struct user_namespace *mnt_userns, + const struct inode *inode, + kuid_t uid) { - if (uid_eq(current_fsuid(), inode->i_uid) && - uid_eq(uid, inode->i_uid)) + kuid_t kuid = i_uid_into_mnt(mnt_userns, inode); + if (uid_eq(current_fsuid(), kuid) && uid_eq(uid, kuid)) return true; - if (capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_CHOWN)) + if (capable_wrt_inode_uidgid(mnt_userns, inode, CAP_CHOWN)) return true; - if (uid_eq(inode->i_uid, INVALID_UID) && + if (uid_eq(kuid, INVALID_UID) && ns_capable(inode->i_sb->s_user_ns, CAP_CHOWN)) return true; return false; } -static bool chgrp_ok(const struct inode *inode, kgid_t gid) +/** + * chgrp_ok - verify permissions to chgrp inode + * @mnt_userns: user namespace of the mount @inode was found from + * @inode: inode to check permissions on + * @gid: gid to chown @inode to + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. + */ +static bool chgrp_ok(struct user_namespace *mnt_userns, + const struct inode *inode, kgid_t gid) { - if (uid_eq(current_fsuid(), inode->i_uid) && - (in_group_p(gid) || gid_eq(gid, inode->i_gid))) + kgid_t kgid = i_gid_into_mnt(mnt_userns, inode); + if (uid_eq(current_fsuid(), i_uid_into_mnt(mnt_userns, inode)) && + (in_group_p(gid) || gid_eq(gid, kgid))) return true; - if (capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_CHOWN)) + if (capable_wrt_inode_uidgid(mnt_userns, inode, CAP_CHOWN)) return true; - if (gid_eq(inode->i_gid, INVALID_GID) && + if (gid_eq(kgid, INVALID_GID) && ns_capable(inode->i_sb->s_user_ns, CAP_CHOWN)) return true; return false; @@ -46,6 +74,7 @@ static bool chgrp_ok(const struct inode *inode, kgid_t gid) /** * setattr_prepare - check if attribute changes to a dentry are allowed + * @mnt_userns: user namespace of the mount the inode was found from * @dentry: dentry to check * @attr: attributes to change * @@ -55,10 +84,17 @@ static bool chgrp_ok(const struct inode *inode, kgid_t gid) * SGID bit from mode if user is not allowed to set it. Also file capabilities * and IMA extended attributes are cleared if ATTR_KILL_PRIV is set. * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. + * * Should be called as the first thing in ->setattr implementations, * possibly after taking additional locks. */ -int setattr_prepare(struct dentry *dentry, struct iattr *attr) +int setattr_prepare(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); unsigned int ia_valid = attr->ia_valid; @@ -78,27 +114,27 @@ int setattr_prepare(struct dentry *dentry, struct iattr *attr) goto kill_priv; /* Make sure a caller can chown. */ - if ((ia_valid & ATTR_UID) && !chown_ok(inode, attr->ia_uid)) + if ((ia_valid & ATTR_UID) && !chown_ok(mnt_userns, inode, attr->ia_uid)) return -EPERM; /* Make sure caller can chgrp. */ - if ((ia_valid & ATTR_GID) && !chgrp_ok(inode, attr->ia_gid)) + if ((ia_valid & ATTR_GID) && !chgrp_ok(mnt_userns, inode, attr->ia_gid)) return -EPERM; /* Make sure a caller can chmod. */ if (ia_valid & ATTR_MODE) { - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(mnt_userns, inode)) return -EPERM; /* Also check the setgid bit! */ - if (!in_group_p((ia_valid & ATTR_GID) ? attr->ia_gid : - inode->i_gid) && - !capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_FSETID)) + if (!in_group_p((ia_valid & ATTR_GID) ? attr->ia_gid : + i_gid_into_mnt(mnt_userns, inode)) && + !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FSETID)) attr->ia_mode &= ~S_ISGID; } /* Check for setting the inode time. */ if (ia_valid & (ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET)) { - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(mnt_userns, inode)) return -EPERM; } @@ -162,20 +198,33 @@ EXPORT_SYMBOL(inode_newsize_ok); /** * setattr_copy - copy simple metadata updates into the generic inode + * @mnt_userns: user namespace of the mount the inode was found from * @inode: the inode to be updated * @attr: the new attributes * * setattr_copy must be called with i_mutex held. * * setattr_copy updates the inode's metadata with that specified - * in attr. Noticeably missing is inode size update, which is more complex + * in attr on idmapped mounts. If file ownership is changed setattr_copy + * doesn't map ia_uid and ia_gid. It will asssume the caller has already + * provided the intended values. Necessary permission checks to determine + * whether or not the S_ISGID property needs to be removed are performed with + * the correct idmapped mount permission helpers. + * Noticeably missing is inode size update, which is more complex * as it requires pagecache updates. * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. + * * The inode is not marked as dirty after this operation. The rationale is * that for "simple" filesystems, the struct inode is the inode storage. * The caller is free to mark the inode dirty afterwards if needed. */ -void setattr_copy(struct inode *inode, const struct iattr *attr) +void setattr_copy(struct user_namespace *mnt_userns, struct inode *inode, + const struct iattr *attr) { unsigned int ia_valid = attr->ia_valid; @@ -191,9 +240,9 @@ void setattr_copy(struct inode *inode, const struct iattr *attr) inode->i_ctime = attr->ia_ctime; if (ia_valid & ATTR_MODE) { umode_t mode = attr->ia_mode; - - if (!in_group_p(inode->i_gid) && - !capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_FSETID)) + kgid_t kgid = i_gid_into_mnt(mnt_userns, inode); + if (!in_group_p(kgid) && + !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FSETID)) mode &= ~S_ISGID; inode->i_mode = mode; } @@ -202,6 +251,7 @@ EXPORT_SYMBOL(setattr_copy); /** * notify_change - modify attributes of a filesytem object + * @mnt_userns: user namespace of the mount the inode was found from * @dentry: object affected * @attr: new attributes * @delegated_inode: returns inode, if the inode is delegated @@ -214,13 +264,23 @@ EXPORT_SYMBOL(setattr_copy); * retry. Because breaking a delegation may take a long time, the * caller should drop the i_mutex before doing so. * + * If file ownership is changed notify_change() doesn't map ia_uid and + * ia_gid. It will asssume the caller has already provided the intended values. + * * Alternatively, a caller may pass NULL for delegated_inode. This may * be appropriate for callers that expect the underlying filesystem not * to be NFS exported. Also, passing NULL is fine for callers holding * the file open for write, as there can be no conflicting delegation in * that case. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. */ -int notify_change(struct dentry * dentry, struct iattr * attr, struct inode **delegated_inode) +int notify_change(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr, struct inode **delegated_inode) { struct inode *inode = dentry->d_inode; umode_t mode = inode->i_mode; @@ -243,9 +303,8 @@ int notify_change(struct dentry * dentry, struct iattr * attr, struct inode **de if (IS_IMMUTABLE(inode)) return -EPERM; - if (!inode_owner_or_capable(&init_user_ns, inode)) { - error = inode_permission(&init_user_ns, inode, - MAY_WRITE); + if (!inode_owner_or_capable(mnt_userns, inode)) { + error = inode_permission(mnt_userns, inode, MAY_WRITE); if (error) return error; } @@ -321,9 +380,11 @@ int notify_change(struct dentry * dentry, struct iattr * attr, struct inode **de /* Don't allow modifications of files with invalid uids or * gids unless those uids & gids are being made valid. */ - if (!(ia_valid & ATTR_UID) && !uid_valid(inode->i_uid)) + if (!(ia_valid & ATTR_UID) && + !uid_valid(i_uid_into_mnt(mnt_userns, inode))) return -EOVERFLOW; - if (!(ia_valid & ATTR_GID) && !gid_valid(inode->i_gid)) + if (!(ia_valid & ATTR_GID) && + !gid_valid(i_gid_into_mnt(mnt_userns, inode))) return -EOVERFLOW; error = security_inode_setattr(dentry, attr); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 07fe8b2f3bab..792191a8705b 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -5054,7 +5054,7 @@ static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) if (btrfs_root_readonly(root)) return -EROFS; - err = setattr_prepare(dentry, attr); + err = setattr_prepare(&init_user_ns, dentry, attr); if (err) return err; @@ -5065,7 +5065,7 @@ static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) } if (attr->ia_valid) { - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); inode_inc_iversion(inode); err = btrfs_dirty_inode(inode); diff --git a/fs/cachefiles/interface.c b/fs/cachefiles/interface.c index 4cea5fbf695e..5efa6a3702c0 100644 --- a/fs/cachefiles/interface.c +++ b/fs/cachefiles/interface.c @@ -470,14 +470,14 @@ static int cachefiles_attr_changed(struct fscache_object *_object) _debug("discard tail %llx", oi_size); newattrs.ia_valid = ATTR_SIZE; newattrs.ia_size = oi_size & PAGE_MASK; - ret = notify_change(object->backer, &newattrs, NULL); + ret = notify_change(&init_user_ns, object->backer, &newattrs, NULL); if (ret < 0) goto truncate_failed; } newattrs.ia_valid = ATTR_SIZE; newattrs.ia_size = ni_size; - ret = notify_change(object->backer, &newattrs, NULL); + ret = notify_change(&init_user_ns, object->backer, &newattrs, NULL); truncate_failed: inode_unlock(d_inode(object->backer)); diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index e8a15ee09bc1..285d3baca27e 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -2247,7 +2247,7 @@ int ceph_setattr(struct dentry *dentry, struct iattr *attr) if (ceph_snap(inode) != CEPH_NOSNAP) return -EROFS; - err = setattr_prepare(dentry, attr); + err = setattr_prepare(&init_user_ns, dentry, attr); if (err != 0) return err; diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index a83b3a8ffaac..27554f71f744 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -2610,7 +2610,7 @@ cifs_setattr_unix(struct dentry *direntry, struct iattr *attrs) if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_PERM) attrs->ia_valid |= ATTR_FORCE; - rc = setattr_prepare(direntry, attrs); + rc = setattr_prepare(&init_user_ns, direntry, attrs); if (rc < 0) goto out; @@ -2715,7 +2715,7 @@ cifs_setattr_unix(struct dentry *direntry, struct iattr *attrs) attrs->ia_size != i_size_read(inode)) truncate_setsize(inode, attrs->ia_size); - setattr_copy(inode, attrs); + setattr_copy(&init_user_ns, inode, attrs); mark_inode_dirty(inode); /* force revalidate when any of these times are set since some @@ -2757,7 +2757,7 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_PERM) attrs->ia_valid |= ATTR_FORCE; - rc = setattr_prepare(direntry, attrs); + rc = setattr_prepare(&init_user_ns, direntry, attrs); if (rc < 0) { free_xid(xid); return rc; @@ -2913,7 +2913,7 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) attrs->ia_size != i_size_read(inode)) truncate_setsize(inode, attrs->ia_size); - setattr_copy(inode, attrs); + setattr_copy(&init_user_ns, inode, attrs); mark_inode_dirty(inode); cifs_setattr_exit: diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index 0b346baf110d..d3ea0c57b075 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -855,7 +855,8 @@ int ecryptfs_truncate(struct dentry *dentry, loff_t new_length) struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry); inode_lock(d_inode(lower_dentry)); - rc = notify_change(lower_dentry, &lower_ia, NULL); + rc = notify_change(&init_user_ns, lower_dentry, + &lower_ia, NULL); inode_unlock(d_inode(lower_dentry)); } return rc; @@ -934,7 +935,7 @@ static int ecryptfs_setattr(struct dentry *dentry, struct iattr *ia) } mutex_unlock(&crypt_stat->cs_mutex); - rc = setattr_prepare(dentry, ia); + rc = setattr_prepare(&init_user_ns, dentry, ia); if (rc) goto out; if (ia->ia_valid & ATTR_SIZE) { @@ -960,7 +961,7 @@ static int ecryptfs_setattr(struct dentry *dentry, struct iattr *ia) lower_ia.ia_valid &= ~ATTR_MODE; inode_lock(d_inode(lower_dentry)); - rc = notify_change(lower_dentry, &lower_ia, NULL); + rc = notify_change(&init_user_ns, lower_dentry, &lower_ia, NULL); inode_unlock(d_inode(lower_dentry)); out: fsstack_copy_attr_all(inode, lower_inode); diff --git a/fs/exfat/file.c b/fs/exfat/file.c index a92478eabfa4..ace35aa8e64b 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -305,7 +305,7 @@ int exfat_setattr(struct dentry *dentry, struct iattr *attr) ATTR_TIMES_SET); } - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); attr->ia_valid = ia_valid; if (error) goto out; @@ -340,7 +340,7 @@ int exfat_setattr(struct dentry *dentry, struct iattr *attr) up_write(&EXFAT_I(inode)->truncate_lock); } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); exfat_truncate_atime(&inode->i_atime); mark_inode_dirty(inode); diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 78c417d3c898..06c0cf28c1a0 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -1669,7 +1669,7 @@ int ext2_setattr(struct dentry *dentry, struct iattr *iattr) struct inode *inode = d_inode(dentry); int error; - error = setattr_prepare(dentry, iattr); + error = setattr_prepare(&init_user_ns, dentry, iattr); if (error) return error; @@ -1689,7 +1689,7 @@ int ext2_setattr(struct dentry *dentry, struct iattr *iattr) if (error) return error; } - setattr_copy(inode, iattr); + setattr_copy(&init_user_ns, inode, iattr); if (iattr->ia_valid & ATTR_MODE) error = posix_acl_chmod(inode, inode->i_mode); mark_inode_dirty(inode); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index c173c8405856..8edfa3e226e6 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -5337,7 +5337,7 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr) ATTR_GID | ATTR_TIMES_SET)))) return -EPERM; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; @@ -5512,7 +5512,7 @@ out_mmap_sem: } if (!error) { - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); } diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 5fc0ff28b5dd..90d7b89176de 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -831,7 +831,8 @@ int f2fs_getattr(const struct path *path, struct kstat *stat, } #ifdef CONFIG_F2FS_FS_POSIX_ACL -static void __setattr_copy(struct inode *inode, const struct iattr *attr) +static void __setattr_copy(struct user_namespace *mnt_userns, struct inode *inode, + const struct iattr *attr) { unsigned int ia_valid = attr->ia_valid; @@ -847,8 +848,9 @@ static void __setattr_copy(struct inode *inode, const struct iattr *attr) inode->i_ctime = attr->ia_ctime; if (ia_valid & ATTR_MODE) { umode_t mode = attr->ia_mode; + kgid_t kgid = i_gid_into_mnt(mnt_userns, inode); - if (!in_group_p(inode->i_gid) && !capable(CAP_FSETID)) + if (!in_group_p(kgid) && !capable(CAP_FSETID)) mode &= ~S_ISGID; set_acl_inode(inode, mode); } @@ -869,7 +871,7 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) !f2fs_is_compress_backend_ready(inode)) return -EOPNOTSUPP; - err = setattr_prepare(dentry, attr); + err = setattr_prepare(&init_user_ns, dentry, attr); if (err) return err; @@ -945,7 +947,7 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) spin_unlock(&F2FS_I(inode)->i_size_lock); } - __setattr_copy(inode, attr); + __setattr_copy(&init_user_ns, inode, attr); if (attr->ia_valid & ATTR_MODE) { err = posix_acl_chmod(inode, f2fs_get_inode_mode(inode)); diff --git a/fs/fat/file.c b/fs/fat/file.c index f9ee27cf4d7c..805b501467e9 100644 --- a/fs/fat/file.c +++ b/fs/fat/file.c @@ -480,7 +480,7 @@ int fat_setattr(struct dentry *dentry, struct iattr *attr) attr->ia_valid &= ~TIMES_SET_FLAGS; } - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); attr->ia_valid = ia_valid; if (error) { if (sbi->options.quiet) @@ -550,7 +550,7 @@ int fat_setattr(struct dentry *dentry, struct iattr *attr) fat_truncate_time(inode, &attr->ia_mtime, S_MTIME); attr->ia_valid &= ~(ATTR_ATIME|ATTR_CTIME|ATTR_MTIME); - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); out: return error; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 7497009a5a45..74fdb6a7ebb3 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -1611,7 +1611,7 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, if (!fc->default_permissions) attr->ia_valid |= ATTR_FORCE; - err = setattr_prepare(dentry, attr); + err = setattr_prepare(&init_user_ns, dentry, attr); if (err) return err; diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 5b2ff0c74b67..59c25181d108 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -1861,7 +1861,7 @@ int gfs2_permission(struct inode *inode, int mask) static int __gfs2_setattr_simple(struct inode *inode, struct iattr *attr) { - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } @@ -1982,7 +1982,7 @@ static int gfs2_setattr(struct dentry *dentry, struct iattr *attr) if (IS_IMMUTABLE(inode) || IS_APPEND(inode)) goto error; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) goto error; diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index f35a37c65e5f..c646218b72bf 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c @@ -608,7 +608,7 @@ int hfs_inode_setattr(struct dentry *dentry, struct iattr * attr) struct hfs_sb_info *hsb = HFS_SB(inode->i_sb); int error; - error = setattr_prepare(dentry, attr); /* basic permission checks */ + error = setattr_prepare(&init_user_ns, dentry, attr); /* basic permission checks */ if (error) return error; @@ -647,7 +647,7 @@ int hfs_inode_setattr(struct dentry *dentry, struct iattr * attr) current_time(inode); } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index 21357046b039..ffa137f8234e 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -246,7 +246,7 @@ static int hfsplus_setattr(struct dentry *dentry, struct iattr *attr) struct inode *inode = d_inode(dentry); int error; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; @@ -264,7 +264,7 @@ static int hfsplus_setattr(struct dentry *dentry, struct iattr *attr) inode->i_mtime = inode->i_ctime = current_time(inode); } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c index b841a05a2b8c..6970e29a5287 100644 --- a/fs/hostfs/hostfs_kern.c +++ b/fs/hostfs/hostfs_kern.c @@ -792,7 +792,7 @@ static int hostfs_setattr(struct dentry *dentry, struct iattr *attr) int fd = HOSTFS_I(inode)->fd; - err = setattr_prepare(dentry, attr); + err = setattr_prepare(&init_user_ns, dentry, attr); if (err) return err; @@ -849,7 +849,7 @@ static int hostfs_setattr(struct dentry *dentry, struct iattr *attr) attr->ia_size != i_size_read(inode)) truncate_setsize(inode, attr->ia_size); - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/hpfs/inode.c b/fs/hpfs/inode.c index eb8b4baf0f2e..8ba2152a78ba 100644 --- a/fs/hpfs/inode.c +++ b/fs/hpfs/inode.c @@ -274,7 +274,7 @@ int hpfs_setattr(struct dentry *dentry, struct iattr *attr) if ((attr->ia_valid & ATTR_SIZE) && attr->ia_size > inode->i_size) goto out_unlock; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) goto out_unlock; @@ -288,7 +288,7 @@ int hpfs_setattr(struct dentry *dentry, struct iattr *attr) hpfs_truncate(inode); } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); hpfs_write_inode(inode); diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 6737929e443c..327e572b4e00 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -761,7 +761,7 @@ static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr) BUG_ON(!inode); - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; @@ -780,7 +780,7 @@ static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr) return error; } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/inode.c b/fs/inode.c index a9ac97a27784..49b512592dcd 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -1912,7 +1912,7 @@ static int __remove_privs(struct dentry *dentry, int kill) * Note we call this on write, so notify_change will not * encounter any conflicting delegations: */ - return notify_change(dentry, &newattrs, NULL); + return notify_change(&init_user_ns, dentry, &newattrs, NULL); } /* diff --git a/fs/jffs2/fs.c b/fs/jffs2/fs.c index 78858f6e9583..67993808f4da 100644 --- a/fs/jffs2/fs.c +++ b/fs/jffs2/fs.c @@ -195,7 +195,7 @@ int jffs2_setattr(struct dentry *dentry, struct iattr *iattr) struct inode *inode = d_inode(dentry); int rc; - rc = setattr_prepare(dentry, iattr); + rc = setattr_prepare(&init_user_ns, dentry, iattr); if (rc) return rc; diff --git a/fs/jfs/file.c b/fs/jfs/file.c index 930d2701f206..ff49876e9c9b 100644 --- a/fs/jfs/file.c +++ b/fs/jfs/file.c @@ -90,7 +90,7 @@ int jfs_setattr(struct dentry *dentry, struct iattr *iattr) struct inode *inode = d_inode(dentry); int rc; - rc = setattr_prepare(dentry, iattr); + rc = setattr_prepare(&init_user_ns, dentry, iattr); if (rc) return rc; @@ -118,7 +118,7 @@ int jfs_setattr(struct dentry *dentry, struct iattr *iattr) jfs_truncate(inode); } - setattr_copy(inode, iattr); + setattr_copy(&init_user_ns, inode, iattr); mark_inode_dirty(inode); if (iattr->ia_valid & ATTR_MODE) diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c index ff5598cc1de0..86bd4c593b78 100644 --- a/fs/kernfs/inode.c +++ b/fs/kernfs/inode.c @@ -122,7 +122,7 @@ int kernfs_iop_setattr(struct dentry *dentry, struct iattr *iattr) return -EINVAL; mutex_lock(&kernfs_mutex); - error = setattr_prepare(dentry, iattr); + error = setattr_prepare(&init_user_ns, dentry, iattr); if (error) goto out; @@ -131,7 +131,7 @@ int kernfs_iop_setattr(struct dentry *dentry, struct iattr *iattr) goto out; /* this ignores size changes */ - setattr_copy(inode, iattr); + setattr_copy(&init_user_ns, inode, iattr); out: mutex_unlock(&kernfs_mutex); diff --git a/fs/libfs.c b/fs/libfs.c index f8b3c02b4f0f..a73fe109403c 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -497,13 +497,13 @@ int simple_setattr(struct dentry *dentry, struct iattr *iattr) struct inode *inode = d_inode(dentry); int error; - error = setattr_prepare(dentry, iattr); + error = setattr_prepare(&init_user_ns, dentry, iattr); if (error) return error; if (iattr->ia_valid & ATTR_SIZE) truncate_setsize(inode, iattr->ia_size); - setattr_copy(inode, iattr); + setattr_copy(&init_user_ns, inode, iattr); mark_inode_dirty(inode); return 0; } diff --git a/fs/minix/file.c b/fs/minix/file.c index c50b0a20fcd9..f07acd268577 100644 --- a/fs/minix/file.c +++ b/fs/minix/file.c @@ -27,7 +27,7 @@ static int minix_setattr(struct dentry *dentry, struct iattr *attr) struct inode *inode = d_inode(dentry); int error; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; @@ -41,7 +41,7 @@ static int minix_setattr(struct dentry *dentry, struct iattr *attr) minix_truncate(inode); } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 9473d048efec..0ea0554d20d1 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -90,7 +90,7 @@ nfsd_proc_setattr(struct svc_rqst *rqstp) if (delta < 0) delta = -delta; if (delta < MAX_TOUCH_TIME_ERROR && - setattr_prepare(fhp->fh_dentry, iap) != 0) { + setattr_prepare(&init_user_ns, fhp->fh_dentry, iap) != 0) { /* * Turn off ATTR_[AM]TIME_SET but leave ATTR_[AM]TIME. * This will cause notify_change to set these times diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 0edf11258aaa..1905b39be1c2 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -448,7 +448,7 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, struct iattr *iap, .ia_size = iap->ia_size, }; - host_err = notify_change(dentry, &size_attr, NULL); + host_err = notify_change(&init_user_ns, dentry, &size_attr, NULL); if (host_err) goto out_unlock; iap->ia_valid &= ~ATTR_SIZE; @@ -463,7 +463,7 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, struct iattr *iap, } iap->ia_valid |= ATTR_CTIME; - host_err = notify_change(dentry, iap, NULL); + host_err = notify_change(&init_user_ns, dentry, iap, NULL); out_unlock: fh_unlock(fhp); diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index 11225a659736..8aad3c48092a 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c @@ -812,7 +812,7 @@ int nilfs_setattr(struct dentry *dentry, struct iattr *iattr) struct super_block *sb = inode->i_sb; int err; - err = setattr_prepare(dentry, iattr); + err = setattr_prepare(&init_user_ns, dentry, iattr); if (err) return err; @@ -827,7 +827,7 @@ int nilfs_setattr(struct dentry *dentry, struct iattr *iattr) nilfs_truncate(inode); } - setattr_copy(inode, iattr); + setattr_copy(&init_user_ns, inode, iattr); mark_inode_dirty(inode); if (iattr->ia_valid & ATTR_MODE) { diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index f7e4cbc26eaf..38f4cf1d4497 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -2866,7 +2866,7 @@ int ntfs_setattr(struct dentry *dentry, struct iattr *attr) int err; unsigned int ia_valid = attr->ia_valid; - err = setattr_prepare(dentry, attr); + err = setattr_prepare(&init_user_ns, dentry, attr); if (err) goto out; /* We do not support NTFS ACLs yet. */ diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c index 37c7d03a6284..9fa66cd1f622 100644 --- a/fs/ocfs2/dlmfs/dlmfs.c +++ b/fs/ocfs2/dlmfs/dlmfs.c @@ -196,11 +196,11 @@ static int dlmfs_file_setattr(struct dentry *dentry, struct iattr *attr) struct inode *inode = d_inode(dentry); attr->ia_valid &= ~ATTR_SIZE; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 0c75619adf54..cabf355b148f 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -1142,7 +1142,7 @@ int ocfs2_setattr(struct dentry *dentry, struct iattr *attr) if (!(attr->ia_valid & OCFS2_VALID_ATTRS)) return 0; - status = setattr_prepare(dentry, attr); + status = setattr_prepare(&init_user_ns, dentry, attr); if (status) return status; @@ -1263,7 +1263,7 @@ int ocfs2_setattr(struct dentry *dentry, struct iattr *attr) } } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); status = ocfs2_mark_inode_dirty(handle, inode, bh); diff --git a/fs/omfs/file.c b/fs/omfs/file.c index 2c7b70ee1388..729339cd7902 100644 --- a/fs/omfs/file.c +++ b/fs/omfs/file.c @@ -348,7 +348,7 @@ static int omfs_setattr(struct dentry *dentry, struct iattr *attr) struct inode *inode = d_inode(dentry); int error; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; @@ -361,7 +361,7 @@ static int omfs_setattr(struct dentry *dentry, struct iattr *attr) omfs_truncate(inode); } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/open.c b/fs/open.c index a6dac6d97988..c3e4dc43dd8d 100644 --- a/fs/open.c +++ b/fs/open.c @@ -61,7 +61,7 @@ int do_truncate(struct dentry *dentry, loff_t length, unsigned int time_attrs, inode_lock(dentry->d_inode); /* Note any delegations or leases have already been broken: */ - ret = notify_change(dentry, &newattrs, NULL); + ret = notify_change(&init_user_ns, dentry, &newattrs, NULL); inode_unlock(dentry->d_inode); return ret; } @@ -580,7 +580,8 @@ retry_deleg: goto out_unlock; newattrs.ia_mode = (mode & S_IALLUGO) | (inode->i_mode & ~S_IALLUGO); newattrs.ia_valid = ATTR_MODE | ATTR_CTIME; - error = notify_change(path->dentry, &newattrs, &delegated_inode); + error = notify_change(&init_user_ns, path->dentry, &newattrs, + &delegated_inode); out_unlock: inode_unlock(inode); if (delegated_inode) { @@ -671,7 +672,8 @@ retry_deleg: inode_lock(inode); error = security_path_chown(path, uid, gid); if (!error) - error = notify_change(path->dentry, &newattrs, &delegated_inode); + error = notify_change(&init_user_ns, path->dentry, &newattrs, + &delegated_inode); inode_unlock(inode); if (delegated_inode) { error = break_deleg_wait(&delegated_inode); diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c index 4c790cc8042d..8ac9491ceb9a 100644 --- a/fs/orangefs/inode.c +++ b/fs/orangefs/inode.c @@ -855,7 +855,7 @@ again: ORANGEFS_I(inode)->attr_uid = current_fsuid(); ORANGEFS_I(inode)->attr_gid = current_fsgid(); } - setattr_copy(inode, iattr); + setattr_copy(&init_user_ns, inode, iattr); spin_unlock(&inode->i_lock); mark_inode_dirty(inode); @@ -876,7 +876,7 @@ int orangefs_setattr(struct dentry *dentry, struct iattr *iattr) int ret; gossip_debug(GOSSIP_INODE_DEBUG, "__orangefs_setattr: called on %pd\n", dentry); - ret = setattr_prepare(dentry, iattr); + ret = setattr_prepare(&init_user_ns, dentry, iattr); if (ret) goto out; ret = __orangefs_setattr(d_inode(dentry), iattr); diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index e5b616c93e11..3e9957ae19fa 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -235,7 +235,7 @@ static int ovl_set_size(struct dentry *upperdentry, struct kstat *stat) .ia_size = stat->size, }; - return notify_change(upperdentry, &attr, NULL); + return notify_change(&init_user_ns, upperdentry, &attr, NULL); } static int ovl_set_timestamps(struct dentry *upperdentry, struct kstat *stat) @@ -247,7 +247,7 @@ static int ovl_set_timestamps(struct dentry *upperdentry, struct kstat *stat) .ia_mtime = stat->mtime, }; - return notify_change(upperdentry, &attr, NULL); + return notify_change(&init_user_ns, upperdentry, &attr, NULL); } int ovl_set_attr(struct dentry *upperdentry, struct kstat *stat) @@ -259,7 +259,7 @@ int ovl_set_attr(struct dentry *upperdentry, struct kstat *stat) .ia_valid = ATTR_MODE, .ia_mode = stat->mode, }; - err = notify_change(upperdentry, &attr, NULL); + err = notify_change(&init_user_ns, upperdentry, &attr, NULL); } if (!err) { struct iattr attr = { @@ -267,7 +267,7 @@ int ovl_set_attr(struct dentry *upperdentry, struct kstat *stat) .ia_uid = stat->uid, .ia_gid = stat->gid, }; - err = notify_change(upperdentry, &attr, NULL); + err = notify_change(&init_user_ns, upperdentry, &attr, NULL); } if (!err) ovl_set_timestamps(upperdentry, stat); diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index 98a23353b19a..29840820a46c 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -508,7 +508,7 @@ static int ovl_create_over_whiteout(struct dentry *dentry, struct inode *inode, .ia_mode = cattr->mode, }; inode_lock(newdentry->d_inode); - err = notify_change(newdentry, &attr, NULL); + err = notify_change(&init_user_ns, newdentry, &attr, NULL); inode_unlock(newdentry->d_inode); if (err) goto out_cleanup; diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index c101ebbb7a77..5aa66881dbd7 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -21,7 +21,7 @@ int ovl_setattr(struct dentry *dentry, struct iattr *attr) struct dentry *upperdentry; const struct cred *old_cred; - err = setattr_prepare(dentry, attr); + err = setattr_prepare(&init_user_ns, dentry, attr); if (err) return err; @@ -79,7 +79,7 @@ int ovl_setattr(struct dentry *dentry, struct iattr *attr) inode_lock(upperdentry->d_inode); old_cred = ovl_override_creds(dentry->d_sb); - err = notify_change(upperdentry, attr, NULL); + err = notify_change(&init_user_ns, upperdentry, attr, NULL); revert_creds(old_cred); if (!err) ovl_copyattr(upperdentry->d_inode, dentry->d_inode); diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 3e925deaa19a..39b2e9aa0e5b 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -804,7 +804,7 @@ retry: /* Clear any inherited mode bits */ inode_lock(work->d_inode); - err = notify_change(work, &attr, NULL); + err = notify_change(&init_user_ns, work, &attr, NULL); inode_unlock(work->d_inode); if (err) goto out_dput; diff --git a/fs/proc/base.c b/fs/proc/base.c index b4ec9293625e..bb4e63a3684f 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -693,11 +693,11 @@ int proc_setattr(struct dentry *dentry, struct iattr *attr) if (attr->ia_valid & ATTR_MODE) return -EPERM; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/proc/generic.c b/fs/proc/generic.c index 6c0a05f55d6b..6d4fabab8aa7 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -121,11 +121,11 @@ static int proc_notify_change(struct dentry *dentry, struct iattr *iattr) struct proc_dir_entry *de = PDE(inode); int error; - error = setattr_prepare(dentry, iattr); + error = setattr_prepare(&init_user_ns, dentry, iattr); if (error) return error; - setattr_copy(inode, iattr); + setattr_copy(&init_user_ns, inode, iattr); mark_inode_dirty(inode); proc_set_user(de, inode->i_uid, inode->i_gid); diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 317899222d7f..ec67dbc1f705 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -821,11 +821,11 @@ static int proc_sys_setattr(struct dentry *dentry, struct iattr *attr) if (attr->ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID)) return -EPERM; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/ramfs/file-nommu.c b/fs/ramfs/file-nommu.c index 355523f4a4bf..f0358fe410d3 100644 --- a/fs/ramfs/file-nommu.c +++ b/fs/ramfs/file-nommu.c @@ -165,7 +165,7 @@ static int ramfs_nommu_setattr(struct dentry *dentry, struct iattr *ia) int ret = 0; /* POSIX UID/GID verification for setting inode attributes */ - ret = setattr_prepare(dentry, ia); + ret = setattr_prepare(&init_user_ns, dentry, ia); if (ret) return ret; @@ -185,7 +185,7 @@ static int ramfs_nommu_setattr(struct dentry *dentry, struct iattr *ia) } } - setattr_copy(inode, ia); + setattr_copy(&init_user_ns, inode, ia); out: ia->ia_valid = old_ia_valid; return ret; diff --git a/fs/reiserfs/inode.c b/fs/reiserfs/inode.c index c76d563dec0e..944f2b487cf8 100644 --- a/fs/reiserfs/inode.c +++ b/fs/reiserfs/inode.c @@ -3288,7 +3288,7 @@ int reiserfs_setattr(struct dentry *dentry, struct iattr *attr) unsigned int ia_valid; int error; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; @@ -3413,7 +3413,7 @@ int reiserfs_setattr(struct dentry *dentry, struct iattr *attr) } if (!error) { - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); } diff --git a/fs/sysv/file.c b/fs/sysv/file.c index 45fc79a18594..ca7e216b7b9e 100644 --- a/fs/sysv/file.c +++ b/fs/sysv/file.c @@ -34,7 +34,7 @@ static int sysv_setattr(struct dentry *dentry, struct iattr *attr) struct inode *inode = d_inode(dentry); int error; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; @@ -47,7 +47,7 @@ static int sysv_setattr(struct dentry *dentry, struct iattr *attr) sysv_truncate(inode); } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c index 2bc7780d2963..76ef392b1e41 100644 --- a/fs/ubifs/file.c +++ b/fs/ubifs/file.c @@ -1265,7 +1265,7 @@ int ubifs_setattr(struct dentry *dentry, struct iattr *attr) dbg_gen("ino %lu, mode %#x, ia_valid %#x", inode->i_ino, inode->i_mode, attr->ia_valid); - err = setattr_prepare(dentry, attr); + err = setattr_prepare(&init_user_ns, dentry, attr); if (err) return err; diff --git a/fs/udf/file.c b/fs/udf/file.c index 3671a40ed3c3..7c7d161315c2 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -259,7 +259,7 @@ static int udf_setattr(struct dentry *dentry, struct iattr *attr) struct super_block *sb = inode->i_sb; int error; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; @@ -282,7 +282,7 @@ static int udf_setattr(struct dentry *dentry, struct iattr *attr) if (attr->ia_valid & ATTR_MODE) udf_update_extra_perms(inode, attr->ia_mode); - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/ufs/inode.c b/fs/ufs/inode.c index c843ec858cf7..6b51f3b20143 100644 --- a/fs/ufs/inode.c +++ b/fs/ufs/inode.c @@ -1217,7 +1217,7 @@ int ufs_setattr(struct dentry *dentry, struct iattr *attr) unsigned int ia_valid = attr->ia_valid; int error; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; @@ -1227,7 +1227,7 @@ int ufs_setattr(struct dentry *dentry, struct iattr *attr) return error; } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/fs/utimes.c b/fs/utimes.c index fd3cc4226224..4572b91ddb91 100644 --- a/fs/utimes.c +++ b/fs/utimes.c @@ -62,7 +62,8 @@ int vfs_utimes(const struct path *path, struct timespec64 *times) } retry_deleg: inode_lock(inode); - error = notify_change(path->dentry, &newattrs, &delegated_inode); + error = notify_change(&init_user_ns, path->dentry, &newattrs, + &delegated_inode); inode_unlock(inode); if (delegated_inode) { error = break_deleg_wait(&delegated_inode); diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 67c8dc9de8aa..08a478d25122 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -637,7 +637,7 @@ xfs_vn_change_ok( if (XFS_FORCED_SHUTDOWN(mp)) return -EIO; - return setattr_prepare(dentry, iattr); + return setattr_prepare(&init_user_ns, dentry, iattr); } /* diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 569525ee8f69..8a1f69677784 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -488,7 +488,7 @@ static int zonefs_inode_setattr(struct dentry *dentry, struct iattr *iattr) if (unlikely(IS_IMMUTABLE(inode))) return -EPERM; - ret = setattr_prepare(dentry, iattr); + ret = setattr_prepare(&init_user_ns, dentry, iattr); if (ret) return ret; @@ -516,7 +516,7 @@ static int zonefs_inode_setattr(struct dentry *dentry, struct iattr *iattr) return ret; } - setattr_copy(inode, iattr); + setattr_copy(&init_user_ns, inode, iattr); return 0; } diff --git a/include/linux/fs.h b/include/linux/fs.h index 2a9d4af6a64d..e3ea1d7c3367 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2809,7 +2809,8 @@ static inline int bmap(struct inode *inode, sector_t *block) } #endif -extern int notify_change(struct dentry *, struct iattr *, struct inode **); +int notify_change(struct user_namespace *, struct dentry *, + struct iattr *, struct inode **); int inode_permission(struct user_namespace *, struct inode *, int); int generic_permission(struct user_namespace *, struct inode *, int); static inline int file_permission(struct file *file, int mask) @@ -3274,9 +3275,10 @@ extern int buffer_migrate_page_norefs(struct address_space *, #define buffer_migrate_page_norefs NULL #endif -extern int setattr_prepare(struct dentry *, struct iattr *); +int setattr_prepare(struct user_namespace *, struct dentry *, struct iattr *); extern int inode_newsize_ok(const struct inode *, loff_t offset); -extern void setattr_copy(struct inode *inode, const struct iattr *attr); +void setattr_copy(struct user_namespace *, struct inode *inode, + const struct iattr *attr); extern int file_update_time(struct file *file); diff --git a/mm/shmem.c b/mm/shmem.c index 1c68c9edba5e..1cb451e131ec 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1087,7 +1087,7 @@ static int shmem_setattr(struct dentry *dentry, struct iattr *attr) struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); int error; - error = setattr_prepare(dentry, attr); + error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; @@ -1141,7 +1141,7 @@ static int shmem_setattr(struct dentry *dentry, struct iattr *attr) } } - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); if (attr->ia_valid & ATTR_MODE) error = posix_acl_chmod(inode, inode->i_mode); return error; -- cgit v1.2.3 From e65ce2a50cf6af216bea6fd80d771fcbb4c0aaa1 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:27 +0100 Subject: acl: handle idmapped mounts The posix acl permission checking helpers determine whether a caller is privileged over an inode according to the acls associated with the inode. Add helpers that make it possible to handle acls on idmapped mounts. The vfs and the filesystems targeted by this first iteration make use of posix_acl_fix_xattr_from_user() and posix_acl_fix_xattr_to_user() to translate basic posix access and default permissions such as the ACL_USER and ACL_GROUP type according to the initial user namespace (or the superblock's user namespace) to and from the caller's current user namespace. Adapt these two helpers to handle idmapped mounts whereby we either map from or into the mount's user namespace depending on in which direction we're translating. Similarly, cap_convert_nscap() is used by the vfs to translate user namespace and non-user namespace aware filesystem capabilities from the superblock's user namespace to the caller's user namespace. Enable it to handle idmapped mounts by accounting for the mount's user namespace. In addition the fileystems targeted in the first iteration of this patch series make use of the posix_acl_chmod() and, posix_acl_update_mode() helpers. Both helpers perform permission checks on the target inode. Let them handle idmapped mounts. These two helpers are called when posix acls are set by the respective filesystems to handle this case we extend the ->set() method to take an additional user namespace argument to pass the mount's user namespace down. Link: https://lore.kernel.org/r/20210121131959.646623-9-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- Documentation/filesystems/locking.rst | 7 ++-- Documentation/filesystems/porting.rst | 2 + fs/9p/acl.c | 4 +- fs/9p/xattr.c | 1 + fs/afs/xattr.c | 2 + fs/btrfs/acl.c | 3 +- fs/btrfs/inode.c | 3 +- fs/btrfs/xattr.c | 2 + fs/ceph/acl.c | 3 +- fs/ceph/inode.c | 2 +- fs/ceph/xattr.c | 1 + fs/cifs/xattr.c | 1 + fs/ecryptfs/inode.c | 1 + fs/ext2/acl.c | 3 +- fs/ext2/inode.c | 2 +- fs/ext2/xattr_security.c | 1 + fs/ext2/xattr_trusted.c | 1 + fs/ext2/xattr_user.c | 1 + fs/ext4/acl.c | 3 +- fs/ext4/inode.c | 2 +- fs/ext4/xattr_hurd.c | 1 + fs/ext4/xattr_security.c | 1 + fs/ext4/xattr_trusted.c | 1 + fs/ext4/xattr_user.c | 1 + fs/f2fs/acl.c | 3 +- fs/f2fs/file.c | 7 ++-- fs/f2fs/xattr.c | 2 + fs/fuse/xattr.c | 2 + fs/gfs2/acl.c | 2 +- fs/gfs2/inode.c | 3 +- fs/gfs2/xattr.c | 1 + fs/hfs/attr.c | 1 + fs/hfsplus/xattr.c | 1 + fs/hfsplus/xattr_security.c | 1 + fs/hfsplus/xattr_trusted.c | 1 + fs/hfsplus/xattr_user.c | 1 + fs/jffs2/acl.c | 3 +- fs/jffs2/fs.c | 2 +- fs/jffs2/security.c | 1 + fs/jffs2/xattr_trusted.c | 1 + fs/jffs2/xattr_user.c | 1 + fs/jfs/acl.c | 2 +- fs/jfs/file.c | 2 +- fs/jfs/xattr.c | 2 + fs/kernfs/inode.c | 2 + fs/nfs/nfs4proc.c | 3 ++ fs/nfsd/nfs2acl.c | 6 ++- fs/nfsd/nfs3acl.c | 6 ++- fs/nfsd/nfs4acl.c | 5 ++- fs/ocfs2/acl.c | 3 +- fs/ocfs2/xattr.c | 3 ++ fs/orangefs/acl.c | 3 +- fs/orangefs/inode.c | 2 +- fs/orangefs/xattr.c | 1 + fs/overlayfs/super.c | 3 ++ fs/posix_acl.c | 79 +++++++++++++++++++++++++---------- fs/reiserfs/xattr_acl.c | 5 ++- fs/reiserfs/xattr_security.c | 3 +- fs/reiserfs/xattr_trusted.c | 3 +- fs/reiserfs/xattr_user.c | 3 +- fs/ubifs/xattr.c | 1 + fs/xattr.c | 14 ++++--- fs/xfs/xfs_acl.c | 3 +- fs/xfs/xfs_iops.c | 2 +- fs/xfs/xfs_xattr.c | 7 ++-- include/linux/capability.h | 3 +- include/linux/posix_acl.h | 11 +++-- include/linux/posix_acl_xattr.h | 12 ++++-- include/linux/xattr.h | 3 +- mm/shmem.c | 3 +- net/socket.c | 1 + security/commoncap.c | 45 ++++++++++++++++---- 72 files changed, 238 insertions(+), 85 deletions(-) (limited to 'fs') diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index c0f2c7586531..b7dcc86c92a4 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -126,9 +126,10 @@ prototypes:: int (*get)(const struct xattr_handler *handler, struct dentry *dentry, struct inode *inode, const char *name, void *buffer, size_t size); - int (*set)(const struct xattr_handler *handler, struct dentry *dentry, - struct inode *inode, const char *name, const void *buffer, - size_t size, int flags); + int (*set)(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, + struct dentry *dentry, struct inode *inode, const char *name, + const void *buffer, size_t size, int flags); locking rules: all may block diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst index 867036aa90b8..de1dcec3b5b8 100644 --- a/Documentation/filesystems/porting.rst +++ b/Documentation/filesystems/porting.rst @@ -717,6 +717,8 @@ be removed. Switch while you still can; the old one won't stay. **mandatory** ->setxattr() and xattr_handler.set() get dentry and inode passed separately. +The xattr_handler.set() gets passed the user namespace of the mount the inode +is seen from so filesystems can idmap the i_uid and i_gid accordingly. dentry might be yet to be attached to inode, so do _not_ use its ->d_inode in the instances. Rationale: !@#!@# security_d_instantiate() needs to be called before we attach dentry to inode and !@#!@##!@$!$#!@#$!@$!@$ smack diff --git a/fs/9p/acl.c b/fs/9p/acl.c index d77b28e8d57a..1c14f18a6ec9 100644 --- a/fs/9p/acl.c +++ b/fs/9p/acl.c @@ -239,6 +239,7 @@ static int v9fs_xattr_get_acl(const struct xattr_handler *handler, } static int v9fs_xattr_set_acl(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) @@ -279,7 +280,8 @@ static int v9fs_xattr_set_acl(const struct xattr_handler *handler, struct iattr iattr = { 0 }; struct posix_acl *old_acl = acl; - retval = posix_acl_update_mode(inode, &iattr.ia_mode, &acl); + retval = posix_acl_update_mode(mnt_userns, inode, + &iattr.ia_mode, &acl); if (retval) goto err_out; if (!acl) { diff --git a/fs/9p/xattr.c b/fs/9p/xattr.c index 87217dd0433e..ee331845e2c7 100644 --- a/fs/9p/xattr.c +++ b/fs/9p/xattr.c @@ -157,6 +157,7 @@ static int v9fs_xattr_handler_get(const struct xattr_handler *handler, } static int v9fs_xattr_handler_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/afs/xattr.c b/fs/afs/xattr.c index 95c573dcda11..c629caae5002 100644 --- a/fs/afs/xattr.c +++ b/fs/afs/xattr.c @@ -120,6 +120,7 @@ static const struct afs_operation_ops afs_store_acl_operation = { * Set a file's AFS3 ACL. */ static int afs_xattr_set_acl(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) @@ -248,6 +249,7 @@ static const struct afs_operation_ops yfs_store_opaque_acl2_operation = { * Set a file's YFS ACL. */ static int afs_xattr_set_yfs(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) diff --git a/fs/btrfs/acl.c b/fs/btrfs/acl.c index a0af1b952c4d..d12a5a8730a8 100644 --- a/fs/btrfs/acl.c +++ b/fs/btrfs/acl.c @@ -113,7 +113,8 @@ int btrfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) umode_t old_mode = inode->i_mode; if (type == ACL_TYPE_ACCESS && acl) { - ret = posix_acl_update_mode(inode, &inode->i_mode, &acl); + ret = posix_acl_update_mode(&init_user_ns, inode, + &inode->i_mode, &acl); if (ret) return ret; } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 792191a8705b..6c18fb1a25af 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -5070,7 +5070,8 @@ static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) err = btrfs_dirty_inode(inode); if (!err && attr->ia_valid & ATTR_MODE) - err = posix_acl_chmod(inode, inode->i_mode); + err = posix_acl_chmod(&init_user_ns, inode, + inode->i_mode); } return err; diff --git a/fs/btrfs/xattr.c b/fs/btrfs/xattr.c index af6246f36a9e..b025102e435f 100644 --- a/fs/btrfs/xattr.c +++ b/fs/btrfs/xattr.c @@ -362,6 +362,7 @@ static int btrfs_xattr_handler_get(const struct xattr_handler *handler, } static int btrfs_xattr_handler_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) @@ -371,6 +372,7 @@ static int btrfs_xattr_handler_set(const struct xattr_handler *handler, } static int btrfs_xattr_handler_set_prop(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/ceph/acl.c b/fs/ceph/acl.c index e0465741c591..52a01ddbc4ac 100644 --- a/fs/ceph/acl.c +++ b/fs/ceph/acl.c @@ -100,7 +100,8 @@ int ceph_set_acl(struct inode *inode, struct posix_acl *acl, int type) case ACL_TYPE_ACCESS: name = XATTR_NAME_POSIX_ACL_ACCESS; if (acl) { - ret = posix_acl_update_mode(inode, &new_mode, &acl); + ret = posix_acl_update_mode(&init_user_ns, inode, + &new_mode, &acl); if (ret) goto out; } diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 285d3baca27e..145e26a4ddbb 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -2262,7 +2262,7 @@ int ceph_setattr(struct dentry *dentry, struct iattr *attr) err = __ceph_setattr(inode, attr); if (err >= 0 && (attr->ia_valid & ATTR_MODE)) - err = posix_acl_chmod(inode, attr->ia_mode); + err = posix_acl_chmod(&init_user_ns, inode, attr->ia_mode); return err; } diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c index 24997982de01..02f59bcb4f27 100644 --- a/fs/ceph/xattr.c +++ b/fs/ceph/xattr.c @@ -1238,6 +1238,7 @@ static int ceph_get_xattr_handler(const struct xattr_handler *handler, } static int ceph_set_xattr_handler(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/cifs/xattr.c b/fs/cifs/xattr.c index 6b658a1172ef..41a611e76bb7 100644 --- a/fs/cifs/xattr.c +++ b/fs/cifs/xattr.c @@ -101,6 +101,7 @@ static int cifs_creation_time_set(unsigned int xid, struct cifs_tcon *pTcon, } static int cifs_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index d3ea0c57b075..ac6472a82567 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -1133,6 +1133,7 @@ static int ecryptfs_xattr_get(const struct xattr_handler *handler, } static int ecryptfs_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/ext2/acl.c b/fs/ext2/acl.c index cf4c77f8dd08..9031f7df2d48 100644 --- a/fs/ext2/acl.c +++ b/fs/ext2/acl.c @@ -223,7 +223,8 @@ ext2_set_acl(struct inode *inode, struct posix_acl *acl, int type) umode_t mode = inode->i_mode; if (type == ACL_TYPE_ACCESS && acl) { - error = posix_acl_update_mode(inode, &mode, &acl); + error = posix_acl_update_mode(&init_user_ns, inode, &mode, + &acl); if (error) return error; update_mode = 1; diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 06c0cf28c1a0..9de813635d8d 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -1691,7 +1691,7 @@ int ext2_setattr(struct dentry *dentry, struct iattr *iattr) } setattr_copy(&init_user_ns, inode, iattr); if (iattr->ia_valid & ATTR_MODE) - error = posix_acl_chmod(inode, inode->i_mode); + error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); mark_inode_dirty(inode); return error; diff --git a/fs/ext2/xattr_security.c b/fs/ext2/xattr_security.c index 9a682e440acb..ebade1f52451 100644 --- a/fs/ext2/xattr_security.c +++ b/fs/ext2/xattr_security.c @@ -19,6 +19,7 @@ ext2_xattr_security_get(const struct xattr_handler *handler, static int ext2_xattr_security_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/ext2/xattr_trusted.c b/fs/ext2/xattr_trusted.c index 49add1107850..18a87d5dd1ab 100644 --- a/fs/ext2/xattr_trusted.c +++ b/fs/ext2/xattr_trusted.c @@ -26,6 +26,7 @@ ext2_xattr_trusted_get(const struct xattr_handler *handler, static int ext2_xattr_trusted_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/ext2/xattr_user.c b/fs/ext2/xattr_user.c index c243a3b4d69d..58092449f8ff 100644 --- a/fs/ext2/xattr_user.c +++ b/fs/ext2/xattr_user.c @@ -30,6 +30,7 @@ ext2_xattr_user_get(const struct xattr_handler *handler, static int ext2_xattr_user_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/ext4/acl.c b/fs/ext4/acl.c index 68aaed48315f..7b0fb66bc04d 100644 --- a/fs/ext4/acl.c +++ b/fs/ext4/acl.c @@ -245,7 +245,8 @@ retry: ext4_fc_start_update(inode); if ((type == ACL_TYPE_ACCESS) && acl) { - error = posix_acl_update_mode(inode, &mode, &acl); + error = posix_acl_update_mode(&init_user_ns, inode, &mode, + &acl); if (error) goto out_stop; if (mode != inode->i_mode) diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 8edfa3e226e6..24ea5851e90a 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -5524,7 +5524,7 @@ out_mmap_sem: ext4_orphan_del(NULL, inode); if (!error && (ia_valid & ATTR_MODE)) - rc = posix_acl_chmod(inode, inode->i_mode); + rc = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); err_out: if (error) diff --git a/fs/ext4/xattr_hurd.c b/fs/ext4/xattr_hurd.c index 8cfa74a56361..c78df5790377 100644 --- a/fs/ext4/xattr_hurd.c +++ b/fs/ext4/xattr_hurd.c @@ -32,6 +32,7 @@ ext4_xattr_hurd_get(const struct xattr_handler *handler, static int ext4_xattr_hurd_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/ext4/xattr_security.c b/fs/ext4/xattr_security.c index 197a9d8a15ef..8213f66f7b2d 100644 --- a/fs/ext4/xattr_security.c +++ b/fs/ext4/xattr_security.c @@ -23,6 +23,7 @@ ext4_xattr_security_get(const struct xattr_handler *handler, static int ext4_xattr_security_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/ext4/xattr_trusted.c b/fs/ext4/xattr_trusted.c index e9389e5d75c3..7c21ffb26d25 100644 --- a/fs/ext4/xattr_trusted.c +++ b/fs/ext4/xattr_trusted.c @@ -30,6 +30,7 @@ ext4_xattr_trusted_get(const struct xattr_handler *handler, static int ext4_xattr_trusted_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/ext4/xattr_user.c b/fs/ext4/xattr_user.c index d4546184b34b..2fe7ff0a479c 100644 --- a/fs/ext4/xattr_user.c +++ b/fs/ext4/xattr_user.c @@ -31,6 +31,7 @@ ext4_xattr_user_get(const struct xattr_handler *handler, static int ext4_xattr_user_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c index 1e5e9b1136ee..6a95bf28f602 100644 --- a/fs/f2fs/acl.c +++ b/fs/f2fs/acl.c @@ -213,7 +213,8 @@ static int __f2fs_set_acl(struct inode *inode, int type, case ACL_TYPE_ACCESS: name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS; if (acl && !ipage) { - error = posix_acl_update_mode(inode, &mode, &acl); + error = posix_acl_update_mode(&init_user_ns, inode, + &mode, &acl); if (error) return error; set_acl_inode(inode, mode); diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 90d7b89176de..6ccdfe0606d9 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -831,8 +831,8 @@ int f2fs_getattr(const struct path *path, struct kstat *stat, } #ifdef CONFIG_F2FS_FS_POSIX_ACL -static void __setattr_copy(struct user_namespace *mnt_userns, struct inode *inode, - const struct iattr *attr) +static void __setattr_copy(struct user_namespace *mnt_userns, + struct inode *inode, const struct iattr *attr) { unsigned int ia_valid = attr->ia_valid; @@ -950,7 +950,8 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) __setattr_copy(&init_user_ns, inode, attr); if (attr->ia_valid & ATTR_MODE) { - err = posix_acl_chmod(inode, f2fs_get_inode_mode(inode)); + err = posix_acl_chmod(&init_user_ns, inode, + f2fs_get_inode_mode(inode)); if (err || is_inode_flag_set(inode, FI_ACL_MODE)) { inode->i_mode = F2FS_I(inode)->i_acl_mode; clear_inode_flag(inode, FI_ACL_MODE); diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index d772bf13a814..10081bf74324 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -64,6 +64,7 @@ static int f2fs_xattr_generic_get(const struct xattr_handler *handler, } static int f2fs_xattr_generic_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) @@ -107,6 +108,7 @@ static int f2fs_xattr_advise_get(const struct xattr_handler *handler, } static int f2fs_xattr_advise_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/fuse/xattr.c b/fs/fuse/xattr.c index cdea18de94f7..1a7d7ace54e1 100644 --- a/fs/fuse/xattr.c +++ b/fs/fuse/xattr.c @@ -188,6 +188,7 @@ static int fuse_xattr_get(const struct xattr_handler *handler, } static int fuse_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) @@ -214,6 +215,7 @@ static int no_xattr_get(const struct xattr_handler *handler, } static int no_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *nodee, const char *name, const void *value, size_t size, int flags) diff --git a/fs/gfs2/acl.c b/fs/gfs2/acl.c index 2e939f5fe751..ce88ef29eef0 100644 --- a/fs/gfs2/acl.c +++ b/fs/gfs2/acl.c @@ -130,7 +130,7 @@ int gfs2_set_acl(struct inode *inode, struct posix_acl *acl, int type) mode = inode->i_mode; if (type == ACL_TYPE_ACCESS && acl) { - ret = posix_acl_update_mode(inode, &mode, &acl); + ret = posix_acl_update_mode(&init_user_ns, inode, &mode, &acl); if (ret) goto unlock; } diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 59c25181d108..728405d15a05 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -1993,7 +1993,8 @@ static int gfs2_setattr(struct dentry *dentry, struct iattr *attr) else { error = gfs2_setattr_simple(inode, attr); if (!error && attr->ia_valid & ATTR_MODE) - error = posix_acl_chmod(inode, inode->i_mode); + error = posix_acl_chmod(&init_user_ns, inode, + inode->i_mode); } error: diff --git a/fs/gfs2/xattr.c b/fs/gfs2/xattr.c index 9d7667bc4292..13969a813410 100644 --- a/fs/gfs2/xattr.c +++ b/fs/gfs2/xattr.c @@ -1214,6 +1214,7 @@ int __gfs2_xattr_set(struct inode *inode, const char *name, } static int gfs2_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/hfs/attr.c b/fs/hfs/attr.c index 74fa62643136..2bd54efaf416 100644 --- a/fs/hfs/attr.c +++ b/fs/hfs/attr.c @@ -121,6 +121,7 @@ static int hfs_xattr_get(const struct xattr_handler *handler, } static int hfs_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/hfsplus/xattr.c b/fs/hfsplus/xattr.c index bb0b27d88e50..4d169c5a2673 100644 --- a/fs/hfsplus/xattr.c +++ b/fs/hfsplus/xattr.c @@ -858,6 +858,7 @@ static int hfsplus_osx_getxattr(const struct xattr_handler *handler, } static int hfsplus_osx_setxattr(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) diff --git a/fs/hfsplus/xattr_security.c b/fs/hfsplus/xattr_security.c index cfbe6a3bfb1e..c1c7a16cbf21 100644 --- a/fs/hfsplus/xattr_security.c +++ b/fs/hfsplus/xattr_security.c @@ -23,6 +23,7 @@ static int hfsplus_security_getxattr(const struct xattr_handler *handler, } static int hfsplus_security_setxattr(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) diff --git a/fs/hfsplus/xattr_trusted.c b/fs/hfsplus/xattr_trusted.c index fbad91e1dada..e150372ec564 100644 --- a/fs/hfsplus/xattr_trusted.c +++ b/fs/hfsplus/xattr_trusted.c @@ -22,6 +22,7 @@ static int hfsplus_trusted_getxattr(const struct xattr_handler *handler, } static int hfsplus_trusted_setxattr(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) diff --git a/fs/hfsplus/xattr_user.c b/fs/hfsplus/xattr_user.c index 74d19faf255e..a6b60b153916 100644 --- a/fs/hfsplus/xattr_user.c +++ b/fs/hfsplus/xattr_user.c @@ -22,6 +22,7 @@ static int hfsplus_user_getxattr(const struct xattr_handler *handler, } static int hfsplus_user_setxattr(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) diff --git a/fs/jffs2/acl.c b/fs/jffs2/acl.c index 093ffbd82395..5f27ac593479 100644 --- a/fs/jffs2/acl.c +++ b/fs/jffs2/acl.c @@ -236,7 +236,8 @@ int jffs2_set_acl(struct inode *inode, struct posix_acl *acl, int type) if (acl) { umode_t mode; - rc = posix_acl_update_mode(inode, &mode, &acl); + rc = posix_acl_update_mode(&init_user_ns, inode, &mode, + &acl); if (rc) return rc; if (inode->i_mode != mode) { diff --git a/fs/jffs2/fs.c b/fs/jffs2/fs.c index 67993808f4da..ee9f51bab4c6 100644 --- a/fs/jffs2/fs.c +++ b/fs/jffs2/fs.c @@ -201,7 +201,7 @@ int jffs2_setattr(struct dentry *dentry, struct iattr *iattr) rc = jffs2_do_setattr(inode, iattr); if (!rc && (iattr->ia_valid & ATTR_MODE)) - rc = posix_acl_chmod(inode, inode->i_mode); + rc = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); return rc; } diff --git a/fs/jffs2/security.c b/fs/jffs2/security.c index c2332e30f218..aef5522551db 100644 --- a/fs/jffs2/security.c +++ b/fs/jffs2/security.c @@ -57,6 +57,7 @@ static int jffs2_security_getxattr(const struct xattr_handler *handler, } static int jffs2_security_setxattr(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) diff --git a/fs/jffs2/xattr_trusted.c b/fs/jffs2/xattr_trusted.c index 5d6030826c52..cc3f24883e7d 100644 --- a/fs/jffs2/xattr_trusted.c +++ b/fs/jffs2/xattr_trusted.c @@ -25,6 +25,7 @@ static int jffs2_trusted_getxattr(const struct xattr_handler *handler, } static int jffs2_trusted_setxattr(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) diff --git a/fs/jffs2/xattr_user.c b/fs/jffs2/xattr_user.c index 9d027b4abcf9..fb945977c013 100644 --- a/fs/jffs2/xattr_user.c +++ b/fs/jffs2/xattr_user.c @@ -25,6 +25,7 @@ static int jffs2_user_getxattr(const struct xattr_handler *handler, } static int jffs2_user_setxattr(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) diff --git a/fs/jfs/acl.c b/fs/jfs/acl.c index 92cc0ac2d1fc..cf79a34bfada 100644 --- a/fs/jfs/acl.c +++ b/fs/jfs/acl.c @@ -101,7 +101,7 @@ int jfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) tid = txBegin(inode->i_sb, 0); mutex_lock(&JFS_IP(inode)->commit_mutex); if (type == ACL_TYPE_ACCESS && acl) { - rc = posix_acl_update_mode(inode, &mode, &acl); + rc = posix_acl_update_mode(&init_user_ns, inode, &mode, &acl); if (rc) goto end_tx; if (mode != inode->i_mode) diff --git a/fs/jfs/file.c b/fs/jfs/file.c index ff49876e9c9b..61c3b0c1fbf6 100644 --- a/fs/jfs/file.c +++ b/fs/jfs/file.c @@ -122,7 +122,7 @@ int jfs_setattr(struct dentry *dentry, struct iattr *iattr) mark_inode_dirty(inode); if (iattr->ia_valid & ATTR_MODE) - rc = posix_acl_chmod(inode, inode->i_mode); + rc = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); return rc; } diff --git a/fs/jfs/xattr.c b/fs/jfs/xattr.c index db41e7803163..f9273f6901c8 100644 --- a/fs/jfs/xattr.c +++ b/fs/jfs/xattr.c @@ -932,6 +932,7 @@ static int jfs_xattr_get(const struct xattr_handler *handler, } static int jfs_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) @@ -950,6 +951,7 @@ static int jfs_xattr_get_os2(const struct xattr_handler *handler, } static int jfs_xattr_set_os2(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c index 86bd4c593b78..7e44052b42e1 100644 --- a/fs/kernfs/inode.c +++ b/fs/kernfs/inode.c @@ -319,6 +319,7 @@ static int kernfs_vfs_xattr_get(const struct xattr_handler *handler, } static int kernfs_vfs_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *suffix, const void *value, size_t size, int flags) @@ -385,6 +386,7 @@ static int kernfs_vfs_user_xattr_rm(struct kernfs_node *kn, } static int kernfs_vfs_user_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *suffix, const void *value, size_t size, int flags) diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 2f4679a62712..a07530cf673d 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -7491,6 +7491,7 @@ nfs4_release_lockowner(struct nfs_server *server, struct nfs4_lock_state *lsp) #define XATTR_NAME_NFSV4_ACL "system.nfs4_acl" static int nfs4_xattr_set_nfs4_acl(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *key, const void *buf, size_t buflen, int flags) @@ -7513,6 +7514,7 @@ static bool nfs4_xattr_list_nfs4_acl(struct dentry *dentry) #ifdef CONFIG_NFS_V4_SECURITY_LABEL static int nfs4_xattr_set_nfs4_label(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *key, const void *buf, size_t buflen, int flags) @@ -7563,6 +7565,7 @@ nfs4_listxattr_nfs4_label(struct inode *inode, char *list, size_t list_len) #ifdef CONFIG_NFS_V4_2 static int nfs4_xattr_set_nfs4_user(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *key, const void *buf, size_t buflen, int flags) diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index b0f66604532a..b83f222558e3 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -113,10 +113,12 @@ static __be32 nfsacld_proc_setacl(struct svc_rqst *rqstp) fh_lock(fh); - error = set_posix_acl(inode, ACL_TYPE_ACCESS, argp->acl_access); + error = set_posix_acl(&init_user_ns, inode, ACL_TYPE_ACCESS, + argp->acl_access); if (error) goto out_drop_lock; - error = set_posix_acl(inode, ACL_TYPE_DEFAULT, argp->acl_default); + error = set_posix_acl(&init_user_ns, inode, ACL_TYPE_DEFAULT, + argp->acl_default); if (error) goto out_drop_lock; diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index 7c30876a31a1..f18ec7e8094d 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -103,10 +103,12 @@ static __be32 nfsd3_proc_setacl(struct svc_rqst *rqstp) fh_lock(fh); - error = set_posix_acl(inode, ACL_TYPE_ACCESS, argp->acl_access); + error = set_posix_acl(&init_user_ns, inode, ACL_TYPE_ACCESS, + argp->acl_access); if (error) goto out_drop_lock; - error = set_posix_acl(inode, ACL_TYPE_DEFAULT, argp->acl_default); + error = set_posix_acl(&init_user_ns, inode, ACL_TYPE_DEFAULT, + argp->acl_default); out_drop_lock: fh_unlock(fh); diff --git a/fs/nfsd/nfs4acl.c b/fs/nfsd/nfs4acl.c index 71292a0d6f09..eaa3a0cf38f1 100644 --- a/fs/nfsd/nfs4acl.c +++ b/fs/nfsd/nfs4acl.c @@ -781,12 +781,13 @@ nfsd4_set_nfs4_acl(struct svc_rqst *rqstp, struct svc_fh *fhp, fh_lock(fhp); - host_error = set_posix_acl(inode, ACL_TYPE_ACCESS, pacl); + host_error = set_posix_acl(&init_user_ns, inode, ACL_TYPE_ACCESS, pacl); if (host_error < 0) goto out_drop_lock; if (S_ISDIR(inode->i_mode)) { - host_error = set_posix_acl(inode, ACL_TYPE_DEFAULT, dpacl); + host_error = set_posix_acl(&init_user_ns, inode, + ACL_TYPE_DEFAULT, dpacl); } out_drop_lock: diff --git a/fs/ocfs2/acl.c b/fs/ocfs2/acl.c index 7b07f5df3a29..990756cee4bd 100644 --- a/fs/ocfs2/acl.c +++ b/fs/ocfs2/acl.c @@ -274,7 +274,8 @@ int ocfs2_iop_set_acl(struct inode *inode, struct posix_acl *acl, int type) if (type == ACL_TYPE_ACCESS && acl) { umode_t mode; - status = posix_acl_update_mode(inode, &mode, &acl); + status = posix_acl_update_mode(&init_user_ns, inode, &mode, + &acl); if (status) goto unlock; diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index 9ccd19d8f7b1..36ae47a4aef6 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -7249,6 +7249,7 @@ static int ocfs2_xattr_security_get(const struct xattr_handler *handler, } static int ocfs2_xattr_security_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) @@ -7321,6 +7322,7 @@ static int ocfs2_xattr_trusted_get(const struct xattr_handler *handler, } static int ocfs2_xattr_trusted_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) @@ -7351,6 +7353,7 @@ static int ocfs2_xattr_user_get(const struct xattr_handler *handler, } static int ocfs2_xattr_user_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/orangefs/acl.c b/fs/orangefs/acl.c index a25e6c890975..628921952d16 100644 --- a/fs/orangefs/acl.c +++ b/fs/orangefs/acl.c @@ -132,7 +132,8 @@ int orangefs_set_acl(struct inode *inode, struct posix_acl *acl, int type) * and "mode" to the new desired value. It is up to * us to propagate the new mode back to the server... */ - error = posix_acl_update_mode(inode, &iattr.ia_mode, &acl); + error = posix_acl_update_mode(&init_user_ns, inode, + &iattr.ia_mode, &acl); if (error) { gossip_err("%s: posix_acl_update_mode err: %d\n", __func__, diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c index 8ac9491ceb9a..563fe9ab8eb2 100644 --- a/fs/orangefs/inode.c +++ b/fs/orangefs/inode.c @@ -861,7 +861,7 @@ again: if (iattr->ia_valid & ATTR_MODE) /* change mod on a file that has ACLs */ - ret = posix_acl_chmod(inode, inode->i_mode); + ret = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); ret = 0; out: diff --git a/fs/orangefs/xattr.c b/fs/orangefs/xattr.c index bdc285aea360..9a5b757fbd2f 100644 --- a/fs/orangefs/xattr.c +++ b/fs/orangefs/xattr.c @@ -526,6 +526,7 @@ out_unlock: } static int orangefs_xattr_set_default(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 39b2e9aa0e5b..e24c995c5fd4 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -980,6 +980,7 @@ ovl_posix_acl_xattr_get(const struct xattr_handler *handler, static int __maybe_unused ovl_posix_acl_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) @@ -1044,6 +1045,7 @@ static int ovl_own_xattr_get(const struct xattr_handler *handler, } static int ovl_own_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) @@ -1059,6 +1061,7 @@ static int ovl_other_xattr_get(const struct xattr_handler *handler, } static int ovl_other_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 9ce8214bfdac..d31b60f5d40d 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -558,8 +558,22 @@ __posix_acl_chmod(struct posix_acl **acl, gfp_t gfp, umode_t mode) } EXPORT_SYMBOL(__posix_acl_chmod); +/** + * posix_acl_chmod - chmod a posix acl + * + * @mnt_userns: user namespace of the mount @inode was found from + * @inode: inode to check permissions on + * @mode: the new mode of @inode + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. + */ int -posix_acl_chmod(struct inode *inode, umode_t mode) + posix_acl_chmod(struct user_namespace *mnt_userns, struct inode *inode, + umode_t mode) { struct posix_acl *acl; int ret = 0; @@ -638,9 +652,10 @@ EXPORT_SYMBOL_GPL(posix_acl_create); /** * posix_acl_update_mode - update mode in set_acl - * @inode: target inode - * @mode_p: mode (pointer) for update - * @acl: acl pointer + * @mnt_userns: user namespace of the mount @inode was found from + * @inode: target inode + * @mode_p: mode (pointer) for update + * @acl: acl pointer * * Update the file mode when setting an ACL: compute the new file permission * bits based on the ACL. In addition, if the ACL is equivalent to the new @@ -649,9 +664,16 @@ EXPORT_SYMBOL_GPL(posix_acl_create); * As with chmod, clear the setgid bit if the caller is not in the owning group * or capable of CAP_FSETID (see inode_change_ok). * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. + * * Called from set_acl inode operations. */ -int posix_acl_update_mode(struct inode *inode, umode_t *mode_p, +int posix_acl_update_mode(struct user_namespace *mnt_userns, + struct inode *inode, umode_t *mode_p, struct posix_acl **acl) { umode_t mode = inode->i_mode; @@ -662,8 +684,8 @@ int posix_acl_update_mode(struct inode *inode, umode_t *mode_p, return error; if (error == 0) *acl = NULL; - if (!in_group_p(inode->i_gid) && - !capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_FSETID)) + if (!in_group_p(i_gid_into_mnt(mnt_userns, inode)) && + !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FSETID)) mode &= ~S_ISGID; *mode_p = mode; return 0; @@ -675,7 +697,8 @@ EXPORT_SYMBOL(posix_acl_update_mode); */ static void posix_acl_fix_xattr_userns( struct user_namespace *to, struct user_namespace *from, - void *value, size_t size) + struct user_namespace *mnt_userns, + void *value, size_t size, bool from_user) { struct posix_acl_xattr_header *header = value; struct posix_acl_xattr_entry *entry = (void *)(header + 1), *end; @@ -700,10 +723,18 @@ static void posix_acl_fix_xattr_userns( switch(le16_to_cpu(entry->e_tag)) { case ACL_USER: uid = make_kuid(from, le32_to_cpu(entry->e_id)); + if (from_user) + uid = kuid_from_mnt(mnt_userns, uid); + else + uid = kuid_into_mnt(mnt_userns, uid); entry->e_id = cpu_to_le32(from_kuid(to, uid)); break; case ACL_GROUP: gid = make_kgid(from, le32_to_cpu(entry->e_id)); + if (from_user) + gid = kgid_from_mnt(mnt_userns, gid); + else + gid = kgid_into_mnt(mnt_userns, gid); entry->e_id = cpu_to_le32(from_kgid(to, gid)); break; default: @@ -712,20 +743,24 @@ static void posix_acl_fix_xattr_userns( } } -void posix_acl_fix_xattr_from_user(void *value, size_t size) +void posix_acl_fix_xattr_from_user(struct user_namespace *mnt_userns, + void *value, size_t size) { struct user_namespace *user_ns = current_user_ns(); - if (user_ns == &init_user_ns) + if ((user_ns == &init_user_ns) && (mnt_userns == &init_user_ns)) return; - posix_acl_fix_xattr_userns(&init_user_ns, user_ns, value, size); + posix_acl_fix_xattr_userns(&init_user_ns, user_ns, mnt_userns, value, + size, true); } -void posix_acl_fix_xattr_to_user(void *value, size_t size) +void posix_acl_fix_xattr_to_user(struct user_namespace *mnt_userns, + void *value, size_t size) { struct user_namespace *user_ns = current_user_ns(); - if (user_ns == &init_user_ns) + if ((user_ns == &init_user_ns) && (mnt_userns == &init_user_ns)) return; - posix_acl_fix_xattr_userns(user_ns, &init_user_ns, value, size); + posix_acl_fix_xattr_userns(user_ns, &init_user_ns, mnt_userns, value, + size, false); } /* @@ -865,7 +900,8 @@ posix_acl_xattr_get(const struct xattr_handler *handler, } int -set_posix_acl(struct inode *inode, int type, struct posix_acl *acl) +set_posix_acl(struct user_namespace *mnt_userns, struct inode *inode, + int type, struct posix_acl *acl) { if (!IS_POSIXACL(inode)) return -EOPNOTSUPP; @@ -874,7 +910,7 @@ set_posix_acl(struct inode *inode, int type, struct posix_acl *acl) if (type == ACL_TYPE_DEFAULT && !S_ISDIR(inode->i_mode)) return acl ? -EACCES : 0; - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(mnt_userns, inode)) return -EPERM; if (acl) { @@ -888,9 +924,10 @@ EXPORT_SYMBOL(set_posix_acl); static int posix_acl_xattr_set(const struct xattr_handler *handler, - struct dentry *unused, struct inode *inode, - const char *name, const void *value, - size_t size, int flags) + struct user_namespace *mnt_userns, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, size_t size, + int flags) { struct posix_acl *acl = NULL; int ret; @@ -900,7 +937,7 @@ posix_acl_xattr_set(const struct xattr_handler *handler, if (IS_ERR(acl)) return PTR_ERR(acl); } - ret = set_posix_acl(inode, handler->flags, acl); + ret = set_posix_acl(mnt_userns, inode, handler->flags, acl); posix_acl_release(acl); return ret; } @@ -934,7 +971,7 @@ int simple_set_acl(struct inode *inode, struct posix_acl *acl, int type) int error; if (type == ACL_TYPE_ACCESS) { - error = posix_acl_update_mode(inode, + error = posix_acl_update_mode(&init_user_ns, inode, &inode->i_mode, &acl); if (error) return error; diff --git a/fs/reiserfs/xattr_acl.c b/fs/reiserfs/xattr_acl.c index ccd40df6eb45..4bf976bc7bad 100644 --- a/fs/reiserfs/xattr_acl.c +++ b/fs/reiserfs/xattr_acl.c @@ -40,7 +40,8 @@ reiserfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) reiserfs_write_unlock(inode->i_sb); if (error == 0) { if (type == ACL_TYPE_ACCESS && acl) { - error = posix_acl_update_mode(inode, &mode, &acl); + error = posix_acl_update_mode(&init_user_ns, inode, + &mode, &acl); if (error) goto unlock; update_mode = 1; @@ -399,5 +400,5 @@ int reiserfs_acl_chmod(struct inode *inode) !reiserfs_posixacl(inode->i_sb)) return 0; - return posix_acl_chmod(inode, inode->i_mode); + return posix_acl_chmod(&init_user_ns, inode, inode->i_mode); } diff --git a/fs/reiserfs/xattr_security.c b/fs/reiserfs/xattr_security.c index 20be9a0e5870..8965c8e5e172 100644 --- a/fs/reiserfs/xattr_security.c +++ b/fs/reiserfs/xattr_security.c @@ -21,7 +21,8 @@ security_get(const struct xattr_handler *handler, struct dentry *unused, } static int -security_set(const struct xattr_handler *handler, struct dentry *unused, +security_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) { diff --git a/fs/reiserfs/xattr_trusted.c b/fs/reiserfs/xattr_trusted.c index 5ed48da3d02b..d853cea2afcd 100644 --- a/fs/reiserfs/xattr_trusted.c +++ b/fs/reiserfs/xattr_trusted.c @@ -20,7 +20,8 @@ trusted_get(const struct xattr_handler *handler, struct dentry *unused, } static int -trusted_set(const struct xattr_handler *handler, struct dentry *unused, +trusted_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) { diff --git a/fs/reiserfs/xattr_user.c b/fs/reiserfs/xattr_user.c index a573ca45bacc..65d9cd10a5ea 100644 --- a/fs/reiserfs/xattr_user.c +++ b/fs/reiserfs/xattr_user.c @@ -18,7 +18,8 @@ user_get(const struct xattr_handler *handler, struct dentry *unused, } static int -user_set(const struct xattr_handler *handler, struct dentry *unused, +user_set(const struct xattr_handler *handler, struct user_namespace *mnt_userns, + struct dentry *unused, struct inode *inode, const char *name, const void *buffer, size_t size, int flags) { diff --git a/fs/ubifs/xattr.c b/fs/ubifs/xattr.c index a0b9b349efe6..8f4135c22ab6 100644 --- a/fs/ubifs/xattr.c +++ b/fs/ubifs/xattr.c @@ -681,6 +681,7 @@ static int xattr_get(const struct xattr_handler *handler, } static int xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/fs/xattr.c b/fs/xattr.c index c669922e1bde..d777025121e0 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -175,7 +175,8 @@ __vfs_setxattr(struct dentry *dentry, struct inode *inode, const char *name, return -EOPNOTSUPP; if (size == 0) value = ""; /* empty EA, do not remove */ - return handler->set(handler, dentry, inode, name, value, size, flags); + return handler->set(handler, &init_user_ns, dentry, inode, name, value, + size, flags); } EXPORT_SYMBOL(__vfs_setxattr); @@ -281,7 +282,7 @@ vfs_setxattr(struct dentry *dentry, const char *name, const void *value, int error; if (size && strcmp(name, XATTR_NAME_CAPS) == 0) { - error = cap_convert_nscap(dentry, &value, size); + error = cap_convert_nscap(&init_user_ns, dentry, &value, size); if (error < 0) return error; size = error; @@ -450,7 +451,8 @@ __vfs_removexattr(struct dentry *dentry, const char *name) return PTR_ERR(handler); if (!handler->set) return -EOPNOTSUPP; - return handler->set(handler, dentry, inode, name, NULL, 0, XATTR_REPLACE); + return handler->set(handler, &init_user_ns, dentry, inode, name, NULL, + 0, XATTR_REPLACE); } EXPORT_SYMBOL(__vfs_removexattr); @@ -548,7 +550,8 @@ setxattr(struct dentry *d, const char __user *name, const void __user *value, } if ((strcmp(kname, XATTR_NAME_POSIX_ACL_ACCESS) == 0) || (strcmp(kname, XATTR_NAME_POSIX_ACL_DEFAULT) == 0)) - posix_acl_fix_xattr_from_user(kvalue, size); + posix_acl_fix_xattr_from_user(&init_user_ns, kvalue, + size); } error = vfs_setxattr(d, kname, kvalue, size, flags); @@ -642,7 +645,8 @@ getxattr(struct dentry *d, const char __user *name, void __user *value, if (error > 0) { if ((strcmp(kname, XATTR_NAME_POSIX_ACL_ACCESS) == 0) || (strcmp(kname, XATTR_NAME_POSIX_ACL_DEFAULT) == 0)) - posix_acl_fix_xattr_to_user(kvalue, error); + posix_acl_fix_xattr_to_user(&init_user_ns, kvalue, + error); if (size && copy_to_user(value, kvalue, error)) error = -EFAULT; } else if (error == -ERANGE && size >= XATTR_SIZE_MAX) { diff --git a/fs/xfs/xfs_acl.c b/fs/xfs/xfs_acl.c index 779cb73b3d00..368351298bd5 100644 --- a/fs/xfs/xfs_acl.c +++ b/fs/xfs/xfs_acl.c @@ -252,7 +252,8 @@ xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) return error; if (type == ACL_TYPE_ACCESS) { - error = posix_acl_update_mode(inode, &mode, &acl); + error = posix_acl_update_mode(&init_user_ns, inode, &mode, + &acl); if (error) return error; set_mode = true; diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 08a478d25122..26d22edef741 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -807,7 +807,7 @@ xfs_setattr_nonsize( * Posix ACL code seems to care about this issue either. */ if (mask & ATTR_MODE) { - error = posix_acl_chmod(inode, inode->i_mode); + error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); if (error) return error; } diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index bca48b308c02..12be32f66dc1 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -38,9 +38,10 @@ xfs_xattr_get(const struct xattr_handler *handler, struct dentry *unused, } static int -xfs_xattr_set(const struct xattr_handler *handler, struct dentry *unused, - struct inode *inode, const char *name, const void *value, - size_t size, int flags) +xfs_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, + struct inode *inode, const char *name, const void *value, + size_t size, int flags) { struct xfs_da_args args = { .dp = XFS_I(inode), diff --git a/include/linux/capability.h b/include/linux/capability.h index 62bfa3ed84d7..da21ef118b04 100644 --- a/include/linux/capability.h +++ b/include/linux/capability.h @@ -273,6 +273,7 @@ static inline bool checkpoint_restore_ns_capable(struct user_namespace *ns) /* audit system wants to get cap info from files as well */ extern int get_vfs_caps_from_disk(const struct dentry *dentry, struct cpu_vfs_cap_data *cpu_caps); -extern int cap_convert_nscap(struct dentry *dentry, const void **ivalue, size_t size); +int cap_convert_nscap(struct user_namespace *mnt_userns, struct dentry *dentry, + const void **ivalue, size_t size); #endif /* !_LINUX_CAPABILITY_H */ diff --git a/include/linux/posix_acl.h b/include/linux/posix_acl.h index 85fb4c0c650a..6dcd8b8f6ab5 100644 --- a/include/linux/posix_acl.h +++ b/include/linux/posix_acl.h @@ -69,13 +69,15 @@ extern int __posix_acl_create(struct posix_acl **, gfp_t, umode_t *); extern int __posix_acl_chmod(struct posix_acl **, gfp_t, umode_t); extern struct posix_acl *get_posix_acl(struct inode *, int); -extern int set_posix_acl(struct inode *, int, struct posix_acl *); +extern int set_posix_acl(struct user_namespace *, struct inode *, int, + struct posix_acl *); #ifdef CONFIG_FS_POSIX_ACL -extern int posix_acl_chmod(struct inode *, umode_t); +int posix_acl_chmod(struct user_namespace *, struct inode *, umode_t); extern int posix_acl_create(struct inode *, umode_t *, struct posix_acl **, struct posix_acl **); -extern int posix_acl_update_mode(struct inode *, umode_t *, struct posix_acl **); +int posix_acl_update_mode(struct user_namespace *, struct inode *, umode_t *, + struct posix_acl **); extern int simple_set_acl(struct inode *, struct posix_acl *, int); extern int simple_acl_create(struct inode *, struct inode *); @@ -95,7 +97,8 @@ static inline void cache_no_acl(struct inode *inode) inode->i_default_acl = NULL; } #else -static inline int posix_acl_chmod(struct inode *inode, umode_t mode) +static inline int posix_acl_chmod(struct user_namespace *mnt_userns, + struct inode *inode, umode_t mode) { return 0; } diff --git a/include/linux/posix_acl_xattr.h b/include/linux/posix_acl_xattr.h index 2387709991b5..060e8d203181 100644 --- a/include/linux/posix_acl_xattr.h +++ b/include/linux/posix_acl_xattr.h @@ -33,13 +33,17 @@ posix_acl_xattr_count(size_t size) } #ifdef CONFIG_FS_POSIX_ACL -void posix_acl_fix_xattr_from_user(void *value, size_t size); -void posix_acl_fix_xattr_to_user(void *value, size_t size); +void posix_acl_fix_xattr_from_user(struct user_namespace *mnt_userns, + void *value, size_t size); +void posix_acl_fix_xattr_to_user(struct user_namespace *mnt_userns, + void *value, size_t size); #else -static inline void posix_acl_fix_xattr_from_user(void *value, size_t size) +static inline void posix_acl_fix_xattr_from_user(struct user_namespace *mnt_userns, + void *value, size_t size) { } -static inline void posix_acl_fix_xattr_to_user(void *value, size_t size) +static inline void posix_acl_fix_xattr_to_user(struct user_namespace *mnt_userns, + void *value, size_t size) { } #endif diff --git a/include/linux/xattr.h b/include/linux/xattr.h index 10b4dc2709f0..260c9bcb0edb 100644 --- a/include/linux/xattr.h +++ b/include/linux/xattr.h @@ -34,7 +34,8 @@ struct xattr_handler { int (*get)(const struct xattr_handler *, struct dentry *dentry, struct inode *inode, const char *name, void *buffer, size_t size); - int (*set)(const struct xattr_handler *, struct dentry *dentry, + int (*set)(const struct xattr_handler *, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *name, const void *buffer, size_t size, int flags); }; diff --git a/mm/shmem.c b/mm/shmem.c index 1cb451e131ec..23b8e9c15a42 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1143,7 +1143,7 @@ static int shmem_setattr(struct dentry *dentry, struct iattr *attr) setattr_copy(&init_user_ns, inode, attr); if (attr->ia_valid & ATTR_MODE) - error = posix_acl_chmod(inode, inode->i_mode); + error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); return error; } @@ -3273,6 +3273,7 @@ static int shmem_xattr_handler_get(const struct xattr_handler *handler, } static int shmem_xattr_handler_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/net/socket.c b/net/socket.c index 33e8b6c4e1d3..c76703c6f480 100644 --- a/net/socket.c +++ b/net/socket.c @@ -334,6 +334,7 @@ static const struct xattr_handler sockfs_xattr_handler = { }; static int sockfs_security_xattr_set(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, struct dentry *dentry, struct inode *inode, const char *suffix, const void *value, size_t size, int flags) diff --git a/security/commoncap.c b/security/commoncap.c index 88ee345f7bfa..c3fd9b86ea9a 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -450,16 +450,33 @@ int cap_inode_getsecurity(struct inode *inode, const char *name, void **buffer, return size; } +/** + * rootid_from_xattr - translate root uid of vfs caps + * + * @value: vfs caps value which may be modified by this function + * @size: size of @ivalue + * @task_ns: user namespace of the caller + * @mnt_userns: user namespace of the mount the inode was found from + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. + */ static kuid_t rootid_from_xattr(const void *value, size_t size, - struct user_namespace *task_ns) + struct user_namespace *task_ns, + struct user_namespace *mnt_userns) { const struct vfs_ns_cap_data *nscap = value; + kuid_t rootkid; uid_t rootid = 0; if (size == XATTR_CAPS_SZ_3) rootid = le32_to_cpu(nscap->rootid); - return make_kuid(task_ns, rootid); + rootkid = make_kuid(task_ns, rootid); + return kuid_from_mnt(mnt_userns, rootkid); } static bool validheader(size_t size, const struct vfs_cap_data *cap) @@ -467,13 +484,27 @@ static bool validheader(size_t size, const struct vfs_cap_data *cap) return is_v2header(size, cap) || is_v3header(size, cap); } -/* +/** + * cap_convert_nscap - check vfs caps + * + * @mnt_userns: user namespace of the mount the inode was found from + * @dentry: used to retrieve inode to check permissions on + * @ivalue: vfs caps value which may be modified by this function + * @size: size of @ivalue + * * User requested a write of security.capability. If needed, update the * xattr to change from v2 to v3, or to fixup the v3 rootid. * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. + * * If all is ok, we return the new size, on error return < 0. */ -int cap_convert_nscap(struct dentry *dentry, const void **ivalue, size_t size) +int cap_convert_nscap(struct user_namespace *mnt_userns, struct dentry *dentry, + const void **ivalue, size_t size) { struct vfs_ns_cap_data *nscap; uid_t nsrootid; @@ -489,14 +520,14 @@ int cap_convert_nscap(struct dentry *dentry, const void **ivalue, size_t size) return -EINVAL; if (!validheader(size, cap)) return -EINVAL; - if (!capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_SETFCAP)) + if (!capable_wrt_inode_uidgid(mnt_userns, inode, CAP_SETFCAP)) return -EPERM; - if (size == XATTR_CAPS_SZ_2) + if (size == XATTR_CAPS_SZ_2 && (mnt_userns == &init_user_ns)) if (ns_capable(inode->i_sb->s_user_ns, CAP_SETFCAP)) /* user is privileged, just write the v2 */ return size; - rootid = rootid_from_xattr(*ivalue, size, task_ns); + rootid = rootid_from_xattr(*ivalue, size, task_ns, mnt_userns); if (!uid_valid(rootid)) return -EINVAL; -- cgit v1.2.3 From c7c7a1a18af4c3bb7749d33e3df3acdf0a95bbb5 Mon Sep 17 00:00:00 2001 From: Tycho Andersen Date: Thu, 21 Jan 2021 14:19:28 +0100 Subject: xattr: handle idmapped mounts When interacting with extended attributes the vfs verifies that the caller is privileged over the inode with which the extended attribute is associated. For posix access and posix default extended attributes a uid or gid can be stored on-disk. Let the functions handle posix extended attributes on idmapped mounts. If the inode is accessed through an idmapped mount we need to map it according to the mount's user namespace. Afterwards the checks are identical to non-idmapped mounts. This has no effect for e.g. security xattrs since they don't store uids or gids and don't perform permission checks on them like posix acls do. Link: https://lore.kernel.org/r/20210121131959.646623-10-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Signed-off-by: Tycho Andersen Signed-off-by: Christian Brauner --- fs/cachefiles/xattr.c | 29 ++++---- fs/ecryptfs/crypto.c | 4 +- fs/ecryptfs/inode.c | 5 +- fs/ecryptfs/mmap.c | 4 +- fs/nfsd/vfs.c | 14 ++-- fs/overlayfs/copy_up.c | 14 ++-- fs/overlayfs/dir.c | 2 +- fs/overlayfs/inode.c | 9 +-- fs/overlayfs/overlayfs.h | 6 +- fs/overlayfs/super.c | 6 +- fs/xattr.c | 120 +++++++++++++++++++--------------- include/linux/xattr.h | 27 +++++--- security/apparmor/domain.c | 4 +- security/commoncap.c | 6 +- security/integrity/evm/evm_crypto.c | 11 ++-- security/integrity/evm/evm_main.c | 4 +- security/integrity/ima/ima_appraise.c | 8 +-- security/selinux/hooks.c | 3 +- security/smack/smack_lsm.c | 8 ++- 19 files changed, 160 insertions(+), 124 deletions(-) (limited to 'fs') diff --git a/fs/cachefiles/xattr.c b/fs/cachefiles/xattr.c index 72e42438f3d7..a591b5e09637 100644 --- a/fs/cachefiles/xattr.c +++ b/fs/cachefiles/xattr.c @@ -39,8 +39,8 @@ int cachefiles_check_object_type(struct cachefiles_object *object) _enter("%p{%s}", object, type); /* attempt to install a type label directly */ - ret = vfs_setxattr(dentry, cachefiles_xattr_cache, type, 2, - XATTR_CREATE); + ret = vfs_setxattr(&init_user_ns, dentry, cachefiles_xattr_cache, type, + 2, XATTR_CREATE); if (ret == 0) { _debug("SET"); /* we succeeded */ goto error; @@ -54,7 +54,8 @@ int cachefiles_check_object_type(struct cachefiles_object *object) } /* read the current type label */ - ret = vfs_getxattr(dentry, cachefiles_xattr_cache, xtype, 3); + ret = vfs_getxattr(&init_user_ns, dentry, cachefiles_xattr_cache, xtype, + 3); if (ret < 0) { if (ret == -ERANGE) goto bad_type_length; @@ -110,9 +111,8 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object, _debug("SET #%u", auxdata->len); clear_bit(FSCACHE_COOKIE_AUX_UPDATED, &object->fscache.cookie->flags); - ret = vfs_setxattr(dentry, cachefiles_xattr_cache, - &auxdata->type, auxdata->len, - XATTR_CREATE); + ret = vfs_setxattr(&init_user_ns, dentry, cachefiles_xattr_cache, + &auxdata->type, auxdata->len, XATTR_CREATE); if (ret < 0 && ret != -ENOMEM) cachefiles_io_error_obj( object, @@ -140,9 +140,8 @@ int cachefiles_update_object_xattr(struct cachefiles_object *object, _debug("SET #%u", auxdata->len); clear_bit(FSCACHE_COOKIE_AUX_UPDATED, &object->fscache.cookie->flags); - ret = vfs_setxattr(dentry, cachefiles_xattr_cache, - &auxdata->type, auxdata->len, - XATTR_REPLACE); + ret = vfs_setxattr(&init_user_ns, dentry, cachefiles_xattr_cache, + &auxdata->type, auxdata->len, XATTR_REPLACE); if (ret < 0 && ret != -ENOMEM) cachefiles_io_error_obj( object, @@ -171,7 +170,7 @@ int cachefiles_check_auxdata(struct cachefiles_object *object) if (!auxbuf) return -ENOMEM; - xlen = vfs_getxattr(dentry, cachefiles_xattr_cache, + xlen = vfs_getxattr(&init_user_ns, dentry, cachefiles_xattr_cache, &auxbuf->type, 512 + 1); ret = -ESTALE; if (xlen < 1 || @@ -213,7 +212,7 @@ int cachefiles_check_object_xattr(struct cachefiles_object *object, } /* read the current type label */ - ret = vfs_getxattr(dentry, cachefiles_xattr_cache, + ret = vfs_getxattr(&init_user_ns, dentry, cachefiles_xattr_cache, &auxbuf->type, 512 + 1); if (ret < 0) { if (ret == -ENODATA) @@ -270,9 +269,9 @@ int cachefiles_check_object_xattr(struct cachefiles_object *object, } /* update the current label */ - ret = vfs_setxattr(dentry, cachefiles_xattr_cache, - &auxdata->type, auxdata->len, - XATTR_REPLACE); + ret = vfs_setxattr(&init_user_ns, dentry, + cachefiles_xattr_cache, &auxdata->type, + auxdata->len, XATTR_REPLACE); if (ret < 0) { cachefiles_io_error_obj(object, "Can't update xattr on %lu" @@ -309,7 +308,7 @@ int cachefiles_remove_object_xattr(struct cachefiles_cache *cache, { int ret; - ret = vfs_removexattr(dentry, cachefiles_xattr_cache); + ret = vfs_removexattr(&init_user_ns, dentry, cachefiles_xattr_cache); if (ret < 0) { if (ret == -ENOENT || ret == -ENODATA) ret = 0; diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c index 0681540c48d9..943e523f4c9d 100644 --- a/fs/ecryptfs/crypto.c +++ b/fs/ecryptfs/crypto.c @@ -1110,8 +1110,8 @@ ecryptfs_write_metadata_to_xattr(struct dentry *ecryptfs_dentry, } inode_lock(lower_inode); - rc = __vfs_setxattr(lower_dentry, lower_inode, ECRYPTFS_XATTR_NAME, - page_virt, size, 0); + rc = __vfs_setxattr(&init_user_ns, lower_dentry, lower_inode, + ECRYPTFS_XATTR_NAME, page_virt, size, 0); if (!rc && ecryptfs_inode) fsstack_copy_attr_all(ecryptfs_inode, lower_inode); inode_unlock(lower_inode); diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index ac6472a82567..b9ccc4085d46 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -1024,7 +1024,8 @@ ecryptfs_setxattr(struct dentry *dentry, struct inode *inode, rc = -EOPNOTSUPP; goto out; } - rc = vfs_setxattr(lower_dentry, name, value, size, flags); + rc = vfs_setxattr(&init_user_ns, lower_dentry, name, value, size, + flags); if (!rc && inode) fsstack_copy_attr_all(inode, d_inode(lower_dentry)); out: @@ -1089,7 +1090,7 @@ static int ecryptfs_removexattr(struct dentry *dentry, struct inode *inode, goto out; } inode_lock(lower_inode); - rc = __vfs_removexattr(lower_dentry, name); + rc = __vfs_removexattr(&init_user_ns, lower_dentry, name); inode_unlock(lower_inode); out: return rc; diff --git a/fs/ecryptfs/mmap.c b/fs/ecryptfs/mmap.c index 019572c6b39a..2f333a40ff4d 100644 --- a/fs/ecryptfs/mmap.c +++ b/fs/ecryptfs/mmap.c @@ -426,8 +426,8 @@ static int ecryptfs_write_inode_size_to_xattr(struct inode *ecryptfs_inode) if (size < 0) size = 8; put_unaligned_be64(i_size_read(ecryptfs_inode), xattr_virt); - rc = __vfs_setxattr(lower_dentry, lower_inode, ECRYPTFS_XATTR_NAME, - xattr_virt, size, 0); + rc = __vfs_setxattr(&init_user_ns, lower_dentry, lower_inode, + ECRYPTFS_XATTR_NAME, xattr_virt, size, 0); inode_unlock(lower_inode); if (rc) printk(KERN_ERR "Error whilst attempting to write inode size " diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 1905b39be1c2..37d85046b4d6 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -499,7 +499,8 @@ int nfsd4_is_junction(struct dentry *dentry) return 0; if (!(inode->i_mode & S_ISVTX)) return 0; - if (vfs_getxattr(dentry, NFSD_JUNCTION_XATTR_NAME, NULL, 0) <= 0) + if (vfs_getxattr(&init_user_ns, dentry, NFSD_JUNCTION_XATTR_NAME, + NULL, 0) <= 0) return 0; return 1; } @@ -2149,7 +2150,7 @@ nfsd_getxattr(struct svc_rqst *rqstp, struct svc_fh *fhp, char *name, inode_lock_shared(inode); - len = vfs_getxattr(dentry, name, NULL, 0); + len = vfs_getxattr(&init_user_ns, dentry, name, NULL, 0); /* * Zero-length attribute, just return. @@ -2176,7 +2177,7 @@ nfsd_getxattr(struct svc_rqst *rqstp, struct svc_fh *fhp, char *name, goto out; } - len = vfs_getxattr(dentry, name, buf, len); + len = vfs_getxattr(&init_user_ns, dentry, name, buf, len); if (len <= 0) { kvfree(buf); buf = NULL; @@ -2283,7 +2284,8 @@ nfsd_removexattr(struct svc_rqst *rqstp, struct svc_fh *fhp, char *name) fh_lock(fhp); - ret = __vfs_removexattr_locked(fhp->fh_dentry, name, NULL); + ret = __vfs_removexattr_locked(&init_user_ns, fhp->fh_dentry, + name, NULL); fh_unlock(fhp); fh_drop_write(fhp); @@ -2307,8 +2309,8 @@ nfsd_setxattr(struct svc_rqst *rqstp, struct svc_fh *fhp, char *name, return nfserrno(ret); fh_lock(fhp); - ret = __vfs_setxattr_locked(fhp->fh_dentry, name, buf, len, flags, - NULL); + ret = __vfs_setxattr_locked(&init_user_ns, fhp->fh_dentry, name, buf, + len, flags, NULL); fh_unlock(fhp); fh_drop_write(fhp); diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index 3e9957ae19fa..f81b836c2256 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -85,9 +85,9 @@ int ovl_copy_xattr(struct super_block *sb, struct dentry *old, if (ovl_is_private_xattr(sb, name)) continue; retry: - size = vfs_getxattr(old, name, value, value_size); + size = vfs_getxattr(&init_user_ns, old, name, value, value_size); if (size == -ERANGE) - size = vfs_getxattr(old, name, NULL, 0); + size = vfs_getxattr(&init_user_ns, old, name, NULL, 0); if (size < 0) { error = size; @@ -114,7 +114,7 @@ retry: error = 0; continue; /* Discard */ } - error = vfs_setxattr(new, name, value, size, 0); + error = vfs_setxattr(&init_user_ns, new, name, value, size, 0); if (error) { if (error != -EOPNOTSUPP || ovl_must_copy_xattr(name)) break; @@ -795,7 +795,7 @@ static ssize_t ovl_getxattr(struct dentry *dentry, char *name, char **value) ssize_t res; char *buf; - res = vfs_getxattr(dentry, name, NULL, 0); + res = vfs_getxattr(&init_user_ns, dentry, name, NULL, 0); if (res == -ENODATA || res == -EOPNOTSUPP) res = 0; @@ -804,7 +804,7 @@ static ssize_t ovl_getxattr(struct dentry *dentry, char *name, char **value) if (!buf) return -ENOMEM; - res = vfs_getxattr(dentry, name, buf, res); + res = vfs_getxattr(&init_user_ns, dentry, name, buf, res); if (res < 0) kfree(buf); else @@ -846,8 +846,8 @@ static int ovl_copy_up_meta_inode_data(struct ovl_copy_up_ctx *c) * don't want that to happen for normal copy-up operation. */ if (capability) { - err = vfs_setxattr(upperpath.dentry, XATTR_NAME_CAPS, - capability, cap_size, 0); + err = vfs_setxattr(&init_user_ns, upperpath.dentry, + XATTR_NAME_CAPS, capability, cap_size, 0); if (err) goto out_free; } diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index 29840820a46c..d75c96cb18c3 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -449,7 +449,7 @@ static int ovl_set_upper_acl(struct dentry *upperdentry, const char *name, if (err < 0) goto out_free; - err = vfs_setxattr(upperdentry, name, buffer, size, XATTR_CREATE); + err = vfs_setxattr(&init_user_ns, upperdentry, name, buffer, size, XATTR_CREATE); out_free: kfree(buffer); return err; diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 5aa66881dbd7..023fde466e3a 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -352,7 +352,7 @@ int ovl_xattr_set(struct dentry *dentry, struct inode *inode, const char *name, goto out; if (!value && !upperdentry) { - err = vfs_getxattr(realdentry, name, NULL, 0); + err = vfs_getxattr(&init_user_ns, realdentry, name, NULL, 0); if (err < 0) goto out_drop_write; } @@ -367,10 +367,11 @@ int ovl_xattr_set(struct dentry *dentry, struct inode *inode, const char *name, old_cred = ovl_override_creds(dentry->d_sb); if (value) - err = vfs_setxattr(realdentry, name, value, size, flags); + err = vfs_setxattr(&init_user_ns, realdentry, name, value, size, + flags); else { WARN_ON(flags != XATTR_REPLACE); - err = vfs_removexattr(realdentry, name); + err = vfs_removexattr(&init_user_ns, realdentry, name); } revert_creds(old_cred); @@ -392,7 +393,7 @@ int ovl_xattr_get(struct dentry *dentry, struct inode *inode, const char *name, ovl_i_dentry_upper(inode) ?: ovl_dentry_lower(dentry); old_cred = ovl_override_creds(dentry->d_sb); - res = vfs_getxattr(realdentry, name, value, size); + res = vfs_getxattr(&init_user_ns, realdentry, name, value, size); revert_creds(old_cred); return res; } diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index b487e48c7fd4..0002834f664a 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -186,7 +186,7 @@ static inline ssize_t ovl_do_getxattr(struct ovl_fs *ofs, struct dentry *dentry, size_t size) { const char *name = ovl_xattr(ofs, ox); - return vfs_getxattr(dentry, name, value, size); + return vfs_getxattr(&init_user_ns, dentry, name, value, size); } static inline int ovl_do_setxattr(struct ovl_fs *ofs, struct dentry *dentry, @@ -194,7 +194,7 @@ static inline int ovl_do_setxattr(struct ovl_fs *ofs, struct dentry *dentry, size_t size) { const char *name = ovl_xattr(ofs, ox); - int err = vfs_setxattr(dentry, name, value, size, 0); + int err = vfs_setxattr(&init_user_ns, dentry, name, value, size, 0); pr_debug("setxattr(%pd2, \"%s\", \"%*pE\", %zu, 0) = %i\n", dentry, name, min((int)size, 48), value, size, err); return err; @@ -204,7 +204,7 @@ static inline int ovl_do_removexattr(struct ovl_fs *ofs, struct dentry *dentry, enum ovl_xattr ox) { const char *name = ovl_xattr(ofs, ox); - int err = vfs_removexattr(dentry, name); + int err = vfs_removexattr(&init_user_ns, dentry, name); pr_debug("removexattr(%pd2, \"%s\") = %i\n", dentry, name, err); return err; } diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index e24c995c5fd4..8168ab2dda11 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -794,11 +794,13 @@ retry: * allowed as upper are limited to "normal" ones, where checking * for the above two errors is sufficient. */ - err = vfs_removexattr(work, XATTR_NAME_POSIX_ACL_DEFAULT); + err = vfs_removexattr(&init_user_ns, work, + XATTR_NAME_POSIX_ACL_DEFAULT); if (err && err != -ENODATA && err != -EOPNOTSUPP) goto out_dput; - err = vfs_removexattr(work, XATTR_NAME_POSIX_ACL_ACCESS); + err = vfs_removexattr(&init_user_ns, work, + XATTR_NAME_POSIX_ACL_ACCESS); if (err && err != -ENODATA && err != -EOPNOTSUPP) goto out_dput; diff --git a/fs/xattr.c b/fs/xattr.c index d777025121e0..a49541713b11 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -83,7 +83,8 @@ xattr_resolve_name(struct inode *inode, const char **name) * because different namespaces have very different rules. */ static int -xattr_permission(struct inode *inode, const char *name, int mask) +xattr_permission(struct user_namespace *mnt_userns, struct inode *inode, + const char *name, int mask) { /* * We can never set or remove an extended attribute on a read-only @@ -128,11 +129,11 @@ xattr_permission(struct inode *inode, const char *name, int mask) return (mask & MAY_WRITE) ? -EPERM : -ENODATA; if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) && (mask & MAY_WRITE) && - !inode_owner_or_capable(&init_user_ns, inode)) + !inode_owner_or_capable(mnt_userns, inode)) return -EPERM; } - return inode_permission(&init_user_ns, inode, mask); + return inode_permission(mnt_userns, inode, mask); } /* @@ -163,8 +164,9 @@ xattr_supported_namespace(struct inode *inode, const char *prefix) EXPORT_SYMBOL(xattr_supported_namespace); int -__vfs_setxattr(struct dentry *dentry, struct inode *inode, const char *name, - const void *value, size_t size, int flags) +__vfs_setxattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct inode *inode, const char *name, const void *value, + size_t size, int flags) { const struct xattr_handler *handler; @@ -175,7 +177,7 @@ __vfs_setxattr(struct dentry *dentry, struct inode *inode, const char *name, return -EOPNOTSUPP; if (size == 0) value = ""; /* empty EA, do not remove */ - return handler->set(handler, &init_user_ns, dentry, inode, name, value, + return handler->set(handler, mnt_userns, dentry, inode, name, value, size, flags); } EXPORT_SYMBOL(__vfs_setxattr); @@ -184,6 +186,7 @@ EXPORT_SYMBOL(__vfs_setxattr); * __vfs_setxattr_noperm - perform setxattr operation without performing * permission checks. * + * @mnt_userns - user namespace of the mount the inode was found from * @dentry - object to perform setxattr on * @name - xattr name to set * @value - value to set @name to @@ -196,8 +199,9 @@ EXPORT_SYMBOL(__vfs_setxattr); * is executed. It also assumes that the caller will make the appropriate * permission checks. */ -int __vfs_setxattr_noperm(struct dentry *dentry, const char *name, - const void *value, size_t size, int flags) +int __vfs_setxattr_noperm(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, + const void *value, size_t size, int flags) { struct inode *inode = dentry->d_inode; int error = -EAGAIN; @@ -207,7 +211,8 @@ int __vfs_setxattr_noperm(struct dentry *dentry, const char *name, if (issec) inode->i_flags &= ~S_NOSEC; if (inode->i_opflags & IOP_XATTR) { - error = __vfs_setxattr(dentry, inode, name, value, size, flags); + error = __vfs_setxattr(mnt_userns, dentry, inode, name, value, + size, flags); if (!error) { fsnotify_xattr(dentry); security_inode_post_setxattr(dentry, name, value, @@ -246,14 +251,14 @@ int __vfs_setxattr_noperm(struct dentry *dentry, const char *name, * a delegation was broken on, NULL if none. */ int -__vfs_setxattr_locked(struct dentry *dentry, const char *name, - const void *value, size_t size, int flags, - struct inode **delegated_inode) +__vfs_setxattr_locked(struct user_namespace *mnt_userns, struct dentry *dentry, + const char *name, const void *value, size_t size, + int flags, struct inode **delegated_inode) { struct inode *inode = dentry->d_inode; int error; - error = xattr_permission(inode, name, MAY_WRITE); + error = xattr_permission(mnt_userns, inode, name, MAY_WRITE); if (error) return error; @@ -265,7 +270,8 @@ __vfs_setxattr_locked(struct dentry *dentry, const char *name, if (error) goto out; - error = __vfs_setxattr_noperm(dentry, name, value, size, flags); + error = __vfs_setxattr_noperm(mnt_userns, dentry, name, value, + size, flags); out: return error; @@ -273,8 +279,8 @@ out: EXPORT_SYMBOL_GPL(__vfs_setxattr_locked); int -vfs_setxattr(struct dentry *dentry, const char *name, const void *value, - size_t size, int flags) +vfs_setxattr(struct user_namespace *mnt_userns, struct dentry *dentry, + const char *name, const void *value, size_t size, int flags) { struct inode *inode = dentry->d_inode; struct inode *delegated_inode = NULL; @@ -282,7 +288,7 @@ vfs_setxattr(struct dentry *dentry, const char *name, const void *value, int error; if (size && strcmp(name, XATTR_NAME_CAPS) == 0) { - error = cap_convert_nscap(&init_user_ns, dentry, &value, size); + error = cap_convert_nscap(mnt_userns, dentry, &value, size); if (error < 0) return error; size = error; @@ -290,8 +296,8 @@ vfs_setxattr(struct dentry *dentry, const char *name, const void *value, retry_deleg: inode_lock(inode); - error = __vfs_setxattr_locked(dentry, name, value, size, flags, - &delegated_inode); + error = __vfs_setxattr_locked(mnt_userns, dentry, name, value, size, + flags, &delegated_inode); inode_unlock(inode); if (delegated_inode) { @@ -341,15 +347,16 @@ out_noalloc: * Returns the result of alloc, if failed, or the getxattr operation. */ ssize_t -vfs_getxattr_alloc(struct dentry *dentry, const char *name, char **xattr_value, - size_t xattr_size, gfp_t flags) +vfs_getxattr_alloc(struct user_namespace *mnt_userns, struct dentry *dentry, + const char *name, char **xattr_value, size_t xattr_size, + gfp_t flags) { const struct xattr_handler *handler; struct inode *inode = dentry->d_inode; char *value = *xattr_value; int error; - error = xattr_permission(inode, name, MAY_READ); + error = xattr_permission(mnt_userns, inode, name, MAY_READ); if (error) return error; @@ -390,12 +397,13 @@ __vfs_getxattr(struct dentry *dentry, struct inode *inode, const char *name, EXPORT_SYMBOL(__vfs_getxattr); ssize_t -vfs_getxattr(struct dentry *dentry, const char *name, void *value, size_t size) +vfs_getxattr(struct user_namespace *mnt_userns, struct dentry *dentry, + const char *name, void *value, size_t size) { struct inode *inode = dentry->d_inode; int error; - error = xattr_permission(inode, name, MAY_READ); + error = xattr_permission(mnt_userns, inode, name, MAY_READ); if (error) return error; @@ -441,7 +449,8 @@ vfs_listxattr(struct dentry *dentry, char *list, size_t size) EXPORT_SYMBOL_GPL(vfs_listxattr); int -__vfs_removexattr(struct dentry *dentry, const char *name) +__vfs_removexattr(struct user_namespace *mnt_userns, struct dentry *dentry, + const char *name) { struct inode *inode = d_inode(dentry); const struct xattr_handler *handler; @@ -451,8 +460,8 @@ __vfs_removexattr(struct dentry *dentry, const char *name) return PTR_ERR(handler); if (!handler->set) return -EOPNOTSUPP; - return handler->set(handler, &init_user_ns, dentry, inode, name, NULL, - 0, XATTR_REPLACE); + return handler->set(handler, mnt_userns, dentry, inode, name, NULL, 0, + XATTR_REPLACE); } EXPORT_SYMBOL(__vfs_removexattr); @@ -466,13 +475,14 @@ EXPORT_SYMBOL(__vfs_removexattr); * a delegation was broken on, NULL if none. */ int -__vfs_removexattr_locked(struct dentry *dentry, const char *name, - struct inode **delegated_inode) +__vfs_removexattr_locked(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, + struct inode **delegated_inode) { struct inode *inode = dentry->d_inode; int error; - error = xattr_permission(inode, name, MAY_WRITE); + error = xattr_permission(mnt_userns, inode, name, MAY_WRITE); if (error) return error; @@ -484,7 +494,7 @@ __vfs_removexattr_locked(struct dentry *dentry, const char *name, if (error) goto out; - error = __vfs_removexattr(dentry, name); + error = __vfs_removexattr(mnt_userns, dentry, name); if (!error) { fsnotify_xattr(dentry); @@ -497,7 +507,8 @@ out: EXPORT_SYMBOL_GPL(__vfs_removexattr_locked); int -vfs_removexattr(struct dentry *dentry, const char *name) +vfs_removexattr(struct user_namespace *mnt_userns, struct dentry *dentry, + const char *name) { struct inode *inode = dentry->d_inode; struct inode *delegated_inode = NULL; @@ -505,7 +516,8 @@ vfs_removexattr(struct dentry *dentry, const char *name) retry_deleg: inode_lock(inode); - error = __vfs_removexattr_locked(dentry, name, &delegated_inode); + error = __vfs_removexattr_locked(mnt_userns, dentry, + name, &delegated_inode); inode_unlock(inode); if (delegated_inode) { @@ -522,8 +534,9 @@ EXPORT_SYMBOL_GPL(vfs_removexattr); * Extended attribute SET operations */ static long -setxattr(struct dentry *d, const char __user *name, const void __user *value, - size_t size, int flags) +setxattr(struct user_namespace *mnt_userns, struct dentry *d, + const char __user *name, const void __user *value, size_t size, + int flags) { int error; void *kvalue = NULL; @@ -550,11 +563,10 @@ setxattr(struct dentry *d, const char __user *name, const void __user *value, } if ((strcmp(kname, XATTR_NAME_POSIX_ACL_ACCESS) == 0) || (strcmp(kname, XATTR_NAME_POSIX_ACL_DEFAULT) == 0)) - posix_acl_fix_xattr_from_user(&init_user_ns, kvalue, - size); + posix_acl_fix_xattr_from_user(mnt_userns, kvalue, size); } - error = vfs_setxattr(d, kname, kvalue, size, flags); + error = vfs_setxattr(mnt_userns, d, kname, kvalue, size, flags); out: kvfree(kvalue); @@ -567,13 +579,15 @@ static int path_setxattr(const char __user *pathname, { struct path path; int error; + retry: error = user_path_at(AT_FDCWD, pathname, lookup_flags, &path); if (error) return error; error = mnt_want_write(path.mnt); if (!error) { - error = setxattr(path.dentry, name, value, size, flags); + error = setxattr(mnt_user_ns(path.mnt), path.dentry, name, + value, size, flags); mnt_drop_write(path.mnt); } path_put(&path); @@ -609,7 +623,9 @@ SYSCALL_DEFINE5(fsetxattr, int, fd, const char __user *, name, audit_file(f.file); error = mnt_want_write_file(f.file); if (!error) { - error = setxattr(f.file->f_path.dentry, name, value, size, flags); + error = setxattr(file_mnt_user_ns(f.file), + f.file->f_path.dentry, name, + value, size, flags); mnt_drop_write_file(f.file); } fdput(f); @@ -620,8 +636,8 @@ SYSCALL_DEFINE5(fsetxattr, int, fd, const char __user *, name, * Extended attribute GET operations */ static ssize_t -getxattr(struct dentry *d, const char __user *name, void __user *value, - size_t size) +getxattr(struct user_namespace *mnt_userns, struct dentry *d, + const char __user *name, void __user *value, size_t size) { ssize_t error; void *kvalue = NULL; @@ -641,12 +657,11 @@ getxattr(struct dentry *d, const char __user *name, void __user *value, return -ENOMEM; } - error = vfs_getxattr(d, kname, kvalue, size); + error = vfs_getxattr(mnt_userns, d, kname, kvalue, size); if (error > 0) { if ((strcmp(kname, XATTR_NAME_POSIX_ACL_ACCESS) == 0) || (strcmp(kname, XATTR_NAME_POSIX_ACL_DEFAULT) == 0)) - posix_acl_fix_xattr_to_user(&init_user_ns, kvalue, - error); + posix_acl_fix_xattr_to_user(mnt_userns, kvalue, error); if (size && copy_to_user(value, kvalue, error)) error = -EFAULT; } else if (error == -ERANGE && size >= XATTR_SIZE_MAX) { @@ -670,7 +685,7 @@ retry: error = user_path_at(AT_FDCWD, pathname, lookup_flags, &path); if (error) return error; - error = getxattr(path.dentry, name, value, size); + error = getxattr(mnt_user_ns(path.mnt), path.dentry, name, value, size); path_put(&path); if (retry_estale(error, lookup_flags)) { lookup_flags |= LOOKUP_REVAL; @@ -700,7 +715,8 @@ SYSCALL_DEFINE4(fgetxattr, int, fd, const char __user *, name, if (!f.file) return error; audit_file(f.file); - error = getxattr(f.file->f_path.dentry, name, value, size); + error = getxattr(file_mnt_user_ns(f.file), f.file->f_path.dentry, + name, value, size); fdput(f); return error; } @@ -784,7 +800,8 @@ SYSCALL_DEFINE3(flistxattr, int, fd, char __user *, list, size_t, size) * Extended attribute REMOVE operations */ static long -removexattr(struct dentry *d, const char __user *name) +removexattr(struct user_namespace *mnt_userns, struct dentry *d, + const char __user *name) { int error; char kname[XATTR_NAME_MAX + 1]; @@ -795,7 +812,7 @@ removexattr(struct dentry *d, const char __user *name) if (error < 0) return error; - return vfs_removexattr(d, kname); + return vfs_removexattr(mnt_userns, d, kname); } static int path_removexattr(const char __user *pathname, @@ -809,7 +826,7 @@ retry: return error; error = mnt_want_write(path.mnt); if (!error) { - error = removexattr(path.dentry, name); + error = removexattr(mnt_user_ns(path.mnt), path.dentry, name); mnt_drop_write(path.mnt); } path_put(&path); @@ -842,7 +859,8 @@ SYSCALL_DEFINE2(fremovexattr, int, fd, const char __user *, name) audit_file(f.file); error = mnt_want_write_file(f.file); if (!error) { - error = removexattr(f.file->f_path.dentry, name); + error = removexattr(file_mnt_user_ns(f.file), + f.file->f_path.dentry, name); mnt_drop_write_file(f.file); } fdput(f); diff --git a/include/linux/xattr.h b/include/linux/xattr.h index 260c9bcb0edb..4c379d23ec6e 100644 --- a/include/linux/xattr.h +++ b/include/linux/xattr.h @@ -16,6 +16,7 @@ #include #include #include +#include #include struct inode; @@ -49,18 +50,26 @@ struct xattr { }; ssize_t __vfs_getxattr(struct dentry *, struct inode *, const char *, void *, size_t); -ssize_t vfs_getxattr(struct dentry *, const char *, void *, size_t); +ssize_t vfs_getxattr(struct user_namespace *, struct dentry *, const char *, + void *, size_t); ssize_t vfs_listxattr(struct dentry *d, char *list, size_t size); -int __vfs_setxattr(struct dentry *, struct inode *, const char *, const void *, size_t, int); -int __vfs_setxattr_noperm(struct dentry *, const char *, const void *, size_t, int); -int __vfs_setxattr_locked(struct dentry *, const char *, const void *, size_t, int, struct inode **); -int vfs_setxattr(struct dentry *, const char *, const void *, size_t, int); -int __vfs_removexattr(struct dentry *, const char *); -int __vfs_removexattr_locked(struct dentry *, const char *, struct inode **); -int vfs_removexattr(struct dentry *, const char *); +int __vfs_setxattr(struct user_namespace *, struct dentry *, struct inode *, + const char *, const void *, size_t, int); +int __vfs_setxattr_noperm(struct user_namespace *, struct dentry *, + const char *, const void *, size_t, int); +int __vfs_setxattr_locked(struct user_namespace *, struct dentry *, + const char *, const void *, size_t, int, + struct inode **); +int vfs_setxattr(struct user_namespace *, struct dentry *, const char *, + const void *, size_t, int); +int __vfs_removexattr(struct user_namespace *, struct dentry *, const char *); +int __vfs_removexattr_locked(struct user_namespace *, struct dentry *, + const char *, struct inode **); +int vfs_removexattr(struct user_namespace *, struct dentry *, const char *); ssize_t generic_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size); -ssize_t vfs_getxattr_alloc(struct dentry *dentry, const char *name, +ssize_t vfs_getxattr_alloc(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, char **xattr_value, size_t size, gfp_t flags); int xattr_supported_namespace(struct inode *inode, const char *prefix); diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index f919ebd042fd..16f184bc48de 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -324,8 +324,8 @@ static int aa_xattrs_match(const struct linux_binprm *bprm, d = bprm->file->f_path.dentry; for (i = 0; i < profile->xattr_count; i++) { - size = vfs_getxattr_alloc(d, profile->xattrs[i], &value, - value_size, GFP_KERNEL); + size = vfs_getxattr_alloc(&init_user_ns, d, profile->xattrs[i], + &value, value_size, GFP_KERNEL); if (size >= 0) { u32 perm; diff --git a/security/commoncap.c b/security/commoncap.c index c3fd9b86ea9a..745dc1f2c97f 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -313,7 +313,7 @@ int cap_inode_killpriv(struct dentry *dentry) { int error; - error = __vfs_removexattr(dentry, XATTR_NAME_CAPS); + error = __vfs_removexattr(&init_user_ns, dentry, XATTR_NAME_CAPS); if (error == -EOPNOTSUPP) error = 0; return error; @@ -386,8 +386,8 @@ int cap_inode_getsecurity(struct inode *inode, const char *name, void **buffer, return -EINVAL; size = sizeof(struct vfs_ns_cap_data); - ret = (int) vfs_getxattr_alloc(dentry, XATTR_NAME_CAPS, - &tmpbuf, size, GFP_NOFS); + ret = (int)vfs_getxattr_alloc(&init_user_ns, dentry, XATTR_NAME_CAPS, + &tmpbuf, size, GFP_NOFS); dput(dentry); if (ret < 0) diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index 168c3b78ac47..f720f78cbbb1 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -222,7 +222,7 @@ static int evm_calc_hmac_or_hash(struct dentry *dentry, ima_present = true; continue; } - size = vfs_getxattr_alloc(dentry, xattr->name, + size = vfs_getxattr_alloc(&init_user_ns, dentry, xattr->name, &xattr_value, xattr_size, GFP_NOFS); if (size == -ENOMEM) { error = -ENOMEM; @@ -275,8 +275,8 @@ static int evm_is_immutable(struct dentry *dentry, struct inode *inode) return 1; /* Do this the hard way */ - rc = vfs_getxattr_alloc(dentry, XATTR_NAME_EVM, (char **)&xattr_data, 0, - GFP_NOFS); + rc = vfs_getxattr_alloc(&init_user_ns, dentry, XATTR_NAME_EVM, + (char **)&xattr_data, 0, GFP_NOFS); if (rc <= 0) { if (rc == -ENODATA) return 0; @@ -319,11 +319,12 @@ int evm_update_evmxattr(struct dentry *dentry, const char *xattr_name, xattr_value_len, &data); if (rc == 0) { data.hdr.xattr.sha1.type = EVM_XATTR_HMAC; - rc = __vfs_setxattr_noperm(dentry, XATTR_NAME_EVM, + rc = __vfs_setxattr_noperm(&init_user_ns, dentry, + XATTR_NAME_EVM, &data.hdr.xattr.data[1], SHA1_DIGEST_SIZE + 1, 0); } else if (rc == -ENODATA && (inode->i_opflags & IOP_XATTR)) { - rc = __vfs_removexattr(dentry, XATTR_NAME_EVM); + rc = __vfs_removexattr(&init_user_ns, dentry, XATTR_NAME_EVM); } return rc; } diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index 76d19146d74b..0de367aaa2d3 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -146,8 +146,8 @@ static enum integrity_status evm_verify_hmac(struct dentry *dentry, /* if status is not PASS, try to check again - against -ENOMEM */ /* first need to know the sig type */ - rc = vfs_getxattr_alloc(dentry, XATTR_NAME_EVM, (char **)&xattr_data, 0, - GFP_NOFS); + rc = vfs_getxattr_alloc(&init_user_ns, dentry, XATTR_NAME_EVM, + (char **)&xattr_data, 0, GFP_NOFS); if (rc <= 0) { evm_status = INTEGRITY_FAIL; if (rc == -ENODATA) { diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 8361941ee0a1..70b643c41c6b 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -94,7 +94,7 @@ static int ima_fix_xattr(struct dentry *dentry, iint->ima_hash->xattr.ng.type = IMA_XATTR_DIGEST_NG; iint->ima_hash->xattr.ng.algo = algo; } - rc = __vfs_setxattr_noperm(dentry, XATTR_NAME_IMA, + rc = __vfs_setxattr_noperm(&init_user_ns, dentry, XATTR_NAME_IMA, &iint->ima_hash->xattr.data[offset], (sizeof(iint->ima_hash->xattr) - offset) + iint->ima_hash->length, 0); @@ -215,8 +215,8 @@ int ima_read_xattr(struct dentry *dentry, { ssize_t ret; - ret = vfs_getxattr_alloc(dentry, XATTR_NAME_IMA, (char **)xattr_value, - 0, GFP_NOFS); + ret = vfs_getxattr_alloc(&init_user_ns, dentry, XATTR_NAME_IMA, + (char **)xattr_value, 0, GFP_NOFS); if (ret == -EOPNOTSUPP) ret = 0; return ret; @@ -520,7 +520,7 @@ void ima_inode_post_setattr(struct dentry *dentry) action = ima_must_appraise(inode, MAY_ACCESS, POST_SETATTR); if (!action) - __vfs_removexattr(dentry, XATTR_NAME_IMA); + __vfs_removexattr(&init_user_ns, dentry, XATTR_NAME_IMA); iint = integrity_iint_find(inode); if (iint) { set_bit(IMA_CHANGE_ATTR, &iint->atomic_flags); diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 9d6d3da2caf2..2efedd7001b2 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -6526,7 +6526,8 @@ static int selinux_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen */ static int selinux_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen) { - return __vfs_setxattr_noperm(dentry, XATTR_NAME_SELINUX, ctx, ctxlen, 0); + return __vfs_setxattr_noperm(&init_user_ns, dentry, XATTR_NAME_SELINUX, + ctx, ctxlen, 0); } static int selinux_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen) diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index f69c3dd9a0c6..746e5743accc 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -3425,7 +3425,7 @@ static void smack_d_instantiate(struct dentry *opt_dentry, struct inode *inode) */ if (isp->smk_flags & SMK_INODE_CHANGED) { isp->smk_flags &= ~SMK_INODE_CHANGED; - rc = __vfs_setxattr(dp, inode, + rc = __vfs_setxattr(&init_user_ns, dp, inode, XATTR_NAME_SMACKTRANSMUTE, TRANS_TRUE, TRANS_TRUE_SIZE, 0); @@ -4597,12 +4597,14 @@ static int smack_secctx_to_secid(const char *secdata, u32 seclen, u32 *secid) static int smack_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen) { - return smack_inode_setsecurity(inode, XATTR_SMACK_SUFFIX, ctx, ctxlen, 0); + return smack_inode_setsecurity(inode, XATTR_SMACK_SUFFIX, ctx, + ctxlen, 0); } static int smack_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen) { - return __vfs_setxattr_noperm(dentry, XATTR_NAME_SMACK, ctx, ctxlen, 0); + return __vfs_setxattr_noperm(&init_user_ns, dentry, XATTR_NAME_SMACK, + ctx, ctxlen, 0); } static int smack_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen) -- cgit v1.2.3 From 71bc356f93a1c589fad13f7487258f89c417976e Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:29 +0100 Subject: commoncap: handle idmapped mounts When interacting with user namespace and non-user namespace aware filesystem capabilities the vfs will perform various security checks to determine whether or not the filesystem capabilities can be used by the caller, whether they need to be removed and so on. The main infrastructure for this resides in the capability codepaths but they are called through the LSM security infrastructure even though they are not technically an LSM or optional. This extends the existing security hooks security_inode_removexattr(), security_inode_killpriv(), security_inode_getsecurity() to pass down the mount's user namespace and makes them aware of idmapped mounts. In order to actually get filesystem capabilities from disk the capability infrastructure exposes the get_vfs_caps_from_disk() helper. For user namespace aware filesystem capabilities a root uid is stored alongside the capabilities. In order to determine whether the caller can make use of the filesystem capability or whether it needs to be ignored it is translated according to the superblock's user namespace. If it can be translated to uid 0 according to that id mapping the caller can use the filesystem capabilities stored on disk. If we are accessing the inode that holds the filesystem capabilities through an idmapped mount we map the root uid according to the mount's user namespace. Afterwards the checks are identical to non-idmapped mounts: reading filesystem caps from disk enforces that the root uid associated with the filesystem capability must have a mapping in the superblock's user namespace and that the caller is either in the same user namespace or is a descendant of the superblock's user namespace. For filesystems that are mountable inside user namespace the caller can just mount the filesystem and won't usually need to idmap it. If they do want to idmap it they can create an idmapped mount and mark it with a user namespace they created and which is thus a descendant of s_user_ns. For filesystems that are not mountable inside user namespaces the descendant rule is trivially true because the s_user_ns will be the initial user namespace. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-11-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Acked-by: James Morris Signed-off-by: Christian Brauner --- fs/attr.c | 2 +- fs/xattr.c | 18 ++++++++----- include/linux/capability.h | 4 ++- include/linux/lsm_hook_defs.h | 15 ++++++----- include/linux/lsm_hooks.h | 1 + include/linux/security.h | 54 +++++++++++++++++++++++-------------- kernel/auditsc.c | 5 ++-- security/commoncap.c | 62 ++++++++++++++++++++++++++++++++++--------- security/security.c | 25 ++++++++++------- security/selinux/hooks.c | 20 ++++++++------ security/smack/smack_lsm.c | 14 +++++----- 11 files changed, 146 insertions(+), 74 deletions(-) (limited to 'fs') diff --git a/fs/attr.c b/fs/attr.c index f4543d2abdfb..c2d3bc0869d4 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -143,7 +143,7 @@ kill_priv: if (ia_valid & ATTR_KILL_PRIV) { int error; - error = security_inode_killpriv(dentry); + error = security_inode_killpriv(mnt_userns, dentry); if (error) return error; } diff --git a/fs/xattr.c b/fs/xattr.c index a49541713b11..79c439d31eaa 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -262,7 +262,8 @@ __vfs_setxattr_locked(struct user_namespace *mnt_userns, struct dentry *dentry, if (error) return error; - error = security_inode_setxattr(dentry, name, value, size, flags); + error = security_inode_setxattr(mnt_userns, dentry, name, value, size, + flags); if (error) goto out; @@ -313,18 +314,20 @@ retry_deleg: EXPORT_SYMBOL_GPL(vfs_setxattr); static ssize_t -xattr_getsecurity(struct inode *inode, const char *name, void *value, - size_t size) +xattr_getsecurity(struct user_namespace *mnt_userns, struct inode *inode, + const char *name, void *value, size_t size) { void *buffer = NULL; ssize_t len; if (!value || !size) { - len = security_inode_getsecurity(inode, name, &buffer, false); + len = security_inode_getsecurity(mnt_userns, inode, name, + &buffer, false); goto out_noalloc; } - len = security_inode_getsecurity(inode, name, &buffer, true); + len = security_inode_getsecurity(mnt_userns, inode, name, &buffer, + true); if (len < 0) return len; if (size < len) { @@ -414,7 +417,8 @@ vfs_getxattr(struct user_namespace *mnt_userns, struct dentry *dentry, if (!strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN)) { const char *suffix = name + XATTR_SECURITY_PREFIX_LEN; - int ret = xattr_getsecurity(inode, suffix, value, size); + int ret = xattr_getsecurity(mnt_userns, inode, suffix, value, + size); /* * Only overwrite the return value if a security module * is actually active. @@ -486,7 +490,7 @@ __vfs_removexattr_locked(struct user_namespace *mnt_userns, if (error) return error; - error = security_inode_removexattr(dentry, name); + error = security_inode_removexattr(mnt_userns, dentry, name); if (error) goto out; diff --git a/include/linux/capability.h b/include/linux/capability.h index da21ef118b04..65efb74c3585 100644 --- a/include/linux/capability.h +++ b/include/linux/capability.h @@ -271,7 +271,9 @@ static inline bool checkpoint_restore_ns_capable(struct user_namespace *ns) } /* audit system wants to get cap info from files as well */ -extern int get_vfs_caps_from_disk(const struct dentry *dentry, struct cpu_vfs_cap_data *cpu_caps); +int get_vfs_caps_from_disk(struct user_namespace *mnt_userns, + const struct dentry *dentry, + struct cpu_vfs_cap_data *cpu_caps); int cap_convert_nscap(struct user_namespace *mnt_userns, struct dentry *dentry, const void **ivalue, size_t size); diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 7aaa753b8608..df4cdad6681e 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -133,17 +133,20 @@ LSM_HOOK(int, 0, inode_follow_link, struct dentry *dentry, struct inode *inode, LSM_HOOK(int, 0, inode_permission, struct inode *inode, int mask) LSM_HOOK(int, 0, inode_setattr, struct dentry *dentry, struct iattr *attr) LSM_HOOK(int, 0, inode_getattr, const struct path *path) -LSM_HOOK(int, 0, inode_setxattr, struct dentry *dentry, const char *name, - const void *value, size_t size, int flags) +LSM_HOOK(int, 0, inode_setxattr, struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, const void *value, + size_t size, int flags) LSM_HOOK(void, LSM_RET_VOID, inode_post_setxattr, struct dentry *dentry, const char *name, const void *value, size_t size, int flags) LSM_HOOK(int, 0, inode_getxattr, struct dentry *dentry, const char *name) LSM_HOOK(int, 0, inode_listxattr, struct dentry *dentry) -LSM_HOOK(int, 0, inode_removexattr, struct dentry *dentry, const char *name) +LSM_HOOK(int, 0, inode_removexattr, struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name) LSM_HOOK(int, 0, inode_need_killpriv, struct dentry *dentry) -LSM_HOOK(int, 0, inode_killpriv, struct dentry *dentry) -LSM_HOOK(int, -EOPNOTSUPP, inode_getsecurity, struct inode *inode, - const char *name, void **buffer, bool alloc) +LSM_HOOK(int, 0, inode_killpriv, struct user_namespace *mnt_userns, + struct dentry *dentry) +LSM_HOOK(int, -EOPNOTSUPP, inode_getsecurity, struct user_namespace *mnt_userns, + struct inode *inode, const char *name, void **buffer, bool alloc) LSM_HOOK(int, -EOPNOTSUPP, inode_setsecurity, struct inode *inode, const char *name, const void *value, size_t size, int flags) LSM_HOOK(int, 0, inode_listsecurity, struct inode *inode, char *buffer, diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index a19adef1f088..98a5e263aabb 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -444,6 +444,7 @@ * @inode_killpriv: * The setuid bit is being removed. Remove similar security labels. * Called with the dentry->d_inode->i_mutex held. + * @mnt_userns: user namespace of the mount * @dentry is the dentry being changed. * Return 0 on success. If error is returned, then the operation * causing setuid bit removal is failed. diff --git a/include/linux/security.h b/include/linux/security.h index c35ea0ffccd9..4e4f6c31e413 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -145,13 +145,16 @@ extern int cap_capset(struct cred *new, const struct cred *old, const kernel_cap_t *inheritable, const kernel_cap_t *permitted); extern int cap_bprm_creds_from_file(struct linux_binprm *bprm, struct file *file); -extern int cap_inode_setxattr(struct dentry *dentry, const char *name, - const void *value, size_t size, int flags); -extern int cap_inode_removexattr(struct dentry *dentry, const char *name); -extern int cap_inode_need_killpriv(struct dentry *dentry); -extern int cap_inode_killpriv(struct dentry *dentry); -extern int cap_inode_getsecurity(struct inode *inode, const char *name, - void **buffer, bool alloc); +int cap_inode_setxattr(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags); +int cap_inode_removexattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name); +int cap_inode_need_killpriv(struct dentry *dentry); +int cap_inode_killpriv(struct user_namespace *mnt_userns, + struct dentry *dentry); +int cap_inode_getsecurity(struct user_namespace *mnt_userns, + struct inode *inode, const char *name, void **buffer, + bool alloc); extern int cap_mmap_addr(unsigned long addr); extern int cap_mmap_file(struct file *file, unsigned long reqprot, unsigned long prot, unsigned long flags); @@ -345,16 +348,21 @@ int security_inode_follow_link(struct dentry *dentry, struct inode *inode, int security_inode_permission(struct inode *inode, int mask); int security_inode_setattr(struct dentry *dentry, struct iattr *attr); int security_inode_getattr(const struct path *path); -int security_inode_setxattr(struct dentry *dentry, const char *name, +int security_inode_setxattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, const void *value, size_t size, int flags); void security_inode_post_setxattr(struct dentry *dentry, const char *name, const void *value, size_t size, int flags); int security_inode_getxattr(struct dentry *dentry, const char *name); int security_inode_listxattr(struct dentry *dentry); -int security_inode_removexattr(struct dentry *dentry, const char *name); +int security_inode_removexattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name); int security_inode_need_killpriv(struct dentry *dentry); -int security_inode_killpriv(struct dentry *dentry); -int security_inode_getsecurity(struct inode *inode, const char *name, void **buffer, bool alloc); +int security_inode_killpriv(struct user_namespace *mnt_userns, + struct dentry *dentry); +int security_inode_getsecurity(struct user_namespace *mnt_userns, + struct inode *inode, const char *name, + void **buffer, bool alloc); int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags); int security_inode_listsecurity(struct inode *inode, char *buffer, size_t buffer_size); void security_inode_getsecid(struct inode *inode, u32 *secid); @@ -831,8 +839,9 @@ static inline int security_inode_getattr(const struct path *path) return 0; } -static inline int security_inode_setxattr(struct dentry *dentry, - const char *name, const void *value, size_t size, int flags) +static inline int security_inode_setxattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, const void *value, + size_t size, int flags) { return cap_inode_setxattr(dentry, name, value, size, flags); } @@ -852,10 +861,11 @@ static inline int security_inode_listxattr(struct dentry *dentry) return 0; } -static inline int security_inode_removexattr(struct dentry *dentry, - const char *name) +static inline int security_inode_removexattr(struct user_namespace *mnt_userns, + struct dentry *dentry, + const char *name) { - return cap_inode_removexattr(dentry, name); + return cap_inode_removexattr(mnt_userns, dentry, name); } static inline int security_inode_need_killpriv(struct dentry *dentry) @@ -863,14 +873,18 @@ static inline int security_inode_need_killpriv(struct dentry *dentry) return cap_inode_need_killpriv(dentry); } -static inline int security_inode_killpriv(struct dentry *dentry) +static inline int security_inode_killpriv(struct user_namespace *mnt_userns, + struct dentry *dentry) { - return cap_inode_killpriv(dentry); + return cap_inode_killpriv(mnt_userns, dentry); } -static inline int security_inode_getsecurity(struct inode *inode, const char *name, void **buffer, bool alloc) +static inline int security_inode_getsecurity(struct user_namespace *mnt_userns, + struct inode *inode, + const char *name, void **buffer, + bool alloc) { - return cap_inode_getsecurity(inode, name, buffer, alloc); + return cap_inode_getsecurity(mnt_userns, inode, name, buffer, alloc); } static inline int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags) diff --git a/kernel/auditsc.c b/kernel/auditsc.c index ce8c9e2279ba..fdfdd71405ff 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -1930,7 +1930,7 @@ static inline int audit_copy_fcaps(struct audit_names *name, if (!dentry) return 0; - rc = get_vfs_caps_from_disk(dentry, &caps); + rc = get_vfs_caps_from_disk(&init_user_ns, dentry, &caps); if (rc) return rc; @@ -2481,7 +2481,8 @@ int __audit_log_bprm_fcaps(struct linux_binprm *bprm, ax->d.next = context->aux; context->aux = (void *)ax; - get_vfs_caps_from_disk(bprm->file->f_path.dentry, &vcaps); + get_vfs_caps_from_disk(&init_user_ns, + bprm->file->f_path.dentry, &vcaps); ax->fcap.permitted = vcaps.permitted; ax->fcap.inheritable = vcaps.inheritable; diff --git a/security/commoncap.c b/security/commoncap.c index 745dc1f2c97f..234b074c2c58 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -303,17 +303,25 @@ int cap_inode_need_killpriv(struct dentry *dentry) /** * cap_inode_killpriv - Erase the security markings on an inode - * @dentry: The inode/dentry to alter + * + * @mnt_userns: user namespace of the mount the inode was found from + * @dentry: The inode/dentry to alter * * Erase the privilege-enhancing security markings on an inode. * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. + * * Returns 0 if successful, -ve on error. */ -int cap_inode_killpriv(struct dentry *dentry) +int cap_inode_killpriv(struct user_namespace *mnt_userns, struct dentry *dentry) { int error; - error = __vfs_removexattr(&init_user_ns, dentry, XATTR_NAME_CAPS); + error = __vfs_removexattr(mnt_userns, dentry, XATTR_NAME_CAPS); if (error == -EOPNOTSUPP) error = 0; return error; @@ -366,7 +374,8 @@ static bool is_v3header(size_t size, const struct vfs_cap_data *cap) * by the integrity subsystem, which really wants the unconverted values - * so that's good. */ -int cap_inode_getsecurity(struct inode *inode, const char *name, void **buffer, +int cap_inode_getsecurity(struct user_namespace *mnt_userns, + struct inode *inode, const char *name, void **buffer, bool alloc) { int size, ret; @@ -386,7 +395,7 @@ int cap_inode_getsecurity(struct inode *inode, const char *name, void **buffer, return -EINVAL; size = sizeof(struct vfs_ns_cap_data); - ret = (int)vfs_getxattr_alloc(&init_user_ns, dentry, XATTR_NAME_CAPS, + ret = (int)vfs_getxattr_alloc(mnt_userns, dentry, XATTR_NAME_CAPS, &tmpbuf, size, GFP_NOFS); dput(dentry); @@ -412,6 +421,9 @@ int cap_inode_getsecurity(struct inode *inode, const char *name, void **buffer, root = le32_to_cpu(nscap->rootid); kroot = make_kuid(fs_ns, root); + /* If this is an idmapped mount shift the kuid. */ + kroot = kuid_into_mnt(mnt_userns, kroot); + /* If the root kuid maps to a valid uid in current ns, then return * this as a nscap. */ mappedroot = from_kuid(current_user_ns(), kroot); @@ -595,10 +607,24 @@ static inline int bprm_caps_from_vfs_caps(struct cpu_vfs_cap_data *caps, return *effective ? ret : 0; } -/* +/** + * get_vfs_caps_from_disk - retrieve vfs caps from disk + * + * @mnt_userns: user namespace of the mount the inode was found from + * @dentry: dentry from which @inode is retrieved + * @cpu_caps: vfs capabilities + * * Extract the on-exec-apply capability sets for an executable file. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. */ -int get_vfs_caps_from_disk(const struct dentry *dentry, struct cpu_vfs_cap_data *cpu_caps) +int get_vfs_caps_from_disk(struct user_namespace *mnt_userns, + const struct dentry *dentry, + struct cpu_vfs_cap_data *cpu_caps) { struct inode *inode = d_backing_inode(dentry); __u32 magic_etc; @@ -654,6 +680,7 @@ int get_vfs_caps_from_disk(const struct dentry *dentry, struct cpu_vfs_cap_data /* Limit the caps to the mounter of the filesystem * or the more limited uid specified in the xattr. */ + rootkuid = kuid_into_mnt(mnt_userns, rootkuid); if (!rootid_owns_currentns(rootkuid)) return -ENODATA; @@ -699,7 +726,8 @@ static int get_file_caps(struct linux_binprm *bprm, struct file *file, if (!current_in_userns(file->f_path.mnt->mnt_sb->s_user_ns)) return 0; - rc = get_vfs_caps_from_disk(file->f_path.dentry, &vcaps); + rc = get_vfs_caps_from_disk(file_mnt_user_ns(file), + file->f_path.dentry, &vcaps); if (rc < 0) { if (rc == -EINVAL) printk(KERN_NOTICE "Invalid argument reading file caps for %s\n", @@ -964,16 +992,25 @@ int cap_inode_setxattr(struct dentry *dentry, const char *name, /** * cap_inode_removexattr - Determine whether an xattr may be removed - * @dentry: The inode/dentry being altered - * @name: The name of the xattr to be changed + * + * @mnt_userns: User namespace of the mount the inode was found from + * @dentry: The inode/dentry being altered + * @name: The name of the xattr to be changed * * Determine whether an xattr may be removed from an inode, returning 0 if * permission is granted, -ve if denied. * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. + * * This is used to make sure security xattrs don't get removed by those who * aren't privileged to remove them. */ -int cap_inode_removexattr(struct dentry *dentry, const char *name) +int cap_inode_removexattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name) { struct user_namespace *user_ns = dentry->d_sb->s_user_ns; @@ -987,8 +1024,7 @@ int cap_inode_removexattr(struct dentry *dentry, const char *name) struct inode *inode = d_backing_inode(dentry); if (!inode) return -EINVAL; - if (!capable_wrt_inode_uidgid(&init_user_ns, inode, - CAP_SETFCAP)) + if (!capable_wrt_inode_uidgid(mnt_userns, inode, CAP_SETFCAP)) return -EPERM; return 0; } diff --git a/security/security.c b/security/security.c index 7b09cfbae94f..698a9f17bad7 100644 --- a/security/security.c +++ b/security/security.c @@ -1280,7 +1280,8 @@ int security_inode_getattr(const struct path *path) return call_int_hook(inode_getattr, 0, path); } -int security_inode_setxattr(struct dentry *dentry, const char *name, +int security_inode_setxattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, const void *value, size_t size, int flags) { int ret; @@ -1291,8 +1292,8 @@ int security_inode_setxattr(struct dentry *dentry, const char *name, * SELinux and Smack integrate the cap call, * so assume that all LSMs supplying this call do so. */ - ret = call_int_hook(inode_setxattr, 1, dentry, name, value, size, - flags); + ret = call_int_hook(inode_setxattr, 1, mnt_userns, dentry, name, value, + size, flags); if (ret == 1) ret = cap_inode_setxattr(dentry, name, value, size, flags); @@ -1327,7 +1328,8 @@ int security_inode_listxattr(struct dentry *dentry) return call_int_hook(inode_listxattr, 0, dentry); } -int security_inode_removexattr(struct dentry *dentry, const char *name) +int security_inode_removexattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name) { int ret; @@ -1337,9 +1339,9 @@ int security_inode_removexattr(struct dentry *dentry, const char *name) * SELinux and Smack integrate the cap call, * so assume that all LSMs supplying this call do so. */ - ret = call_int_hook(inode_removexattr, 1, dentry, name); + ret = call_int_hook(inode_removexattr, 1, mnt_userns, dentry, name); if (ret == 1) - ret = cap_inode_removexattr(dentry, name); + ret = cap_inode_removexattr(mnt_userns, dentry, name); if (ret) return ret; ret = ima_inode_removexattr(dentry, name); @@ -1353,12 +1355,15 @@ int security_inode_need_killpriv(struct dentry *dentry) return call_int_hook(inode_need_killpriv, 0, dentry); } -int security_inode_killpriv(struct dentry *dentry) +int security_inode_killpriv(struct user_namespace *mnt_userns, + struct dentry *dentry) { - return call_int_hook(inode_killpriv, 0, dentry); + return call_int_hook(inode_killpriv, 0, mnt_userns, dentry); } -int security_inode_getsecurity(struct inode *inode, const char *name, void **buffer, bool alloc) +int security_inode_getsecurity(struct user_namespace *mnt_userns, + struct inode *inode, const char *name, + void **buffer, bool alloc) { struct security_hook_list *hp; int rc; @@ -1369,7 +1374,7 @@ int security_inode_getsecurity(struct inode *inode, const char *name, void **buf * Only one module will provide an attribute with a given name. */ hlist_for_each_entry(hp, &security_hook_heads.inode_getsecurity, list) { - rc = hp->hook.inode_getsecurity(inode, name, buffer, alloc); + rc = hp->hook.inode_getsecurity(mnt_userns, inode, name, buffer, alloc); if (rc != LSM_RET_DEFAULT(inode_getsecurity)) return rc; } diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 2efedd7001b2..9719dd124221 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -3119,7 +3119,8 @@ static bool has_cap_mac_admin(bool audit) return true; } -static int selinux_inode_setxattr(struct dentry *dentry, const char *name, +static int selinux_inode_setxattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, const void *value, size_t size, int flags) { struct inode *inode = d_backing_inode(dentry); @@ -3140,13 +3141,13 @@ static int selinux_inode_setxattr(struct dentry *dentry, const char *name, } if (!selinux_initialized(&selinux_state)) - return (inode_owner_or_capable(&init_user_ns, inode) ? 0 : -EPERM); + return (inode_owner_or_capable(mnt_userns, inode) ? 0 : -EPERM); sbsec = inode->i_sb->s_security; if (!(sbsec->flags & SBLABEL_MNT)) return -EOPNOTSUPP; - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(mnt_userns, inode)) return -EPERM; ad.type = LSM_AUDIT_DATA_DENTRY; @@ -3267,10 +3268,11 @@ static int selinux_inode_listxattr(struct dentry *dentry) return dentry_has_perm(cred, dentry, FILE__GETATTR); } -static int selinux_inode_removexattr(struct dentry *dentry, const char *name) +static int selinux_inode_removexattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name) { if (strcmp(name, XATTR_NAME_SELINUX)) { - int rc = cap_inode_removexattr(dentry, name); + int rc = cap_inode_removexattr(mnt_userns, dentry, name); if (rc) return rc; @@ -3336,7 +3338,9 @@ static int selinux_path_notify(const struct path *path, u64 mask, * * Permission check is handled by selinux_inode_getxattr hook. */ -static int selinux_inode_getsecurity(struct inode *inode, const char *name, void **buffer, bool alloc) +static int selinux_inode_getsecurity(struct user_namespace *mnt_userns, + struct inode *inode, const char *name, + void **buffer, bool alloc) { u32 size; int error; @@ -6533,8 +6537,8 @@ static int selinux_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen) static int selinux_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen) { int len = 0; - len = selinux_inode_getsecurity(inode, XATTR_SELINUX_SUFFIX, - ctx, true); + len = selinux_inode_getsecurity(&init_user_ns, inode, + XATTR_SELINUX_SUFFIX, ctx, true); if (len < 0) return len; *ctxlen = len; diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index 746e5743accc..12a45e61c1a5 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -1240,7 +1240,8 @@ static int smack_inode_getattr(const struct path *path) * * Returns 0 if access is permitted, an error code otherwise */ -static int smack_inode_setxattr(struct dentry *dentry, const char *name, +static int smack_inode_setxattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, const void *value, size_t size, int flags) { struct smk_audit_info ad; @@ -1362,7 +1363,8 @@ static int smack_inode_getxattr(struct dentry *dentry, const char *name) * * Returns 0 if access is permitted, an error code otherwise */ -static int smack_inode_removexattr(struct dentry *dentry, const char *name) +static int smack_inode_removexattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name) { struct inode_smack *isp; struct smk_audit_info ad; @@ -1377,7 +1379,7 @@ static int smack_inode_removexattr(struct dentry *dentry, const char *name) if (!smack_privileged(CAP_MAC_ADMIN)) rc = -EPERM; } else - rc = cap_inode_removexattr(dentry, name); + rc = cap_inode_removexattr(mnt_userns, dentry, name); if (rc != 0) return rc; @@ -1420,9 +1422,9 @@ static int smack_inode_removexattr(struct dentry *dentry, const char *name) * * Returns the size of the attribute or an error code */ -static int smack_inode_getsecurity(struct inode *inode, - const char *name, void **buffer, - bool alloc) +static int smack_inode_getsecurity(struct user_namespace *mnt_userns, + struct inode *inode, const char *name, + void **buffer, bool alloc) { struct socket_smack *ssp; struct socket *sock; -- cgit v1.2.3 From 0d56a4518d5eaf595a24ab2202e171330bb2ed72 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:30 +0100 Subject: stat: handle idmapped mounts The generic_fillattr() helper fills in the basic attributes associated with an inode. Enable it to handle idmapped mounts. If the inode is accessed through an idmapped mount map it into the mount's user namespace before we store the uid and gid. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-12-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Signed-off-by: Christian Brauner --- fs/9p/vfs_inode.c | 4 ++-- fs/9p/vfs_inode_dotl.c | 4 ++-- fs/afs/inode.c | 2 +- fs/btrfs/inode.c | 2 +- fs/ceph/inode.c | 2 +- fs/cifs/inode.c | 2 +- fs/coda/inode.c | 2 +- fs/ecryptfs/inode.c | 4 ++-- fs/erofs/inode.c | 2 +- fs/exfat/file.c | 2 +- fs/ext2/inode.c | 2 +- fs/ext4/inode.c | 2 +- fs/f2fs/file.c | 2 +- fs/fat/file.c | 2 +- fs/fuse/dir.c | 2 +- fs/gfs2/inode.c | 2 +- fs/hfsplus/inode.c | 2 +- fs/kernfs/inode.c | 2 +- fs/libfs.c | 4 ++-- fs/minix/inode.c | 2 +- fs/nfs/inode.c | 2 +- fs/nfs/namespace.c | 2 +- fs/ocfs2/file.c | 2 +- fs/orangefs/inode.c | 2 +- fs/proc/base.c | 4 ++-- fs/proc/generic.c | 2 +- fs/proc/proc_net.c | 2 +- fs/proc/proc_sysctl.c | 2 +- fs/proc/root.c | 2 +- fs/stat.c | 20 ++++++++++++++------ fs/sysv/itree.c | 2 +- fs/ubifs/dir.c | 2 +- fs/udf/symlink.c | 2 +- fs/vboxsf/utils.c | 2 +- include/linux/fs.h | 2 +- mm/shmem.c | 2 +- 36 files changed, 54 insertions(+), 46 deletions(-) (limited to 'fs') diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index 9c3ff6e9ab82..c21b146c8d91 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -1027,7 +1027,7 @@ v9fs_vfs_getattr(const struct path *path, struct kstat *stat, p9_debug(P9_DEBUG_VFS, "dentry: %p\n", dentry); v9ses = v9fs_dentry2v9ses(dentry); if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) { - generic_fillattr(d_inode(dentry), stat); + generic_fillattr(&init_user_ns, d_inode(dentry), stat); return 0; } fid = v9fs_fid_lookup(dentry); @@ -1040,7 +1040,7 @@ v9fs_vfs_getattr(const struct path *path, struct kstat *stat, return PTR_ERR(st); v9fs_stat2inode(st, d_inode(dentry), dentry->d_sb, 0); - generic_fillattr(d_inode(dentry), stat); + generic_fillattr(&init_user_ns, d_inode(dentry), stat); p9stat_free(st); kfree(st); diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c index 302553101fcb..984f28315d2a 100644 --- a/fs/9p/vfs_inode_dotl.c +++ b/fs/9p/vfs_inode_dotl.c @@ -468,7 +468,7 @@ v9fs_vfs_getattr_dotl(const struct path *path, struct kstat *stat, p9_debug(P9_DEBUG_VFS, "dentry: %p\n", dentry); v9ses = v9fs_dentry2v9ses(dentry); if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) { - generic_fillattr(d_inode(dentry), stat); + generic_fillattr(&init_user_ns, d_inode(dentry), stat); return 0; } fid = v9fs_fid_lookup(dentry); @@ -485,7 +485,7 @@ v9fs_vfs_getattr_dotl(const struct path *path, struct kstat *stat, return PTR_ERR(st); v9fs_stat2inode_dotl(st, d_inode(dentry), 0); - generic_fillattr(d_inode(dentry), stat); + generic_fillattr(&init_user_ns, d_inode(dentry), stat); /* Change block size to what the server returned */ stat->blksize = st->st_blksize; diff --git a/fs/afs/inode.c b/fs/afs/inode.c index b0d7b892090d..795ee5cb3817 100644 --- a/fs/afs/inode.c +++ b/fs/afs/inode.c @@ -745,7 +745,7 @@ int afs_getattr(const struct path *path, struct kstat *stat, do { read_seqbegin_or_lock(&vnode->cb_lock, &seq); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); if (test_bit(AFS_VNODE_SILLY_DELETED, &vnode->flags) && stat->nlink > 0) stat->nlink -= 1; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 6c18fb1a25af..a63faed171de 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8842,7 +8842,7 @@ static int btrfs_getattr(const struct path *path, struct kstat *stat, STATX_ATTR_IMMUTABLE | STATX_ATTR_NODUMP); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); stat->dev = BTRFS_I(inode)->root->anon_dev; spin_lock(&BTRFS_I(inode)->lock); diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 145e26a4ddbb..179a2bb88538 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -2385,7 +2385,7 @@ int ceph_getattr(const struct path *path, struct kstat *stat, return err; } - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); stat->ino = ceph_present_inode(inode); /* diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index 27554f71f744..374abce7efaf 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -2408,7 +2408,7 @@ int cifs_getattr(const struct path *path, struct kstat *stat, return rc; } - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); stat->blksize = cifs_sb->ctx->bsize; stat->ino = CIFS_I(inode)->uniqueid; diff --git a/fs/coda/inode.c b/fs/coda/inode.c index b1c70e2b9b1e..4d113e191cb8 100644 --- a/fs/coda/inode.c +++ b/fs/coda/inode.c @@ -256,7 +256,7 @@ int coda_getattr(const struct path *path, struct kstat *stat, { int err = coda_revalidate_inode(d_inode(path->dentry)); if (!err) - generic_fillattr(d_inode(path->dentry), stat); + generic_fillattr(&init_user_ns, d_inode(path->dentry), stat); return err; } diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index b9ccc4085d46..385b5e8741c0 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -977,7 +977,7 @@ static int ecryptfs_getattr_link(const struct path *path, struct kstat *stat, mount_crypt_stat = &ecryptfs_superblock_to_private( dentry->d_sb)->mount_crypt_stat; - generic_fillattr(d_inode(dentry), stat); + generic_fillattr(&init_user_ns, d_inode(dentry), stat); if (mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) { char *target; size_t targetsiz; @@ -1005,7 +1005,7 @@ static int ecryptfs_getattr(const struct path *path, struct kstat *stat, if (!rc) { fsstack_copy_attr_all(d_inode(dentry), ecryptfs_inode_to_lower(d_inode(dentry))); - generic_fillattr(d_inode(dentry), stat); + generic_fillattr(&init_user_ns, d_inode(dentry), stat); stat->blocks = lower_stat.blocks; } return rc; diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c index 3e21c0e8adae..083818063ac6 100644 --- a/fs/erofs/inode.c +++ b/fs/erofs/inode.c @@ -343,7 +343,7 @@ int erofs_getattr(const struct path *path, struct kstat *stat, stat->attributes_mask |= (STATX_ATTR_COMPRESSED | STATX_ATTR_IMMUTABLE); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); return 0; } diff --git a/fs/exfat/file.c b/fs/exfat/file.c index ace35aa8e64b..e9705b3295d3 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -273,7 +273,7 @@ int exfat_getattr(const struct path *path, struct kstat *stat, struct inode *inode = d_backing_inode(path->dentry); struct exfat_inode_info *ei = EXFAT_I(inode); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); exfat_truncate_atime(&stat->atime); stat->result_mask |= STATX_BTIME; stat->btime.tv_sec = ei->i_crtime.tv_sec; diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 9de813635d8d..3d8acafca8ce 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -1660,7 +1660,7 @@ int ext2_getattr(const struct path *path, struct kstat *stat, STATX_ATTR_IMMUTABLE | STATX_ATTR_NODUMP); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); return 0; } diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 24ea5851e90a..3a303d3f8423 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -5571,7 +5571,7 @@ int ext4_getattr(const struct path *path, struct kstat *stat, STATX_ATTR_NODUMP | STATX_ATTR_VERITY); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); return 0; } diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 6ccdfe0606d9..44cd0dbdbb5d 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -820,7 +820,7 @@ int f2fs_getattr(const struct path *path, struct kstat *stat, STATX_ATTR_NODUMP | STATX_ATTR_VERITY); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); /* we need to show initial sectors used for inline_data/dentries */ if ((S_ISREG(inode->i_mode) && f2fs_has_inline_data(inode)) || diff --git a/fs/fat/file.c b/fs/fat/file.c index 805b501467e9..f7e04f533d31 100644 --- a/fs/fat/file.c +++ b/fs/fat/file.c @@ -398,7 +398,7 @@ int fat_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { struct inode *inode = d_inode(path->dentry); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); stat->blksize = MSDOS_SB(inode->i_sb)->cluster_size; if (MSDOS_SB(inode->i_sb)->options.nfs == FAT_NFS_NOSTALE_RO) { diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 74fdb6a7ebb3..d2e318ed9b26 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -1087,7 +1087,7 @@ static int fuse_update_get_attr(struct inode *inode, struct file *file, forget_all_cached_acls(inode); err = fuse_do_getattr(inode, stat, file); } else if (stat) { - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); stat->mode = fi->orig_i_mode; stat->ino = fi->orig_ino; } diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 728405d15a05..226b5b1dc1fa 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -2050,7 +2050,7 @@ static int gfs2_getattr(const struct path *path, struct kstat *stat, STATX_ATTR_IMMUTABLE | STATX_ATTR_NODUMP); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); if (gfs2_holder_initialized(&gh)) gfs2_glock_dq_uninit(&gh); diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index ffa137f8234e..642e067d8fe8 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -286,7 +286,7 @@ int hfsplus_getattr(const struct path *path, struct kstat *stat, stat->attributes_mask |= STATX_ATTR_APPEND | STATX_ATTR_IMMUTABLE | STATX_ATTR_NODUMP; - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); return 0; } diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c index 7e44052b42e1..032d3d7546d8 100644 --- a/fs/kernfs/inode.c +++ b/fs/kernfs/inode.c @@ -193,7 +193,7 @@ int kernfs_iop_getattr(const struct path *path, struct kstat *stat, kernfs_refresh_inode(kn, inode); mutex_unlock(&kernfs_mutex); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); return 0; } diff --git a/fs/libfs.c b/fs/libfs.c index a73fe109403c..508e9ea8e6f3 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -31,7 +31,7 @@ int simple_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); stat->blocks = inode->i_mapping->nrpages << (PAGE_SHIFT - 9); return 0; } @@ -1304,7 +1304,7 @@ static int empty_dir_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); return 0; } diff --git a/fs/minix/inode.c b/fs/minix/inode.c index 34f546404aa1..91c81d2fc90d 100644 --- a/fs/minix/inode.c +++ b/fs/minix/inode.c @@ -658,7 +658,7 @@ int minix_getattr(const struct path *path, struct kstat *stat, struct super_block *sb = path->dentry->d_sb; struct inode *inode = d_inode(path->dentry); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); if (INODE_VERSION(inode) == MINIX_V1) stat->blocks = (BLOCK_SIZE / 512) * V1_minix_blocks(stat->size, sb); else diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 522aa10a1a3e..cab123ec1664 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -857,7 +857,7 @@ out_no_revalidate: /* Only return attributes that were revalidated. */ stat->result_mask &= request_mask; out_no_update: - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); stat->ino = nfs_compat_user_ino64(NFS_FILEID(inode)); if (S_ISDIR(inode->i_mode)) stat->blksize = NFS_SERVER(inode)->dtsize; diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c index 2bcbe38afe2e..55fc711e368b 100644 --- a/fs/nfs/namespace.c +++ b/fs/nfs/namespace.c @@ -213,7 +213,7 @@ nfs_namespace_getattr(const struct path *path, struct kstat *stat, { if (NFS_FH(d_inode(path->dentry))->size != 0) return nfs_getattr(path, stat, request_mask, query_flags); - generic_fillattr(d_inode(path->dentry), stat); + generic_fillattr(&init_user_ns, d_inode(path->dentry), stat); return 0; } diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index cabf355b148f..a070d4c9b6ed 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -1313,7 +1313,7 @@ int ocfs2_getattr(const struct path *path, struct kstat *stat, goto bail; } - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); /* * If there is inline data in the inode, the inode will normally not * have data blocks allocated (it may have an external xattr block). diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c index 563fe9ab8eb2..b94032f77e61 100644 --- a/fs/orangefs/inode.c +++ b/fs/orangefs/inode.c @@ -903,7 +903,7 @@ int orangefs_getattr(const struct path *path, struct kstat *stat, ret = orangefs_inode_getattr(inode, request_mask & STATX_SIZE ? ORANGEFS_GETATTR_SIZE : 0); if (ret == 0) { - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); /* override block size reported to stat */ if (!(request_mask & STATX_SIZE)) diff --git a/fs/proc/base.c b/fs/proc/base.c index bb4e63a3684f..d45aa68c1f17 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -1934,7 +1934,7 @@ int pid_getattr(const struct path *path, struct kstat *stat, struct proc_fs_info *fs_info = proc_sb_info(inode->i_sb); struct task_struct *task; - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); stat->uid = GLOBAL_ROOT_UID; stat->gid = GLOBAL_ROOT_GID; @@ -3803,7 +3803,7 @@ static int proc_task_getattr(const struct path *path, struct kstat *stat, { struct inode *inode = d_inode(path->dentry); struct task_struct *p = get_proc_task(inode); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); if (p) { stat->nlink += get_nr_threads(p); diff --git a/fs/proc/generic.c b/fs/proc/generic.c index 6d4fabab8aa7..0db96a761149 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -145,7 +145,7 @@ static int proc_getattr(const struct path *path, struct kstat *stat, } } - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); return 0; } diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c index 18601042af99..4aef49ccf571 100644 --- a/fs/proc/proc_net.c +++ b/fs/proc/proc_net.c @@ -297,7 +297,7 @@ static int proc_tgid_net_getattr(const struct path *path, struct kstat *stat, net = get_proc_task_net(inode); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); if (net != NULL) { stat->nlink = net->proc_net->nlink; diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index ec67dbc1f705..87c828348140 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -840,7 +840,7 @@ static int proc_sys_getattr(const struct path *path, struct kstat *stat, if (IS_ERR(head)) return PTR_ERR(head); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); if (table) stat->mode = (stat->mode & S_IFMT) | table->mode; diff --git a/fs/proc/root.c b/fs/proc/root.c index 5e444d4f9717..244e4b6f15ef 100644 --- a/fs/proc/root.c +++ b/fs/proc/root.c @@ -311,7 +311,7 @@ void __init proc_root_init(void) static int proc_root_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { - generic_fillattr(d_inode(path->dentry), stat); + generic_fillattr(&init_user_ns, d_inode(path->dentry), stat); stat->nlink = proc_root.nlink + nr_processes(); return 0; } diff --git a/fs/stat.c b/fs/stat.c index dacecdda2e79..2c471c2fd766 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -26,21 +26,29 @@ /** * generic_fillattr - Fill in the basic attributes from the inode struct - * @inode: Inode to use as the source - * @stat: Where to fill in the attributes + * @mnt_userns: user namespace of the mount the inode was found from + * @inode: Inode to use as the source + * @stat: Where to fill in the attributes * * Fill in the basic attributes in the kstat structure from data that's to be * found on the VFS inode structure. This is the default if no getattr inode * operation is supplied. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then + * take care to map the inode according to @mnt_userns before filling in the + * uid and gid filds. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply passs init_user_ns. */ -void generic_fillattr(struct inode *inode, struct kstat *stat) +void generic_fillattr(struct user_namespace *mnt_userns, struct inode *inode, + struct kstat *stat) { stat->dev = inode->i_sb->s_dev; stat->ino = inode->i_ino; stat->mode = inode->i_mode; stat->nlink = inode->i_nlink; - stat->uid = inode->i_uid; - stat->gid = inode->i_gid; + stat->uid = i_uid_into_mnt(mnt_userns, inode); + stat->gid = i_gid_into_mnt(mnt_userns, inode); stat->rdev = inode->i_rdev; stat->size = i_size_read(inode); stat->atime = inode->i_atime; @@ -87,7 +95,7 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat, return inode->i_op->getattr(path, stat, request_mask, query_flags); - generic_fillattr(inode, stat); + generic_fillattr(mnt_user_ns(path->mnt), inode, stat); return 0; } EXPORT_SYMBOL(vfs_getattr_nosec); diff --git a/fs/sysv/itree.c b/fs/sysv/itree.c index bcb67b0cabe7..83cffab6955f 100644 --- a/fs/sysv/itree.c +++ b/fs/sysv/itree.c @@ -445,7 +445,7 @@ int sysv_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { struct super_block *s = path->dentry->d_sb; - generic_fillattr(d_inode(path->dentry), stat); + generic_fillattr(&init_user_ns, d_inode(path->dentry), stat); stat->blocks = (s->s_blocksize / 512) * sysv_nblocks(s, stat->size); stat->blksize = s->s_blocksize; return 0; diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c index 694e7714545b..a8881ed61620 100644 --- a/fs/ubifs/dir.c +++ b/fs/ubifs/dir.c @@ -1589,7 +1589,7 @@ int ubifs_getattr(const struct path *path, struct kstat *stat, STATX_ATTR_ENCRYPTED | STATX_ATTR_IMMUTABLE); - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); stat->blksize = UBIFS_BLOCK_SIZE; stat->size = ui->ui_size; diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index c973db239604..54a44d1f023c 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -159,7 +159,7 @@ static int udf_symlink_getattr(const struct path *path, struct kstat *stat, struct inode *inode = d_backing_inode(dentry); struct page *page; - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); page = read_mapping_page(inode->i_mapping, 0, NULL); if (IS_ERR(page)) return PTR_ERR(page); diff --git a/fs/vboxsf/utils.c b/fs/vboxsf/utils.c index 018057546067..d2cd1c99f48e 100644 --- a/fs/vboxsf/utils.c +++ b/fs/vboxsf/utils.c @@ -233,7 +233,7 @@ int vboxsf_getattr(const struct path *path, struct kstat *kstat, if (err) return err; - generic_fillattr(d_inode(dentry), kstat); + generic_fillattr(&init_user_ns, d_inode(dentry), kstat); return 0; } diff --git a/include/linux/fs.h b/include/linux/fs.h index e3ea1d7c3367..182641d8322f 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3154,7 +3154,7 @@ extern int __page_symlink(struct inode *inode, const char *symname, int len, extern int page_symlink(struct inode *inode, const char *symname, int len); extern const struct inode_operations page_symlink_inode_operations; extern void kfree_link(void *); -extern void generic_fillattr(struct inode *, struct kstat *); +void generic_fillattr(struct user_namespace *, struct inode *, struct kstat *); extern int vfs_getattr_nosec(const struct path *, struct kstat *, u32, unsigned int); extern int vfs_getattr(const struct path *, struct kstat *, u32, unsigned int); void __inode_add_bytes(struct inode *inode, loff_t bytes); diff --git a/mm/shmem.c b/mm/shmem.c index 23b8e9c15a42..339d5530d3a9 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1072,7 +1072,7 @@ static int shmem_getattr(const struct path *path, struct kstat *stat, shmem_recalc_inode(inode); spin_unlock_irq(&info->lock); } - generic_fillattr(inode, stat); + generic_fillattr(&init_user_ns, inode, stat); if (is_huge_enabled(sb_info)) stat->blksize = HPAGE_PMD_SIZE; -- cgit v1.2.3 From ba73d98745be1c10dc3cce68e8d7b95012d07d05 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:31 +0100 Subject: namei: handle idmapped mounts in may_*() helpers The may_follow_link(), may_linkat(), may_lookup(), may_open(), may_o_create(), may_create_in_sticky(), may_delete(), and may_create() helpers determine whether the caller is privileged enough to perform the associated operations. Let them handle idmapped mounts by mapping the inode or fsids according to the mount's user namespace. Afterwards the checks are identical to non-idmapped inodes. The patch takes care to retrieve the mount's user namespace right before performing permission checks and passing it down into the fileystem so the user namespace can't change in between by someone idmapping a mount that is currently not idmapped. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-13-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Signed-off-by: Christian Brauner --- fs/btrfs/ioctl.c | 5 +- fs/init.c | 2 +- fs/inode.c | 2 +- fs/internal.h | 2 +- fs/namei.c | 148 +++++++++++++++++++++++++++++++++-------------------- fs/xattr.c | 2 +- include/linux/fs.h | 14 +++-- 7 files changed, 108 insertions(+), 67 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 1f763c60415b..56f53d692fa2 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -927,8 +927,9 @@ static int btrfs_may_delete(struct inode *dir, struct dentry *victim, int isdir) return error; if (IS_APPEND(dir)) return -EPERM; - if (check_sticky(dir, d_inode(victim)) || IS_APPEND(d_inode(victim)) || - IS_IMMUTABLE(d_inode(victim)) || IS_SWAPFILE(d_inode(victim))) + if (check_sticky(&init_user_ns, dir, d_inode(victim)) || + IS_APPEND(d_inode(victim)) || IS_IMMUTABLE(d_inode(victim)) || + IS_SWAPFILE(d_inode(victim))) return -EPERM; if (isdir) { if (!d_is_dir(victim)) diff --git a/fs/init.c b/fs/init.c index 02723bea8499..891284f8a443 100644 --- a/fs/init.c +++ b/fs/init.c @@ -181,7 +181,7 @@ int __init init_link(const char *oldname, const char *newname) error = -EXDEV; if (old_path.mnt != new_path.mnt) goto out_dput; - error = may_linkat(&old_path); + error = may_linkat(&init_user_ns, &old_path); if (unlikely(error)) goto out_dput; error = security_path_link(old_path.dentry, &new_path, new_dentry); diff --git a/fs/inode.c b/fs/inode.c index 49b512592dcd..46116ef44c9f 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -1796,7 +1796,7 @@ bool atime_needs_update(const struct path *path, struct inode *inode) /* Atime updates will likely cause i_uid and i_gid to be written * back improprely if their true value is unknown to the vfs. */ - if (HAS_UNMAPPED_ID(inode)) + if (HAS_UNMAPPED_ID(mnt_user_ns(mnt), inode)) return false; if (IS_NOATIME(inode)) diff --git a/fs/internal.h b/fs/internal.h index 77c50befbfbe..6c8a4eddc7e6 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -73,7 +73,7 @@ extern int vfs_path_lookup(struct dentry *, struct vfsmount *, const char *, unsigned int, struct path *); long do_rmdir(int dfd, struct filename *name); long do_unlinkat(int dfd, struct filename *name); -int may_linkat(struct path *link); +int may_linkat(struct user_namespace *mnt_userns, struct path *link); int do_renameat2(int olddfd, struct filename *oldname, int newdfd, struct filename *newname, unsigned int flags); diff --git a/fs/namei.c b/fs/namei.c index 04b001ddade3..93fa7d803fb2 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -506,7 +506,7 @@ int inode_permission(struct user_namespace *mnt_userns, * written back improperly if their true value is unknown * to the vfs. */ - if (HAS_UNMAPPED_ID(inode)) + if (HAS_UNMAPPED_ID(mnt_userns, inode)) return -EACCES; } @@ -1004,11 +1004,16 @@ int sysctl_protected_regular __read_mostly; */ static inline int may_follow_link(struct nameidata *nd, const struct inode *inode) { + struct user_namespace *mnt_userns; + kuid_t i_uid; + if (!sysctl_protected_symlinks) return 0; + mnt_userns = mnt_user_ns(nd->path.mnt); + i_uid = i_uid_into_mnt(mnt_userns, inode); /* Allowed if owner and follower match. */ - if (uid_eq(current_cred()->fsuid, inode->i_uid)) + if (uid_eq(current_cred()->fsuid, i_uid)) return 0; /* Allowed if parent directory not sticky and world-writable. */ @@ -1016,7 +1021,7 @@ static inline int may_follow_link(struct nameidata *nd, const struct inode *inod return 0; /* Allowed if parent directory and link owner match. */ - if (uid_valid(nd->dir_uid) && uid_eq(nd->dir_uid, inode->i_uid)) + if (uid_valid(nd->dir_uid) && uid_eq(nd->dir_uid, i_uid)) return 0; if (nd->flags & LOOKUP_RCU) @@ -1029,6 +1034,7 @@ static inline int may_follow_link(struct nameidata *nd, const struct inode *inod /** * safe_hardlink_source - Check for safe hardlink conditions + * @mnt_userns: user namespace of the mount the inode was found from * @inode: the source inode to hardlink from * * Return false if at least one of the following conditions: @@ -1039,7 +1045,8 @@ static inline int may_follow_link(struct nameidata *nd, const struct inode *inod * * Otherwise returns true. */ -static bool safe_hardlink_source(struct inode *inode) +static bool safe_hardlink_source(struct user_namespace *mnt_userns, + struct inode *inode) { umode_t mode = inode->i_mode; @@ -1056,7 +1063,7 @@ static bool safe_hardlink_source(struct inode *inode) return false; /* Hardlinking to unreadable or unwritable sources is dangerous. */ - if (inode_permission(&init_user_ns, inode, MAY_READ | MAY_WRITE)) + if (inode_permission(mnt_userns, inode, MAY_READ | MAY_WRITE)) return false; return true; @@ -1064,6 +1071,7 @@ static bool safe_hardlink_source(struct inode *inode) /** * may_linkat - Check permissions for creating a hardlink + * @mnt_userns: user namespace of the mount the inode was found from * @link: the source to hardlink from * * Block hardlink when all of: @@ -1072,14 +1080,21 @@ static bool safe_hardlink_source(struct inode *inode) * - hardlink source is unsafe (see safe_hardlink_source() above) * - not CAP_FOWNER in a namespace with the inode owner uid mapped * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. + * * Returns 0 if successful, -ve on error. */ -int may_linkat(struct path *link) +int may_linkat(struct user_namespace *mnt_userns, struct path *link) { struct inode *inode = link->dentry->d_inode; /* Inode writeback is not safe when the uid or gid are invalid. */ - if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid)) + if (!uid_valid(i_uid_into_mnt(mnt_userns, inode)) || + !gid_valid(i_gid_into_mnt(mnt_userns, inode))) return -EOVERFLOW; if (!sysctl_protected_hardlinks) @@ -1088,8 +1103,8 @@ int may_linkat(struct path *link) /* Source inode owner (or CAP_FOWNER) can hardlink all they like, * otherwise, it must be a safe source. */ - if (safe_hardlink_source(inode) || - inode_owner_or_capable(&init_user_ns, inode)) + if (safe_hardlink_source(mnt_userns, inode) || + inode_owner_or_capable(mnt_userns, inode)) return 0; audit_log_path_denied(AUDIT_ANOM_LINK, "linkat"); @@ -1100,6 +1115,7 @@ int may_linkat(struct path *link) * may_create_in_sticky - Check whether an O_CREAT open in a sticky directory * should be allowed, or not, on files that already * exist. + * @mnt_userns: user namespace of the mount the inode was found from * @dir_mode: mode bits of directory * @dir_uid: owner of directory * @inode: the inode of the file to open @@ -1115,16 +1131,25 @@ int may_linkat(struct path *link) * the directory doesn't have to be world writable: being group writable will * be enough. * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. + * * Returns 0 if the open is allowed, -ve on error. */ -static int may_create_in_sticky(umode_t dir_mode, kuid_t dir_uid, - struct inode * const inode) +static int may_create_in_sticky(struct user_namespace *mnt_userns, + struct nameidata *nd, struct inode *const inode) { + umode_t dir_mode = nd->dir_mode; + kuid_t dir_uid = nd->dir_uid; + if ((!sysctl_protected_fifos && S_ISFIFO(inode->i_mode)) || (!sysctl_protected_regular && S_ISREG(inode->i_mode)) || likely(!(dir_mode & S_ISVTX)) || - uid_eq(inode->i_uid, dir_uid) || - uid_eq(current_fsuid(), inode->i_uid)) + uid_eq(i_uid_into_mnt(mnt_userns, inode), dir_uid) || + uid_eq(current_fsuid(), i_uid_into_mnt(mnt_userns, inode))) return 0; if (likely(dir_mode & 0002) || @@ -1614,17 +1639,18 @@ static struct dentry *lookup_slow(const struct qstr *name, return res; } -static inline int may_lookup(struct nameidata *nd) +static inline int may_lookup(struct user_namespace *mnt_userns, + struct nameidata *nd) { if (nd->flags & LOOKUP_RCU) { - int err = inode_permission(&init_user_ns, nd->inode, + int err = inode_permission(mnt_userns, nd->inode, MAY_EXEC | MAY_NOT_BLOCK); if (err != -ECHILD) return err; if (unlazy_walk(nd)) return -ECHILD; } - return inode_permission(&init_user_ns, nd->inode, MAY_EXEC); + return inode_permission(mnt_userns, nd->inode, MAY_EXEC); } static int reserve_stack(struct nameidata *nd, struct path *link, unsigned seq) @@ -2177,7 +2203,7 @@ static int link_path_walk(const char *name, struct nameidata *nd) u64 hash_len; int type; - err = may_lookup(nd); + err = may_lookup(&init_user_ns, nd); if (err) return err; @@ -2225,7 +2251,7 @@ static int link_path_walk(const char *name, struct nameidata *nd) OK: /* pathname or trailing symlink, done */ if (!depth) { - nd->dir_uid = nd->inode->i_uid; + nd->dir_uid = i_uid_into_mnt(&init_user_ns, nd->inode); nd->dir_mode = nd->inode->i_mode; nd->flags &= ~LOOKUP_PARENT; return 0; @@ -2703,15 +2729,16 @@ int user_path_at_empty(int dfd, const char __user *name, unsigned flags, } EXPORT_SYMBOL(user_path_at_empty); -int __check_sticky(struct inode *dir, struct inode *inode) +int __check_sticky(struct user_namespace *mnt_userns, struct inode *dir, + struct inode *inode) { kuid_t fsuid = current_fsuid(); - if (uid_eq(inode->i_uid, fsuid)) + if (uid_eq(i_uid_into_mnt(mnt_userns, inode), fsuid)) return 0; - if (uid_eq(dir->i_uid, fsuid)) + if (uid_eq(i_uid_into_mnt(mnt_userns, dir), fsuid)) return 0; - return !capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_FOWNER); + return !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FOWNER); } EXPORT_SYMBOL(__check_sticky); @@ -2735,7 +2762,8 @@ EXPORT_SYMBOL(__check_sticky); * 11. We don't allow removal of NFS sillyrenamed files; it's handled by * nfs_async_unlink(). */ -static int may_delete(struct inode *dir, struct dentry *victim, bool isdir) +static int may_delete(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *victim, bool isdir) { struct inode *inode = d_backing_inode(victim); int error; @@ -2747,19 +2775,21 @@ static int may_delete(struct inode *dir, struct dentry *victim, bool isdir) BUG_ON(victim->d_parent->d_inode != dir); /* Inode writeback is not safe when the uid or gid are invalid. */ - if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid)) + if (!uid_valid(i_uid_into_mnt(mnt_userns, inode)) || + !gid_valid(i_gid_into_mnt(mnt_userns, inode))) return -EOVERFLOW; audit_inode_child(dir, victim, AUDIT_TYPE_CHILD_DELETE); - error = inode_permission(&init_user_ns, dir, MAY_WRITE | MAY_EXEC); + error = inode_permission(mnt_userns, dir, MAY_WRITE | MAY_EXEC); if (error) return error; if (IS_APPEND(dir)) return -EPERM; - if (check_sticky(dir, inode) || IS_APPEND(inode) || - IS_IMMUTABLE(inode) || IS_SWAPFILE(inode) || HAS_UNMAPPED_ID(inode)) + if (check_sticky(mnt_userns, dir, inode) || IS_APPEND(inode) || + IS_IMMUTABLE(inode) || IS_SWAPFILE(inode) || + HAS_UNMAPPED_ID(mnt_userns, inode)) return -EPERM; if (isdir) { if (!d_is_dir(victim)) @@ -2784,7 +2814,8 @@ static int may_delete(struct inode *dir, struct dentry *victim, bool isdir) * 4. We should have write and exec permissions on dir * 5. We can't do it if dir is immutable (done in permission()) */ -static inline int may_create(struct inode *dir, struct dentry *child) +static inline int may_create(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *child) { struct user_namespace *s_user_ns; audit_inode_child(dir, child, AUDIT_TYPE_CHILD_CREATE); @@ -2793,10 +2824,10 @@ static inline int may_create(struct inode *dir, struct dentry *child) if (IS_DEADDIR(dir)) return -ENOENT; s_user_ns = dir->i_sb->s_user_ns; - if (!kuid_has_mapping(s_user_ns, current_fsuid()) || - !kgid_has_mapping(s_user_ns, current_fsgid())) + if (!kuid_has_mapping(s_user_ns, fsuid_into_mnt(mnt_userns)) || + !kgid_has_mapping(s_user_ns, fsgid_into_mnt(mnt_userns))) return -EOVERFLOW; - return inode_permission(&init_user_ns, dir, MAY_WRITE | MAY_EXEC); + return inode_permission(mnt_userns, dir, MAY_WRITE | MAY_EXEC); } /* @@ -2846,7 +2877,7 @@ EXPORT_SYMBOL(unlock_rename); int vfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool want_excl) { - int error = may_create(dir, dentry); + int error = may_create(&init_user_ns, dir, dentry); if (error) return error; @@ -2869,7 +2900,7 @@ int vfs_mkobj(struct dentry *dentry, umode_t mode, void *arg) { struct inode *dir = dentry->d_parent->d_inode; - int error = may_create(dir, dentry); + int error = may_create(&init_user_ns, dir, dentry); if (error) return error; @@ -2891,7 +2922,8 @@ bool may_open_dev(const struct path *path) !(path->mnt->mnt_sb->s_iflags & SB_I_NODEV); } -static int may_open(const struct path *path, int acc_mode, int flag) +static int may_open(struct user_namespace *mnt_userns, const struct path *path, + int acc_mode, int flag) { struct dentry *dentry = path->dentry; struct inode *inode = dentry->d_inode; @@ -2926,7 +2958,7 @@ static int may_open(const struct path *path, int acc_mode, int flag) break; } - error = inode_permission(&init_user_ns, inode, MAY_OPEN | acc_mode); + error = inode_permission(mnt_userns, inode, MAY_OPEN | acc_mode); if (error) return error; @@ -2941,7 +2973,7 @@ static int may_open(const struct path *path, int acc_mode, int flag) } /* O_NOATIME can only be set by the owner or superuser */ - if (flag & O_NOATIME && !inode_owner_or_capable(&init_user_ns, inode)) + if (flag & O_NOATIME && !inode_owner_or_capable(mnt_userns, inode)) return -EPERM; return 0; @@ -2976,7 +3008,9 @@ static inline int open_to_namei_flags(int flag) return flag; } -static int may_o_create(const struct path *dir, struct dentry *dentry, umode_t mode) +static int may_o_create(struct user_namespace *mnt_userns, + const struct path *dir, struct dentry *dentry, + umode_t mode) { struct user_namespace *s_user_ns; int error = security_path_mknod(dir, dentry, mode, 0); @@ -2984,11 +3018,11 @@ static int may_o_create(const struct path *dir, struct dentry *dentry, umode_t m return error; s_user_ns = dir->dentry->d_sb->s_user_ns; - if (!kuid_has_mapping(s_user_ns, current_fsuid()) || - !kgid_has_mapping(s_user_ns, current_fsgid())) + if (!kuid_has_mapping(s_user_ns, fsuid_into_mnt(mnt_userns)) || + !kgid_has_mapping(s_user_ns, fsgid_into_mnt(mnt_userns))) return -EOVERFLOW; - error = inode_permission(&init_user_ns, dir->dentry->d_inode, + error = inode_permission(mnt_userns, dir->dentry->d_inode, MAY_WRITE | MAY_EXEC); if (error) return error; @@ -3121,7 +3155,8 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file, if (!IS_POSIXACL(dir->d_inode)) mode &= ~current_umask(); if (likely(got_write)) - create_error = may_o_create(&nd->path, dentry, mode); + create_error = may_o_create(&init_user_ns, &nd->path, + dentry, mode); else create_error = -EROFS; } @@ -3282,7 +3317,7 @@ static int do_open(struct nameidata *nd, return -EEXIST; if (d_is_dir(nd->path.dentry)) return -EISDIR; - error = may_create_in_sticky(nd->dir_mode, nd->dir_uid, + error = may_create_in_sticky(&init_user_ns, nd, d_backing_inode(nd->path.dentry)); if (unlikely(error)) return error; @@ -3302,7 +3337,7 @@ static int do_open(struct nameidata *nd, return error; do_truncate = true; } - error = may_open(&nd->path, acc_mode, open_flag); + error = may_open(&init_user_ns, &nd->path, acc_mode, open_flag); if (!error && !(file->f_mode & FMODE_OPENED)) error = vfs_open(&nd->path, file); if (!error) @@ -3377,7 +3412,7 @@ static int do_tmpfile(struct nameidata *nd, unsigned flags, path.dentry = child; audit_inode(nd->name, child, 0); /* Don't check for other permissions, the inode was just created */ - error = may_open(&path, 0, op->open_flag); + error = may_open(&init_user_ns, &path, 0, op->open_flag); if (error) goto out2; file->f_path.mnt = path.mnt; @@ -3584,7 +3619,7 @@ EXPORT_SYMBOL(user_path_create); int vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) { bool is_whiteout = S_ISCHR(mode) && dev == WHITEOUT_DEV; - int error = may_create(dir, dentry); + int error = may_create(&init_user_ns, dir, dentry); if (error) return error; @@ -3685,7 +3720,7 @@ SYSCALL_DEFINE3(mknod, const char __user *, filename, umode_t, mode, unsigned, d int vfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) { - int error = may_create(dir, dentry); + int error = may_create(&init_user_ns, dir, dentry); unsigned max_links = dir->i_sb->s_max_links; if (error) @@ -3746,7 +3781,7 @@ SYSCALL_DEFINE2(mkdir, const char __user *, pathname, umode_t, mode) int vfs_rmdir(struct inode *dir, struct dentry *dentry) { - int error = may_delete(dir, dentry, 1); + int error = may_delete(&init_user_ns, dir, dentry, 1); if (error) return error; @@ -3868,7 +3903,7 @@ SYSCALL_DEFINE1(rmdir, const char __user *, pathname) int vfs_unlink(struct inode *dir, struct dentry *dentry, struct inode **delegated_inode) { struct inode *target = dentry->d_inode; - int error = may_delete(dir, dentry, 0); + int error = may_delete(&init_user_ns, dir, dentry, 0); if (error) return error; @@ -4000,7 +4035,7 @@ SYSCALL_DEFINE1(unlink, const char __user *, pathname) int vfs_symlink(struct inode *dir, struct dentry *dentry, const char *oldname) { - int error = may_create(dir, dentry); + int error = may_create(&init_user_ns, dir, dentry); if (error) return error; @@ -4089,7 +4124,7 @@ int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_de if (!inode) return -ENOENT; - error = may_create(dir, new_dentry); + error = may_create(&init_user_ns, dir, new_dentry); if (error) return error; @@ -4106,7 +4141,7 @@ int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_de * be writen back improperly if their true value is unknown to * the vfs. */ - if (HAS_UNMAPPED_ID(inode)) + if (HAS_UNMAPPED_ID(&init_user_ns, inode)) return -EPERM; if (!dir->i_op->link) return -EPERM; @@ -4188,7 +4223,7 @@ retry: error = -EXDEV; if (old_path.mnt != new_path.mnt) goto out_dput; - error = may_linkat(&old_path); + error = may_linkat(&init_user_ns, &old_path); if (unlikely(error)) goto out_dput; error = security_path_link(old_path.dentry, &new_path, new_dentry); @@ -4281,6 +4316,7 @@ int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode **delegated_inode, unsigned int flags) { int error; + struct user_namespace *mnt_userns = &init_user_ns; bool is_dir = d_is_dir(old_dentry); struct inode *source = old_dentry->d_inode; struct inode *target = new_dentry->d_inode; @@ -4291,19 +4327,19 @@ int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, if (source == target) return 0; - error = may_delete(old_dir, old_dentry, is_dir); + error = may_delete(mnt_userns, old_dir, old_dentry, is_dir); if (error) return error; if (!target) { - error = may_create(new_dir, new_dentry); + error = may_create(mnt_userns, new_dir, new_dentry); } else { new_is_dir = d_is_dir(new_dentry); if (!(flags & RENAME_EXCHANGE)) - error = may_delete(new_dir, new_dentry, is_dir); + error = may_delete(mnt_userns, new_dir, new_dentry, is_dir); else - error = may_delete(new_dir, new_dentry, new_is_dir); + error = may_delete(mnt_userns, new_dir, new_dentry, new_is_dir); } if (error) return error; diff --git a/fs/xattr.c b/fs/xattr.c index 79c439d31eaa..b3444e06cded 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -98,7 +98,7 @@ xattr_permission(struct user_namespace *mnt_userns, struct inode *inode, * to be writen back improperly if their true value is * unknown to the vfs. */ - if (HAS_UNMAPPED_ID(inode)) + if (HAS_UNMAPPED_ID(mnt_userns, inode)) return -EPERM; } diff --git a/include/linux/fs.h b/include/linux/fs.h index 182641d8322f..a27884af7222 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2083,9 +2083,11 @@ static inline bool sb_rdonly(const struct super_block *sb) { return sb->s_flags #define IS_WHITEOUT(inode) (S_ISCHR(inode->i_mode) && \ (inode)->i_rdev == WHITEOUT_DEV) -static inline bool HAS_UNMAPPED_ID(struct inode *inode) +static inline bool HAS_UNMAPPED_ID(struct user_namespace *mnt_userns, + struct inode *inode) { - return !uid_valid(inode->i_uid) || !gid_valid(inode->i_gid); + return !uid_valid(i_uid_into_mnt(mnt_userns, inode)) || + !gid_valid(i_gid_into_mnt(mnt_userns, inode)); } static inline enum rw_hint file_write_hint(struct file *file) @@ -2823,7 +2825,8 @@ static inline int path_permission(const struct path *path, int mask) return inode_permission(mnt_user_ns(path->mnt), d_inode(path->dentry), mask); } -extern int __check_sticky(struct inode *dir, struct inode *inode); +int __check_sticky(struct user_namespace *mnt_userns, struct inode *dir, + struct inode *inode); static inline bool execute_ok(struct inode *inode) { @@ -3442,12 +3445,13 @@ static inline bool is_sxid(umode_t mode) return (mode & S_ISUID) || ((mode & S_ISGID) && (mode & S_IXGRP)); } -static inline int check_sticky(struct inode *dir, struct inode *inode) +static inline int check_sticky(struct user_namespace *mnt_userns, + struct inode *dir, struct inode *inode) { if (!(dir->i_mode & S_ISVTX)) return 0; - return __check_sticky(dir, inode); + return __check_sticky(mnt_userns, dir, inode); } static inline void inode_has_no_xattr(struct inode *inode) -- cgit v1.2.3 From 9fe61450972d3900bffb1dc26a17ebb9cdd92db2 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:32 +0100 Subject: namei: introduce struct renamedata In order to handle idmapped mounts we will extend the vfs rename helper to take two new arguments in follow up patches. Since this operations already takes a bunch of arguments add a simple struct renamedata and make the current helper use it before we extend it. Link: https://lore.kernel.org/r/20210121131959.646623-14-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/cachefiles/namei.c | 9 +++++++-- fs/ecryptfs/inode.c | 10 +++++++--- fs/namei.c | 21 +++++++++++++++------ fs/nfsd/vfs.c | 8 +++++++- fs/overlayfs/overlayfs.h | 9 ++++++++- include/linux/fs.h | 12 +++++++++++- 6 files changed, 55 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index ecc8ecbbfa5a..7b987de0babe 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -412,9 +412,14 @@ try_again: if (ret < 0) { cachefiles_io_error(cache, "Rename security error %d", ret); } else { + struct renamedata rd = { + .old_dir = d_inode(dir), + .old_dentry = rep, + .new_dir = d_inode(cache->graveyard), + .new_dentry = grave, + }; trace_cachefiles_rename(object, rep, grave, why); - ret = vfs_rename(d_inode(dir), rep, - d_inode(cache->graveyard), grave, NULL, 0); + ret = vfs_rename(&rd); if (ret != 0 && ret != -ENOMEM) cachefiles_io_error(cache, "Rename failed with error %d", ret); diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index 385b5e8741c0..ff48abb09679 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -590,6 +590,7 @@ ecryptfs_rename(struct inode *old_dir, struct dentry *old_dentry, struct dentry *lower_new_dir_dentry; struct dentry *trap; struct inode *target_inode; + struct renamedata rd = {}; if (flags) return -EINVAL; @@ -619,9 +620,12 @@ ecryptfs_rename(struct inode *old_dir, struct dentry *old_dentry, rc = -ENOTEMPTY; goto out_lock; } - rc = vfs_rename(d_inode(lower_old_dir_dentry), lower_old_dentry, - d_inode(lower_new_dir_dentry), lower_new_dentry, - NULL, 0); + + rd.old_dir = d_inode(lower_old_dir_dentry); + rd.old_dentry = lower_old_dentry; + rd.new_dir = d_inode(lower_new_dir_dentry); + rd.new_dentry = lower_new_dentry; + rc = vfs_rename(&rd); if (rc) goto out_lock; if (target_inode) diff --git a/fs/namei.c b/fs/namei.c index 93fa7d803fb2..38ab51881247 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -4311,12 +4311,15 @@ SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname * ->i_mutex on parents, which works but leads to some truly excessive * locking]. */ -int vfs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - struct inode **delegated_inode, unsigned int flags) +int vfs_rename(struct renamedata *rd) { int error; struct user_namespace *mnt_userns = &init_user_ns; + struct inode *old_dir = rd->old_dir, *new_dir = rd->new_dir; + struct dentry *old_dentry = rd->old_dentry; + struct dentry *new_dentry = rd->new_dentry; + struct inode **delegated_inode = rd->delegated_inode; + unsigned int flags = rd->flags; bool is_dir = d_is_dir(old_dentry); struct inode *source = old_dentry->d_inode; struct inode *target = new_dentry->d_inode; @@ -4442,6 +4445,7 @@ EXPORT_SYMBOL(vfs_rename); int do_renameat2(int olddfd, struct filename *from, int newdfd, struct filename *to, unsigned int flags) { + struct renamedata rd; struct dentry *old_dentry, *new_dentry; struct dentry *trap; struct path old_path, new_path; @@ -4545,9 +4549,14 @@ retry_deleg: &new_path, new_dentry, flags); if (error) goto exit5; - error = vfs_rename(old_path.dentry->d_inode, old_dentry, - new_path.dentry->d_inode, new_dentry, - &delegated_inode, flags); + + rd.old_dir = old_path.dentry->d_inode; + rd.old_dentry = old_dentry; + rd.new_dir = new_path.dentry->d_inode; + rd.new_dentry = new_dentry; + rd.delegated_inode = &delegated_inode; + rd.flags = flags; + error = vfs_rename(&rd); exit5: dput(new_dentry); exit4: diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 37d85046b4d6..f7d83ff2b44e 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -1798,7 +1798,13 @@ retry: close_cached = true; goto out_dput_old; } else { - host_err = vfs_rename(fdir, odentry, tdir, ndentry, NULL, 0); + struct renamedata rd = { + .old_dir = fdir, + .old_dentry = odentry, + .new_dir = tdir, + .new_dentry = ndentry, + }; + host_err = vfs_rename(&rd); if (!host_err) { host_err = commit_metadata(tfhp); if (!host_err) diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index 0002834f664a..426899681df7 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -214,9 +214,16 @@ static inline int ovl_do_rename(struct inode *olddir, struct dentry *olddentry, unsigned int flags) { int err; + struct renamedata rd = { + .old_dir = olddir, + .old_dentry = olddentry, + .new_dir = newdir, + .new_dentry = newdentry, + .flags = flags, + }; pr_debug("rename(%pd2, %pd2, 0x%x)\n", olddentry, newdentry, flags); - err = vfs_rename(olddir, olddentry, newdir, newdentry, NULL, flags); + err = vfs_rename(&rd); if (err) { pr_debug("...rename(%pd2, %pd2, ...) = %i\n", olddentry, newdentry, err); diff --git a/include/linux/fs.h b/include/linux/fs.h index a27884af7222..430e457f67f1 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1775,7 +1775,17 @@ extern int vfs_symlink(struct inode *, struct dentry *, const char *); extern int vfs_link(struct dentry *, struct inode *, struct dentry *, struct inode **); extern int vfs_rmdir(struct inode *, struct dentry *); extern int vfs_unlink(struct inode *, struct dentry *, struct inode **); -extern int vfs_rename(struct inode *, struct dentry *, struct inode *, struct dentry *, struct inode **, unsigned int); + +struct renamedata { + struct inode *old_dir; + struct dentry *old_dentry; + struct inode *new_dir; + struct dentry *new_dentry; + struct inode **delegated_inode; + unsigned int flags; +} __randomize_layout; + +int vfs_rename(struct renamedata *); static inline int vfs_whiteout(struct inode *dir, struct dentry *dentry) { -- cgit v1.2.3 From 6521f8917082928a4cb637eb64b77b5f2f5b30fc Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:33 +0100 Subject: namei: prepare for idmapped mounts The various vfs_*() helpers are called by filesystems or by the vfs itself to perform core operations such as create, link, mkdir, mknod, rename, rmdir, tmpfile and unlink. Enable them to handle idmapped mounts. If the inode is accessed through an idmapped mount map it into the mount's user namespace and pass it down. Afterwards the checks and operations are identical to non-idmapped mounts. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-15-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- drivers/base/devtmpfs.c | 11 ++- fs/cachefiles/namei.c | 12 ++- fs/ecryptfs/inode.c | 33 ++++--- fs/init.c | 14 +-- fs/namei.c | 227 ++++++++++++++++++++++++++++++++++++++--------- fs/nfsd/nfs4recover.c | 6 +- fs/nfsd/vfs.c | 19 ++-- fs/overlayfs/dir.c | 4 +- fs/overlayfs/overlayfs.h | 20 +++-- include/linux/fs.h | 32 ++++--- ipc/mqueue.c | 3 +- net/unix/af_unix.c | 3 +- 12 files changed, 279 insertions(+), 105 deletions(-) (limited to 'fs') diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c index 2e0c3cdb4184..653c8c6ac7a7 100644 --- a/drivers/base/devtmpfs.c +++ b/drivers/base/devtmpfs.c @@ -162,7 +162,7 @@ static int dev_mkdir(const char *name, umode_t mode) if (IS_ERR(dentry)) return PTR_ERR(dentry); - err = vfs_mkdir(d_inode(path.dentry), dentry, mode); + err = vfs_mkdir(&init_user_ns, d_inode(path.dentry), dentry, mode); if (!err) /* mark as kernel-created inode */ d_inode(dentry)->i_private = &thread; @@ -212,7 +212,8 @@ static int handle_create(const char *nodename, umode_t mode, kuid_t uid, if (IS_ERR(dentry)) return PTR_ERR(dentry); - err = vfs_mknod(d_inode(path.dentry), dentry, mode, dev->devt); + err = vfs_mknod(&init_user_ns, d_inode(path.dentry), dentry, mode, + dev->devt); if (!err) { struct iattr newattrs; @@ -242,7 +243,8 @@ static int dev_rmdir(const char *name) return PTR_ERR(dentry); if (d_really_is_positive(dentry)) { if (d_inode(dentry)->i_private == &thread) - err = vfs_rmdir(d_inode(parent.dentry), dentry); + err = vfs_rmdir(&init_user_ns, d_inode(parent.dentry), + dentry); else err = -EPERM; } else { @@ -330,7 +332,8 @@ static int handle_remove(const char *nodename, struct device *dev) inode_lock(d_inode(dentry)); notify_change(&init_user_ns, dentry, &newattrs, NULL); inode_unlock(d_inode(dentry)); - err = vfs_unlink(d_inode(parent.dentry), dentry, NULL); + err = vfs_unlink(&init_user_ns, d_inode(parent.dentry), + dentry, NULL); if (!err || err == -ENOENT) deleted = 1; } diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index 7b987de0babe..7bf0732ae25c 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -311,7 +311,8 @@ static int cachefiles_bury_object(struct cachefiles_cache *cache, cachefiles_io_error(cache, "Unlink security error"); } else { trace_cachefiles_unlink(object, rep, why); - ret = vfs_unlink(d_inode(dir), rep, NULL); + ret = vfs_unlink(&init_user_ns, d_inode(dir), rep, + NULL); if (preemptive) cachefiles_mark_object_buried(cache, rep, why); @@ -413,8 +414,10 @@ try_again: cachefiles_io_error(cache, "Rename security error %d", ret); } else { struct renamedata rd = { + .old_mnt_userns = &init_user_ns, .old_dir = d_inode(dir), .old_dentry = rep, + .new_mnt_userns = &init_user_ns, .new_dir = d_inode(cache->graveyard), .new_dentry = grave, }; @@ -566,7 +569,7 @@ lookup_again: if (ret < 0) goto create_error; start = jiffies; - ret = vfs_mkdir(d_inode(dir), next, 0); + ret = vfs_mkdir(&init_user_ns, d_inode(dir), next, 0); cachefiles_hist(cachefiles_mkdir_histogram, start); if (!key) trace_cachefiles_mkdir(object, next, ret); @@ -602,7 +605,8 @@ lookup_again: if (ret < 0) goto create_error; start = jiffies; - ret = vfs_create(d_inode(dir), next, S_IFREG, true); + ret = vfs_create(&init_user_ns, d_inode(dir), next, + S_IFREG, true); cachefiles_hist(cachefiles_create_histogram, start); trace_cachefiles_create(object, next, ret); if (ret < 0) @@ -796,7 +800,7 @@ retry: ret = security_path_mkdir(&path, subdir, 0700); if (ret < 0) goto mkdir_error; - ret = vfs_mkdir(d_inode(dir), subdir, 0700); + ret = vfs_mkdir(&init_user_ns, d_inode(dir), subdir, 0700); if (ret < 0) goto mkdir_error; diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index ff48abb09679..73e3d47e7b2d 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -141,7 +141,8 @@ static int ecryptfs_do_unlink(struct inode *dir, struct dentry *dentry, else if (d_unhashed(lower_dentry)) rc = -EINVAL; else - rc = vfs_unlink(lower_dir_inode, lower_dentry, NULL); + rc = vfs_unlink(&init_user_ns, lower_dir_inode, lower_dentry, + NULL); if (rc) { printk(KERN_ERR "Error in vfs_unlink; rc = [%d]\n", rc); goto out_unlock; @@ -180,7 +181,8 @@ ecryptfs_do_create(struct inode *directory_inode, lower_dentry = ecryptfs_dentry_to_lower(ecryptfs_dentry); lower_dir_dentry = lock_parent(lower_dentry); - rc = vfs_create(d_inode(lower_dir_dentry), lower_dentry, mode, true); + rc = vfs_create(&init_user_ns, d_inode(lower_dir_dentry), lower_dentry, + mode, true); if (rc) { printk(KERN_ERR "%s: Failure to create dentry in lower fs; " "rc = [%d]\n", __func__, rc); @@ -190,7 +192,8 @@ ecryptfs_do_create(struct inode *directory_inode, inode = __ecryptfs_get_inode(d_inode(lower_dentry), directory_inode->i_sb); if (IS_ERR(inode)) { - vfs_unlink(d_inode(lower_dir_dentry), lower_dentry, NULL); + vfs_unlink(&init_user_ns, d_inode(lower_dir_dentry), + lower_dentry, NULL); goto out_lock; } fsstack_copy_attr_times(directory_inode, d_inode(lower_dir_dentry)); @@ -436,8 +439,8 @@ static int ecryptfs_link(struct dentry *old_dentry, struct inode *dir, dget(lower_old_dentry); dget(lower_new_dentry); lower_dir_dentry = lock_parent(lower_new_dentry); - rc = vfs_link(lower_old_dentry, d_inode(lower_dir_dentry), - lower_new_dentry, NULL); + rc = vfs_link(lower_old_dentry, &init_user_ns, + d_inode(lower_dir_dentry), lower_new_dentry, NULL); if (rc || d_really_is_negative(lower_new_dentry)) goto out_lock; rc = ecryptfs_interpose(lower_new_dentry, new_dentry, dir->i_sb); @@ -481,7 +484,7 @@ static int ecryptfs_symlink(struct inode *dir, struct dentry *dentry, strlen(symname)); if (rc) goto out_lock; - rc = vfs_symlink(d_inode(lower_dir_dentry), lower_dentry, + rc = vfs_symlink(&init_user_ns, d_inode(lower_dir_dentry), lower_dentry, encoded_symname); kfree(encoded_symname); if (rc || d_really_is_negative(lower_dentry)) @@ -507,7 +510,8 @@ static int ecryptfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode lower_dentry = ecryptfs_dentry_to_lower(dentry); lower_dir_dentry = lock_parent(lower_dentry); - rc = vfs_mkdir(d_inode(lower_dir_dentry), lower_dentry, mode); + rc = vfs_mkdir(&init_user_ns, d_inode(lower_dir_dentry), lower_dentry, + mode); if (rc || d_really_is_negative(lower_dentry)) goto out; rc = ecryptfs_interpose(lower_dentry, dentry, dir->i_sb); @@ -541,7 +545,7 @@ static int ecryptfs_rmdir(struct inode *dir, struct dentry *dentry) else if (d_unhashed(lower_dentry)) rc = -EINVAL; else - rc = vfs_rmdir(lower_dir_inode, lower_dentry); + rc = vfs_rmdir(&init_user_ns, lower_dir_inode, lower_dentry); if (!rc) { clear_nlink(d_inode(dentry)); fsstack_copy_attr_times(dir, lower_dir_inode); @@ -563,7 +567,8 @@ ecryptfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev lower_dentry = ecryptfs_dentry_to_lower(dentry); lower_dir_dentry = lock_parent(lower_dentry); - rc = vfs_mknod(d_inode(lower_dir_dentry), lower_dentry, mode, dev); + rc = vfs_mknod(&init_user_ns, d_inode(lower_dir_dentry), lower_dentry, + mode, dev); if (rc || d_really_is_negative(lower_dentry)) goto out; rc = ecryptfs_interpose(lower_dentry, dentry, dir->i_sb); @@ -621,10 +626,12 @@ ecryptfs_rename(struct inode *old_dir, struct dentry *old_dentry, goto out_lock; } - rd.old_dir = d_inode(lower_old_dir_dentry); - rd.old_dentry = lower_old_dentry; - rd.new_dir = d_inode(lower_new_dir_dentry); - rd.new_dentry = lower_new_dentry; + rd.old_mnt_userns = &init_user_ns; + rd.old_dir = d_inode(lower_old_dir_dentry); + rd.old_dentry = lower_old_dentry; + rd.new_mnt_userns = &init_user_ns; + rd.new_dir = d_inode(lower_new_dir_dentry); + rd.new_dentry = lower_new_dentry; rc = vfs_rename(&rd); if (rc) goto out_lock; diff --git a/fs/init.c b/fs/init.c index 891284f8a443..e65452750fa5 100644 --- a/fs/init.c +++ b/fs/init.c @@ -157,8 +157,8 @@ int __init init_mknod(const char *filename, umode_t mode, unsigned int dev) mode &= ~current_umask(); error = security_path_mknod(&path, dentry, mode, dev); if (!error) - error = vfs_mknod(path.dentry->d_inode, dentry, mode, - new_decode_dev(dev)); + error = vfs_mknod(&init_user_ns, path.dentry->d_inode, dentry, + mode, new_decode_dev(dev)); done_path_create(&path, dentry); return error; } @@ -187,8 +187,8 @@ int __init init_link(const char *oldname, const char *newname) error = security_path_link(old_path.dentry, &new_path, new_dentry); if (error) goto out_dput; - error = vfs_link(old_path.dentry, new_path.dentry->d_inode, new_dentry, - NULL); + error = vfs_link(old_path.dentry, &init_user_ns, + new_path.dentry->d_inode, new_dentry, NULL); out_dput: done_path_create(&new_path, new_dentry); out: @@ -207,7 +207,8 @@ int __init init_symlink(const char *oldname, const char *newname) return PTR_ERR(dentry); error = security_path_symlink(&path, dentry, oldname); if (!error) - error = vfs_symlink(path.dentry->d_inode, dentry, oldname); + error = vfs_symlink(&init_user_ns, path.dentry->d_inode, dentry, + oldname); done_path_create(&path, dentry); return error; } @@ -230,7 +231,8 @@ int __init init_mkdir(const char *pathname, umode_t mode) mode &= ~current_umask(); error = security_path_mkdir(&path, dentry, mode); if (!error) - error = vfs_mkdir(path.dentry->d_inode, dentry, mode); + error = vfs_mkdir(&init_user_ns, path.dentry->d_inode, dentry, + mode); done_path_create(&path, dentry); return error; } diff --git a/fs/namei.c b/fs/namei.c index 38ab51881247..5c9f6f8e90c4 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -2874,10 +2874,26 @@ void unlock_rename(struct dentry *p1, struct dentry *p2) } EXPORT_SYMBOL(unlock_rename); -int vfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool want_excl) +/** + * vfs_create - create new file + * @mnt_userns: user namespace of the mount the inode was found from + * @dir: inode of @dentry + * @dentry: pointer to dentry of the base directory + * @mode: mode of the new file + * @want_excl: whether the file must not yet exist + * + * Create a new file. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. + */ +int vfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool want_excl) { - int error = may_create(&init_user_ns, dir, dentry); + int error = may_create(mnt_userns, dir, dentry); if (error) return error; @@ -3353,7 +3369,23 @@ static int do_open(struct nameidata *nd, return error; } -struct dentry *vfs_tmpfile(struct dentry *dentry, umode_t mode, int open_flag) +/** + * vfs_tmpfile - create tmpfile + * @mnt_userns: user namespace of the mount the inode was found from + * @dentry: pointer to dentry of the base directory + * @mode: mode of the new tmpfile + * @open_flags: flags + * + * Create a temporary file. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. + */ +struct dentry *vfs_tmpfile(struct user_namespace *mnt_userns, + struct dentry *dentry, umode_t mode, int open_flag) { struct dentry *child = NULL; struct inode *dir = dentry->d_inode; @@ -3361,7 +3393,7 @@ struct dentry *vfs_tmpfile(struct dentry *dentry, umode_t mode, int open_flag) int error; /* we want directory to be writable */ - error = inode_permission(&init_user_ns, dir, MAY_WRITE | MAY_EXEC); + error = inode_permission(mnt_userns, dir, MAY_WRITE | MAY_EXEC); if (error) goto out_err; error = -EOPNOTSUPP; @@ -3396,6 +3428,7 @@ static int do_tmpfile(struct nameidata *nd, unsigned flags, const struct open_flags *op, struct file *file) { + struct user_namespace *mnt_userns; struct dentry *child; struct path path; int error = path_lookupat(nd, flags | LOOKUP_DIRECTORY, &path); @@ -3404,7 +3437,8 @@ static int do_tmpfile(struct nameidata *nd, unsigned flags, error = mnt_want_write(path.mnt); if (unlikely(error)) goto out; - child = vfs_tmpfile(path.dentry, op->mode, op->open_flag); + mnt_userns = mnt_user_ns(path.mnt); + child = vfs_tmpfile(mnt_userns, path.dentry, op->mode, op->open_flag); error = PTR_ERR(child); if (IS_ERR(child)) goto out2; @@ -3616,10 +3650,27 @@ inline struct dentry *user_path_create(int dfd, const char __user *pathname, } EXPORT_SYMBOL(user_path_create); -int vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +/** + * vfs_mknod - create device node or file + * @mnt_userns: user namespace of the mount the inode was found from + * @dir: inode of @dentry + * @dentry: pointer to dentry of the base directory + * @mode: mode of the new device node or file + * @dev: device number of device to create + * + * Create a device node or file. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. + */ +int vfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) { bool is_whiteout = S_ISCHR(mode) && dev == WHITEOUT_DEV; - int error = may_create(&init_user_ns, dir, dentry); + int error = may_create(mnt_userns, dir, dentry); if (error) return error; @@ -3666,6 +3717,7 @@ static int may_mknod(umode_t mode) static long do_mknodat(int dfd, const char __user *filename, umode_t mode, unsigned int dev) { + struct user_namespace *mnt_userns; struct dentry *dentry; struct path path; int error; @@ -3684,18 +3736,22 @@ retry: error = security_path_mknod(&path, dentry, mode, dev); if (error) goto out; + + mnt_userns = mnt_user_ns(path.mnt); switch (mode & S_IFMT) { case 0: case S_IFREG: - error = vfs_create(path.dentry->d_inode,dentry,mode,true); + error = vfs_create(mnt_userns, path.dentry->d_inode, + dentry, mode, true); if (!error) ima_post_path_mknod(dentry); break; case S_IFCHR: case S_IFBLK: - error = vfs_mknod(path.dentry->d_inode,dentry,mode, - new_decode_dev(dev)); + error = vfs_mknod(mnt_userns, path.dentry->d_inode, + dentry, mode, new_decode_dev(dev)); break; case S_IFIFO: case S_IFSOCK: - error = vfs_mknod(path.dentry->d_inode,dentry,mode,0); + error = vfs_mknod(mnt_userns, path.dentry->d_inode, + dentry, mode, 0); break; } out: @@ -3718,9 +3774,25 @@ SYSCALL_DEFINE3(mknod, const char __user *, filename, umode_t, mode, unsigned, d return do_mknodat(AT_FDCWD, filename, mode, dev); } -int vfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +/** + * vfs_mkdir - create directory + * @mnt_userns: user namespace of the mount the inode was found from + * @dir: inode of @dentry + * @dentry: pointer to dentry of the base directory + * @mode: mode of the new directory + * + * Create a directory. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. + */ +int vfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { - int error = may_create(&init_user_ns, dir, dentry); + int error = may_create(mnt_userns, dir, dentry); unsigned max_links = dir->i_sb->s_max_links; if (error) @@ -3759,8 +3831,11 @@ retry: if (!IS_POSIXACL(path.dentry->d_inode)) mode &= ~current_umask(); error = security_path_mkdir(&path, dentry, mode); - if (!error) - error = vfs_mkdir(path.dentry->d_inode, dentry, mode); + if (!error) { + struct user_namespace *mnt_userns; + mnt_userns = mnt_user_ns(path.mnt); + error = vfs_mkdir(mnt_userns, path.dentry->d_inode, dentry, mode); + } done_path_create(&path, dentry); if (retry_estale(error, lookup_flags)) { lookup_flags |= LOOKUP_REVAL; @@ -3779,9 +3854,24 @@ SYSCALL_DEFINE2(mkdir, const char __user *, pathname, umode_t, mode) return do_mkdirat(AT_FDCWD, pathname, mode); } -int vfs_rmdir(struct inode *dir, struct dentry *dentry) +/** + * vfs_rmdir - remove directory + * @mnt_userns: user namespace of the mount the inode was found from + * @dir: inode of @dentry + * @dentry: pointer to dentry of the base directory + * + * Remove a directory. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. + */ +int vfs_rmdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry) { - int error = may_delete(&init_user_ns, dir, dentry, 1); + int error = may_delete(mnt_userns, dir, dentry, 1); if (error) return error; @@ -3821,6 +3911,7 @@ EXPORT_SYMBOL(vfs_rmdir); long do_rmdir(int dfd, struct filename *name) { + struct user_namespace *mnt_userns; int error = 0; struct dentry *dentry; struct path path; @@ -3861,7 +3952,8 @@ retry: error = security_path_rmdir(&path, dentry); if (error) goto exit3; - error = vfs_rmdir(path.dentry->d_inode, dentry); + mnt_userns = mnt_user_ns(path.mnt); + error = vfs_rmdir(mnt_userns, path.dentry->d_inode, dentry); exit3: dput(dentry); exit2: @@ -3884,6 +3976,7 @@ SYSCALL_DEFINE1(rmdir, const char __user *, pathname) /** * vfs_unlink - unlink a filesystem object + * @mnt_userns: user namespace of the mount the inode was found from * @dir: parent directory * @dentry: victim * @delegated_inode: returns victim inode, if the inode is delegated. @@ -3899,11 +3992,18 @@ SYSCALL_DEFINE1(rmdir, const char __user *, pathname) * Alternatively, a caller may pass NULL for delegated_inode. This may * be appropriate for callers that expect the underlying filesystem not * to be NFS exported. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. */ -int vfs_unlink(struct inode *dir, struct dentry *dentry, struct inode **delegated_inode) +int vfs_unlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, struct inode **delegated_inode) { struct inode *target = dentry->d_inode; - int error = may_delete(&init_user_ns, dir, dentry, 0); + int error = may_delete(mnt_userns, dir, dentry, 0); if (error) return error; @@ -3974,6 +4074,8 @@ retry_deleg: dentry = __lookup_hash(&last, path.dentry, lookup_flags); error = PTR_ERR(dentry); if (!IS_ERR(dentry)) { + struct user_namespace *mnt_userns; + /* Why not before? Because we want correct error value */ if (last.name[last.len]) goto slashes; @@ -3984,7 +4086,8 @@ retry_deleg: error = security_path_unlink(&path, dentry); if (error) goto exit2; - error = vfs_unlink(path.dentry->d_inode, dentry, &delegated_inode); + mnt_userns = mnt_user_ns(path.mnt); + error = vfs_unlink(mnt_userns, path.dentry->d_inode, dentry, &delegated_inode); exit2: dput(dentry); } @@ -4033,9 +4136,25 @@ SYSCALL_DEFINE1(unlink, const char __user *, pathname) return do_unlinkat(AT_FDCWD, getname(pathname)); } -int vfs_symlink(struct inode *dir, struct dentry *dentry, const char *oldname) +/** + * vfs_symlink - create symlink + * @mnt_userns: user namespace of the mount the inode was found from + * @dir: inode of @dentry + * @dentry: pointer to dentry of the base directory + * @oldname: name of the file to link to + * + * Create a symlink. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. + */ +int vfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *oldname) { - int error = may_create(&init_user_ns, dir, dentry); + int error = may_create(mnt_userns, dir, dentry); if (error) return error; @@ -4073,8 +4192,13 @@ retry: goto out_putname; error = security_path_symlink(&path, dentry, from->name); - if (!error) - error = vfs_symlink(path.dentry->d_inode, dentry, from->name); + if (!error) { + struct user_namespace *mnt_userns; + + mnt_userns = mnt_user_ns(path.mnt); + error = vfs_symlink(mnt_userns, path.dentry->d_inode, dentry, + from->name); + } done_path_create(&path, dentry); if (retry_estale(error, lookup_flags)) { lookup_flags |= LOOKUP_REVAL; @@ -4099,6 +4223,7 @@ SYSCALL_DEFINE2(symlink, const char __user *, oldname, const char __user *, newn /** * vfs_link - create a new link * @old_dentry: object to be linked + * @mnt_userns: the user namespace of the mount * @dir: new parent * @new_dentry: where to create the new link * @delegated_inode: returns inode needing a delegation break @@ -4114,8 +4239,16 @@ SYSCALL_DEFINE2(symlink, const char __user *, oldname, const char __user *, newn * Alternatively, a caller may pass NULL for delegated_inode. This may * be appropriate for callers that expect the underlying filesystem not * to be NFS exported. + * + * If the inode has been found through an idmapped mount the user namespace of + * the vfsmount must be passed through @mnt_userns. This function will then take + * care to map the inode according to @mnt_userns before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply passs init_user_ns. */ -int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry, struct inode **delegated_inode) +int vfs_link(struct dentry *old_dentry, struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *new_dentry, + struct inode **delegated_inode) { struct inode *inode = old_dentry->d_inode; unsigned max_links = dir->i_sb->s_max_links; @@ -4124,7 +4257,7 @@ int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_de if (!inode) return -ENOENT; - error = may_create(&init_user_ns, dir, new_dentry); + error = may_create(mnt_userns, dir, new_dentry); if (error) return error; @@ -4141,7 +4274,7 @@ int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_de * be writen back improperly if their true value is unknown to * the vfs. */ - if (HAS_UNMAPPED_ID(&init_user_ns, inode)) + if (HAS_UNMAPPED_ID(mnt_userns, inode)) return -EPERM; if (!dir->i_op->link) return -EPERM; @@ -4188,6 +4321,7 @@ EXPORT_SYMBOL(vfs_link); static int do_linkat(int olddfd, const char __user *oldname, int newdfd, const char __user *newname, int flags) { + struct user_namespace *mnt_userns; struct dentry *new_dentry; struct path old_path, new_path; struct inode *delegated_inode = NULL; @@ -4229,7 +4363,9 @@ retry: error = security_path_link(old_path.dentry, &new_path, new_dentry); if (error) goto out_dput; - error = vfs_link(old_path.dentry, new_path.dentry->d_inode, new_dentry, &delegated_inode); + mnt_userns = mnt_user_ns(new_path.mnt); + error = vfs_link(old_path.dentry, mnt_userns, new_path.dentry->d_inode, + new_dentry, &delegated_inode); out_dput: done_path_create(&new_path, new_dentry); if (delegated_inode) { @@ -4263,12 +4399,14 @@ SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname /** * vfs_rename - rename a filesystem object - * @old_dir: parent of source - * @old_dentry: source - * @new_dir: parent of destination - * @new_dentry: destination - * @delegated_inode: returns an inode needing a delegation break - * @flags: rename flags + * @old_mnt_userns: old user namespace of the mount the inode was found from + * @old_dir: parent of source + * @old_dentry: source + * @new_mnt_userns: new user namespace of the mount the inode was found from + * @new_dir: parent of destination + * @new_dentry: destination + * @delegated_inode: returns an inode needing a delegation break + * @flags: rename flags * * The caller must hold multiple mutexes--see lock_rename()). * @@ -4314,7 +4452,6 @@ SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname int vfs_rename(struct renamedata *rd) { int error; - struct user_namespace *mnt_userns = &init_user_ns; struct inode *old_dir = rd->old_dir, *new_dir = rd->new_dir; struct dentry *old_dentry = rd->old_dentry; struct dentry *new_dentry = rd->new_dentry; @@ -4330,19 +4467,21 @@ int vfs_rename(struct renamedata *rd) if (source == target) return 0; - error = may_delete(mnt_userns, old_dir, old_dentry, is_dir); + error = may_delete(rd->old_mnt_userns, old_dir, old_dentry, is_dir); if (error) return error; if (!target) { - error = may_create(mnt_userns, new_dir, new_dentry); + error = may_create(rd->new_mnt_userns, new_dir, new_dentry); } else { new_is_dir = d_is_dir(new_dentry); if (!(flags & RENAME_EXCHANGE)) - error = may_delete(mnt_userns, new_dir, new_dentry, is_dir); + error = may_delete(rd->new_mnt_userns, new_dir, + new_dentry, is_dir); else - error = may_delete(mnt_userns, new_dir, new_dentry, new_is_dir); + error = may_delete(rd->new_mnt_userns, new_dir, + new_dentry, new_is_dir); } if (error) return error; @@ -4356,13 +4495,13 @@ int vfs_rename(struct renamedata *rd) */ if (new_dir != old_dir) { if (is_dir) { - error = inode_permission(&init_user_ns, source, + error = inode_permission(rd->old_mnt_userns, source, MAY_WRITE); if (error) return error; } if ((flags & RENAME_EXCHANGE) && new_is_dir) { - error = inode_permission(&init_user_ns, target, + error = inode_permission(rd->new_mnt_userns, target, MAY_WRITE); if (error) return error; @@ -4552,8 +4691,10 @@ retry_deleg: rd.old_dir = old_path.dentry->d_inode; rd.old_dentry = old_dentry; + rd.old_mnt_userns = mnt_user_ns(old_path.mnt); rd.new_dir = new_path.dentry->d_inode; rd.new_dentry = new_dentry; + rd.new_mnt_userns = mnt_user_ns(new_path.mnt); rd.delegated_inode = &delegated_inode; rd.flags = flags; error = vfs_rename(&rd); diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index 186fa2c2c6ba..891395c6c7d3 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -233,7 +233,7 @@ nfsd4_create_clid_dir(struct nfs4_client *clp) * as well be forgiving and just succeed silently. */ goto out_put; - status = vfs_mkdir(d_inode(dir), dentry, S_IRWXU); + status = vfs_mkdir(&init_user_ns, d_inode(dir), dentry, S_IRWXU); out_put: dput(dentry); out_unlock: @@ -353,7 +353,7 @@ nfsd4_unlink_clid_dir(char *name, int namlen, struct nfsd_net *nn) status = -ENOENT; if (d_really_is_negative(dentry)) goto out; - status = vfs_rmdir(d_inode(dir), dentry); + status = vfs_rmdir(&init_user_ns, d_inode(dir), dentry); out: dput(dentry); out_unlock: @@ -443,7 +443,7 @@ purge_old(struct dentry *parent, struct dentry *child, struct nfsd_net *nn) if (nfs4_has_reclaimed_state(name, nn)) goto out_free; - status = vfs_rmdir(d_inode(parent), child); + status = vfs_rmdir(&init_user_ns, d_inode(parent), child); if (status) printk("failed to remove client recovery directory %pd\n", child); diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index f7d83ff2b44e..fab873178140 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -1255,12 +1255,12 @@ nfsd_create_locked(struct svc_rqst *rqstp, struct svc_fh *fhp, host_err = 0; switch (type) { case S_IFREG: - host_err = vfs_create(dirp, dchild, iap->ia_mode, true); + host_err = vfs_create(&init_user_ns, dirp, dchild, iap->ia_mode, true); if (!host_err) nfsd_check_ignore_resizing(iap); break; case S_IFDIR: - host_err = vfs_mkdir(dirp, dchild, iap->ia_mode); + host_err = vfs_mkdir(&init_user_ns, dirp, dchild, iap->ia_mode); if (!host_err && unlikely(d_unhashed(dchild))) { struct dentry *d; d = lookup_one_len(dchild->d_name.name, @@ -1288,7 +1288,8 @@ nfsd_create_locked(struct svc_rqst *rqstp, struct svc_fh *fhp, case S_IFBLK: case S_IFIFO: case S_IFSOCK: - host_err = vfs_mknod(dirp, dchild, iap->ia_mode, rdev); + host_err = vfs_mknod(&init_user_ns, dirp, dchild, + iap->ia_mode, rdev); break; default: printk(KERN_WARNING "nfsd: bad file type %o in nfsd_create\n", @@ -1486,7 +1487,7 @@ do_nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp, if (!IS_POSIXACL(dirp)) iap->ia_mode &= ~current_umask(); - host_err = vfs_create(dirp, dchild, iap->ia_mode, true); + host_err = vfs_create(&init_user_ns, dirp, dchild, iap->ia_mode, true); if (host_err < 0) { fh_drop_write(fhp); goto out_nfserr; @@ -1610,7 +1611,7 @@ nfsd_symlink(struct svc_rqst *rqstp, struct svc_fh *fhp, if (IS_ERR(dnew)) goto out_nfserr; - host_err = vfs_symlink(d_inode(dentry), dnew, path); + host_err = vfs_symlink(&init_user_ns, d_inode(dentry), dnew, path); err = nfserrno(host_err); if (!err) err = nfserrno(commit_metadata(fhp)); @@ -1678,7 +1679,7 @@ nfsd_link(struct svc_rqst *rqstp, struct svc_fh *ffhp, err = nfserr_noent; if (d_really_is_negative(dold)) goto out_dput; - host_err = vfs_link(dold, dirp, dnew, NULL); + host_err = vfs_link(dold, &init_user_ns, dirp, dnew, NULL); if (!host_err) { err = nfserrno(commit_metadata(ffhp)); if (!err) @@ -1799,8 +1800,10 @@ retry: goto out_dput_old; } else { struct renamedata rd = { + .old_mnt_userns = &init_user_ns, .old_dir = fdir, .old_dentry = odentry, + .new_mnt_userns = &init_user_ns, .new_dir = tdir, .new_dentry = ndentry, }; @@ -1891,9 +1894,9 @@ nfsd_unlink(struct svc_rqst *rqstp, struct svc_fh *fhp, int type, if (type != S_IFDIR) { if (rdentry->d_sb->s_export_op->flags & EXPORT_OP_CLOSE_BEFORE_UNLINK) nfsd_close_cached_files(rdentry); - host_err = vfs_unlink(dirp, rdentry, NULL); + host_err = vfs_unlink(&init_user_ns, dirp, rdentry, NULL); } else { - host_err = vfs_rmdir(dirp, rdentry); + host_err = vfs_rmdir(&init_user_ns, dirp, rdentry); } if (!host_err) diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index d75c96cb18c3..6904cc2ed7bb 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -821,9 +821,9 @@ static int ovl_remove_upper(struct dentry *dentry, bool is_dir, goto out_dput_upper; if (is_dir) - err = vfs_rmdir(dir, upper); + err = vfs_rmdir(&init_user_ns, dir, upper); else - err = vfs_unlink(dir, upper, NULL); + err = vfs_unlink(&init_user_ns, dir, upper, NULL); ovl_dir_modified(dentry->d_parent, ovl_type_origin(dentry)); /* diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index 426899681df7..5e9eb46e741a 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -123,7 +123,7 @@ static inline const char *ovl_xattr(struct ovl_fs *ofs, enum ovl_xattr ox) static inline int ovl_do_rmdir(struct inode *dir, struct dentry *dentry) { - int err = vfs_rmdir(dir, dentry); + int err = vfs_rmdir(&init_user_ns, dir, dentry); pr_debug("rmdir(%pd2) = %i\n", dentry, err); return err; @@ -131,7 +131,7 @@ static inline int ovl_do_rmdir(struct inode *dir, struct dentry *dentry) static inline int ovl_do_unlink(struct inode *dir, struct dentry *dentry) { - int err = vfs_unlink(dir, dentry, NULL); + int err = vfs_unlink(&init_user_ns, dir, dentry, NULL); pr_debug("unlink(%pd2) = %i\n", dentry, err); return err; @@ -140,7 +140,7 @@ static inline int ovl_do_unlink(struct inode *dir, struct dentry *dentry) static inline int ovl_do_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry) { - int err = vfs_link(old_dentry, dir, new_dentry, NULL); + int err = vfs_link(old_dentry, &init_user_ns, dir, new_dentry, NULL); pr_debug("link(%pd2, %pd2) = %i\n", old_dentry, new_dentry, err); return err; @@ -149,7 +149,7 @@ static inline int ovl_do_link(struct dentry *old_dentry, struct inode *dir, static inline int ovl_do_create(struct inode *dir, struct dentry *dentry, umode_t mode) { - int err = vfs_create(dir, dentry, mode, true); + int err = vfs_create(&init_user_ns, dir, dentry, mode, true); pr_debug("create(%pd2, 0%o) = %i\n", dentry, mode, err); return err; @@ -158,7 +158,7 @@ static inline int ovl_do_create(struct inode *dir, struct dentry *dentry, static inline int ovl_do_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) { - int err = vfs_mkdir(dir, dentry, mode); + int err = vfs_mkdir(&init_user_ns, dir, dentry, mode); pr_debug("mkdir(%pd2, 0%o) = %i\n", dentry, mode, err); return err; } @@ -166,7 +166,7 @@ static inline int ovl_do_mkdir(struct inode *dir, struct dentry *dentry, static inline int ovl_do_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) { - int err = vfs_mknod(dir, dentry, mode, dev); + int err = vfs_mknod(&init_user_ns, dir, dentry, mode, dev); pr_debug("mknod(%pd2, 0%o, 0%o) = %i\n", dentry, mode, dev, err); return err; @@ -175,7 +175,7 @@ static inline int ovl_do_mknod(struct inode *dir, struct dentry *dentry, static inline int ovl_do_symlink(struct inode *dir, struct dentry *dentry, const char *oldname) { - int err = vfs_symlink(dir, dentry, oldname); + int err = vfs_symlink(&init_user_ns, dir, dentry, oldname); pr_debug("symlink(\"%s\", %pd2) = %i\n", oldname, dentry, err); return err; @@ -215,8 +215,10 @@ static inline int ovl_do_rename(struct inode *olddir, struct dentry *olddentry, { int err; struct renamedata rd = { + .old_mnt_userns = &init_user_ns, .old_dir = olddir, .old_dentry = olddentry, + .new_mnt_userns = &init_user_ns, .new_dir = newdir, .new_dentry = newdentry, .flags = flags, @@ -233,14 +235,14 @@ static inline int ovl_do_rename(struct inode *olddir, struct dentry *olddentry, static inline int ovl_do_whiteout(struct inode *dir, struct dentry *dentry) { - int err = vfs_whiteout(dir, dentry); + int err = vfs_whiteout(&init_user_ns, dir, dentry); pr_debug("whiteout(%pd2) = %i\n", dentry, err); return err; } static inline struct dentry *ovl_do_tmpfile(struct dentry *dentry, umode_t mode) { - struct dentry *ret = vfs_tmpfile(dentry, mode, 0); + struct dentry *ret = vfs_tmpfile(&init_user_ns, dentry, mode, 0); int err = PTR_ERR_OR_ZERO(ret); pr_debug("tmpfile(%pd2, 0%o) = %i\n", dentry, mode, err); diff --git a/include/linux/fs.h b/include/linux/fs.h index 430e457f67f1..29d7b2fe7de4 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1768,17 +1768,25 @@ bool inode_owner_or_capable(struct user_namespace *mnt_userns, /* * VFS helper functions.. */ -extern int vfs_create(struct inode *, struct dentry *, umode_t, bool); -extern int vfs_mkdir(struct inode *, struct dentry *, umode_t); -extern int vfs_mknod(struct inode *, struct dentry *, umode_t, dev_t); -extern int vfs_symlink(struct inode *, struct dentry *, const char *); -extern int vfs_link(struct dentry *, struct inode *, struct dentry *, struct inode **); -extern int vfs_rmdir(struct inode *, struct dentry *); -extern int vfs_unlink(struct inode *, struct dentry *, struct inode **); +int vfs_create(struct user_namespace *, struct inode *, + struct dentry *, umode_t, bool); +int vfs_mkdir(struct user_namespace *, struct inode *, + struct dentry *, umode_t); +int vfs_mknod(struct user_namespace *, struct inode *, struct dentry *, + umode_t, dev_t); +int vfs_symlink(struct user_namespace *, struct inode *, + struct dentry *, const char *); +int vfs_link(struct dentry *, struct user_namespace *, struct inode *, + struct dentry *, struct inode **); +int vfs_rmdir(struct user_namespace *, struct inode *, struct dentry *); +int vfs_unlink(struct user_namespace *, struct inode *, struct dentry *, + struct inode **); struct renamedata { + struct user_namespace *old_mnt_userns; struct inode *old_dir; struct dentry *old_dentry; + struct user_namespace *new_mnt_userns; struct inode *new_dir; struct dentry *new_dentry; struct inode **delegated_inode; @@ -1787,13 +1795,15 @@ struct renamedata { int vfs_rename(struct renamedata *); -static inline int vfs_whiteout(struct inode *dir, struct dentry *dentry) +static inline int vfs_whiteout(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry) { - return vfs_mknod(dir, dentry, S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); + return vfs_mknod(mnt_userns, dir, dentry, S_IFCHR | WHITEOUT_MODE, + WHITEOUT_DEV); } -extern struct dentry *vfs_tmpfile(struct dentry *dentry, umode_t mode, - int open_flag); +struct dentry *vfs_tmpfile(struct user_namespace *mnt_userns, + struct dentry *dentry, umode_t mode, int open_flag); int vfs_mkobj(struct dentry *, umode_t, int (*f)(struct dentry *, umode_t, void *), diff --git a/ipc/mqueue.c b/ipc/mqueue.c index 693f01fe1216..fcd56e077733 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -965,7 +965,8 @@ SYSCALL_DEFINE1(mq_unlink, const char __user *, u_name) err = -ENOENT; } else { ihold(inode); - err = vfs_unlink(d_inode(dentry->d_parent), dentry, NULL); + err = vfs_unlink(&init_user_ns, d_inode(dentry->d_parent), + dentry, NULL); } dput(dentry); diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 18453d15dddf..9a1f3c04402e 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -996,7 +996,8 @@ static int unix_mknod(const char *sun_path, umode_t mode, struct path *res) */ err = security_path_mknod(&path, dentry, mode, 0); if (!err) { - err = vfs_mknod(d_inode(path.dentry), dentry, mode, 0); + err = vfs_mknod(&init_user_ns, d_inode(path.dentry), dentry, + mode, 0); if (!err) { res->mnt = mntget(path.mnt); res->dentry = dget(dentry); -- cgit v1.2.3 From 643fe55a0679ae5582a1a2a1df86dc240292cd1b Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:34 +0100 Subject: open: handle idmapped mounts in do_truncate() When truncating files the vfs will verify that the caller is privileged over the inode. Extend it to handle idmapped mounts. If the inode is accessed through an idmapped mount it is mapped according to the mount's user namespace. Afterwards the permissions checks are identical to non-idmapped mounts. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-16-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/coredump.c | 10 +++++++--- fs/inode.c | 7 ++++--- fs/namei.c | 2 +- fs/open.c | 16 +++++++++------- include/linux/fs.h | 4 ++-- 5 files changed, 23 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/coredump.c b/fs/coredump.c index a2f6ecc8e345..ae778937a1ff 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -703,6 +703,7 @@ void do_coredump(const kernel_siginfo_t *siginfo) goto close_fail; } } else { + struct user_namespace *mnt_userns; struct inode *inode; int open_flags = O_CREAT | O_RDWR | O_NOFOLLOW | O_LARGEFILE | O_EXCL; @@ -780,13 +781,15 @@ void do_coredump(const kernel_siginfo_t *siginfo) * a process dumps core while its cwd is e.g. on a vfat * filesystem. */ - if (!uid_eq(inode->i_uid, current_fsuid())) + mnt_userns = file_mnt_user_ns(cprm.file); + if (!uid_eq(i_uid_into_mnt(mnt_userns, inode), current_fsuid())) goto close_fail; if ((inode->i_mode & 0677) != 0600) goto close_fail; if (!(cprm.file->f_mode & FMODE_CAN_WRITE)) goto close_fail; - if (do_truncate(cprm.file->f_path.dentry, 0, 0, cprm.file)) + if (do_truncate(mnt_userns, cprm.file->f_path.dentry, + 0, 0, cprm.file)) goto close_fail; } @@ -931,7 +934,8 @@ void dump_truncate(struct coredump_params *cprm) if (file->f_op->llseek && file->f_op->llseek != no_llseek) { offset = file->f_op->llseek(file, 0, SEEK_CUR); if (i_size_read(file->f_mapping->host) < offset) - do_truncate(file->f_path.dentry, offset, 0, file); + do_truncate(file_mnt_user_ns(file), file->f_path.dentry, + offset, 0, file); } } EXPORT_SYMBOL(dump_truncate); diff --git a/fs/inode.c b/fs/inode.c index 46116ef44c9f..08151968c9ef 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -1903,7 +1903,8 @@ int dentry_needs_remove_privs(struct dentry *dentry) return mask; } -static int __remove_privs(struct dentry *dentry, int kill) +static int __remove_privs(struct user_namespace *mnt_userns, + struct dentry *dentry, int kill) { struct iattr newattrs; @@ -1912,7 +1913,7 @@ static int __remove_privs(struct dentry *dentry, int kill) * Note we call this on write, so notify_change will not * encounter any conflicting delegations: */ - return notify_change(&init_user_ns, dentry, &newattrs, NULL); + return notify_change(mnt_userns, dentry, &newattrs, NULL); } /* @@ -1939,7 +1940,7 @@ int file_remove_privs(struct file *file) if (kill < 0) return kill; if (kill) - error = __remove_privs(dentry, kill); + error = __remove_privs(file_mnt_user_ns(file), dentry, kill); if (!error) inode_has_no_xattr(inode); diff --git a/fs/namei.c b/fs/namei.c index 5c9f6f8e90c4..c8c083daf368 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -3009,7 +3009,7 @@ static int handle_truncate(struct file *filp) if (!error) error = security_path_truncate(path); if (!error) { - error = do_truncate(path->dentry, 0, + error = do_truncate(&init_user_ns, path->dentry, 0, ATTR_MTIME|ATTR_CTIME|ATTR_OPEN, filp); } diff --git a/fs/open.c b/fs/open.c index c3e4dc43dd8d..8b3f3eb652d0 100644 --- a/fs/open.c +++ b/fs/open.c @@ -35,8 +35,8 @@ #include "internal.h" -int do_truncate(struct dentry *dentry, loff_t length, unsigned int time_attrs, - struct file *filp) +int do_truncate(struct user_namespace *mnt_userns, struct dentry *dentry, + loff_t length, unsigned int time_attrs, struct file *filp) { int ret; struct iattr newattrs; @@ -61,13 +61,14 @@ int do_truncate(struct dentry *dentry, loff_t length, unsigned int time_attrs, inode_lock(dentry->d_inode); /* Note any delegations or leases have already been broken: */ - ret = notify_change(&init_user_ns, dentry, &newattrs, NULL); + ret = notify_change(mnt_userns, dentry, &newattrs, NULL); inode_unlock(dentry->d_inode); return ret; } long vfs_truncate(const struct path *path, loff_t length) { + struct user_namespace *mnt_userns; struct inode *inode; long error; @@ -83,7 +84,8 @@ long vfs_truncate(const struct path *path, loff_t length) if (error) goto out; - error = inode_permission(&init_user_ns, inode, MAY_WRITE); + mnt_userns = mnt_user_ns(path->mnt); + error = inode_permission(mnt_userns, inode, MAY_WRITE); if (error) goto mnt_drop_write_and_out; @@ -107,7 +109,7 @@ long vfs_truncate(const struct path *path, loff_t length) if (!error) error = security_path_truncate(path); if (!error) - error = do_truncate(path->dentry, length, 0, NULL); + error = do_truncate(mnt_userns, path->dentry, length, 0, NULL); put_write_and_out: put_write_access(inode); @@ -186,13 +188,13 @@ long do_sys_ftruncate(unsigned int fd, loff_t length, int small) /* Check IS_APPEND on real upper inode */ if (IS_APPEND(file_inode(f.file))) goto out_putf; - sb_start_write(inode->i_sb); error = locks_verify_truncate(inode, f.file, length); if (!error) error = security_path_truncate(&f.file->f_path); if (!error) - error = do_truncate(dentry, length, ATTR_MTIME|ATTR_CTIME, f.file); + error = do_truncate(file_mnt_user_ns(f.file), dentry, length, + ATTR_MTIME | ATTR_CTIME, f.file); sb_end_write(inode->i_sb); out_putf: fdput(f); diff --git a/include/linux/fs.h b/include/linux/fs.h index 29d7b2fe7de4..f0601cca1930 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2593,8 +2593,8 @@ static inline struct user_namespace *file_mnt_user_ns(struct file *file) return mnt_user_ns(file->f_path.mnt); } extern long vfs_truncate(const struct path *, loff_t); -extern int do_truncate(struct dentry *, loff_t start, unsigned int time_attrs, - struct file *filp); +int do_truncate(struct user_namespace *, struct dentry *, loff_t start, + unsigned int time_attrs, struct file *filp); extern int vfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len); extern long do_sys_open(int dfd, const char __user *filename, int flags, -- cgit v1.2.3 From b8b546a061884d98ba6ac6b4a0d1ee6566031cf5 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:35 +0100 Subject: open: handle idmapped mounts For core file operations such as changing directories or chrooting, determining file access, changing mode or ownership the vfs will verify that the caller is privileged over the inode. Extend the various helpers to handle idmapped mounts. If the inode is accessed through an idmapped mount map it into the mount's user namespace. Afterwards the permissions checks are identical to non-idmapped mounts. When changing file ownership we need to map the uid and gid from the mount's user namespace. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-17-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: James Morris Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/open.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/open.c b/fs/open.c index 8b3f3eb652d0..4ec3979d0466 100644 --- a/fs/open.c +++ b/fs/open.c @@ -438,7 +438,7 @@ retry: goto out_path_release; } - res = inode_permission(&init_user_ns, inode, mode | MAY_ACCESS); + res = inode_permission(mnt_user_ns(path.mnt), inode, mode | MAY_ACCESS); /* SuS v2 requires we report a read only fs too */ if (res || !(mode & S_IWOTH) || special_file(inode->i_mode)) goto out_path_release; @@ -582,8 +582,8 @@ retry_deleg: goto out_unlock; newattrs.ia_mode = (mode & S_IALLUGO) | (inode->i_mode & ~S_IALLUGO); newattrs.ia_valid = ATTR_MODE | ATTR_CTIME; - error = notify_change(&init_user_ns, path->dentry, &newattrs, - &delegated_inode); + error = notify_change(mnt_user_ns(path->mnt), path->dentry, + &newattrs, &delegated_inode); out_unlock: inode_unlock(inode); if (delegated_inode) { @@ -644,6 +644,7 @@ SYSCALL_DEFINE2(chmod, const char __user *, filename, umode_t, mode) int chown_common(const struct path *path, uid_t user, gid_t group) { + struct user_namespace *mnt_userns; struct inode *inode = path->dentry->d_inode; struct inode *delegated_inode = NULL; int error; @@ -654,6 +655,10 @@ int chown_common(const struct path *path, uid_t user, gid_t group) uid = make_kuid(current_user_ns(), user); gid = make_kgid(current_user_ns(), group); + mnt_userns = mnt_user_ns(path->mnt); + uid = kuid_from_mnt(mnt_userns, uid); + gid = kgid_from_mnt(mnt_userns, gid); + retry_deleg: newattrs.ia_valid = ATTR_CTIME; if (user != (uid_t) -1) { @@ -674,7 +679,7 @@ retry_deleg: inode_lock(inode); error = security_path_chown(path, uid, gid); if (!error) - error = notify_change(&init_user_ns, path->dentry, &newattrs, + error = notify_change(mnt_userns, path->dentry, &newattrs, &delegated_inode); inode_unlock(inode); if (delegated_inode) { -- cgit v1.2.3 From d06c26f196dd5b196dfa5ff9703cf6c1ce921b9d Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:37 +0100 Subject: utimes: handle idmapped mounts Enable the vfs_utimes() helper to handle idmapped mounts by passing down the mount's user namespace. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-19-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Signed-off-by: Christian Brauner --- fs/utimes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/utimes.c b/fs/utimes.c index 4572b91ddb91..39f356017635 100644 --- a/fs/utimes.c +++ b/fs/utimes.c @@ -62,7 +62,7 @@ int vfs_utimes(const struct path *path, struct timespec64 *times) } retry_deleg: inode_lock(inode); - error = notify_change(&init_user_ns, path->dentry, &newattrs, + error = notify_change(mnt_user_ns(path->mnt), path->dentry, &newattrs, &delegated_inode); inode_unlock(inode); if (delegated_inode) { -- cgit v1.2.3 From 9eccd12ce7f1c063a5ff03d8b8456b2b9761c6b6 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:38 +0100 Subject: fcntl: handle idmapped mounts Enable the setfl() helper to handle idmapped mounts by passing down the mount's user namespace. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-20-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Signed-off-by: Christian Brauner --- fs/fcntl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/fcntl.c b/fs/fcntl.c index 74d99731fd43..f6ac5285060d 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -46,7 +47,7 @@ static int setfl(int fd, struct file * filp, unsigned long arg) /* O_NOATIME can only be set by the owner or superuser */ if ((arg & O_NOATIME) && !(filp->f_flags & O_NOATIME)) - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(file_mnt_user_ns(filp), inode)) return -EPERM; /* required for strict SunOS emulation */ -- cgit v1.2.3 From b816dd5ddeae3252bb4255105b4c641e4c841501 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:39 +0100 Subject: init: handle idmapped mounts Enable the init helpers to handle idmapped mounts by passing down the mount's user namespace. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-21-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/init.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/init.c b/fs/init.c index e65452750fa5..5c36adaa9b44 100644 --- a/fs/init.c +++ b/fs/init.c @@ -157,8 +157,8 @@ int __init init_mknod(const char *filename, umode_t mode, unsigned int dev) mode &= ~current_umask(); error = security_path_mknod(&path, dentry, mode, dev); if (!error) - error = vfs_mknod(&init_user_ns, path.dentry->d_inode, dentry, - mode, new_decode_dev(dev)); + error = vfs_mknod(mnt_user_ns(path.mnt), path.dentry->d_inode, + dentry, mode, new_decode_dev(dev)); done_path_create(&path, dentry); return error; } @@ -167,6 +167,7 @@ int __init init_link(const char *oldname, const char *newname) { struct dentry *new_dentry; struct path old_path, new_path; + struct user_namespace *mnt_userns; int error; error = kern_path(oldname, 0, &old_path); @@ -181,14 +182,15 @@ int __init init_link(const char *oldname, const char *newname) error = -EXDEV; if (old_path.mnt != new_path.mnt) goto out_dput; - error = may_linkat(&init_user_ns, &old_path); + mnt_userns = mnt_user_ns(new_path.mnt); + error = may_linkat(mnt_userns, &old_path); if (unlikely(error)) goto out_dput; error = security_path_link(old_path.dentry, &new_path, new_dentry); if (error) goto out_dput; - error = vfs_link(old_path.dentry, &init_user_ns, - new_path.dentry->d_inode, new_dentry, NULL); + error = vfs_link(old_path.dentry, mnt_userns, new_path.dentry->d_inode, + new_dentry, NULL); out_dput: done_path_create(&new_path, new_dentry); out: @@ -207,8 +209,8 @@ int __init init_symlink(const char *oldname, const char *newname) return PTR_ERR(dentry); error = security_path_symlink(&path, dentry, oldname); if (!error) - error = vfs_symlink(&init_user_ns, path.dentry->d_inode, dentry, - oldname); + error = vfs_symlink(mnt_user_ns(path.mnt), path.dentry->d_inode, + dentry, oldname); done_path_create(&path, dentry); return error; } @@ -231,8 +233,8 @@ int __init init_mkdir(const char *pathname, umode_t mode) mode &= ~current_umask(); error = security_path_mkdir(&path, dentry, mode); if (!error) - error = vfs_mkdir(&init_user_ns, path.dentry->d_inode, dentry, - mode); + error = vfs_mkdir(mnt_user_ns(path.mnt), path.dentry->d_inode, + dentry, mode); done_path_create(&path, dentry); return error; } -- cgit v1.2.3 From 0f5d220b42fba0ab17a7353787be3c29bea92e30 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:40 +0100 Subject: ioctl: handle idmapped mounts Enable generic ioctls to handle idmapped mounts by passing down the mount's user namespace. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-22-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Signed-off-by: Christian Brauner --- fs/remap_range.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/remap_range.c b/fs/remap_range.c index 29a4a4dbfe12..e4a5fdd7ad7b 100644 --- a/fs/remap_range.c +++ b/fs/remap_range.c @@ -432,13 +432,16 @@ EXPORT_SYMBOL(vfs_clone_file_range); /* Check whether we are allowed to dedupe the destination file */ static bool allow_file_dedupe(struct file *file) { + struct user_namespace *mnt_userns = file_mnt_user_ns(file); + struct inode *inode = file_inode(file); + if (capable(CAP_SYS_ADMIN)) return true; if (file->f_mode & FMODE_WRITE) return true; - if (uid_eq(current_fsuid(), file_inode(file)->i_uid)) + if (uid_eq(current_fsuid(), i_uid_into_mnt(mnt_userns, inode))) return true; - if (!inode_permission(&init_user_ns, file_inode(file), MAY_WRITE)) + if (!inode_permission(mnt_userns, inode, MAY_WRITE)) return true; return false; } -- cgit v1.2.3 From 435ac6214e00eab191296cd6cfdc316aa07e7794 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:41 +0100 Subject: would_dump: handle idmapped mounts When determining whether or not to create a coredump the vfs will verify that the caller is privileged over the inode. Make the would_dump() helper handle idmapped mounts by passing down the mount's user namespace of the exec file. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-23-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/exec.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/exec.c b/fs/exec.c index a8ec371cd3cd..d803227805f6 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1404,15 +1404,15 @@ EXPORT_SYMBOL(begin_new_exec); void would_dump(struct linux_binprm *bprm, struct file *file) { struct inode *inode = file_inode(file); - if (inode_permission(&init_user_ns, inode, MAY_READ) < 0) { + struct user_namespace *mnt_userns = file_mnt_user_ns(file); + if (inode_permission(mnt_userns, inode, MAY_READ) < 0) { struct user_namespace *old, *user_ns; bprm->interp_flags |= BINPRM_FLAGS_ENFORCE_NONDUMP; /* Ensure mm->user_ns contains the executable */ user_ns = old = bprm->mm->user_ns; while ((user_ns != &init_user_ns) && - !privileged_wrt_inode_uidgid(user_ns, &init_user_ns, - inode)) + !privileged_wrt_inode_uidgid(user_ns, mnt_userns, inode)) user_ns = user_ns->parent; if (old != user_ns) { -- cgit v1.2.3 From 1ab29965b30599c797266eb3b970b4e6c8de3672 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:42 +0100 Subject: exec: handle idmapped mounts When executing a setuid binary the kernel will verify in bprm_fill_uid() that the inode has a mapping in the caller's user namespace before setting the callers uid and gid. Let bprm_fill_uid() handle idmapped mounts. If the inode is accessed through an idmapped mount it is mapped according to the mount's user namespace. Afterwards the checks are identical to non-idmapped mounts. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-24-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: James Morris Signed-off-by: Christian Brauner --- fs/exec.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/exec.c b/fs/exec.c index d803227805f6..48d1e8b1610b 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1580,6 +1580,7 @@ static void check_unsafe_exec(struct linux_binprm *bprm) static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file) { /* Handle suid and sgid on files */ + struct user_namespace *mnt_userns; struct inode *inode; unsigned int mode; kuid_t uid; @@ -1596,13 +1597,15 @@ static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file) if (!(mode & (S_ISUID|S_ISGID))) return; + mnt_userns = file_mnt_user_ns(file); + /* Be careful if suid/sgid is set */ inode_lock(inode); /* reload atomically mode/uid/gid now that lock held */ mode = inode->i_mode; - uid = inode->i_uid; - gid = inode->i_gid; + uid = i_uid_into_mnt(mnt_userns, inode); + gid = i_gid_into_mnt(mnt_userns, inode); inode_unlock(inode); /* We ignore suid/sgid if there are no mappings for them in the ns */ -- cgit v1.2.3 From 549c7297717c32ee53f156cd949e055e601f67bb Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:43 +0100 Subject: fs: make helpers idmap mount aware Extend some inode methods with an additional user namespace argument. A filesystem that is aware of idmapped mounts will receive the user namespace the mount has been marked with. This can be used for additional permission checking and also to enable filesystems to translate between uids and gids if they need to. We have implemented all relevant helpers in earlier patches. As requested we simply extend the exisiting inode method instead of introducing new ones. This is a little more code churn but it's mostly mechanical and doesnt't leave us with additional inode methods. Link: https://lore.kernel.org/r/20210121131959.646623-25-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- Documentation/filesystems/vfs.rst | 19 ++++++----- arch/powerpc/platforms/cell/spufs/inode.c | 3 +- drivers/android/binderfs.c | 6 ++-- fs/9p/acl.c | 4 +-- fs/9p/v9fs.h | 3 +- fs/9p/v9fs_vfs.h | 3 +- fs/9p/vfs_inode.c | 26 ++++++++------- fs/9p/vfs_inode_dotl.c | 31 +++++++++-------- fs/adfs/adfs.h | 3 +- fs/adfs/inode.c | 3 +- fs/affs/affs.h | 24 +++++++++----- fs/affs/inode.c | 3 +- fs/affs/namei.c | 15 +++++---- fs/afs/dir.c | 34 ++++++++++--------- fs/afs/inode.c | 7 ++-- fs/afs/internal.h | 7 ++-- fs/afs/security.c | 3 +- fs/attr.c | 4 +-- fs/autofs/root.c | 17 ++++++---- fs/bad_inode.c | 36 ++++++++++++-------- fs/bfs/dir.c | 10 +++--- fs/btrfs/acl.c | 3 +- fs/btrfs/ctree.h | 3 +- fs/btrfs/inode.c | 33 +++++++++++-------- fs/ceph/acl.c | 3 +- fs/ceph/dir.c | 23 ++++++------- fs/ceph/inode.c | 10 +++--- fs/ceph/super.h | 12 ++++--- fs/cifs/cifsfs.c | 3 +- fs/cifs/cifsfs.h | 25 ++++++++------ fs/cifs/dir.c | 8 ++--- fs/cifs/inode.c | 16 +++++---- fs/cifs/link.c | 3 +- fs/coda/coda_linux.h | 8 +++-- fs/coda/dir.c | 18 ++++++---- fs/coda/inode.c | 7 ++-- fs/coda/pioctl.c | 6 ++-- fs/configfs/configfs_internal.h | 6 ++-- fs/configfs/dir.c | 3 +- fs/configfs/inode.c | 5 +-- fs/configfs/symlink.c | 3 +- fs/debugfs/inode.c | 9 ++--- fs/ecryptfs/inode.c | 30 ++++++++++------- fs/efivarfs/inode.c | 4 +-- fs/erofs/inode.c | 5 +-- fs/erofs/internal.h | 5 +-- fs/exfat/exfat_fs.h | 8 +++-- fs/exfat/file.c | 8 +++-- fs/exfat/namei.c | 14 ++++---- fs/ext2/acl.c | 3 +- fs/ext2/acl.h | 3 +- fs/ext2/ext2.h | 5 +-- fs/ext2/inode.c | 7 ++-- fs/ext2/namei.c | 22 ++++++++----- fs/ext4/acl.c | 3 +- fs/ext4/acl.h | 3 +- fs/ext4/ext4.h | 9 +++-- fs/ext4/inode.c | 12 ++++--- fs/ext4/namei.c | 19 ++++++----- fs/f2fs/acl.c | 3 +- fs/f2fs/acl.h | 3 +- fs/f2fs/f2fs.h | 7 ++-- fs/f2fs/file.c | 7 ++-- fs/f2fs/namei.c | 21 +++++++----- fs/fat/fat.h | 6 ++-- fs/fat/file.c | 9 ++--- fs/fat/namei_msdos.c | 10 +++--- fs/fat/namei_vfat.c | 13 ++++---- fs/fuse/acl.c | 3 +- fs/fuse/dir.c | 37 ++++++++++++--------- fs/fuse/fuse_i.h | 4 +-- fs/gfs2/acl.c | 3 +- fs/gfs2/acl.h | 3 +- fs/gfs2/file.c | 2 +- fs/gfs2/inode.c | 53 +++++++++++++++++------------ fs/gfs2/inode.h | 3 +- fs/hfs/dir.c | 13 ++++---- fs/hfs/hfs_fs.h | 3 +- fs/hfs/inode.c | 6 ++-- fs/hfsplus/dir.c | 22 +++++++------ fs/hfsplus/hfsplus_fs.h | 5 +-- fs/hfsplus/inode.c | 8 +++-- fs/hostfs/hostfs_kern.c | 23 ++++++++----- fs/hpfs/hpfs_fn.h | 2 +- fs/hpfs/inode.c | 3 +- fs/hpfs/namei.c | 20 ++++++----- fs/hugetlbfs/inode.c | 29 +++++++++------- fs/jffs2/acl.c | 3 +- fs/jffs2/acl.h | 3 +- fs/jffs2/dir.c | 33 +++++++++++-------- fs/jffs2/fs.c | 3 +- fs/jffs2/os-linux.h | 2 +- fs/jfs/acl.c | 3 +- fs/jfs/file.c | 3 +- fs/jfs/jfs_acl.h | 3 +- fs/jfs/jfs_inode.h | 2 +- fs/jfs/namei.c | 21 ++++++------ fs/kernfs/dir.c | 6 ++-- fs/kernfs/inode.c | 9 +++-- fs/kernfs/kernfs-internal.h | 9 +++-- fs/libfs.c | 31 +++++++++-------- fs/minix/file.c | 3 +- fs/minix/inode.c | 4 +-- fs/minix/minix.h | 3 +- fs/minix/namei.c | 24 ++++++++------ fs/namei.c | 55 ++++++++++++++++++------------- fs/nfs/dir.c | 23 ++++++++----- fs/nfs/inode.c | 7 ++-- fs/nfs/internal.h | 14 +++++--- fs/nfs/namespace.c | 13 +++++--- fs/nfs/nfs3_fs.h | 3 +- fs/nfs/nfs3acl.c | 3 +- fs/nilfs2/inode.c | 6 ++-- fs/nilfs2/namei.c | 19 ++++++----- fs/nilfs2/nilfs.h | 6 ++-- fs/ntfs/inode.c | 4 ++- fs/ntfs/inode.h | 3 +- fs/ocfs2/acl.c | 3 +- fs/ocfs2/acl.h | 3 +- fs/ocfs2/dlmfs/dlmfs.c | 9 +++-- fs/ocfs2/file.c | 10 +++--- fs/ocfs2/file.h | 11 ++++--- fs/ocfs2/namei.c | 19 +++++++---- fs/omfs/dir.c | 13 ++++---- fs/omfs/file.c | 3 +- fs/orangefs/acl.c | 3 +- fs/orangefs/inode.c | 10 +++--- fs/orangefs/namei.c | 12 ++++--- fs/orangefs/orangefs-kernel.h | 13 +++++--- fs/overlayfs/dir.c | 21 ++++++------ fs/overlayfs/inode.c | 10 +++--- fs/overlayfs/overlayfs.h | 10 +++--- fs/overlayfs/super.c | 2 +- fs/posix_acl.c | 9 ++--- fs/proc/base.c | 16 +++++---- fs/proc/fd.c | 3 +- fs/proc/fd.h | 3 +- fs/proc/generic.c | 6 ++-- fs/proc/internal.h | 6 ++-- fs/proc/proc_net.c | 3 +- fs/proc/proc_sysctl.c | 9 +++-- fs/proc/root.c | 3 +- fs/ramfs/file-nommu.c | 5 +-- fs/ramfs/inode.c | 16 +++++---- fs/reiserfs/acl.h | 3 +- fs/reiserfs/inode.c | 3 +- fs/reiserfs/namei.c | 19 ++++++----- fs/reiserfs/reiserfs.h | 3 +- fs/reiserfs/xattr.c | 11 ++++--- fs/reiserfs/xattr.h | 3 +- fs/reiserfs/xattr_acl.c | 3 +- fs/stat.c | 8 +++-- fs/sysv/file.c | 3 +- fs/sysv/itree.c | 4 +-- fs/sysv/namei.c | 21 +++++++----- fs/sysv/sysv.h | 3 +- fs/tracefs/inode.c | 4 ++- fs/ubifs/dir.c | 26 ++++++++------- fs/ubifs/file.c | 3 +- fs/ubifs/ubifs.h | 5 +-- fs/udf/file.c | 3 +- fs/udf/namei.c | 24 +++++++------- fs/udf/symlink.c | 5 +-- fs/ufs/inode.c | 3 +- fs/ufs/namei.c | 19 ++++++----- fs/ufs/ufs.h | 3 +- fs/vboxsf/dir.c | 12 ++++--- fs/vboxsf/utils.c | 7 ++-- fs/vboxsf/vfsmod.h | 8 +++-- fs/xfs/xfs_acl.c | 3 +- fs/xfs/xfs_acl.h | 3 +- fs/xfs/xfs_iops.c | 52 ++++++++++++++++------------- fs/zonefs/super.c | 3 +- include/linux/fs.h | 39 ++++++++++++++-------- include/linux/nfs_fs.h | 7 ++-- include/linux/posix_acl.h | 3 +- ipc/mqueue.c | 4 +-- kernel/bpf/inode.c | 7 ++-- mm/shmem.c | 39 ++++++++++++++-------- net/socket.c | 5 +-- security/apparmor/apparmorfs.c | 3 +- security/integrity/evm/evm_secfs.c | 2 +- 182 files changed, 1121 insertions(+), 756 deletions(-) (limited to 'fs') diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index ca52c82e5bb5..98290ef311ca 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -415,28 +415,29 @@ As of kernel 2.6.22, the following members are defined: .. code-block:: c struct inode_operations { - int (*create) (struct inode *,struct dentry *, umode_t, bool); + int (*create) (struct user_namespace *, struct inode *,struct dentry *, umode_t, bool); struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int); int (*link) (struct dentry *,struct inode *,struct dentry *); int (*unlink) (struct inode *,struct dentry *); - int (*symlink) (struct inode *,struct dentry *,const char *); - int (*mkdir) (struct inode *,struct dentry *,umode_t); + int (*symlink) (struct user_namespace *, struct inode *,struct dentry *,const char *); + int (*mkdir) (struct user_namespace *, struct inode *,struct dentry *,umode_t); int (*rmdir) (struct inode *,struct dentry *); - int (*mknod) (struct inode *,struct dentry *,umode_t,dev_t); - int (*rename) (struct inode *, struct dentry *, + int (*mknod) (struct user_namespace *, struct inode *,struct dentry *,umode_t,dev_t); + int (*rename) (struct user_namespace *, struct inode *, struct dentry *, struct inode *, struct dentry *, unsigned int); int (*readlink) (struct dentry *, char __user *,int); const char *(*get_link) (struct dentry *, struct inode *, struct delayed_call *); - int (*permission) (struct inode *, int); + int (*permission) (struct user_namespace *, struct inode *, int); int (*get_acl)(struct inode *, int); - int (*setattr) (struct dentry *, struct iattr *); - int (*getattr) (const struct path *, struct kstat *, u32, unsigned int); + int (*setattr) (struct user_namespace *, struct dentry *, struct iattr *); + int (*getattr) (struct user_namespace *, const struct path *, struct kstat *, u32, unsigned int); ssize_t (*listxattr) (struct dentry *, char *, size_t); void (*update_time)(struct inode *, struct timespec *, int); int (*atomic_open)(struct inode *, struct dentry *, struct file *, unsigned open_flag, umode_t create_mode); - int (*tmpfile) (struct inode *, struct dentry *, umode_t); + int (*tmpfile) (struct user_namespace *, struct inode *, struct dentry *, umode_t); + int (*set_acl)(struct user_namespace *, struct inode *, struct posix_acl *, int); }; Again, all methods are called without any locks being held, unless diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c index 3de526eb2275..b83a3670bd74 100644 --- a/arch/powerpc/platforms/cell/spufs/inode.c +++ b/arch/powerpc/platforms/cell/spufs/inode.c @@ -91,7 +91,8 @@ out: } static int -spufs_setattr(struct dentry *dentry, struct iattr *attr) +spufs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); diff --git a/drivers/android/binderfs.c b/drivers/android/binderfs.c index 7b4f154f07e6..e80ba93c62a9 100644 --- a/drivers/android/binderfs.c +++ b/drivers/android/binderfs.c @@ -355,7 +355,8 @@ static inline bool is_binderfs_control_device(const struct dentry *dentry) return info->control_dentry == dentry; } -static int binderfs_rename(struct inode *old_dir, struct dentry *old_dentry, +static int binderfs_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { @@ -363,7 +364,8 @@ static int binderfs_rename(struct inode *old_dir, struct dentry *old_dentry, is_binderfs_control_device(new_dentry)) return -EPERM; - return simple_rename(old_dir, old_dentry, new_dir, new_dentry, flags); + return simple_rename(&init_user_ns, old_dir, old_dentry, new_dir, + new_dentry, flags); } static int binderfs_unlink(struct inode *dir, struct dentry *dentry) diff --git a/fs/9p/acl.c b/fs/9p/acl.c index 1c14f18a6ec9..bb1b286c49ae 100644 --- a/fs/9p/acl.c +++ b/fs/9p/acl.c @@ -280,7 +280,7 @@ static int v9fs_xattr_set_acl(const struct xattr_handler *handler, struct iattr iattr = { 0 }; struct posix_acl *old_acl = acl; - retval = posix_acl_update_mode(mnt_userns, inode, + retval = posix_acl_update_mode(&init_user_ns, inode, &iattr.ia_mode, &acl); if (retval) goto err_out; @@ -299,7 +299,7 @@ static int v9fs_xattr_set_acl(const struct xattr_handler *handler, * What is the following setxattr update the * mode ? */ - v9fs_vfs_setattr_dotl(dentry, &iattr); + v9fs_vfs_setattr_dotl(&init_user_ns, dentry, &iattr); } break; case ACL_TYPE_DEFAULT: diff --git a/fs/9p/v9fs.h b/fs/9p/v9fs.h index 7b763776306e..4ca56c5dd637 100644 --- a/fs/9p/v9fs.h +++ b/fs/9p/v9fs.h @@ -135,7 +135,8 @@ extern struct dentry *v9fs_vfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags); extern int v9fs_vfs_unlink(struct inode *i, struct dentry *d); extern int v9fs_vfs_rmdir(struct inode *i, struct dentry *d); -extern int v9fs_vfs_rename(struct inode *old_dir, struct dentry *old_dentry, +extern int v9fs_vfs_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags); extern struct inode *v9fs_inode_from_fid(struct v9fs_session_info *v9ses, diff --git a/fs/9p/v9fs_vfs.h b/fs/9p/v9fs_vfs.h index fd2a2b040250..d44ade76966a 100644 --- a/fs/9p/v9fs_vfs.h +++ b/fs/9p/v9fs_vfs.h @@ -59,7 +59,8 @@ void v9fs_inode2stat(struct inode *inode, struct p9_wstat *stat); int v9fs_uflags2omode(int uflags, int extended); void v9fs_blank_wstat(struct p9_wstat *wstat); -int v9fs_vfs_setattr_dotl(struct dentry *, struct iattr *); +int v9fs_vfs_setattr_dotl(struct user_namespace *, struct dentry *, + struct iattr *); int v9fs_file_fsync_dotl(struct file *filp, loff_t start, loff_t end, int datasync); int v9fs_refresh_inode(struct p9_fid *fid, struct inode *inode); diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index c21b146c8d91..648eb4c4cf7f 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -676,8 +676,8 @@ error: */ static int -v9fs_vfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +v9fs_vfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct v9fs_session_info *v9ses = v9fs_inode2v9ses(dir); u32 perm = unixmode2p9mode(v9ses, mode); @@ -702,7 +702,8 @@ v9fs_vfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, * */ -static int v9fs_vfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int v9fs_vfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { int err; u32 perm; @@ -907,9 +908,9 @@ int v9fs_vfs_rmdir(struct inode *i, struct dentry *d) */ int -v9fs_vfs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +v9fs_vfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { int retval; struct inode *old_inode; @@ -1016,8 +1017,8 @@ done: */ static int -v9fs_vfs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +v9fs_vfs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { struct dentry *dentry = path->dentry; struct v9fs_session_info *v9ses; @@ -1054,7 +1055,8 @@ v9fs_vfs_getattr(const struct path *path, struct kstat *stat, * */ -static int v9fs_vfs_setattr(struct dentry *dentry, struct iattr *iattr) +static int v9fs_vfs_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *iattr) { int retval, use_dentry = 0; struct v9fs_session_info *v9ses; @@ -1295,7 +1297,8 @@ static int v9fs_vfs_mkspecial(struct inode *dir, struct dentry *dentry, */ static int -v9fs_vfs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) +v9fs_vfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { p9_debug(P9_DEBUG_VFS, " %lu,%pd,%s\n", dir->i_ino, dentry, symname); @@ -1348,7 +1351,8 @@ v9fs_vfs_link(struct dentry *old_dentry, struct inode *dir, */ static int -v9fs_vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) +v9fs_vfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct v9fs_session_info *v9ses = v9fs_inode2v9ses(dir); int retval; diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c index 984f28315d2a..1dc7af046615 100644 --- a/fs/9p/vfs_inode_dotl.c +++ b/fs/9p/vfs_inode_dotl.c @@ -33,8 +33,8 @@ #include "acl.h" static int -v9fs_vfs_mknod_dotl(struct inode *dir, struct dentry *dentry, umode_t omode, - dev_t rdev); +v9fs_vfs_mknod_dotl(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t omode, dev_t rdev); /** * v9fs_get_fsgid_for_create - Helper function to get the gid for creating a @@ -218,10 +218,10 @@ int v9fs_open_to_dotl_flags(int flags) */ static int -v9fs_vfs_create_dotl(struct inode *dir, struct dentry *dentry, umode_t omode, - bool excl) +v9fs_vfs_create_dotl(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t omode, bool excl) { - return v9fs_vfs_mknod_dotl(dir, dentry, omode, 0); + return v9fs_vfs_mknod_dotl(mnt_userns, dir, dentry, omode, 0); } static int @@ -367,8 +367,9 @@ err_clunk_old_fid: * */ -static int v9fs_vfs_mkdir_dotl(struct inode *dir, - struct dentry *dentry, umode_t omode) +static int v9fs_vfs_mkdir_dotl(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + umode_t omode) { int err; struct v9fs_session_info *v9ses; @@ -457,8 +458,9 @@ error: } static int -v9fs_vfs_getattr_dotl(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +v9fs_vfs_getattr_dotl(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags) { struct dentry *dentry = path->dentry; struct v9fs_session_info *v9ses; @@ -540,7 +542,8 @@ static int v9fs_mapped_iattr_valid(int iattr_valid) * */ -int v9fs_vfs_setattr_dotl(struct dentry *dentry, struct iattr *iattr) +int v9fs_vfs_setattr_dotl(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *iattr) { int retval, use_dentry = 0; struct p9_fid *fid = NULL; @@ -684,8 +687,8 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode, } static int -v9fs_vfs_symlink_dotl(struct inode *dir, struct dentry *dentry, - const char *symname) +v9fs_vfs_symlink_dotl(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { int err; kgid_t gid; @@ -824,8 +827,8 @@ v9fs_vfs_link_dotl(struct dentry *old_dentry, struct inode *dir, * */ static int -v9fs_vfs_mknod_dotl(struct inode *dir, struct dentry *dentry, umode_t omode, - dev_t rdev) +v9fs_vfs_mknod_dotl(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t omode, dev_t rdev) { int err; kgid_t gid; diff --git a/fs/adfs/adfs.h b/fs/adfs/adfs.h index 699c4fa8b78b..06b7c92343ad 100644 --- a/fs/adfs/adfs.h +++ b/fs/adfs/adfs.h @@ -144,7 +144,8 @@ struct adfs_discmap { /* Inode stuff */ struct inode *adfs_iget(struct super_block *sb, struct object_info *obj); int adfs_write_inode(struct inode *inode, struct writeback_control *wbc); -int adfs_notify_change(struct dentry *dentry, struct iattr *attr); +int adfs_notify_change(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr); /* map.c */ int adfs_map_lookup(struct super_block *sb, u32 frag_id, unsigned int offset); diff --git a/fs/adfs/inode.c b/fs/adfs/inode.c index 278dcee6ae22..fb7ee026d101 100644 --- a/fs/adfs/inode.c +++ b/fs/adfs/inode.c @@ -292,7 +292,8 @@ out: * later. */ int -adfs_notify_change(struct dentry *dentry, struct iattr *attr) +adfs_notify_change(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); struct super_block *sb = inode->i_sb; diff --git a/fs/affs/affs.h b/fs/affs/affs.h index a755bef7c4c7..bfa89e131ead 100644 --- a/fs/affs/affs.h +++ b/fs/affs/affs.h @@ -167,27 +167,33 @@ extern const struct export_operations affs_export_ops; extern int affs_hash_name(struct super_block *sb, const u8 *name, unsigned int len); extern struct dentry *affs_lookup(struct inode *dir, struct dentry *dentry, unsigned int); extern int affs_unlink(struct inode *dir, struct dentry *dentry); -extern int affs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool); -extern int affs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode); +extern int affs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool); +extern int affs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode); extern int affs_rmdir(struct inode *dir, struct dentry *dentry); extern int affs_link(struct dentry *olddentry, struct inode *dir, struct dentry *dentry); -extern int affs_symlink(struct inode *dir, struct dentry *dentry, - const char *symname); -extern int affs_rename2(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags); +extern int affs_symlink(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + const char *symname); +extern int affs_rename2(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags); /* inode.c */ extern struct inode *affs_new_inode(struct inode *dir); -extern int affs_notify_change(struct dentry *dentry, struct iattr *attr); +extern int affs_notify_change(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr); extern void affs_evict_inode(struct inode *inode); extern struct inode *affs_iget(struct super_block *sb, unsigned long ino); extern int affs_write_inode(struct inode *inode, struct writeback_control *wbc); -extern int affs_add_entry(struct inode *dir, struct inode *inode, struct dentry *dentry, s32 type); +extern int affs_add_entry(struct inode *dir, struct inode *inode, + struct dentry *dentry, s32 type); /* file.c */ diff --git a/fs/affs/inode.c b/fs/affs/inode.c index 767e5bdfb703..2352a75bd9d6 100644 --- a/fs/affs/inode.c +++ b/fs/affs/inode.c @@ -216,7 +216,8 @@ affs_write_inode(struct inode *inode, struct writeback_control *wbc) } int -affs_notify_change(struct dentry *dentry, struct iattr *attr) +affs_notify_change(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); int error; diff --git a/fs/affs/namei.c b/fs/affs/namei.c index 41c5749f4db7..9ad22befce28 100644 --- a/fs/affs/namei.c +++ b/fs/affs/namei.c @@ -242,7 +242,8 @@ affs_unlink(struct inode *dir, struct dentry *dentry) } int -affs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) +affs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct super_block *sb = dir->i_sb; struct inode *inode; @@ -273,7 +274,8 @@ affs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) } int -affs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +affs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode *inode; int error; @@ -311,7 +313,8 @@ affs_rmdir(struct inode *dir, struct dentry *dentry) } int -affs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) +affs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { struct super_block *sb = dir->i_sb; struct buffer_head *bh; @@ -498,9 +501,9 @@ done: return retval; } -int affs_rename2(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +int affs_rename2(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE)) diff --git a/fs/afs/dir.c b/fs/afs/dir.c index 7bd659ad959e..714fcca9af99 100644 --- a/fs/afs/dir.c +++ b/fs/afs/dir.c @@ -28,18 +28,19 @@ static int afs_lookup_one_filldir(struct dir_context *ctx, const char *name, int loff_t fpos, u64 ino, unsigned dtype); static int afs_lookup_filldir(struct dir_context *ctx, const char *name, int nlen, loff_t fpos, u64 ino, unsigned dtype); -static int afs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl); -static int afs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode); +static int afs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl); +static int afs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode); static int afs_rmdir(struct inode *dir, struct dentry *dentry); static int afs_unlink(struct inode *dir, struct dentry *dentry); static int afs_link(struct dentry *from, struct inode *dir, struct dentry *dentry); -static int afs_symlink(struct inode *dir, struct dentry *dentry, - const char *content); -static int afs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags); +static int afs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *content); +static int afs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags); static int afs_dir_releasepage(struct page *page, gfp_t gfp_flags); static void afs_dir_invalidatepage(struct page *page, unsigned int offset, unsigned int length); @@ -1325,7 +1326,8 @@ static const struct afs_operation_ops afs_mkdir_operation = { /* * create a directory on an AFS filesystem */ -static int afs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int afs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct afs_operation *op; struct afs_vnode *dvnode = AFS_FS_I(dir); @@ -1619,8 +1621,8 @@ static const struct afs_operation_ops afs_create_operation = { /* * create a regular file on an AFS filesystem */ -static int afs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int afs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct afs_operation *op; struct afs_vnode *dvnode = AFS_FS_I(dir); @@ -1741,8 +1743,8 @@ static const struct afs_operation_ops afs_symlink_operation = { /* * create a symlink in an AFS filesystem */ -static int afs_symlink(struct inode *dir, struct dentry *dentry, - const char *content) +static int afs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *content) { struct afs_operation *op; struct afs_vnode *dvnode = AFS_FS_I(dir); @@ -1876,9 +1878,9 @@ static const struct afs_operation_ops afs_rename_operation = { /* * rename a file in an AFS filesystem and/or move it between directories */ -static int afs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int afs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct afs_operation *op; struct afs_vnode *orig_dvnode, *new_dvnode, *vnode; diff --git a/fs/afs/inode.c b/fs/afs/inode.c index 795ee5cb3817..1156b2df28d3 100644 --- a/fs/afs/inode.c +++ b/fs/afs/inode.c @@ -734,8 +734,8 @@ error_unlock: /* * read the attributes of an inode */ -int afs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags) +int afs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); struct afs_vnode *vnode = AFS_FS_I(inode); @@ -857,7 +857,8 @@ static const struct afs_operation_ops afs_setattr_operation = { /* * set the attributes of an inode */ -int afs_setattr(struct dentry *dentry, struct iattr *attr) +int afs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct afs_operation *op; struct afs_vnode *vnode = AFS_FS_I(d_inode(dentry)); diff --git a/fs/afs/internal.h b/fs/afs/internal.h index 0d150a29e39e..b626e38e9ab5 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -1149,8 +1149,9 @@ extern struct inode *afs_iget(struct afs_operation *, struct afs_vnode_param *); extern struct inode *afs_root_iget(struct super_block *, struct key *); extern bool afs_check_validity(struct afs_vnode *); extern int afs_validate(struct afs_vnode *, struct key *); -extern int afs_getattr(const struct path *, struct kstat *, u32, unsigned int); -extern int afs_setattr(struct dentry *, struct iattr *); +extern int afs_getattr(struct user_namespace *mnt_userns, const struct path *, + struct kstat *, u32, unsigned int); +extern int afs_setattr(struct user_namespace *mnt_userns, struct dentry *, struct iattr *); extern void afs_evict_inode(struct inode *); extern int afs_drop_inode(struct inode *); @@ -1361,7 +1362,7 @@ extern void afs_zap_permits(struct rcu_head *); extern struct key *afs_request_key(struct afs_cell *); extern struct key *afs_request_key_rcu(struct afs_cell *); extern int afs_check_permit(struct afs_vnode *, struct key *, afs_access_t *); -extern int afs_permission(struct inode *, int); +extern int afs_permission(struct user_namespace *, struct inode *, int); extern void __exit afs_clean_up_permit_cache(void); /* diff --git a/fs/afs/security.c b/fs/afs/security.c index 9cf3102f370c..3c7a8fc4f93f 100644 --- a/fs/afs/security.c +++ b/fs/afs/security.c @@ -396,7 +396,8 @@ int afs_check_permit(struct afs_vnode *vnode, struct key *key, * - AFS ACLs are attached to directories only, and a file is controlled by its * parent directory's ACL */ -int afs_permission(struct inode *inode, int mask) +int afs_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask) { struct afs_vnode *vnode = AFS_FS_I(inode); afs_access_t access; diff --git a/fs/attr.c b/fs/attr.c index c2d3bc0869d4..41abd0d973d8 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -395,9 +395,9 @@ int notify_change(struct user_namespace *mnt_userns, struct dentry *dentry, return error; if (inode->i_op->setattr) - error = inode->i_op->setattr(dentry, attr); + error = inode->i_op->setattr(mnt_userns, dentry, attr); else - error = simple_setattr(dentry, attr); + error = simple_setattr(mnt_userns, dentry, attr); if (!error) { fsnotify_change(dentry, ia_valid); diff --git a/fs/autofs/root.c b/fs/autofs/root.c index 5aaa1732bf1e..91fe4548c256 100644 --- a/fs/autofs/root.c +++ b/fs/autofs/root.c @@ -10,10 +10,12 @@ #include "autofs_i.h" -static int autofs_dir_symlink(struct inode *, struct dentry *, const char *); +static int autofs_dir_symlink(struct user_namespace *, struct inode *, + struct dentry *, const char *); static int autofs_dir_unlink(struct inode *, struct dentry *); static int autofs_dir_rmdir(struct inode *, struct dentry *); -static int autofs_dir_mkdir(struct inode *, struct dentry *, umode_t); +static int autofs_dir_mkdir(struct user_namespace *, struct inode *, + struct dentry *, umode_t); static long autofs_root_ioctl(struct file *, unsigned int, unsigned long); #ifdef CONFIG_COMPAT static long autofs_root_compat_ioctl(struct file *, @@ -524,9 +526,9 @@ static struct dentry *autofs_lookup(struct inode *dir, return NULL; } -static int autofs_dir_symlink(struct inode *dir, - struct dentry *dentry, - const char *symname) +static int autofs_dir_symlink(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + const char *symname) { struct autofs_sb_info *sbi = autofs_sbi(dir->i_sb); struct autofs_info *ino = autofs_dentry_ino(dentry); @@ -715,8 +717,9 @@ static int autofs_dir_rmdir(struct inode *dir, struct dentry *dentry) return 0; } -static int autofs_dir_mkdir(struct inode *dir, - struct dentry *dentry, umode_t mode) +static int autofs_dir_mkdir(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + umode_t mode) { struct autofs_sb_info *sbi = autofs_sbi(dir->i_sb); struct autofs_info *ino = autofs_dentry_ino(dentry); diff --git a/fs/bad_inode.c b/fs/bad_inode.c index 54f0ce444272..48e16144c1f7 100644 --- a/fs/bad_inode.c +++ b/fs/bad_inode.c @@ -27,8 +27,9 @@ static const struct file_operations bad_file_ops = .open = bad_file_open, }; -static int bad_inode_create (struct inode *dir, struct dentry *dentry, - umode_t mode, bool excl) +static int bad_inode_create(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + umode_t mode, bool excl) { return -EIO; } @@ -50,14 +51,15 @@ static int bad_inode_unlink(struct inode *dir, struct dentry *dentry) return -EIO; } -static int bad_inode_symlink (struct inode *dir, struct dentry *dentry, - const char *symname) +static int bad_inode_symlink(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + const char *symname) { return -EIO; } -static int bad_inode_mkdir(struct inode *dir, struct dentry *dentry, - umode_t mode) +static int bad_inode_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { return -EIO; } @@ -67,13 +69,14 @@ static int bad_inode_rmdir (struct inode *dir, struct dentry *dentry) return -EIO; } -static int bad_inode_mknod (struct inode *dir, struct dentry *dentry, - umode_t mode, dev_t rdev) +static int bad_inode_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { return -EIO; } -static int bad_inode_rename2(struct inode *old_dir, struct dentry *old_dentry, +static int bad_inode_rename2(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { @@ -86,18 +89,21 @@ static int bad_inode_readlink(struct dentry *dentry, char __user *buffer, return -EIO; } -static int bad_inode_permission(struct inode *inode, int mask) +static int bad_inode_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { return -EIO; } -static int bad_inode_getattr(const struct path *path, struct kstat *stat, +static int bad_inode_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { return -EIO; } -static int bad_inode_setattr(struct dentry *direntry, struct iattr *attrs) +static int bad_inode_setattr(struct user_namespace *mnt_userns, + struct dentry *direntry, struct iattr *attrs) { return -EIO; } @@ -140,13 +146,15 @@ static int bad_inode_atomic_open(struct inode *inode, struct dentry *dentry, return -EIO; } -static int bad_inode_tmpfile(struct inode *inode, struct dentry *dentry, +static int bad_inode_tmpfile(struct user_namespace *mnt_userns, + struct inode *inode, struct dentry *dentry, umode_t mode) { return -EIO; } -static int bad_inode_set_acl(struct inode *inode, struct posix_acl *acl, +static int bad_inode_set_acl(struct user_namespace *mnt_userns, + struct inode *inode, struct posix_acl *acl, int type) { return -EIO; diff --git a/fs/bfs/dir.c b/fs/bfs/dir.c index be1335a8d25b..34d4f68f786b 100644 --- a/fs/bfs/dir.c +++ b/fs/bfs/dir.c @@ -75,8 +75,8 @@ const struct file_operations bfs_dir_operations = { .llseek = generic_file_llseek, }; -static int bfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int bfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { int err; struct inode *inode; @@ -199,9 +199,9 @@ out_brelse: return error; } -static int bfs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int bfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct inode *old_inode, *new_inode; struct buffer_head *old_bh, *new_bh; diff --git a/fs/btrfs/acl.c b/fs/btrfs/acl.c index d12a5a8730a8..d95eb5c8cb37 100644 --- a/fs/btrfs/acl.c +++ b/fs/btrfs/acl.c @@ -107,7 +107,8 @@ out: return ret; } -int btrfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int btrfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { int ret; umode_t old_mode = inode->i_mode; diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index e6e37591f1de..9c0b43853cd2 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -3625,7 +3625,8 @@ static inline int __btrfs_fs_compat_ro(struct btrfs_fs_info *fs_info, u64 flag) /* acl.c */ #ifdef CONFIG_BTRFS_FS_POSIX_ACL struct posix_acl *btrfs_get_acl(struct inode *inode, int type); -int btrfs_set_acl(struct inode *inode, struct posix_acl *acl, int type); +int btrfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); int btrfs_init_acl(struct btrfs_trans_handle *trans, struct inode *inode, struct inode *dir); #else diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index a63faed171de..c0b11db98e5e 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -5045,7 +5045,8 @@ static int btrfs_setsize(struct inode *inode, struct iattr *attr) return ret; } -static int btrfs_setattr(struct dentry *dentry, struct iattr *attr) +static int btrfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); struct btrfs_root *root = BTRFS_I(inode)->root; @@ -6352,8 +6353,8 @@ static int btrfs_add_nondir(struct btrfs_trans_handle *trans, return err; } -static int btrfs_mknod(struct inode *dir, struct dentry *dentry, - umode_t mode, dev_t rdev) +static int btrfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); struct btrfs_trans_handle *trans; @@ -6416,8 +6417,8 @@ out_unlock: return err; } -static int btrfs_create(struct inode *dir, struct dentry *dentry, - umode_t mode, bool excl) +static int btrfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); struct btrfs_trans_handle *trans; @@ -6561,7 +6562,8 @@ fail: return err; } -static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int btrfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); struct inode *inode = NULL; @@ -8816,7 +8818,8 @@ fail: return -ENOMEM; } -static int btrfs_getattr(const struct path *path, struct kstat *stat, +static int btrfs_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { u64 delalloc_bytes; @@ -9333,9 +9336,9 @@ out_notrans: return ret; } -static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int btrfs_rename2(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) return -EINVAL; @@ -9543,8 +9546,8 @@ out: return ret; } -static int btrfs_symlink(struct inode *dir, struct dentry *dentry, - const char *symname) +static int btrfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); struct btrfs_trans_handle *trans; @@ -9878,7 +9881,8 @@ static int btrfs_set_page_dirty(struct page *page) return __set_page_dirty_nobuffers(page); } -static int btrfs_permission(struct inode *inode, int mask) +static int btrfs_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { struct btrfs_root *root = BTRFS_I(inode)->root; umode_t mode = inode->i_mode; @@ -9893,7 +9897,8 @@ static int btrfs_permission(struct inode *inode, int mask) return generic_permission(&init_user_ns, inode, mask); } -static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) +static int btrfs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); struct btrfs_trans_handle *trans; diff --git a/fs/ceph/acl.c b/fs/ceph/acl.c index 52a01ddbc4ac..529af59d9fd3 100644 --- a/fs/ceph/acl.c +++ b/fs/ceph/acl.c @@ -82,7 +82,8 @@ retry: return acl; } -int ceph_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int ceph_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { int ret = 0, size = 0; const char *name = NULL; diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index 858ee7362ff5..83d9358854fb 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -823,8 +823,8 @@ int ceph_handle_notrace_create(struct inode *dir, struct dentry *dentry) return PTR_ERR(result); } -static int ceph_mknod(struct inode *dir, struct dentry *dentry, - umode_t mode, dev_t rdev) +static int ceph_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb); struct ceph_mds_request *req; @@ -878,14 +878,14 @@ out: return err; } -static int ceph_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int ceph_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { - return ceph_mknod(dir, dentry, mode, 0); + return ceph_mknod(mnt_userns, dir, dentry, mode, 0); } -static int ceph_symlink(struct inode *dir, struct dentry *dentry, - const char *dest) +static int ceph_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *dest) { struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb); struct ceph_mds_request *req; @@ -937,7 +937,8 @@ out: return err; } -static int ceph_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int ceph_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb); struct ceph_mds_request *req; @@ -1183,9 +1184,9 @@ out: return err; } -static int ceph_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int ceph_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(old_dir->i_sb); struct ceph_mds_request *req; diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 179a2bb88538..d6ece56d40e8 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -2238,7 +2238,8 @@ int __ceph_setattr(struct inode *inode, struct iattr *attr) /* * setattr */ -int ceph_setattr(struct dentry *dentry, struct iattr *attr) +int ceph_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); struct ceph_fs_client *fsc = ceph_inode_to_client(inode); @@ -2321,7 +2322,8 @@ int __ceph_do_getattr(struct inode *inode, struct page *locked_page, * Check inode permissions. We verify we have a valid value for * the AUTH cap, then call the generic handler. */ -int ceph_permission(struct inode *inode, int mask) +int ceph_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask) { int err; @@ -2368,8 +2370,8 @@ static int statx_to_caps(u32 want, umode_t mode) * Get all the attributes. If we have sufficient caps for the requested attrs, * then we can avoid talking to the MDS at all. */ -int ceph_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +int ceph_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { struct inode *inode = d_inode(path->dentry); struct ceph_inode_info *ci = ceph_inode(inode); diff --git a/fs/ceph/super.h b/fs/ceph/super.h index b62d8fee3b86..1ef0a2a15817 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -973,10 +973,13 @@ static inline int ceph_do_getattr(struct inode *inode, int mask, bool force) { return __ceph_do_getattr(inode, NULL, mask, force); } -extern int ceph_permission(struct inode *inode, int mask); +extern int ceph_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask); extern int __ceph_setattr(struct inode *inode, struct iattr *attr); -extern int ceph_setattr(struct dentry *dentry, struct iattr *attr); -extern int ceph_getattr(const struct path *path, struct kstat *stat, +extern int ceph_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr); +extern int ceph_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags); /* xattr.c */ @@ -1037,7 +1040,8 @@ void ceph_release_acl_sec_ctx(struct ceph_acl_sec_ctx *as_ctx); #ifdef CONFIG_CEPH_FS_POSIX_ACL struct posix_acl *ceph_get_acl(struct inode *, int); -int ceph_set_acl(struct inode *inode, struct posix_acl *acl, int type); +int ceph_set_acl(struct user_namespace *mnt_userns, + struct inode *inode, struct posix_acl *acl, int type); int ceph_pre_init_acls(struct inode *dir, umode_t *mode, struct ceph_acl_sec_ctx *as_ctx); void ceph_init_inode_acls(struct inode *inode, diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index ce14e6f8adb6..39e51dcf796f 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -305,7 +305,8 @@ static long cifs_fallocate(struct file *file, int mode, loff_t off, loff_t len) return -EOPNOTSUPP; } -static int cifs_permission(struct inode *inode, int mask) +static int cifs_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { struct cifs_sb_info *cifs_sb; diff --git a/fs/cifs/cifsfs.h b/fs/cifs/cifsfs.h index 2307bb0f6147..71e9c6abb2a6 100644 --- a/fs/cifs/cifsfs.h +++ b/fs/cifs/cifsfs.h @@ -62,19 +62,22 @@ extern void cifs_sb_deactive(struct super_block *sb); /* Functions related to inodes */ extern const struct inode_operations cifs_dir_inode_ops; extern struct inode *cifs_root_iget(struct super_block *); -extern int cifs_create(struct inode *, struct dentry *, umode_t, - bool excl); +extern int cifs_create(struct user_namespace *, struct inode *, + struct dentry *, umode_t, bool excl); extern int cifs_atomic_open(struct inode *, struct dentry *, struct file *, unsigned, umode_t); extern struct dentry *cifs_lookup(struct inode *, struct dentry *, unsigned int); extern int cifs_unlink(struct inode *dir, struct dentry *dentry); extern int cifs_hardlink(struct dentry *, struct inode *, struct dentry *); -extern int cifs_mknod(struct inode *, struct dentry *, umode_t, dev_t); -extern int cifs_mkdir(struct inode *, struct dentry *, umode_t); +extern int cifs_mknod(struct user_namespace *, struct inode *, struct dentry *, + umode_t, dev_t); +extern int cifs_mkdir(struct user_namespace *, struct inode *, struct dentry *, + umode_t); extern int cifs_rmdir(struct inode *, struct dentry *); -extern int cifs_rename2(struct inode *, struct dentry *, struct inode *, - struct dentry *, unsigned int); +extern int cifs_rename2(struct user_namespace *, struct inode *, + struct dentry *, struct inode *, struct dentry *, + unsigned int); extern int cifs_revalidate_file_attr(struct file *filp); extern int cifs_revalidate_dentry_attr(struct dentry *); extern int cifs_revalidate_file(struct file *filp); @@ -82,8 +85,10 @@ extern int cifs_revalidate_dentry(struct dentry *); extern int cifs_invalidate_mapping(struct inode *inode); extern int cifs_revalidate_mapping(struct inode *inode); extern int cifs_zap_mapping(struct inode *inode); -extern int cifs_getattr(const struct path *, struct kstat *, u32, unsigned int); -extern int cifs_setattr(struct dentry *, struct iattr *); +extern int cifs_getattr(struct user_namespace *, const struct path *, + struct kstat *, u32, unsigned int); +extern int cifs_setattr(struct user_namespace *, struct dentry *, + struct iattr *); extern int cifs_fiemap(struct inode *, struct fiemap_extent_info *, u64 start, u64 len); @@ -132,8 +137,8 @@ extern struct vfsmount *cifs_dfs_d_automount(struct path *path); /* Functions related to symlinks */ extern const char *cifs_get_link(struct dentry *, struct inode *, struct delayed_call *); -extern int cifs_symlink(struct inode *inode, struct dentry *direntry, - const char *symname); +extern int cifs_symlink(struct user_namespace *mnt_userns, struct inode *inode, + struct dentry *direntry, const char *symname); #ifdef CONFIG_CIFS_XATTR extern const struct xattr_handler *cifs_xattr_handlers[]; diff --git a/fs/cifs/dir.c b/fs/cifs/dir.c index 68900f1629bf..68f4f8536e6a 100644 --- a/fs/cifs/dir.c +++ b/fs/cifs/dir.c @@ -567,8 +567,8 @@ out_free_xid: return rc; } -int cifs_create(struct inode *inode, struct dentry *direntry, umode_t mode, - bool excl) +int cifs_create(struct user_namespace *mnt_userns, struct inode *inode, + struct dentry *direntry, umode_t mode, bool excl) { int rc; unsigned int xid = get_xid(); @@ -611,8 +611,8 @@ out_free_xid: return rc; } -int cifs_mknod(struct inode *inode, struct dentry *direntry, umode_t mode, - dev_t device_number) +int cifs_mknod(struct user_namespace *mnt_userns, struct inode *inode, + struct dentry *direntry, umode_t mode, dev_t device_number) { int rc = -EPERM; unsigned int xid; diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index 374abce7efaf..3e9c7bb23f26 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -1857,7 +1857,8 @@ posix_mkdir_get_info: goto posix_mkdir_out; } -int cifs_mkdir(struct inode *inode, struct dentry *direntry, umode_t mode) +int cifs_mkdir(struct user_namespace *mnt_userns, struct inode *inode, + struct dentry *direntry, umode_t mode) { int rc = 0; unsigned int xid; @@ -2067,9 +2068,9 @@ do_rename_exit: } int -cifs_rename2(struct inode *source_dir, struct dentry *source_dentry, - struct inode *target_dir, struct dentry *target_dentry, - unsigned int flags) +cifs_rename2(struct user_namespace *mnt_userns, struct inode *source_dir, + struct dentry *source_dentry, struct inode *target_dir, + struct dentry *target_dentry, unsigned int flags) { char *from_name = NULL; char *to_name = NULL; @@ -2370,8 +2371,8 @@ int cifs_revalidate_dentry(struct dentry *dentry) return cifs_revalidate_mapping(inode); } -int cifs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { struct dentry *dentry = path->dentry; struct cifs_sb_info *cifs_sb = CIFS_SB(dentry->d_sb); @@ -2923,7 +2924,8 @@ cifs_setattr_exit: } int -cifs_setattr(struct dentry *direntry, struct iattr *attrs) +cifs_setattr(struct user_namespace *mnt_userns, struct dentry *direntry, + struct iattr *attrs) { struct cifs_sb_info *cifs_sb = CIFS_SB(direntry->d_sb); struct cifs_tcon *pTcon = cifs_sb_master_tcon(cifs_sb); diff --git a/fs/cifs/link.c b/fs/cifs/link.c index 94dab4309fbb..7c5878a645d9 100644 --- a/fs/cifs/link.c +++ b/fs/cifs/link.c @@ -661,7 +661,8 @@ cifs_get_link(struct dentry *direntry, struct inode *inode, } int -cifs_symlink(struct inode *inode, struct dentry *direntry, const char *symname) +cifs_symlink(struct user_namespace *mnt_userns, struct inode *inode, + struct dentry *direntry, const char *symname) { int rc = -EOPNOTSUPP; unsigned int xid; diff --git a/fs/coda/coda_linux.h b/fs/coda/coda_linux.h index d5ebd36fb2cc..e7b27754ce78 100644 --- a/fs/coda/coda_linux.h +++ b/fs/coda/coda_linux.h @@ -46,10 +46,12 @@ extern const struct file_operations coda_ioctl_operations; /* operations shared over more than one file */ int coda_open(struct inode *i, struct file *f); int coda_release(struct inode *i, struct file *f); -int coda_permission(struct inode *inode, int mask); +int coda_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask); int coda_revalidate_inode(struct inode *); -int coda_getattr(const struct path *, struct kstat *, u32, unsigned int); -int coda_setattr(struct dentry *, struct iattr *); +int coda_getattr(struct user_namespace *, const struct path *, struct kstat *, + u32, unsigned int); +int coda_setattr(struct user_namespace *, struct dentry *, struct iattr *); /* this file: heloers */ char *coda_f2s(struct CodaFid *f); diff --git a/fs/coda/dir.c b/fs/coda/dir.c index ca40c2556ba6..d69989c1bac3 100644 --- a/fs/coda/dir.c +++ b/fs/coda/dir.c @@ -73,7 +73,8 @@ static struct dentry *coda_lookup(struct inode *dir, struct dentry *entry, unsig } -int coda_permission(struct inode *inode, int mask) +int coda_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask) { int error; @@ -132,7 +133,8 @@ static inline void coda_dir_drop_nlink(struct inode *dir) } /* creation routines: create, mknod, mkdir, link, symlink */ -static int coda_create(struct inode *dir, struct dentry *de, umode_t mode, bool excl) +static int coda_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *de, umode_t mode, bool excl) { int error; const char *name=de->d_name.name; @@ -164,7 +166,8 @@ err_out: return error; } -static int coda_mkdir(struct inode *dir, struct dentry *de, umode_t mode) +static int coda_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *de, umode_t mode) { struct inode *inode; struct coda_vattr attrs; @@ -225,7 +228,8 @@ static int coda_link(struct dentry *source_de, struct inode *dir_inode, } -static int coda_symlink(struct inode *dir_inode, struct dentry *de, +static int coda_symlink(struct user_namespace *mnt_userns, + struct inode *dir_inode, struct dentry *de, const char *symname) { const char *name = de->d_name.name; @@ -291,9 +295,9 @@ static int coda_rmdir(struct inode *dir, struct dentry *de) } /* rename */ -static int coda_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int coda_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { const char *old_name = old_dentry->d_name.name; const char *new_name = new_dentry->d_name.name; diff --git a/fs/coda/inode.c b/fs/coda/inode.c index 4d113e191cb8..d9f1bd7153df 100644 --- a/fs/coda/inode.c +++ b/fs/coda/inode.c @@ -251,8 +251,8 @@ static void coda_evict_inode(struct inode *inode) coda_cache_clear_inode(inode); } -int coda_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +int coda_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { int err = coda_revalidate_inode(d_inode(path->dentry)); if (!err) @@ -260,7 +260,8 @@ int coda_getattr(const struct path *path, struct kstat *stat, return err; } -int coda_setattr(struct dentry *de, struct iattr *iattr) +int coda_setattr(struct user_namespace *mnt_userns, struct dentry *de, + struct iattr *iattr) { struct inode *inode = d_inode(de); struct coda_vattr vattr; diff --git a/fs/coda/pioctl.c b/fs/coda/pioctl.c index 3aec27e5eb82..cb9fd59a688c 100644 --- a/fs/coda/pioctl.c +++ b/fs/coda/pioctl.c @@ -24,7 +24,8 @@ #include "coda_linux.h" /* pioctl ops */ -static int coda_ioctl_permission(struct inode *inode, int mask); +static int coda_ioctl_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask); static long coda_pioctl(struct file *filp, unsigned int cmd, unsigned long user_data); @@ -40,7 +41,8 @@ const struct file_operations coda_ioctl_operations = { }; /* the coda pioctl inode ops */ -static int coda_ioctl_permission(struct inode *inode, int mask) +static int coda_ioctl_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { return (mask & MAY_EXEC) ? -EACCES : 0; } diff --git a/fs/configfs/configfs_internal.h b/fs/configfs/configfs_internal.h index 22dce2d35a4b..9a3aed249692 100644 --- a/fs/configfs/configfs_internal.h +++ b/fs/configfs/configfs_internal.h @@ -79,7 +79,8 @@ extern void configfs_hash_and_remove(struct dentry * dir, const char * name); extern const unsigned char * configfs_get_name(struct configfs_dirent *sd); extern void configfs_drop_dentry(struct configfs_dirent *sd, struct dentry *parent); -extern int configfs_setattr(struct dentry *dentry, struct iattr *iattr); +extern int configfs_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *iattr); extern struct dentry *configfs_pin_fs(void); extern void configfs_release_fs(void); @@ -92,7 +93,8 @@ extern const struct inode_operations configfs_root_inode_operations; extern const struct inode_operations configfs_symlink_inode_operations; extern const struct dentry_operations configfs_dentry_ops; -extern int configfs_symlink(struct inode *dir, struct dentry *dentry, +extern int configfs_symlink(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, const char *symname); extern int configfs_unlink(struct inode *dir, struct dentry *dentry); diff --git a/fs/configfs/dir.c b/fs/configfs/dir.c index b839dd1b459f..b6098e02e20b 100644 --- a/fs/configfs/dir.c +++ b/fs/configfs/dir.c @@ -1268,7 +1268,8 @@ out_root_unlock: } EXPORT_SYMBOL(configfs_depend_item_unlocked); -static int configfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int configfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { int ret = 0; int module_got = 0; diff --git a/fs/configfs/inode.c b/fs/configfs/inode.c index 8bd6a883c94c..42c348bb2903 100644 --- a/fs/configfs/inode.c +++ b/fs/configfs/inode.c @@ -40,7 +40,8 @@ static const struct inode_operations configfs_inode_operations ={ .setattr = configfs_setattr, }; -int configfs_setattr(struct dentry * dentry, struct iattr * iattr) +int configfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr) { struct inode * inode = d_inode(dentry); struct configfs_dirent * sd = dentry->d_fsdata; @@ -67,7 +68,7 @@ int configfs_setattr(struct dentry * dentry, struct iattr * iattr) } /* attributes were changed atleast once in past */ - error = simple_setattr(dentry, iattr); + error = simple_setattr(mnt_userns, dentry, iattr); if (error) return error; diff --git a/fs/configfs/symlink.c b/fs/configfs/symlink.c index 8ca36394fa30..77c854364e60 100644 --- a/fs/configfs/symlink.c +++ b/fs/configfs/symlink.c @@ -139,7 +139,8 @@ static int get_target(const char *symname, struct path *path, } -int configfs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) +int configfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { int ret; struct path path; diff --git a/fs/debugfs/inode.c b/fs/debugfs/inode.c index 2fcf66473436..c35249497b9b 100644 --- a/fs/debugfs/inode.c +++ b/fs/debugfs/inode.c @@ -42,13 +42,14 @@ static unsigned int debugfs_allow = DEFAULT_DEBUGFS_ALLOW_BITS; * so that we can use the file mode as part of a heuristic to determine whether * to lock down individual files. */ -static int debugfs_setattr(struct dentry *dentry, struct iattr *ia) +static int debugfs_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *ia) { int ret = security_locked_down(LOCKDOWN_DEBUGFS); if (ret && (ia->ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID))) return ret; - return simple_setattr(dentry, ia); + return simple_setattr(&init_user_ns, dentry, ia); } static const struct inode_operations debugfs_file_inode_operations = { @@ -775,8 +776,8 @@ struct dentry *debugfs_rename(struct dentry *old_dir, struct dentry *old_dentry, take_dentry_name_snapshot(&old_name, old_dentry); - error = simple_rename(d_inode(old_dir), old_dentry, d_inode(new_dir), - dentry, 0); + error = simple_rename(&init_user_ns, d_inode(old_dir), old_dentry, + d_inode(new_dir), dentry, 0); if (error) { release_dentry_name_snapshot(&old_name); goto exit; diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index 73e3d47e7b2d..55da9a91f51a 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -257,7 +257,8 @@ out: * Returns zero on success; non-zero on error condition */ static int -ecryptfs_create(struct inode *directory_inode, struct dentry *ecryptfs_dentry, +ecryptfs_create(struct user_namespace *mnt_userns, + struct inode *directory_inode, struct dentry *ecryptfs_dentry, umode_t mode, bool excl) { struct inode *ecryptfs_inode; @@ -463,7 +464,8 @@ static int ecryptfs_unlink(struct inode *dir, struct dentry *dentry) return ecryptfs_do_unlink(dir, dentry, d_inode(dentry)); } -static int ecryptfs_symlink(struct inode *dir, struct dentry *dentry, +static int ecryptfs_symlink(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, const char *symname) { int rc; @@ -502,7 +504,8 @@ out_lock: return rc; } -static int ecryptfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int ecryptfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { int rc; struct dentry *lower_dentry; @@ -559,7 +562,8 @@ static int ecryptfs_rmdir(struct inode *dir, struct dentry *dentry) } static int -ecryptfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +ecryptfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) { int rc; struct dentry *lower_dentry; @@ -584,9 +588,9 @@ out: } static int -ecryptfs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +ecryptfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { int rc; struct dentry *lower_old_dentry; @@ -874,7 +878,8 @@ int ecryptfs_truncate(struct dentry *dentry, loff_t new_length) } static int -ecryptfs_permission(struct inode *inode, int mask) +ecryptfs_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask) { return inode_permission(&init_user_ns, ecryptfs_inode_to_lower(inode), mask); @@ -892,7 +897,8 @@ ecryptfs_permission(struct inode *inode, int mask) * All other metadata changes will be passed right to the lower filesystem, * and we will just update our inode to look like the lower. */ -static int ecryptfs_setattr(struct dentry *dentry, struct iattr *ia) +static int ecryptfs_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *ia) { int rc = 0; struct dentry *lower_dentry; @@ -979,7 +985,8 @@ out: return rc; } -static int ecryptfs_getattr_link(const struct path *path, struct kstat *stat, +static int ecryptfs_getattr_link(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { struct dentry *dentry = path->dentry; @@ -1004,7 +1011,8 @@ static int ecryptfs_getattr_link(const struct path *path, struct kstat *stat, return rc; } -static int ecryptfs_getattr(const struct path *path, struct kstat *stat, +static int ecryptfs_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { struct dentry *dentry = path->dentry; diff --git a/fs/efivarfs/inode.c b/fs/efivarfs/inode.c index 0297ad95eb5c..14e2947975fd 100644 --- a/fs/efivarfs/inode.c +++ b/fs/efivarfs/inode.c @@ -66,8 +66,8 @@ bool efivarfs_valid_name(const char *str, int len) return uuid_is_valid(s); } -static int efivarfs_create(struct inode *dir, struct dentry *dentry, - umode_t mode, bool excl) +static int efivarfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct inode *inode = NULL; struct efivar_entry *var; diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c index 083818063ac6..119fdce1b520 100644 --- a/fs/erofs/inode.c +++ b/fs/erofs/inode.c @@ -331,8 +331,9 @@ struct inode *erofs_iget(struct super_block *sb, return inode; } -int erofs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags) +int erofs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) { struct inode *const inode = d_inode(path->dentry); diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 67a7ec945686..351dae524a0c 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -373,8 +373,9 @@ extern const struct inode_operations erofs_symlink_iops; extern const struct inode_operations erofs_fast_symlink_iops; struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid, bool dir); -int erofs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags); +int erofs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags); /* namei.c */ extern const struct inode_operations erofs_dir_iops; diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h index b8f0e829ecbd..d905bb9cd2ca 100644 --- a/fs/exfat/exfat_fs.h +++ b/fs/exfat/exfat_fs.h @@ -416,9 +416,11 @@ int exfat_count_used_clusters(struct super_block *sb, unsigned int *ret_count); extern const struct file_operations exfat_file_operations; int __exfat_truncate(struct inode *inode, loff_t new_size); void exfat_truncate(struct inode *inode, loff_t size); -int exfat_setattr(struct dentry *dentry, struct iattr *attr); -int exfat_getattr(const struct path *path, struct kstat *stat, - unsigned int request_mask, unsigned int query_flags); +int exfat_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr); +int exfat_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, unsigned int request_mask, + unsigned int query_flags); int exfat_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); /* namei.c */ diff --git a/fs/exfat/file.c b/fs/exfat/file.c index e9705b3295d3..3aa6eb4de5e3 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -267,8 +267,9 @@ write_size: mutex_unlock(&sbi->s_lock); } -int exfat_getattr(const struct path *path, struct kstat *stat, - unsigned int request_mask, unsigned int query_flags) +int exfat_getattr(struct user_namespace *mnt_uerns, const struct path *path, + struct kstat *stat, unsigned int request_mask, + unsigned int query_flags) { struct inode *inode = d_backing_inode(path->dentry); struct exfat_inode_info *ei = EXFAT_I(inode); @@ -282,7 +283,8 @@ int exfat_getattr(const struct path *path, struct kstat *stat, return 0; } -int exfat_setattr(struct dentry *dentry, struct iattr *attr) +int exfat_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct exfat_sb_info *sbi = EXFAT_SB(dentry->d_sb); struct inode *inode = dentry->d_inode; diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c index 2932b23a3b6c..d9e8ec689c55 100644 --- a/fs/exfat/namei.c +++ b/fs/exfat/namei.c @@ -541,8 +541,8 @@ out: return ret; } -static int exfat_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int exfat_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct super_block *sb = dir->i_sb; struct inode *inode; @@ -827,7 +827,8 @@ unlock: return err; } -static int exfat_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int exfat_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct super_block *sb = dir->i_sb; struct inode *inode; @@ -1318,9 +1319,10 @@ out: return ret; } -static int exfat_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int exfat_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) { struct inode *old_inode, *new_inode; struct super_block *sb = old_dir->i_sb; diff --git a/fs/ext2/acl.c b/fs/ext2/acl.c index 9031f7df2d48..b9a9db98e94b 100644 --- a/fs/ext2/acl.c +++ b/fs/ext2/acl.c @@ -216,7 +216,8 @@ __ext2_set_acl(struct inode *inode, struct posix_acl *acl, int type) * inode->i_mutex: down */ int -ext2_set_acl(struct inode *inode, struct posix_acl *acl, int type) +ext2_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { int error; int update_mode = 0; diff --git a/fs/ext2/acl.h b/fs/ext2/acl.h index 0f01c759daac..917db5f6630a 100644 --- a/fs/ext2/acl.h +++ b/fs/ext2/acl.h @@ -56,7 +56,8 @@ static inline int ext2_acl_count(size_t size) /* acl.c */ extern struct posix_acl *ext2_get_acl(struct inode *inode, int type); -extern int ext2_set_acl(struct inode *inode, struct posix_acl *acl, int type); +extern int ext2_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); extern int ext2_init_acl (struct inode *, struct inode *); #else diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h index 2a4175fbaf5e..3309fb2d327a 100644 --- a/fs/ext2/ext2.h +++ b/fs/ext2/ext2.h @@ -764,8 +764,9 @@ extern struct inode *ext2_iget (struct super_block *, unsigned long); extern int ext2_write_inode (struct inode *, struct writeback_control *); extern void ext2_evict_inode(struct inode *); extern int ext2_get_block(struct inode *, sector_t, struct buffer_head *, int); -extern int ext2_setattr (struct dentry *, struct iattr *); -extern int ext2_getattr (const struct path *, struct kstat *, u32, unsigned int); +extern int ext2_setattr (struct user_namespace *, struct dentry *, struct iattr *); +extern int ext2_getattr (struct user_namespace *, const struct path *, + struct kstat *, u32, unsigned int); extern void ext2_set_inode_flags(struct inode *inode); extern int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len); diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 3d8acafca8ce..68178b2234bd 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -1638,8 +1638,8 @@ int ext2_write_inode(struct inode *inode, struct writeback_control *wbc) return __ext2_write_inode(inode, wbc->sync_mode == WB_SYNC_ALL); } -int ext2_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags) +int ext2_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); struct ext2_inode_info *ei = EXT2_I(inode); @@ -1664,7 +1664,8 @@ int ext2_getattr(const struct path *path, struct kstat *stat, return 0; } -int ext2_setattr(struct dentry *dentry, struct iattr *iattr) +int ext2_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr) { struct inode *inode = d_inode(dentry); int error; diff --git a/fs/ext2/namei.c b/fs/ext2/namei.c index ea980f1e2e99..3367384d344d 100644 --- a/fs/ext2/namei.c +++ b/fs/ext2/namei.c @@ -100,7 +100,9 @@ struct dentry *ext2_get_parent(struct dentry *child) * If the create succeeds, we fill in the inode information * with d_instantiate(). */ -static int ext2_create (struct inode * dir, struct dentry * dentry, umode_t mode, bool excl) +static int ext2_create (struct user_namespace * mnt_userns, + struct inode * dir, struct dentry * dentry, + umode_t mode, bool excl) { struct inode *inode; int err; @@ -118,7 +120,8 @@ static int ext2_create (struct inode * dir, struct dentry * dentry, umode_t mode return ext2_add_nondir(dentry, inode); } -static int ext2_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) +static int ext2_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode *inode = ext2_new_inode(dir, mode, NULL); if (IS_ERR(inode)) @@ -131,7 +134,8 @@ static int ext2_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) return 0; } -static int ext2_mknod (struct inode * dir, struct dentry *dentry, umode_t mode, dev_t rdev) +static int ext2_mknod (struct user_namespace * mnt_userns, struct inode * dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct inode * inode; int err; @@ -151,8 +155,8 @@ static int ext2_mknod (struct inode * dir, struct dentry *dentry, umode_t mode, return err; } -static int ext2_symlink (struct inode * dir, struct dentry * dentry, - const char * symname) +static int ext2_symlink (struct user_namespace * mnt_userns, struct inode * dir, + struct dentry * dentry, const char * symname) { struct super_block * sb = dir->i_sb; int err = -ENAMETOOLONG; @@ -225,7 +229,8 @@ static int ext2_link (struct dentry * old_dentry, struct inode * dir, return err; } -static int ext2_mkdir(struct inode * dir, struct dentry * dentry, umode_t mode) +static int ext2_mkdir(struct user_namespace * mnt_userns, + struct inode * dir, struct dentry * dentry, umode_t mode) { struct inode * inode; int err; @@ -315,8 +320,9 @@ static int ext2_rmdir (struct inode * dir, struct dentry *dentry) return err; } -static int ext2_rename (struct inode * old_dir, struct dentry * old_dentry, - struct inode * new_dir, struct dentry * new_dentry, +static int ext2_rename (struct user_namespace * mnt_userns, + struct inode * old_dir, struct dentry * old_dentry, + struct inode * new_dir, struct dentry * new_dentry, unsigned int flags) { struct inode * old_inode = d_inode(old_dentry); diff --git a/fs/ext4/acl.c b/fs/ext4/acl.c index 7b0fb66bc04d..059434e0f36c 100644 --- a/fs/ext4/acl.c +++ b/fs/ext4/acl.c @@ -222,7 +222,8 @@ __ext4_set_acl(handle_t *handle, struct inode *inode, int type, } int -ext4_set_acl(struct inode *inode, struct posix_acl *acl, int type) +ext4_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { handle_t *handle; int error, credits, retries = 0; diff --git a/fs/ext4/acl.h b/fs/ext4/acl.h index 9b63f5416a2f..84b8942a57f2 100644 --- a/fs/ext4/acl.h +++ b/fs/ext4/acl.h @@ -56,7 +56,8 @@ static inline int ext4_acl_count(size_t size) /* acl.c */ struct posix_acl *ext4_get_acl(struct inode *inode, int type); -int ext4_set_acl(struct inode *inode, struct posix_acl *acl, int type); +int ext4_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); extern int ext4_init_acl(handle_t *, struct inode *, struct inode *); #else /* CONFIG_EXT4_FS_POSIX_ACL */ diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 2866d249f3d2..3c750f5e8ebd 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -2877,11 +2877,14 @@ extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, __ext4_iget((sb), (ino), (flags), __func__, __LINE__) extern int ext4_write_inode(struct inode *, struct writeback_control *); -extern int ext4_setattr(struct dentry *, struct iattr *); -extern int ext4_getattr(const struct path *, struct kstat *, u32, unsigned int); +extern int ext4_setattr(struct user_namespace *, struct dentry *, + struct iattr *); +extern int ext4_getattr(struct user_namespace *, const struct path *, + struct kstat *, u32, unsigned int); extern void ext4_evict_inode(struct inode *); extern void ext4_clear_inode(struct inode *); -extern int ext4_file_getattr(const struct path *, struct kstat *, u32, unsigned int); +extern int ext4_file_getattr(struct user_namespace *, const struct path *, + struct kstat *, u32, unsigned int); extern int ext4_sync_inode(handle_t *, struct inode *); extern void ext4_dirty_inode(struct inode *, int); extern int ext4_change_inode_journal_flag(struct inode *, int); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 3a303d3f8423..ce45535336fa 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -5319,7 +5319,8 @@ static void ext4_wait_for_tail_page_commit(struct inode *inode) * * Called with inode->i_mutex down. */ -int ext4_setattr(struct dentry *dentry, struct iattr *attr) +int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); int error, rc = 0; @@ -5535,8 +5536,8 @@ err_out: return error; } -int ext4_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags) +int ext4_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); struct ext4_inode *raw_inode; @@ -5575,13 +5576,14 @@ int ext4_getattr(const struct path *path, struct kstat *stat, return 0; } -int ext4_file_getattr(const struct path *path, struct kstat *stat, +int ext4_file_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); u64 delalloc_blocks; - ext4_getattr(path, stat, request_mask, query_flags); + ext4_getattr(&init_user_ns, path, stat, request_mask, query_flags); /* * If there is inline data in the inode, the inode will normally not diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index cf652ba3e74d..13dff80aedcb 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -2596,8 +2596,8 @@ static int ext4_add_nondir(handle_t *handle, * If the create succeeds, we fill in the inode information * with d_instantiate(). */ -static int ext4_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int ext4_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { handle_t *handle; struct inode *inode; @@ -2631,8 +2631,8 @@ retry: return err; } -static int ext4_mknod(struct inode *dir, struct dentry *dentry, - umode_t mode, dev_t rdev) +static int ext4_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { handle_t *handle; struct inode *inode; @@ -2665,7 +2665,8 @@ retry: return err; } -static int ext4_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) +static int ext4_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { handle_t *handle; struct inode *inode; @@ -2774,7 +2775,8 @@ out: return err; } -static int ext4_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int ext4_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { handle_t *handle; struct inode *inode; @@ -3292,7 +3294,7 @@ out_trace: return retval; } -static int ext4_symlink(struct inode *dir, +static int ext4_symlink(struct user_namespace *mnt_userns, struct inode *dir, struct dentry *dentry, const char *symname) { handle_t *handle; @@ -4085,7 +4087,8 @@ end_rename: return retval; } -static int ext4_rename2(struct inode *old_dir, struct dentry *old_dentry, +static int ext4_rename2(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c index 6a95bf28f602..a19e86c9adac 100644 --- a/fs/f2fs/acl.c +++ b/fs/f2fs/acl.c @@ -249,7 +249,8 @@ static int __f2fs_set_acl(struct inode *inode, int type, return error; } -int f2fs_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int f2fs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) return -EIO; diff --git a/fs/f2fs/acl.h b/fs/f2fs/acl.h index 124868c13f80..986fd1bc780b 100644 --- a/fs/f2fs/acl.h +++ b/fs/f2fs/acl.h @@ -34,7 +34,8 @@ struct f2fs_acl_header { #ifdef CONFIG_F2FS_FS_POSIX_ACL extern struct posix_acl *f2fs_get_acl(struct inode *, int); -extern int f2fs_set_acl(struct inode *, struct posix_acl *, int); +extern int f2fs_set_acl(struct user_namespace *, struct inode *, + struct posix_acl *, int); extern int f2fs_init_acl(struct inode *, struct inode *, struct page *, struct page *); #else diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index bb11759191dc..c9002b1933f0 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -3135,9 +3135,10 @@ void f2fs_truncate_data_blocks(struct dnode_of_data *dn); int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock); int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock); int f2fs_truncate(struct inode *inode); -int f2fs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags); -int f2fs_setattr(struct dentry *dentry, struct iattr *attr); +int f2fs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); +int f2fs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr); int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end); void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count); int f2fs_precache_extents(struct inode *inode); diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 44cd0dbdbb5d..8f1e97e7d242 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -783,8 +783,8 @@ int f2fs_truncate(struct inode *inode) return 0; } -int f2fs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags) +int f2fs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); struct f2fs_inode_info *fi = F2FS_I(inode); @@ -859,7 +859,8 @@ static void __setattr_copy(struct user_namespace *mnt_userns, #define __setattr_copy setattr_copy #endif -int f2fs_setattr(struct dentry *dentry, struct iattr *attr) +int f2fs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); int err; diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index ad98926bacac..c061a67e43a3 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -314,8 +314,8 @@ static void set_compress_inode(struct f2fs_sb_info *sbi, struct inode *inode, } } -static int f2fs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int f2fs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode; @@ -637,8 +637,8 @@ static const char *f2fs_get_link(struct dentry *dentry, return link; } -static int f2fs_symlink(struct inode *dir, struct dentry *dentry, - const char *symname) +static int f2fs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode; @@ -717,7 +717,8 @@ out_free_encrypted_link: return err; } -static int f2fs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int f2fs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode; @@ -770,8 +771,8 @@ static int f2fs_rmdir(struct inode *dir, struct dentry *dentry) return -ENOTEMPTY; } -static int f2fs_mknod(struct inode *dir, struct dentry *dentry, - umode_t mode, dev_t rdev) +static int f2fs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode; @@ -874,7 +875,8 @@ out: return err; } -static int f2fs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) +static int f2fs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); @@ -1247,7 +1249,8 @@ out: return err; } -static int f2fs_rename2(struct inode *old_dir, struct dentry *old_dentry, +static int f2fs_rename2(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { diff --git a/fs/fat/fat.h b/fs/fat/fat.h index 922a0c6ba46c..02d4d4234956 100644 --- a/fs/fat/fat.h +++ b/fs/fat/fat.h @@ -397,9 +397,11 @@ extern long fat_generic_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); extern const struct file_operations fat_file_operations; extern const struct inode_operations fat_file_inode_operations; -extern int fat_setattr(struct dentry *dentry, struct iattr *attr); +extern int fat_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr); extern void fat_truncate_blocks(struct inode *inode, loff_t offset); -extern int fat_getattr(const struct path *path, struct kstat *stat, +extern int fat_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags); extern int fat_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); diff --git a/fs/fat/file.c b/fs/fat/file.c index f7e04f533d31..dd73d1b70c55 100644 --- a/fs/fat/file.c +++ b/fs/fat/file.c @@ -95,7 +95,7 @@ static int fat_ioctl_set_attributes(struct file *file, u32 __user *user_attr) goto out_unlock_inode; /* This MUST be done before doing anything irreversible... */ - err = fat_setattr(file->f_path.dentry, &ia); + err = fat_setattr(file_mnt_user_ns(file), file->f_path.dentry, &ia); if (err) goto out_unlock_inode; @@ -394,8 +394,8 @@ void fat_truncate_blocks(struct inode *inode, loff_t offset) fat_flush_inodes(inode->i_sb, inode, NULL); } -int fat_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +int fat_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { struct inode *inode = d_inode(path->dentry); generic_fillattr(&init_user_ns, inode, stat); @@ -466,7 +466,8 @@ static int fat_allow_set_time(struct msdos_sb_info *sbi, struct inode *inode) /* valid file mode bits */ #define FAT_VALID_MODE (S_IFREG | S_IFDIR | S_IRWXUGO) -int fat_setattr(struct dentry *dentry, struct iattr *attr) +int fat_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct msdos_sb_info *sbi = MSDOS_SB(dentry->d_sb); struct inode *inode = d_inode(dentry); diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c index 9d062886fbc1..a8f3375d9d10 100644 --- a/fs/fat/namei_msdos.c +++ b/fs/fat/namei_msdos.c @@ -261,8 +261,8 @@ static int msdos_add_entry(struct inode *dir, const unsigned char *name, } /***** Create a file */ -static int msdos_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int msdos_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct super_block *sb = dir->i_sb; struct inode *inode = NULL; @@ -339,7 +339,8 @@ out: } /***** Make a directory */ -static int msdos_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int msdos_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct super_block *sb = dir->i_sb; struct fat_slot_info sinfo; @@ -593,7 +594,8 @@ error_inode: } /***** Rename, a wrapper for rename_same_dir & rename_diff_dir */ -static int msdos_rename(struct inode *old_dir, struct dentry *old_dentry, +static int msdos_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c index 0cdd0fb9f742..23936ecf79a5 100644 --- a/fs/fat/namei_vfat.c +++ b/fs/fat/namei_vfat.c @@ -756,8 +756,8 @@ error: return ERR_PTR(err); } -static int vfat_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int vfat_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct super_block *sb = dir->i_sb; struct inode *inode; @@ -846,7 +846,8 @@ out: return err; } -static int vfat_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int vfat_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct super_block *sb = dir->i_sb; struct inode *inode; @@ -892,9 +893,9 @@ out: return err; } -static int vfat_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int vfat_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct buffer_head *dotdot_bh; struct msdos_dir_entry *dotdot_de; diff --git a/fs/fuse/acl.c b/fs/fuse/acl.c index f529075a2ce8..e9c0f916349d 100644 --- a/fs/fuse/acl.c +++ b/fs/fuse/acl.c @@ -50,7 +50,8 @@ struct posix_acl *fuse_get_acl(struct inode *inode, int type) return acl; } -int fuse_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int fuse_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { struct fuse_conn *fc = get_fuse_conn(inode); const char *name; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index d2e318ed9b26..06a18700a845 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -605,7 +605,8 @@ out_err: return err; } -static int fuse_mknod(struct inode *, struct dentry *, umode_t, dev_t); +static int fuse_mknod(struct user_namespace *, struct inode *, struct dentry *, + umode_t, dev_t); static int fuse_atomic_open(struct inode *dir, struct dentry *entry, struct file *file, unsigned flags, umode_t mode) @@ -645,7 +646,7 @@ out_dput: return err; mknod: - err = fuse_mknod(dir, entry, mode, 0); + err = fuse_mknod(&init_user_ns, dir, entry, mode, 0); if (err) goto out_dput; no_open: @@ -715,8 +716,8 @@ static int create_new_entry(struct fuse_mount *fm, struct fuse_args *args, return err; } -static int fuse_mknod(struct inode *dir, struct dentry *entry, umode_t mode, - dev_t rdev) +static int fuse_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *entry, umode_t mode, dev_t rdev) { struct fuse_mknod_in inarg; struct fuse_mount *fm = get_fuse_mount(dir); @@ -738,13 +739,14 @@ static int fuse_mknod(struct inode *dir, struct dentry *entry, umode_t mode, return create_new_entry(fm, &args, dir, entry, mode); } -static int fuse_create(struct inode *dir, struct dentry *entry, umode_t mode, - bool excl) +static int fuse_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *entry, umode_t mode, bool excl) { - return fuse_mknod(dir, entry, mode, 0); + return fuse_mknod(&init_user_ns, dir, entry, mode, 0); } -static int fuse_mkdir(struct inode *dir, struct dentry *entry, umode_t mode) +static int fuse_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *entry, umode_t mode) { struct fuse_mkdir_in inarg; struct fuse_mount *fm = get_fuse_mount(dir); @@ -765,8 +767,8 @@ static int fuse_mkdir(struct inode *dir, struct dentry *entry, umode_t mode) return create_new_entry(fm, &args, dir, entry, S_IFDIR); } -static int fuse_symlink(struct inode *dir, struct dentry *entry, - const char *link) +static int fuse_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *entry, const char *link) { struct fuse_mount *fm = get_fuse_mount(dir); unsigned len = strlen(link) + 1; @@ -908,9 +910,9 @@ static int fuse_rename_common(struct inode *olddir, struct dentry *oldent, return err; } -static int fuse_rename2(struct inode *olddir, struct dentry *oldent, - struct inode *newdir, struct dentry *newent, - unsigned int flags) +static int fuse_rename2(struct user_namespace *mnt_userns, struct inode *olddir, + struct dentry *oldent, struct inode *newdir, + struct dentry *newent, unsigned int flags) { struct fuse_conn *fc = get_fuse_conn(olddir); int err; @@ -1249,7 +1251,8 @@ static int fuse_perm_getattr(struct inode *inode, int mask) * access request is sent. Execute permission is still checked * locally based on file mode. */ -static int fuse_permission(struct inode *inode, int mask) +static int fuse_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { struct fuse_conn *fc = get_fuse_conn(inode); bool refreshed = false; @@ -1757,7 +1760,8 @@ error: return err; } -static int fuse_setattr(struct dentry *entry, struct iattr *attr) +static int fuse_setattr(struct user_namespace *mnt_userns, struct dentry *entry, + struct iattr *attr) { struct inode *inode = d_inode(entry); struct fuse_conn *fc = get_fuse_conn(inode); @@ -1819,7 +1823,8 @@ static int fuse_setattr(struct dentry *entry, struct iattr *attr) return ret; } -static int fuse_getattr(const struct path *path, struct kstat *stat, +static int fuse_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { struct inode *inode = d_inode(path->dentry); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 7c4b8cb93f9f..68cca8d4db6e 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1180,8 +1180,8 @@ extern const struct xattr_handler *fuse_no_acl_xattr_handlers[]; struct posix_acl; struct posix_acl *fuse_get_acl(struct inode *inode, int type); -int fuse_set_acl(struct inode *inode, struct posix_acl *acl, int type); - +int fuse_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); /* readdir.c */ int fuse_readdir(struct file *file, struct dir_context *ctx); diff --git a/fs/gfs2/acl.c b/fs/gfs2/acl.c index ce88ef29eef0..9165d70ead07 100644 --- a/fs/gfs2/acl.c +++ b/fs/gfs2/acl.c @@ -106,7 +106,8 @@ out: return error; } -int gfs2_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int gfs2_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { struct gfs2_inode *ip = GFS2_I(inode); struct gfs2_holder gh; diff --git a/fs/gfs2/acl.h b/fs/gfs2/acl.h index 61353a1501c5..eccc6a43326c 100644 --- a/fs/gfs2/acl.h +++ b/fs/gfs2/acl.h @@ -13,6 +13,7 @@ extern struct posix_acl *gfs2_get_acl(struct inode *inode, int type); extern int __gfs2_set_acl(struct inode *inode, struct posix_acl *acl, int type); -extern int gfs2_set_acl(struct inode *inode, struct posix_acl *acl, int type); +extern int gfs2_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); #endif /* __ACL_DOT_H__ */ diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index 1d994bdfffaa..8f5523822788 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -256,7 +256,7 @@ static int do_gfs2_set_flags(struct file *filp, u32 reqflags, u32 mask, !capable(CAP_LINUX_IMMUTABLE)) goto out; if (!IS_IMMUTABLE(inode)) { - error = gfs2_permission(inode, MAY_WRITE); + error = gfs2_permission(&init_user_ns, inode, MAY_WRITE); if (error) goto out; } diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 226b5b1dc1fa..cfac2c1e67fa 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -325,7 +325,7 @@ struct inode *gfs2_lookupi(struct inode *dir, const struct qstr *name, } if (!is_root) { - error = gfs2_permission(dir, MAY_EXEC); + error = gfs2_permission(&init_user_ns, dir, MAY_EXEC); if (error) goto out; } @@ -355,7 +355,8 @@ static int create_ok(struct gfs2_inode *dip, const struct qstr *name, { int error; - error = gfs2_permission(&dip->i_inode, MAY_WRITE | MAY_EXEC); + error = gfs2_permission(&init_user_ns, &dip->i_inode, + MAY_WRITE | MAY_EXEC); if (error) return error; @@ -843,8 +844,8 @@ fail: * Returns: errno */ -static int gfs2_create(struct inode *dir, struct dentry *dentry, - umode_t mode, bool excl) +static int gfs2_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { return gfs2_create_inode(dir, dentry, NULL, S_IFREG | mode, 0, NULL, 0, excl); } @@ -951,7 +952,7 @@ static int gfs2_link(struct dentry *old_dentry, struct inode *dir, if (inode->i_nlink == 0) goto out_gunlock; - error = gfs2_permission(dir, MAY_WRITE | MAY_EXEC); + error = gfs2_permission(&init_user_ns, dir, MAY_WRITE | MAY_EXEC); if (error) goto out_gunlock; @@ -1068,7 +1069,8 @@ static int gfs2_unlink_ok(struct gfs2_inode *dip, const struct qstr *name, if (IS_APPEND(&dip->i_inode)) return -EPERM; - error = gfs2_permission(&dip->i_inode, MAY_WRITE | MAY_EXEC); + error = gfs2_permission(&init_user_ns, &dip->i_inode, + MAY_WRITE | MAY_EXEC); if (error) return error; @@ -1204,8 +1206,8 @@ out_inodes: * Returns: errno */ -static int gfs2_symlink(struct inode *dir, struct dentry *dentry, - const char *symname) +static int gfs2_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { unsigned int size; @@ -1225,7 +1227,8 @@ static int gfs2_symlink(struct inode *dir, struct dentry *dentry, * Returns: errno */ -static int gfs2_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int gfs2_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { unsigned dsize = gfs2_max_stuffed_size(GFS2_I(dir)); return gfs2_create_inode(dir, dentry, NULL, S_IFDIR | mode, 0, NULL, dsize, 0); @@ -1240,8 +1243,8 @@ static int gfs2_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) * */ -static int gfs2_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, - dev_t dev) +static int gfs2_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) { return gfs2_create_inode(dir, dentry, NULL, mode, dev, NULL, 0, 0); } @@ -1490,7 +1493,8 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, } } } else { - error = gfs2_permission(ndir, MAY_WRITE | MAY_EXEC); + error = gfs2_permission(&init_user_ns, ndir, + MAY_WRITE | MAY_EXEC); if (error) goto out_gunlock; @@ -1525,7 +1529,8 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, /* Check out the dir to be renamed */ if (dir_rename) { - error = gfs2_permission(d_inode(odentry), MAY_WRITE); + error = gfs2_permission(&init_user_ns, d_inode(odentry), + MAY_WRITE); if (error) goto out_gunlock; } @@ -1688,12 +1693,14 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry, goto out_gunlock; if (S_ISDIR(old_mode)) { - error = gfs2_permission(odentry->d_inode, MAY_WRITE); + error = gfs2_permission(&init_user_ns, odentry->d_inode, + MAY_WRITE); if (error) goto out_gunlock; } if (S_ISDIR(new_mode)) { - error = gfs2_permission(ndentry->d_inode, MAY_WRITE); + error = gfs2_permission(&init_user_ns, ndentry->d_inode, + MAY_WRITE); if (error) goto out_gunlock; } @@ -1747,9 +1754,9 @@ out: return error; } -static int gfs2_rename2(struct inode *odir, struct dentry *odentry, - struct inode *ndir, struct dentry *ndentry, - unsigned int flags) +static int gfs2_rename2(struct user_namespace *mnt_userns, struct inode *odir, + struct dentry *odentry, struct inode *ndir, + struct dentry *ndentry, unsigned int flags) { flags &= ~RENAME_NOREPLACE; @@ -1833,7 +1840,8 @@ out: * Returns: errno */ -int gfs2_permission(struct inode *inode, int mask) +int gfs2_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask) { struct gfs2_inode *ip; struct gfs2_holder i_gh; @@ -1963,7 +1971,8 @@ out: * Returns: errno */ -static int gfs2_setattr(struct dentry *dentry, struct iattr *attr) +static int gfs2_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); struct gfs2_inode *ip = GFS2_I(inode); @@ -2008,6 +2017,7 @@ out: /** * gfs2_getattr - Read out an inode's attributes + * @mnt_userns: user namespace of the mount the inode was found from * @path: Object to query * @stat: The inode's stats * @request_mask: Mask of STATX_xxx flags indicating the caller's interests @@ -2022,7 +2032,8 @@ out: * Returns: errno */ -static int gfs2_getattr(const struct path *path, struct kstat *stat, +static int gfs2_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { struct inode *inode = d_inode(path->dentry); diff --git a/fs/gfs2/inode.h b/fs/gfs2/inode.h index 8073b8d2c7fa..c447bd5b3017 100644 --- a/fs/gfs2/inode.h +++ b/fs/gfs2/inode.h @@ -99,7 +99,8 @@ extern int gfs2_inode_refresh(struct gfs2_inode *ip); extern struct inode *gfs2_lookupi(struct inode *dir, const struct qstr *name, int is_root); -extern int gfs2_permission(struct inode *inode, int mask); +extern int gfs2_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask); extern int gfs2_setattr_simple(struct inode *inode, struct iattr *attr); extern struct inode *gfs2_lookup_simple(struct inode *dip, const char *name); extern void gfs2_dinode_out(const struct gfs2_inode *ip, void *buf); diff --git a/fs/hfs/dir.c b/fs/hfs/dir.c index 3bf2ae0e467c..527f6e46cbe8 100644 --- a/fs/hfs/dir.c +++ b/fs/hfs/dir.c @@ -189,8 +189,8 @@ static int hfs_dir_release(struct inode *inode, struct file *file) * a directory and return a corresponding inode, given the inode for * the directory and the name (and its length) of the new file. */ -static int hfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int hfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct inode *inode; int res; @@ -219,7 +219,8 @@ static int hfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, * in a directory, given the inode for the parent directory and the * name (and its length) of the new directory. */ -static int hfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int hfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode *inode; int res; @@ -279,9 +280,9 @@ static int hfs_remove(struct inode *dir, struct dentry *dentry) * new file/directory. * XXX: how do you handle must_be dir? */ -static int hfs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int hfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { int res; diff --git a/fs/hfs/hfs_fs.h b/fs/hfs/hfs_fs.h index f71c384064c8..b8eb0322a3e5 100644 --- a/fs/hfs/hfs_fs.h +++ b/fs/hfs/hfs_fs.h @@ -204,7 +204,8 @@ extern const struct address_space_operations hfs_btree_aops; extern struct inode *hfs_new_inode(struct inode *, const struct qstr *, umode_t); extern void hfs_inode_write_fork(struct inode *, struct hfs_extent *, __be32 *, __be32 *); extern int hfs_write_inode(struct inode *, struct writeback_control *); -extern int hfs_inode_setattr(struct dentry *, struct iattr *); +extern int hfs_inode_setattr(struct user_namespace *, struct dentry *, + struct iattr *); extern void hfs_inode_read_fork(struct inode *inode, struct hfs_extent *ext, __be32 log_size, __be32 phys_size, u32 clump_size); extern struct inode *hfs_iget(struct super_block *, struct hfs_cat_key *, hfs_cat_rec *); diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index c646218b72bf..3fc5cb346586 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c @@ -602,13 +602,15 @@ static int hfs_file_release(struct inode *inode, struct file *file) * correspond to the same HFS file. */ -int hfs_inode_setattr(struct dentry *dentry, struct iattr * attr) +int hfs_inode_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); struct hfs_sb_info *hsb = HFS_SB(inode->i_sb); int error; - error = setattr_prepare(&init_user_ns, dentry, attr); /* basic permission checks */ + error = setattr_prepare(&init_user_ns, dentry, + attr); /* basic permission checks */ if (error) return error; diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c index 29a9dcfbe81f..03e6c046faf4 100644 --- a/fs/hfsplus/dir.c +++ b/fs/hfsplus/dir.c @@ -434,8 +434,8 @@ out: return res; } -static int hfsplus_symlink(struct inode *dir, struct dentry *dentry, - const char *symname) +static int hfsplus_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { struct hfsplus_sb_info *sbi = HFSPLUS_SB(dir->i_sb); struct inode *inode; @@ -476,8 +476,8 @@ out: return res; } -static int hfsplus_mknod(struct inode *dir, struct dentry *dentry, - umode_t mode, dev_t rdev) +static int hfsplus_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct hfsplus_sb_info *sbi = HFSPLUS_SB(dir->i_sb); struct inode *inode; @@ -517,18 +517,20 @@ out: return res; } -static int hfsplus_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int hfsplus_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { - return hfsplus_mknod(dir, dentry, mode, 0); + return hfsplus_mknod(&init_user_ns, dir, dentry, mode, 0); } -static int hfsplus_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int hfsplus_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { - return hfsplus_mknod(dir, dentry, mode | S_IFDIR, 0); + return hfsplus_mknod(&init_user_ns, dir, dentry, mode | S_IFDIR, 0); } -static int hfsplus_rename(struct inode *old_dir, struct dentry *old_dentry, +static int hfsplus_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h index a92de5199ec3..12b20479ed2b 100644 --- a/fs/hfsplus/hfsplus_fs.h +++ b/fs/hfsplus/hfsplus_fs.h @@ -488,8 +488,9 @@ void hfsplus_inode_write_fork(struct inode *inode, struct hfsplus_fork_raw *fork); int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd); int hfsplus_cat_write_inode(struct inode *inode); -int hfsplus_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags); +int hfsplus_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags); int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index 642e067d8fe8..7a937de9b2ad 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -241,7 +241,8 @@ static int hfsplus_file_release(struct inode *inode, struct file *file) return 0; } -static int hfsplus_setattr(struct dentry *dentry, struct iattr *attr) +static int hfsplus_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); int error; @@ -270,8 +271,9 @@ static int hfsplus_setattr(struct dentry *dentry, struct iattr *attr) return 0; } -int hfsplus_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags) +int hfsplus_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); struct hfsplus_inode_info *hip = HFSPLUS_I(inode); diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c index 6970e29a5287..7c918cd816a3 100644 --- a/fs/hostfs/hostfs_kern.c +++ b/fs/hostfs/hostfs_kern.c @@ -555,8 +555,8 @@ static int read_name(struct inode *ino, char *name) return 0; } -static int hostfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int hostfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct inode *inode; char *name; @@ -654,8 +654,8 @@ static int hostfs_unlink(struct inode *ino, struct dentry *dentry) return err; } -static int hostfs_symlink(struct inode *ino, struct dentry *dentry, - const char *to) +static int hostfs_symlink(struct user_namespace *mnt_userns, struct inode *ino, + struct dentry *dentry, const char *to) { char *file; int err; @@ -667,7 +667,8 @@ static int hostfs_symlink(struct inode *ino, struct dentry *dentry, return err; } -static int hostfs_mkdir(struct inode *ino, struct dentry *dentry, umode_t mode) +static int hostfs_mkdir(struct user_namespace *mnt_userns, struct inode *ino, + struct dentry *dentry, umode_t mode) { char *file; int err; @@ -691,7 +692,8 @@ static int hostfs_rmdir(struct inode *ino, struct dentry *dentry) return err; } -static int hostfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +static int hostfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) { struct inode *inode; char *name; @@ -729,7 +731,8 @@ static int hostfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, return err; } -static int hostfs_rename2(struct inode *old_dir, struct dentry *old_dentry, +static int hostfs_rename2(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { @@ -757,7 +760,8 @@ static int hostfs_rename2(struct inode *old_dir, struct dentry *old_dentry, return err; } -static int hostfs_permission(struct inode *ino, int desired) +static int hostfs_permission(struct user_namespace *mnt_userns, + struct inode *ino, int desired) { char *name; int r = 0, w = 0, x = 0, err; @@ -783,7 +787,8 @@ static int hostfs_permission(struct inode *ino, int desired) return err; } -static int hostfs_setattr(struct dentry *dentry, struct iattr *attr) +static int hostfs_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); struct hostfs_iattr attrs; diff --git a/fs/hpfs/hpfs_fn.h b/fs/hpfs/hpfs_fn.h index 1cca83218fb5..167ec6884642 100644 --- a/fs/hpfs/hpfs_fn.h +++ b/fs/hpfs/hpfs_fn.h @@ -280,7 +280,7 @@ void hpfs_init_inode(struct inode *); void hpfs_read_inode(struct inode *); void hpfs_write_inode(struct inode *); void hpfs_write_inode_nolock(struct inode *); -int hpfs_setattr(struct dentry *, struct iattr *); +int hpfs_setattr(struct user_namespace *, struct dentry *, struct iattr *); void hpfs_write_if_changed(struct inode *); void hpfs_evict_inode(struct inode *); diff --git a/fs/hpfs/inode.c b/fs/hpfs/inode.c index 8ba2152a78ba..82208cc28ebd 100644 --- a/fs/hpfs/inode.c +++ b/fs/hpfs/inode.c @@ -257,7 +257,8 @@ void hpfs_write_inode_nolock(struct inode *i) brelse(bh); } -int hpfs_setattr(struct dentry *dentry, struct iattr *attr) +int hpfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); int error = -EINVAL; diff --git a/fs/hpfs/namei.c b/fs/hpfs/namei.c index 1aee39160ac5..d73f8a67168e 100644 --- a/fs/hpfs/namei.c +++ b/fs/hpfs/namei.c @@ -20,7 +20,8 @@ static void hpfs_update_directory_times(struct inode *dir) hpfs_write_inode_nolock(dir); } -static int hpfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int hpfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { const unsigned char *name = dentry->d_name.name; unsigned len = dentry->d_name.len; @@ -128,7 +129,8 @@ bail: return err; } -static int hpfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) +static int hpfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { const unsigned char *name = dentry->d_name.name; unsigned len = dentry->d_name.len; @@ -215,7 +217,8 @@ bail: return err; } -static int hpfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) +static int hpfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { const unsigned char *name = dentry->d_name.name; unsigned len = dentry->d_name.len; @@ -289,7 +292,8 @@ bail: return err; } -static int hpfs_symlink(struct inode *dir, struct dentry *dentry, const char *symlink) +static int hpfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symlink) { const unsigned char *name = dentry->d_name.name; unsigned len = dentry->d_name.len; @@ -506,10 +510,10 @@ fail: const struct address_space_operations hpfs_symlink_aops = { .readpage = hpfs_symlink_readpage }; - -static int hpfs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) + +static int hpfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { const unsigned char *old_name = old_dentry->d_name.name; unsigned old_len = old_dentry->d_name.len; diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 327e572b4e00..c5c32eb59498 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -751,7 +751,8 @@ out: return error; } -static int hugetlbfs_setattr(struct dentry *dentry, struct iattr *attr) +static int hugetlbfs_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); struct hstate *h = hstate_inode(inode); @@ -898,33 +899,39 @@ static int do_hugetlbfs_mknod(struct inode *dir, return error; } -static int hugetlbfs_mknod(struct inode *dir, - struct dentry *dentry, umode_t mode, dev_t dev) +static int hugetlbfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) { return do_hugetlbfs_mknod(dir, dentry, mode, dev, false); } -static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int hugetlbfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { - int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0); + int retval = hugetlbfs_mknod(&init_user_ns, dir, dentry, + mode | S_IFDIR, 0); if (!retval) inc_nlink(dir); return retval; } -static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) +static int hugetlbfs_create(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + umode_t mode, bool excl) { - return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0); + return hugetlbfs_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0); } -static int hugetlbfs_tmpfile(struct inode *dir, - struct dentry *dentry, umode_t mode) +static int hugetlbfs_tmpfile(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + umode_t mode) { return do_hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0, true); } -static int hugetlbfs_symlink(struct inode *dir, - struct dentry *dentry, const char *symname) +static int hugetlbfs_symlink(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, + const char *symname) { struct inode *inode; int error = -ENOSPC; diff --git a/fs/jffs2/acl.c b/fs/jffs2/acl.c index 5f27ac593479..55a79df70d24 100644 --- a/fs/jffs2/acl.c +++ b/fs/jffs2/acl.c @@ -226,7 +226,8 @@ static int __jffs2_set_acl(struct inode *inode, int xprefix, struct posix_acl *a return rc; } -int jffs2_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int jffs2_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { int rc, xprefix; diff --git a/fs/jffs2/acl.h b/fs/jffs2/acl.h index 12d0271bdde3..62c50da9d493 100644 --- a/fs/jffs2/acl.h +++ b/fs/jffs2/acl.h @@ -28,7 +28,8 @@ struct jffs2_acl_header { #ifdef CONFIG_JFFS2_FS_POSIX_ACL struct posix_acl *jffs2_get_acl(struct inode *inode, int type); -int jffs2_set_acl(struct inode *inode, struct posix_acl *acl, int type); +int jffs2_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); extern int jffs2_init_acl_pre(struct inode *, struct inode *, umode_t *); extern int jffs2_init_acl_post(struct inode *); diff --git a/fs/jffs2/dir.c b/fs/jffs2/dir.c index 776493713153..c0aabbcbfd58 100644 --- a/fs/jffs2/dir.c +++ b/fs/jffs2/dir.c @@ -24,18 +24,21 @@ static int jffs2_readdir (struct file *, struct dir_context *); -static int jffs2_create (struct inode *,struct dentry *,umode_t, - bool); +static int jffs2_create (struct user_namespace *, struct inode *, + struct dentry *, umode_t, bool); static struct dentry *jffs2_lookup (struct inode *,struct dentry *, unsigned int); static int jffs2_link (struct dentry *,struct inode *,struct dentry *); static int jffs2_unlink (struct inode *,struct dentry *); -static int jffs2_symlink (struct inode *,struct dentry *,const char *); -static int jffs2_mkdir (struct inode *,struct dentry *,umode_t); +static int jffs2_symlink (struct user_namespace *, struct inode *, + struct dentry *, const char *); +static int jffs2_mkdir (struct user_namespace *, struct inode *,struct dentry *, + umode_t); static int jffs2_rmdir (struct inode *,struct dentry *); -static int jffs2_mknod (struct inode *,struct dentry *,umode_t,dev_t); -static int jffs2_rename (struct inode *, struct dentry *, - struct inode *, struct dentry *, +static int jffs2_mknod (struct user_namespace *, struct inode *,struct dentry *, + umode_t,dev_t); +static int jffs2_rename (struct user_namespace *, struct inode *, + struct dentry *, struct inode *, struct dentry *, unsigned int); const struct file_operations jffs2_dir_operations = @@ -157,8 +160,8 @@ static int jffs2_readdir(struct file *file, struct dir_context *ctx) /***********************************************************************/ -static int jffs2_create(struct inode *dir_i, struct dentry *dentry, - umode_t mode, bool excl) +static int jffs2_create(struct user_namespace *mnt_userns, struct inode *dir_i, + struct dentry *dentry, umode_t mode, bool excl) { struct jffs2_raw_inode *ri; struct jffs2_inode_info *f, *dir_f; @@ -276,7 +279,8 @@ static int jffs2_link (struct dentry *old_dentry, struct inode *dir_i, struct de /***********************************************************************/ -static int jffs2_symlink (struct inode *dir_i, struct dentry *dentry, const char *target) +static int jffs2_symlink (struct user_namespace *mnt_userns, struct inode *dir_i, + struct dentry *dentry, const char *target) { struct jffs2_inode_info *f, *dir_f; struct jffs2_sb_info *c; @@ -438,7 +442,8 @@ static int jffs2_symlink (struct inode *dir_i, struct dentry *dentry, const char } -static int jffs2_mkdir (struct inode *dir_i, struct dentry *dentry, umode_t mode) +static int jffs2_mkdir (struct user_namespace *mnt_userns, struct inode *dir_i, + struct dentry *dentry, umode_t mode) { struct jffs2_inode_info *f, *dir_f; struct jffs2_sb_info *c; @@ -609,7 +614,8 @@ static int jffs2_rmdir (struct inode *dir_i, struct dentry *dentry) return ret; } -static int jffs2_mknod (struct inode *dir_i, struct dentry *dentry, umode_t mode, dev_t rdev) +static int jffs2_mknod (struct user_namespace *mnt_userns, struct inode *dir_i, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct jffs2_inode_info *f, *dir_f; struct jffs2_sb_info *c; @@ -756,7 +762,8 @@ static int jffs2_mknod (struct inode *dir_i, struct dentry *dentry, umode_t mode return ret; } -static int jffs2_rename (struct inode *old_dir_i, struct dentry *old_dentry, +static int jffs2_rename (struct user_namespace *mnt_userns, + struct inode *old_dir_i, struct dentry *old_dentry, struct inode *new_dir_i, struct dentry *new_dentry, unsigned int flags) { diff --git a/fs/jffs2/fs.c b/fs/jffs2/fs.c index ee9f51bab4c6..2ac410477c4f 100644 --- a/fs/jffs2/fs.c +++ b/fs/jffs2/fs.c @@ -190,7 +190,8 @@ int jffs2_do_setattr (struct inode *inode, struct iattr *iattr) return 0; } -int jffs2_setattr(struct dentry *dentry, struct iattr *iattr) +int jffs2_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr) { struct inode *inode = d_inode(dentry); int rc; diff --git a/fs/jffs2/os-linux.h b/fs/jffs2/os-linux.h index ef1cfa61549e..173eccac691d 100644 --- a/fs/jffs2/os-linux.h +++ b/fs/jffs2/os-linux.h @@ -164,7 +164,7 @@ long jffs2_ioctl(struct file *, unsigned int, unsigned long); extern const struct inode_operations jffs2_symlink_inode_operations; /* fs.c */ -int jffs2_setattr (struct dentry *, struct iattr *); +int jffs2_setattr (struct user_namespace *, struct dentry *, struct iattr *); int jffs2_do_setattr (struct inode *, struct iattr *); struct inode *jffs2_iget(struct super_block *, unsigned long); void jffs2_evict_inode (struct inode *); diff --git a/fs/jfs/acl.c b/fs/jfs/acl.c index cf79a34bfada..43c285c3d2a7 100644 --- a/fs/jfs/acl.c +++ b/fs/jfs/acl.c @@ -91,7 +91,8 @@ out: return rc; } -int jfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int jfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { int rc; tid_t tid; diff --git a/fs/jfs/file.c b/fs/jfs/file.c index 61c3b0c1fbf6..28b70e7c7dd4 100644 --- a/fs/jfs/file.c +++ b/fs/jfs/file.c @@ -85,7 +85,8 @@ static int jfs_release(struct inode *inode, struct file *file) return 0; } -int jfs_setattr(struct dentry *dentry, struct iattr *iattr) +int jfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr) { struct inode *inode = d_inode(dentry); int rc; diff --git a/fs/jfs/jfs_acl.h b/fs/jfs/jfs_acl.h index 9f8f92dd6f84..7ae389a7a366 100644 --- a/fs/jfs/jfs_acl.h +++ b/fs/jfs/jfs_acl.h @@ -8,7 +8,8 @@ #ifdef CONFIG_JFS_POSIX_ACL struct posix_acl *jfs_get_acl(struct inode *inode, int type); -int jfs_set_acl(struct inode *inode, struct posix_acl *acl, int type); +int jfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); int jfs_init_acl(tid_t, struct inode *, struct inode *); #else diff --git a/fs/jfs/jfs_inode.h b/fs/jfs/jfs_inode.h index 70a0d12e427e..01daa0cb0ae5 100644 --- a/fs/jfs/jfs_inode.h +++ b/fs/jfs/jfs_inode.h @@ -26,7 +26,7 @@ extern struct dentry *jfs_fh_to_parent(struct super_block *sb, struct fid *fid, int fh_len, int fh_type); extern void jfs_set_inode_flags(struct inode *); extern int jfs_get_block(struct inode *, sector_t, struct buffer_head *, int); -extern int jfs_setattr(struct dentry *, struct iattr *); +extern int jfs_setattr(struct user_namespace *, struct dentry *, struct iattr *); extern const struct address_space_operations jfs_aops; extern const struct inode_operations jfs_dir_inode_operations; diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c index 7a55d14cc1af..9abed0d750e5 100644 --- a/fs/jfs/namei.c +++ b/fs/jfs/namei.c @@ -59,8 +59,8 @@ static inline void free_ea_wmap(struct inode *inode) * RETURN: Errors from subroutines * */ -static int jfs_create(struct inode *dip, struct dentry *dentry, umode_t mode, - bool excl) +static int jfs_create(struct user_namespace *mnt_userns, struct inode *dip, + struct dentry *dentry, umode_t mode, bool excl) { int rc = 0; tid_t tid; /* transaction id */ @@ -192,7 +192,8 @@ static int jfs_create(struct inode *dip, struct dentry *dentry, umode_t mode, * note: * EACCES: user needs search+write permission on the parent directory */ -static int jfs_mkdir(struct inode *dip, struct dentry *dentry, umode_t mode) +static int jfs_mkdir(struct user_namespace *mnt_userns, struct inode *dip, + struct dentry *dentry, umode_t mode) { int rc = 0; tid_t tid; /* transaction id */ @@ -868,8 +869,8 @@ static int jfs_link(struct dentry *old_dentry, * an intermediate result whose length exceeds PATH_MAX [XPG4.2] */ -static int jfs_symlink(struct inode *dip, struct dentry *dentry, - const char *name) +static int jfs_symlink(struct user_namespace *mnt_userns, struct inode *dip, + struct dentry *dentry, const char *name) { int rc; tid_t tid; @@ -1058,9 +1059,9 @@ static int jfs_symlink(struct inode *dip, struct dentry *dentry, * * FUNCTION: rename a file or directory */ -static int jfs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int jfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct btstack btstack; ino_t ino; @@ -1344,8 +1345,8 @@ static int jfs_rename(struct inode *old_dir, struct dentry *old_dentry, * * FUNCTION: Create a special file (device) */ -static int jfs_mknod(struct inode *dir, struct dentry *dentry, - umode_t mode, dev_t rdev) +static int jfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct jfs_inode_info *jfs_ip; struct btstack btstack; diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index 7a53eed69fef..7e0e62deab53 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -1110,7 +1110,8 @@ static struct dentry *kernfs_iop_lookup(struct inode *dir, return ret; } -static int kernfs_iop_mkdir(struct inode *dir, struct dentry *dentry, +static int kernfs_iop_mkdir(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, umode_t mode) { struct kernfs_node *parent = dir->i_private; @@ -1147,7 +1148,8 @@ static int kernfs_iop_rmdir(struct inode *dir, struct dentry *dentry) return ret; } -static int kernfs_iop_rename(struct inode *old_dir, struct dentry *old_dentry, +static int kernfs_iop_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c index 032d3d7546d8..d73950fc3d57 100644 --- a/fs/kernfs/inode.c +++ b/fs/kernfs/inode.c @@ -112,7 +112,8 @@ int kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr) return ret; } -int kernfs_iop_setattr(struct dentry *dentry, struct iattr *iattr) +int kernfs_iop_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr) { struct inode *inode = d_inode(dentry); struct kernfs_node *kn = inode->i_private; @@ -183,7 +184,8 @@ static void kernfs_refresh_inode(struct kernfs_node *kn, struct inode *inode) set_nlink(inode, kn->dir.subdirs + 2); } -int kernfs_iop_getattr(const struct path *path, struct kstat *stat, +int kernfs_iop_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); @@ -272,7 +274,8 @@ void kernfs_evict_inode(struct inode *inode) kernfs_put(kn); } -int kernfs_iop_permission(struct inode *inode, int mask) +int kernfs_iop_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { struct kernfs_node *kn; diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h index 7ee97ef59184..ccc3b44f6306 100644 --- a/fs/kernfs/kernfs-internal.h +++ b/fs/kernfs/kernfs-internal.h @@ -89,9 +89,12 @@ extern struct kmem_cache *kernfs_node_cache, *kernfs_iattrs_cache; */ extern const struct xattr_handler *kernfs_xattr_handlers[]; void kernfs_evict_inode(struct inode *inode); -int kernfs_iop_permission(struct inode *inode, int mask); -int kernfs_iop_setattr(struct dentry *dentry, struct iattr *iattr); -int kernfs_iop_getattr(const struct path *path, struct kstat *stat, +int kernfs_iop_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask); +int kernfs_iop_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr); +int kernfs_iop_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags); ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size); int __kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr); diff --git a/fs/libfs.c b/fs/libfs.c index 508e9ea8e6f3..967aefda6ee3 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -27,8 +27,9 @@ #include "internal.h" -int simple_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags) +int simple_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); generic_fillattr(&init_user_ns, inode, stat); @@ -447,9 +448,9 @@ int simple_rmdir(struct inode *dir, struct dentry *dentry) } EXPORT_SYMBOL(simple_rmdir); -int simple_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +int simple_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct inode *inode = d_inode(old_dentry); int they_are_dirs = d_is_dir(old_dentry); @@ -492,18 +493,19 @@ EXPORT_SYMBOL(simple_rename); * on simple regular filesystems. Anything that needs to change on-disk * or wire state on size changes needs its own setattr method. */ -int simple_setattr(struct dentry *dentry, struct iattr *iattr) +int simple_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr) { struct inode *inode = d_inode(dentry); int error; - error = setattr_prepare(&init_user_ns, dentry, iattr); + error = setattr_prepare(mnt_userns, dentry, iattr); if (error) return error; if (iattr->ia_valid & ATTR_SIZE) truncate_setsize(inode, iattr->ia_size); - setattr_copy(&init_user_ns, inode, iattr); + setattr_copy(mnt_userns, inode, iattr); mark_inode_dirty(inode); return 0; } @@ -1300,7 +1302,8 @@ static struct dentry *empty_dir_lookup(struct inode *dir, struct dentry *dentry, return ERR_PTR(-ENOENT); } -static int empty_dir_getattr(const struct path *path, struct kstat *stat, +static int empty_dir_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); @@ -1308,7 +1311,8 @@ static int empty_dir_getattr(const struct path *path, struct kstat *stat, return 0; } -static int empty_dir_setattr(struct dentry *dentry, struct iattr *attr) +static int empty_dir_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { return -EPERM; } @@ -1318,14 +1322,9 @@ static ssize_t empty_dir_listxattr(struct dentry *dentry, char *list, size_t siz return -EOPNOTSUPP; } -static int empty_dir_permission(struct inode *inode, int mask) -{ - return generic_permission(&init_user_ns, inode, mask); -} - static const struct inode_operations empty_dir_inode_operations = { .lookup = empty_dir_lookup, - .permission = empty_dir_permission, + .permission = generic_permission, .setattr = empty_dir_setattr, .getattr = empty_dir_getattr, .listxattr = empty_dir_listxattr, diff --git a/fs/minix/file.c b/fs/minix/file.c index f07acd268577..6a7bd2d9eec0 100644 --- a/fs/minix/file.c +++ b/fs/minix/file.c @@ -22,7 +22,8 @@ const struct file_operations minix_file_operations = { .splice_read = generic_file_splice_read, }; -static int minix_setattr(struct dentry *dentry, struct iattr *attr) +static int minix_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); int error; diff --git a/fs/minix/inode.c b/fs/minix/inode.c index 91c81d2fc90d..a532a99bbe81 100644 --- a/fs/minix/inode.c +++ b/fs/minix/inode.c @@ -652,8 +652,8 @@ static int minix_write_inode(struct inode *inode, struct writeback_control *wbc) return err; } -int minix_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +int minix_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { struct super_block *sb = path->dentry->d_sb; struct inode *inode = d_inode(path->dentry); diff --git a/fs/minix/minix.h b/fs/minix/minix.h index 168d45d3de73..202173368025 100644 --- a/fs/minix/minix.h +++ b/fs/minix/minix.h @@ -51,7 +51,8 @@ extern unsigned long minix_count_free_inodes(struct super_block *sb); extern int minix_new_block(struct inode * inode); extern void minix_free_block(struct inode *inode, unsigned long block); extern unsigned long minix_count_free_blocks(struct super_block *sb); -extern int minix_getattr(const struct path *, struct kstat *, u32, unsigned int); +extern int minix_getattr(struct user_namespace *, const struct path *, + struct kstat *, u32, unsigned int); extern int minix_prepare_chunk(struct page *page, loff_t pos, unsigned len); extern void V1_minix_truncate(struct inode *); diff --git a/fs/minix/namei.c b/fs/minix/namei.c index 1a6084d2b02e..937fa5fae2b8 100644 --- a/fs/minix/namei.c +++ b/fs/minix/namei.c @@ -33,7 +33,8 @@ static struct dentry *minix_lookup(struct inode * dir, struct dentry *dentry, un return d_splice_alias(inode, dentry); } -static int minix_mknod(struct inode * dir, struct dentry *dentry, umode_t mode, dev_t rdev) +static int minix_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { int error; struct inode *inode; @@ -51,7 +52,8 @@ static int minix_mknod(struct inode * dir, struct dentry *dentry, umode_t mode, return error; } -static int minix_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) +static int minix_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { int error; struct inode *inode = minix_new_inode(dir, mode, &error); @@ -63,14 +65,14 @@ static int minix_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) return error; } -static int minix_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int minix_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { - return minix_mknod(dir, dentry, mode, 0); + return minix_mknod(mnt_userns, dir, dentry, mode, 0); } -static int minix_symlink(struct inode * dir, struct dentry *dentry, - const char * symname) +static int minix_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { int err = -ENAMETOOLONG; int i = strlen(symname)+1; @@ -109,7 +111,8 @@ static int minix_link(struct dentry * old_dentry, struct inode * dir, return add_nondir(dentry, inode); } -static int minix_mkdir(struct inode * dir, struct dentry *dentry, umode_t mode) +static int minix_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode * inode; int err; @@ -181,8 +184,9 @@ static int minix_rmdir(struct inode * dir, struct dentry *dentry) return err; } -static int minix_rename(struct inode * old_dir, struct dentry *old_dentry, - struct inode * new_dir, struct dentry *new_dentry, +static int minix_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { struct inode * old_inode = d_inode(old_dentry); diff --git a/fs/namei.c b/fs/namei.c index c8c083daf368..d9ceb75ac169 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -443,7 +443,7 @@ static inline int do_inode_permission(struct user_namespace *mnt_userns, { if (unlikely(!(inode->i_opflags & IOP_FASTPERM))) { if (likely(inode->i_op->permission)) - return inode->i_op->permission(inode, mask); + return inode->i_op->permission(mnt_userns, inode, mask); /* This gets set once for the inode lifetime */ spin_lock(&inode->i_lock); @@ -2199,11 +2199,13 @@ static int link_path_walk(const char *name, struct nameidata *nd) /* At this point we know we have a real path component. */ for(;;) { + struct user_namespace *mnt_userns; const char *link; u64 hash_len; int type; - err = may_lookup(&init_user_ns, nd); + mnt_userns = mnt_user_ns(nd->path.mnt); + err = may_lookup(mnt_userns, nd); if (err) return err; @@ -2251,7 +2253,7 @@ static int link_path_walk(const char *name, struct nameidata *nd) OK: /* pathname or trailing symlink, done */ if (!depth) { - nd->dir_uid = i_uid_into_mnt(&init_user_ns, nd->inode); + nd->dir_uid = i_uid_into_mnt(mnt_userns, nd->inode); nd->dir_mode = nd->inode->i_mode; nd->flags &= ~LOOKUP_PARENT; return 0; @@ -2904,7 +2906,7 @@ int vfs_create(struct user_namespace *mnt_userns, struct inode *dir, error = security_inode_create(dir, dentry, mode); if (error) return error; - error = dir->i_op->create(dir, dentry, mode, want_excl); + error = dir->i_op->create(mnt_userns, dir, dentry, mode, want_excl); if (!error) fsnotify_create(dir, dentry); return error; @@ -2995,7 +2997,7 @@ static int may_open(struct user_namespace *mnt_userns, const struct path *path, return 0; } -static int handle_truncate(struct file *filp) +static int handle_truncate(struct user_namespace *mnt_userns, struct file *filp) { const struct path *path = &filp->f_path; struct inode *inode = path->dentry->d_inode; @@ -3009,7 +3011,7 @@ static int handle_truncate(struct file *filp) if (!error) error = security_path_truncate(path); if (!error) { - error = do_truncate(&init_user_ns, path->dentry, 0, + error = do_truncate(mnt_userns, path->dentry, 0, ATTR_MTIME|ATTR_CTIME|ATTR_OPEN, filp); } @@ -3118,6 +3120,7 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file, const struct open_flags *op, bool got_write) { + struct user_namespace *mnt_userns; struct dentry *dir = nd->path.dentry; struct inode *dir_inode = dir->d_inode; int open_flag = op->open_flag; @@ -3165,13 +3168,14 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file, */ if (unlikely(!got_write)) open_flag &= ~O_TRUNC; + mnt_userns = mnt_user_ns(nd->path.mnt); if (open_flag & O_CREAT) { if (open_flag & O_EXCL) open_flag &= ~O_TRUNC; if (!IS_POSIXACL(dir->d_inode)) mode &= ~current_umask(); if (likely(got_write)) - create_error = may_o_create(&init_user_ns, &nd->path, + create_error = may_o_create(mnt_userns, &nd->path, dentry, mode); else create_error = -EROFS; @@ -3207,8 +3211,9 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file, error = -EACCES; goto out_dput; } - error = dir_inode->i_op->create(dir_inode, dentry, mode, - open_flag & O_EXCL); + + error = dir_inode->i_op->create(mnt_userns, dir_inode, dentry, + mode, open_flag & O_EXCL); if (error) goto out_dput; } @@ -3316,6 +3321,7 @@ finish_lookup: static int do_open(struct nameidata *nd, struct file *file, const struct open_flags *op) { + struct user_namespace *mnt_userns; int open_flag = op->open_flag; bool do_truncate; int acc_mode; @@ -3328,12 +3334,13 @@ static int do_open(struct nameidata *nd, } if (!(file->f_mode & FMODE_CREATED)) audit_inode(nd->name, nd->path.dentry, 0); + mnt_userns = mnt_user_ns(nd->path.mnt); if (open_flag & O_CREAT) { if ((open_flag & O_EXCL) && !(file->f_mode & FMODE_CREATED)) return -EEXIST; if (d_is_dir(nd->path.dentry)) return -EISDIR; - error = may_create_in_sticky(&init_user_ns, nd, + error = may_create_in_sticky(mnt_userns, nd, d_backing_inode(nd->path.dentry)); if (unlikely(error)) return error; @@ -3353,13 +3360,13 @@ static int do_open(struct nameidata *nd, return error; do_truncate = true; } - error = may_open(&init_user_ns, &nd->path, acc_mode, open_flag); + error = may_open(mnt_userns, &nd->path, acc_mode, open_flag); if (!error && !(file->f_mode & FMODE_OPENED)) error = vfs_open(&nd->path, file); if (!error) error = ima_file_check(file, op->acc_mode); if (!error && do_truncate) - error = handle_truncate(file); + error = handle_truncate(mnt_userns, file); if (unlikely(error > 0)) { WARN_ON(1); error = -EINVAL; @@ -3403,7 +3410,7 @@ struct dentry *vfs_tmpfile(struct user_namespace *mnt_userns, child = d_alloc(dentry, &slash_name); if (unlikely(!child)) goto out_err; - error = dir->i_op->tmpfile(dir, child, mode); + error = dir->i_op->tmpfile(mnt_userns, dir, child, mode); if (error) goto out_err; error = -ENOENT; @@ -3446,7 +3453,7 @@ static int do_tmpfile(struct nameidata *nd, unsigned flags, path.dentry = child; audit_inode(nd->name, child, 0); /* Don't check for other permissions, the inode was just created */ - error = may_open(&init_user_ns, &path, 0, op->open_flag); + error = may_open(mnt_userns, &path, 0, op->open_flag); if (error) goto out2; file->f_path.mnt = path.mnt; @@ -3690,7 +3697,7 @@ int vfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, if (error) return error; - error = dir->i_op->mknod(dir, dentry, mode, dev); + error = dir->i_op->mknod(mnt_userns, dir, dentry, mode, dev); if (!error) fsnotify_create(dir, dentry); return error; @@ -3809,7 +3816,7 @@ int vfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, if (max_links && dir->i_nlink >= max_links) return -EMLINK; - error = dir->i_op->mkdir(dir, dentry, mode); + error = dir->i_op->mkdir(mnt_userns, dir, dentry, mode); if (!error) fsnotify_mkdir(dir, dentry); return error; @@ -3834,7 +3841,8 @@ retry: if (!error) { struct user_namespace *mnt_userns; mnt_userns = mnt_user_ns(path.mnt); - error = vfs_mkdir(mnt_userns, path.dentry->d_inode, dentry, mode); + error = vfs_mkdir(mnt_userns, path.dentry->d_inode, dentry, + mode); } done_path_create(&path, dentry); if (retry_estale(error, lookup_flags)) { @@ -4087,7 +4095,8 @@ retry_deleg: if (error) goto exit2; mnt_userns = mnt_user_ns(path.mnt); - error = vfs_unlink(mnt_userns, path.dentry->d_inode, dentry, &delegated_inode); + error = vfs_unlink(mnt_userns, path.dentry->d_inode, dentry, + &delegated_inode); exit2: dput(dentry); } @@ -4166,7 +4175,7 @@ int vfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, if (error) return error; - error = dir->i_op->symlink(dir, dentry, oldname); + error = dir->i_op->symlink(mnt_userns, dir, dentry, oldname); if (!error) fsnotify_create(dir, dentry); return error; @@ -4357,13 +4366,13 @@ retry: error = -EXDEV; if (old_path.mnt != new_path.mnt) goto out_dput; - error = may_linkat(&init_user_ns, &old_path); + mnt_userns = mnt_user_ns(new_path.mnt); + error = may_linkat(mnt_userns, &old_path); if (unlikely(error)) goto out_dput; error = security_path_link(old_path.dentry, &new_path, new_dentry); if (error) goto out_dput; - mnt_userns = mnt_user_ns(new_path.mnt); error = vfs_link(old_path.dentry, mnt_userns, new_path.dentry->d_inode, new_dentry, &delegated_inode); out_dput: @@ -4542,8 +4551,8 @@ int vfs_rename(struct renamedata *rd) if (error) goto out; } - error = old_dir->i_op->rename(old_dir, old_dentry, - new_dir, new_dentry, flags); + error = old_dir->i_op->rename(rd->new_mnt_userns, old_dir, old_dentry, + new_dir, new_dentry, flags); if (error) goto out; diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 727e01a84503..19a9f434442f 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2095,8 +2095,8 @@ EXPORT_SYMBOL_GPL(nfs_instantiate); * that the operation succeeded on the server, but an error in the * reply path made it appear to have failed. */ -int nfs_create(struct inode *dir, struct dentry *dentry, - umode_t mode, bool excl) +int nfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct iattr attr; int open_flags = excl ? O_CREAT | O_EXCL : O_CREAT; @@ -2124,7 +2124,8 @@ EXPORT_SYMBOL_GPL(nfs_create); * See comments for nfs_proc_create regarding failed operations. */ int -nfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) +nfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct iattr attr; int status; @@ -2150,7 +2151,8 @@ EXPORT_SYMBOL_GPL(nfs_mknod); /* * See comments for nfs_proc_create regarding failed operations. */ -int nfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +int nfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct iattr attr; int error; @@ -2295,7 +2297,8 @@ EXPORT_SYMBOL_GPL(nfs_unlink); * now have a new file handle and can instantiate an in-core NFS inode * and move the raw page into its mapping. */ -int nfs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) +int nfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { struct page *page; char *kaddr; @@ -2398,9 +2401,9 @@ EXPORT_SYMBOL_GPL(nfs_link); * If these conditions are met, we can drop the dentries before doing * the rename. */ -int nfs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +int nfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct inode *old_inode = d_inode(old_dentry); struct inode *new_inode = d_inode(new_dentry); @@ -2939,7 +2942,9 @@ static int nfs_execute_ok(struct inode *inode, int mask) return ret; } -int nfs_permission(struct inode *inode, int mask) +int nfs_permission(struct user_namespace *mnt_userns, + struct inode *inode, + int mask) { const struct cred *cred = current_cred(); int res = 0; diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index cab123ec1664..447e95974386 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -594,7 +594,8 @@ EXPORT_SYMBOL_GPL(nfs_fhget); #define NFS_VALID_ATTRS (ATTR_MODE|ATTR_UID|ATTR_GID|ATTR_SIZE|ATTR_ATIME|ATTR_ATIME_SET|ATTR_MTIME|ATTR_MTIME_SET|ATTR_FILE|ATTR_OPEN) int -nfs_setattr(struct dentry *dentry, struct iattr *attr) +nfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); struct nfs_fattr *fattr; @@ -787,8 +788,8 @@ static bool nfs_need_revalidate_inode(struct inode *inode) return false; } -int nfs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags) +int nfs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); struct nfs_server *server = NFS_SERVER(inode); diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 62d3189745cd..25fb43b69e5a 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -378,14 +378,18 @@ extern unsigned long nfs_access_cache_count(struct shrinker *shrink, extern unsigned long nfs_access_cache_scan(struct shrinker *shrink, struct shrink_control *sc); struct dentry *nfs_lookup(struct inode *, struct dentry *, unsigned int); -int nfs_create(struct inode *, struct dentry *, umode_t, bool); -int nfs_mkdir(struct inode *, struct dentry *, umode_t); +int nfs_create(struct user_namespace *, struct inode *, struct dentry *, + umode_t, bool); +int nfs_mkdir(struct user_namespace *, struct inode *, struct dentry *, + umode_t); int nfs_rmdir(struct inode *, struct dentry *); int nfs_unlink(struct inode *, struct dentry *); -int nfs_symlink(struct inode *, struct dentry *, const char *); +int nfs_symlink(struct user_namespace *, struct inode *, struct dentry *, + const char *); int nfs_link(struct dentry *, struct inode *, struct dentry *); -int nfs_mknod(struct inode *, struct dentry *, umode_t, dev_t); -int nfs_rename(struct inode *, struct dentry *, +int nfs_mknod(struct user_namespace *, struct inode *, struct dentry *, umode_t, + dev_t); +int nfs_rename(struct user_namespace *, struct inode *, struct dentry *, struct inode *, struct dentry *, unsigned int); /* file.c */ diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c index 55fc711e368b..93e60e921f92 100644 --- a/fs/nfs/namespace.c +++ b/fs/nfs/namespace.c @@ -208,20 +208,23 @@ out_fc: } static int -nfs_namespace_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags) +nfs_namespace_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) { if (NFS_FH(d_inode(path->dentry))->size != 0) - return nfs_getattr(path, stat, request_mask, query_flags); + return nfs_getattr(mnt_userns, path, stat, request_mask, + query_flags); generic_fillattr(&init_user_ns, d_inode(path->dentry), stat); return 0; } static int -nfs_namespace_setattr(struct dentry *dentry, struct iattr *attr) +nfs_namespace_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { if (NFS_FH(d_inode(dentry))->size != 0) - return nfs_setattr(dentry, attr); + return nfs_setattr(mnt_userns, dentry, attr); return -EACCES; } diff --git a/fs/nfs/nfs3_fs.h b/fs/nfs/nfs3_fs.h index 1b950b66b3bb..c8a192802dda 100644 --- a/fs/nfs/nfs3_fs.h +++ b/fs/nfs/nfs3_fs.h @@ -12,7 +12,8 @@ */ #ifdef CONFIG_NFS_V3_ACL extern struct posix_acl *nfs3_get_acl(struct inode *inode, int type); -extern int nfs3_set_acl(struct inode *inode, struct posix_acl *acl, int type); +extern int nfs3_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); extern int nfs3_proc_setacls(struct inode *inode, struct posix_acl *acl, struct posix_acl *dfacl); extern ssize_t nfs3_listxattr(struct dentry *, char *, size_t); diff --git a/fs/nfs/nfs3acl.c b/fs/nfs/nfs3acl.c index c6c863382f37..5604e807fc01 100644 --- a/fs/nfs/nfs3acl.c +++ b/fs/nfs/nfs3acl.c @@ -251,7 +251,8 @@ int nfs3_proc_setacls(struct inode *inode, struct posix_acl *acl, } -int nfs3_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int nfs3_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { struct posix_acl *orig = acl, *dfacl = NULL, *alloc; int status; diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index 8aad3c48092a..2e8eb263cf0f 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c @@ -805,7 +805,8 @@ void nilfs_evict_inode(struct inode *inode) */ } -int nilfs_setattr(struct dentry *dentry, struct iattr *iattr) +int nilfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr) { struct nilfs_transaction_info ti; struct inode *inode = d_inode(dentry); @@ -843,7 +844,8 @@ out_err: return err; } -int nilfs_permission(struct inode *inode, int mask) +int nilfs_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask) { struct nilfs_root *root = NILFS_I(inode)->i_root; diff --git a/fs/nilfs2/namei.c b/fs/nilfs2/namei.c index a6ec7961d4f5..ecace5f96a95 100644 --- a/fs/nilfs2/namei.c +++ b/fs/nilfs2/namei.c @@ -72,8 +72,8 @@ nilfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) * If the create succeeds, we fill in the inode information * with d_instantiate(). */ -static int nilfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int nilfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct inode *inode; struct nilfs_transaction_info ti; @@ -100,7 +100,8 @@ static int nilfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, } static int -nilfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) +nilfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct inode *inode; struct nilfs_transaction_info ti; @@ -124,8 +125,8 @@ nilfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) return err; } -static int nilfs_symlink(struct inode *dir, struct dentry *dentry, - const char *symname) +static int nilfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { struct nilfs_transaction_info ti; struct super_block *sb = dir->i_sb; @@ -201,7 +202,8 @@ static int nilfs_link(struct dentry *old_dentry, struct inode *dir, return err; } -static int nilfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int nilfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode *inode; struct nilfs_transaction_info ti; @@ -338,8 +340,9 @@ static int nilfs_rmdir(struct inode *dir, struct dentry *dentry) return err; } -static int nilfs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, +static int nilfs_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { struct inode *old_inode = d_inode(old_dentry); diff --git a/fs/nilfs2/nilfs.h b/fs/nilfs2/nilfs.h index f8450ee3fd06..c4a45a081ade 100644 --- a/fs/nilfs2/nilfs.h +++ b/fs/nilfs2/nilfs.h @@ -267,9 +267,11 @@ extern struct inode *nilfs_iget_for_gc(struct super_block *sb, extern void nilfs_update_inode(struct inode *, struct buffer_head *, int); extern void nilfs_truncate(struct inode *); extern void nilfs_evict_inode(struct inode *); -extern int nilfs_setattr(struct dentry *, struct iattr *); +extern int nilfs_setattr(struct user_namespace *, struct dentry *, + struct iattr *); extern void nilfs_write_failed(struct address_space *mapping, loff_t to); -int nilfs_permission(struct inode *inode, int mask); +int nilfs_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask); int nilfs_load_inode_block(struct inode *inode, struct buffer_head **pbh); extern int nilfs_inode_dirty(struct inode *); int nilfs_set_file_dirty(struct inode *inode, unsigned int nr_dirty); diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index 38f4cf1d4497..4435dbbc0b63 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -2848,6 +2848,7 @@ void ntfs_truncate_vfs(struct inode *vi) { /** * ntfs_setattr - called from notify_change() when an attribute is being changed + * @mnt_userns: user namespace of the mount the inode was found from * @dentry: dentry whose attributes to change * @attr: structure describing the attributes and the changes * @@ -2860,7 +2861,8 @@ void ntfs_truncate_vfs(struct inode *vi) { * * Called with ->i_mutex held. */ -int ntfs_setattr(struct dentry *dentry, struct iattr *attr) +int ntfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *vi = d_inode(dentry); int err; diff --git a/fs/ntfs/inode.h b/fs/ntfs/inode.h index 363e4e820673..6f78ee00f57f 100644 --- a/fs/ntfs/inode.h +++ b/fs/ntfs/inode.h @@ -289,7 +289,8 @@ extern int ntfs_show_options(struct seq_file *sf, struct dentry *root); extern int ntfs_truncate(struct inode *vi); extern void ntfs_truncate_vfs(struct inode *vi); -extern int ntfs_setattr(struct dentry *dentry, struct iattr *attr); +extern int ntfs_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr); extern int __ntfs_write_inode(struct inode *vi, int sync); diff --git a/fs/ocfs2/acl.c b/fs/ocfs2/acl.c index 990756cee4bd..5259badabb56 100644 --- a/fs/ocfs2/acl.c +++ b/fs/ocfs2/acl.c @@ -262,7 +262,8 @@ static int ocfs2_set_acl(handle_t *handle, return ret; } -int ocfs2_iop_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int ocfs2_iop_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { struct buffer_head *bh = NULL; int status, had_lock; diff --git a/fs/ocfs2/acl.h b/fs/ocfs2/acl.h index 127b13432146..4e86450917b2 100644 --- a/fs/ocfs2/acl.h +++ b/fs/ocfs2/acl.h @@ -19,7 +19,8 @@ struct ocfs2_acl_entry { }; struct posix_acl *ocfs2_iop_get_acl(struct inode *inode, int type); -int ocfs2_iop_set_acl(struct inode *inode, struct posix_acl *acl, int type); +int ocfs2_iop_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); extern int ocfs2_acl_chmod(struct inode *, struct buffer_head *); extern int ocfs2_init_acl(handle_t *, struct inode *, struct inode *, struct buffer_head *, struct buffer_head *, diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c index 9fa66cd1f622..b2870f1a31df 100644 --- a/fs/ocfs2/dlmfs/dlmfs.c +++ b/fs/ocfs2/dlmfs/dlmfs.c @@ -190,7 +190,8 @@ static int dlmfs_file_release(struct inode *inode, * We do ->setattr() just to override size changes. Our size is the size * of the LVB and nothing else. */ -static int dlmfs_file_setattr(struct dentry *dentry, struct iattr *attr) +static int dlmfs_file_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { int error; struct inode *inode = d_inode(dentry); @@ -395,7 +396,8 @@ static struct inode *dlmfs_get_inode(struct inode *parent, * File creation. Allocate an inode, and we're done.. */ /* SMP-safe */ -static int dlmfs_mkdir(struct inode * dir, +static int dlmfs_mkdir(struct user_namespace * mnt_userns, + struct inode * dir, struct dentry * dentry, umode_t mode) { @@ -443,7 +445,8 @@ bail: return status; } -static int dlmfs_create(struct inode *dir, +static int dlmfs_create(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index a070d4c9b6ed..e3039d973acd 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -1112,7 +1112,8 @@ out: return ret; } -int ocfs2_setattr(struct dentry *dentry, struct iattr *attr) +int ocfs2_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { int status = 0, size_change; int inode_locked = 0; @@ -1298,8 +1299,8 @@ bail: return status; } -int ocfs2_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +int ocfs2_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { struct inode *inode = d_inode(path->dentry); struct super_block *sb = path->dentry->d_sb; @@ -1330,7 +1331,8 @@ bail: return err; } -int ocfs2_permission(struct inode *inode, int mask) +int ocfs2_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask) { int ret, had_lock; struct ocfs2_lock_holder oh; diff --git a/fs/ocfs2/file.h b/fs/ocfs2/file.h index 4832cbceba5b..8536cec5f122 100644 --- a/fs/ocfs2/file.h +++ b/fs/ocfs2/file.h @@ -51,10 +51,13 @@ int ocfs2_extend_no_holes(struct inode *inode, struct buffer_head *di_bh, u64 new_i_size, u64 zero_to); int ocfs2_zero_extend(struct inode *inode, struct buffer_head *di_bh, loff_t zero_to); -int ocfs2_setattr(struct dentry *dentry, struct iattr *attr); -int ocfs2_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags); -int ocfs2_permission(struct inode *inode, int mask); +int ocfs2_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr); +int ocfs2_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); +int ocfs2_permission(struct user_namespace *mnt_userns, + struct inode *inode, + int mask); int ocfs2_should_update_atime(struct inode *inode, struct vfsmount *vfsmnt); diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c index 908b79e1082b..3abdd36da2e2 100644 --- a/fs/ocfs2/namei.c +++ b/fs/ocfs2/namei.c @@ -221,7 +221,8 @@ static void ocfs2_cleanup_add_entry_failure(struct ocfs2_super *osb, iput(inode); } -static int ocfs2_mknod(struct inode *dir, +static int ocfs2_mknod(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) @@ -645,7 +646,8 @@ static int ocfs2_mknod_locked(struct ocfs2_super *osb, return status; } -static int ocfs2_mkdir(struct inode *dir, +static int ocfs2_mkdir(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, umode_t mode) { @@ -653,14 +655,15 @@ static int ocfs2_mkdir(struct inode *dir, trace_ocfs2_mkdir(dir, dentry, dentry->d_name.len, dentry->d_name.name, OCFS2_I(dir)->ip_blkno, mode); - ret = ocfs2_mknod(dir, dentry, mode | S_IFDIR, 0); + ret = ocfs2_mknod(&init_user_ns, dir, dentry, mode | S_IFDIR, 0); if (ret) mlog_errno(ret); return ret; } -static int ocfs2_create(struct inode *dir, +static int ocfs2_create(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) @@ -669,7 +672,7 @@ static int ocfs2_create(struct inode *dir, trace_ocfs2_create(dir, dentry, dentry->d_name.len, dentry->d_name.name, (unsigned long long)OCFS2_I(dir)->ip_blkno, mode); - ret = ocfs2_mknod(dir, dentry, mode | S_IFREG, 0); + ret = ocfs2_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0); if (ret) mlog_errno(ret); @@ -1195,7 +1198,8 @@ static void ocfs2_double_unlock(struct inode *inode1, struct inode *inode2) ocfs2_inode_unlock(inode2, 1); } -static int ocfs2_rename(struct inode *old_dir, +static int ocfs2_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, @@ -1784,7 +1788,8 @@ bail: return status; } -static int ocfs2_symlink(struct inode *dir, +static int ocfs2_symlink(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, const char *symname) { diff --git a/fs/omfs/dir.c b/fs/omfs/dir.c index a0f45651f3b7..c219f91f44e9 100644 --- a/fs/omfs/dir.c +++ b/fs/omfs/dir.c @@ -279,13 +279,14 @@ out_free_inode: return err; } -static int omfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int omfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { return omfs_add_node(dir, dentry, mode | S_IFDIR); } -static int omfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int omfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { return omfs_add_node(dir, dentry, mode | S_IFREG); } @@ -369,9 +370,9 @@ static bool omfs_fill_chain(struct inode *dir, struct dir_context *ctx, return true; } -static int omfs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int omfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct inode *new_inode = d_inode(new_dentry); struct inode *old_inode = d_inode(old_dentry); diff --git a/fs/omfs/file.c b/fs/omfs/file.c index 729339cd7902..11e733aab25d 100644 --- a/fs/omfs/file.c +++ b/fs/omfs/file.c @@ -343,7 +343,8 @@ const struct file_operations omfs_file_operations = { .splice_read = generic_file_splice_read, }; -static int omfs_setattr(struct dentry *dentry, struct iattr *attr) +static int omfs_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); int error; diff --git a/fs/orangefs/acl.c b/fs/orangefs/acl.c index 628921952d16..18852b9ed82b 100644 --- a/fs/orangefs/acl.c +++ b/fs/orangefs/acl.c @@ -116,7 +116,8 @@ out: return error; } -int orangefs_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int orangefs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { int error; struct iattr iattr; diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c index b94032f77e61..5079cfafa8d7 100644 --- a/fs/orangefs/inode.c +++ b/fs/orangefs/inode.c @@ -871,7 +871,8 @@ out: /* * Change attributes of an object referenced by dentry. */ -int orangefs_setattr(struct dentry *dentry, struct iattr *iattr) +int orangefs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr) { int ret; gossip_debug(GOSSIP_INODE_DEBUG, "__orangefs_setattr: called on %pd\n", @@ -890,8 +891,8 @@ out: /* * Obtain attributes of an object given a dentry */ -int orangefs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +int orangefs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { int ret; struct inode *inode = path->dentry->d_inode; @@ -919,7 +920,8 @@ int orangefs_getattr(const struct path *path, struct kstat *stat, return ret; } -int orangefs_permission(struct inode *inode, int mask) +int orangefs_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { int ret; diff --git a/fs/orangefs/namei.c b/fs/orangefs/namei.c index 3e7cf3d0a494..600e8eee541f 100644 --- a/fs/orangefs/namei.c +++ b/fs/orangefs/namei.c @@ -15,7 +15,8 @@ /* * Get a newly allocated inode to go with a negative dentry. */ -static int orangefs_create(struct inode *dir, +static int orangefs_create(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, umode_t mode, bool exclusive) @@ -215,7 +216,8 @@ static int orangefs_unlink(struct inode *dir, struct dentry *dentry) return ret; } -static int orangefs_symlink(struct inode *dir, +static int orangefs_symlink(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, const char *symname) { @@ -303,7 +305,8 @@ out: return ret; } -static int orangefs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int orangefs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct orangefs_inode_s *parent = ORANGEFS_I(dir); struct orangefs_kernel_op_s *new_op; @@ -372,7 +375,8 @@ out: return ret; } -static int orangefs_rename(struct inode *old_dir, +static int orangefs_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, diff --git a/fs/orangefs/orangefs-kernel.h b/fs/orangefs/orangefs-kernel.h index e12aeb9623d6..0e6b97682e41 100644 --- a/fs/orangefs/orangefs-kernel.h +++ b/fs/orangefs/orangefs-kernel.h @@ -107,7 +107,9 @@ extern int orangefs_init_acl(struct inode *inode, struct inode *dir); extern const struct xattr_handler *orangefs_xattr_handlers[]; extern struct posix_acl *orangefs_get_acl(struct inode *inode, int type); -extern int orangefs_set_acl(struct inode *inode, struct posix_acl *acl, int type); +extern int orangefs_set_acl(struct user_namespace *mnt_userns, + struct inode *inode, struct posix_acl *acl, + int type); /* * orangefs data structures @@ -359,12 +361,13 @@ struct inode *orangefs_new_inode(struct super_block *sb, struct orangefs_object_kref *ref); int __orangefs_setattr(struct inode *, struct iattr *); -int orangefs_setattr(struct dentry *, struct iattr *); +int orangefs_setattr(struct user_namespace *, struct dentry *, struct iattr *); -int orangefs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags); +int orangefs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); -int orangefs_permission(struct inode *inode, int mask); +int orangefs_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask); int orangefs_update_time(struct inode *, struct timespec64 *, int); diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index 6904cc2ed7bb..8b3be7342a8c 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -650,19 +650,20 @@ out: return err; } -static int ovl_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int ovl_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { return ovl_create_object(dentry, (mode & 07777) | S_IFREG, 0, NULL); } -static int ovl_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int ovl_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { return ovl_create_object(dentry, (mode & 07777) | S_IFDIR, 0, NULL); } -static int ovl_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, - dev_t rdev) +static int ovl_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { /* Don't allow creation of "whiteout" on overlay */ if (S_ISCHR(mode) && rdev == WHITEOUT_DEV) @@ -671,8 +672,8 @@ static int ovl_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, return ovl_create_object(dentry, mode, rdev, NULL); } -static int ovl_symlink(struct inode *dir, struct dentry *dentry, - const char *link) +static int ovl_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *link) { return ovl_create_object(dentry, S_IFLNK, 0, link); } @@ -1069,9 +1070,9 @@ static int ovl_set_redirect(struct dentry *dentry, bool samedir) return err; } -static int ovl_rename(struct inode *olddir, struct dentry *old, - struct inode *newdir, struct dentry *new, - unsigned int flags) +static int ovl_rename(struct user_namespace *mnt_userns, struct inode *olddir, + struct dentry *old, struct inode *newdir, + struct dentry *new, unsigned int flags) { int err; struct dentry *old_upperdir; diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index 023fde466e3a..e78d45dfeaee 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -14,7 +14,8 @@ #include "overlayfs.h" -int ovl_setattr(struct dentry *dentry, struct iattr *attr) +int ovl_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { int err; bool full_copy_up = false; @@ -154,8 +155,8 @@ static int ovl_map_dev_ino(struct dentry *dentry, struct kstat *stat, int fsid) return 0; } -int ovl_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +int ovl_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { struct dentry *dentry = path->dentry; enum ovl_path_type type; @@ -277,7 +278,8 @@ out: return err; } -int ovl_permission(struct inode *inode, int mask) +int ovl_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { struct inode *upperinode = ovl_inode_upper(inode); struct inode *realinode = upperinode ?: ovl_inode_lower(inode); diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index 5e9eb46e741a..78b9d93a33c9 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -444,10 +444,12 @@ int ovl_set_nlink_lower(struct dentry *dentry); unsigned int ovl_get_nlink(struct ovl_fs *ofs, struct dentry *lowerdentry, struct dentry *upperdentry, unsigned int fallback); -int ovl_setattr(struct dentry *dentry, struct iattr *attr); -int ovl_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags); -int ovl_permission(struct inode *inode, int mask); +int ovl_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr); +int ovl_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); +int ovl_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask); int ovl_xattr_set(struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags); int ovl_xattr_get(struct dentry *dentry, struct inode *inode, const char *name, diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 8168ab2dda11..c04612b19054 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1023,7 +1023,7 @@ ovl_posix_acl_xattr_set(const struct xattr_handler *handler, !capable_wrt_inode_uidgid(&init_user_ns, inode, CAP_FSETID)) { struct iattr iattr = { .ia_valid = ATTR_KILL_SGID }; - err = ovl_setattr(dentry, &iattr); + err = ovl_setattr(&init_user_ns, dentry, &iattr); if (err) return err; } diff --git a/fs/posix_acl.c b/fs/posix_acl.c index d31b60f5d40d..f3309a7edb49 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -593,7 +593,7 @@ int ret = __posix_acl_chmod(&acl, GFP_KERNEL, mode); if (ret) return ret; - ret = inode->i_op->set_acl(inode, acl, ACL_TYPE_ACCESS); + ret = inode->i_op->set_acl(mnt_userns, inode, acl, ACL_TYPE_ACCESS); posix_acl_release(acl); return ret; } @@ -918,7 +918,7 @@ set_posix_acl(struct user_namespace *mnt_userns, struct inode *inode, if (ret) return ret; } - return inode->i_op->set_acl(inode, acl, type); + return inode->i_op->set_acl(mnt_userns, inode, acl, type); } EXPORT_SYMBOL(set_posix_acl); @@ -966,12 +966,13 @@ const struct xattr_handler posix_acl_default_xattr_handler = { }; EXPORT_SYMBOL_GPL(posix_acl_default_xattr_handler); -int simple_set_acl(struct inode *inode, struct posix_acl *acl, int type) +int simple_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { int error; if (type == ACL_TYPE_ACCESS) { - error = posix_acl_update_mode(&init_user_ns, inode, + error = posix_acl_update_mode(mnt_userns, inode, &inode->i_mode, &acl); if (error) return error; diff --git a/fs/proc/base.c b/fs/proc/base.c index d45aa68c1f17..56bf14316122 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -685,7 +685,8 @@ static int proc_fd_access_allowed(struct inode *inode) return allowed; } -int proc_setattr(struct dentry *dentry, struct iattr *attr) +int proc_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { int error; struct inode *inode = d_inode(dentry); @@ -726,7 +727,8 @@ static bool has_pid_permissions(struct proc_fs_info *fs_info, } -static int proc_pid_permission(struct inode *inode, int mask) +static int proc_pid_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { struct proc_fs_info *fs_info = proc_sb_info(inode->i_sb); struct task_struct *task; @@ -1927,8 +1929,8 @@ out_unlock: return NULL; } -int pid_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int query_flags) +int pid_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); struct proc_fs_info *fs_info = proc_sb_info(inode->i_sb); @@ -3473,7 +3475,8 @@ int proc_pid_readdir(struct file *file, struct dir_context *ctx) * This function makes sure that the node is always accessible for members of * same thread group. */ -static int proc_tid_comm_permission(struct inode *inode, int mask) +static int proc_tid_comm_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { bool is_same_tgroup; struct task_struct *task; @@ -3798,7 +3801,8 @@ static int proc_task_readdir(struct file *file, struct dir_context *ctx) return 0; } -static int proc_task_getattr(const struct path *path, struct kstat *stat, +static int proc_task_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); diff --git a/fs/proc/fd.c b/fs/proc/fd.c index d6e76461e135..07fc4fad2602 100644 --- a/fs/proc/fd.c +++ b/fs/proc/fd.c @@ -276,7 +276,8 @@ static struct dentry *proc_lookupfd(struct inode *dir, struct dentry *dentry, * /proc/pid/fd needs a special permission handler so that a process can still * access /proc/self/fd after it has executed a setuid(). */ -int proc_fd_permission(struct inode *inode, int mask) +int proc_fd_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { struct task_struct *p; int rv; diff --git a/fs/proc/fd.h b/fs/proc/fd.h index f371a602bf58..c5a921a06a0b 100644 --- a/fs/proc/fd.h +++ b/fs/proc/fd.h @@ -10,7 +10,8 @@ extern const struct inode_operations proc_fd_inode_operations; extern const struct file_operations proc_fdinfo_operations; extern const struct inode_operations proc_fdinfo_inode_operations; -extern int proc_fd_permission(struct inode *inode, int mask); +extern int proc_fd_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask); static inline unsigned int proc_fd(struct inode *inode) { diff --git a/fs/proc/generic.c b/fs/proc/generic.c index 0db96a761149..bc86aa87cc41 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -115,7 +115,8 @@ static bool pde_subdir_insert(struct proc_dir_entry *dir, return true; } -static int proc_notify_change(struct dentry *dentry, struct iattr *iattr) +static int proc_notify_change(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *iattr) { struct inode *inode = d_inode(dentry); struct proc_dir_entry *de = PDE(inode); @@ -133,7 +134,8 @@ static int proc_notify_change(struct dentry *dentry, struct iattr *iattr) return 0; } -static int proc_getattr(const struct path *path, struct kstat *stat, +static int proc_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); diff --git a/fs/proc/internal.h b/fs/proc/internal.h index f60b379dcdc7..03415f3fb3a8 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -162,8 +162,10 @@ extern int proc_pid_statm(struct seq_file *, struct pid_namespace *, * base.c */ extern const struct dentry_operations pid_dentry_operations; -extern int pid_getattr(const struct path *, struct kstat *, u32, unsigned int); -extern int proc_setattr(struct dentry *, struct iattr *); +extern int pid_getattr(struct user_namespace *, const struct path *, + struct kstat *, u32, unsigned int); +extern int proc_setattr(struct user_namespace *, struct dentry *, + struct iattr *); extern void proc_pid_evict_inode(struct proc_inode *); extern struct inode *proc_pid_make_inode(struct super_block *, struct task_struct *, umode_t); extern void pid_update_inode(struct task_struct *, struct inode *); diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c index 4aef49ccf571..15c2e55d2ed2 100644 --- a/fs/proc/proc_net.c +++ b/fs/proc/proc_net.c @@ -289,7 +289,8 @@ static struct dentry *proc_tgid_net_lookup(struct inode *dir, return de; } -static int proc_tgid_net_getattr(const struct path *path, struct kstat *stat, +static int proc_tgid_net_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 87c828348140..2daac06727d0 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -785,7 +785,8 @@ out: return 0; } -static int proc_sys_permission(struct inode *inode, int mask) +static int proc_sys_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask) { /* * sysctl entries that are not writeable, @@ -813,7 +814,8 @@ static int proc_sys_permission(struct inode *inode, int mask) return error; } -static int proc_sys_setattr(struct dentry *dentry, struct iattr *attr) +static int proc_sys_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); int error; @@ -830,7 +832,8 @@ static int proc_sys_setattr(struct dentry *dentry, struct iattr *attr) return 0; } -static int proc_sys_getattr(const struct path *path, struct kstat *stat, +static int proc_sys_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); diff --git a/fs/proc/root.c b/fs/proc/root.c index 244e4b6f15ef..c7e3b1350ef8 100644 --- a/fs/proc/root.c +++ b/fs/proc/root.c @@ -308,7 +308,8 @@ void __init proc_root_init(void) register_filesystem(&proc_fs_type); } -static int proc_root_getattr(const struct path *path, struct kstat *stat, +static int proc_root_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { generic_fillattr(&init_user_ns, d_inode(path->dentry), stat); diff --git a/fs/ramfs/file-nommu.c b/fs/ramfs/file-nommu.c index f0358fe410d3..ba3525ccc27e 100644 --- a/fs/ramfs/file-nommu.c +++ b/fs/ramfs/file-nommu.c @@ -22,7 +22,7 @@ #include #include "internal.h" -static int ramfs_nommu_setattr(struct dentry *, struct iattr *); +static int ramfs_nommu_setattr(struct user_namespace *, struct dentry *, struct iattr *); static unsigned long ramfs_nommu_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, @@ -158,7 +158,8 @@ static int ramfs_nommu_resize(struct inode *inode, loff_t newsize, loff_t size) * handle a change of attributes * - we're specifically interested in a change of size */ -static int ramfs_nommu_setattr(struct dentry *dentry, struct iattr *ia) +static int ramfs_nommu_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *ia) { struct inode *inode = d_inode(dentry); unsigned int old_ia_valid = ia->ia_valid; diff --git a/fs/ramfs/inode.c b/fs/ramfs/inode.c index 3fd4326f36b5..3c2658c8fde0 100644 --- a/fs/ramfs/inode.c +++ b/fs/ramfs/inode.c @@ -101,7 +101,8 @@ struct inode *ramfs_get_inode(struct super_block *sb, */ /* SMP-safe */ static int -ramfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +ramfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) { struct inode * inode = ramfs_get_inode(dir->i_sb, dir, mode, dev); int error = -ENOSPC; @@ -115,20 +116,23 @@ ramfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) return error; } -static int ramfs_mkdir(struct inode * dir, struct dentry * dentry, umode_t mode) +static int ramfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { - int retval = ramfs_mknod(dir, dentry, mode | S_IFDIR, 0); + int retval = ramfs_mknod(&init_user_ns, dir, dentry, mode | S_IFDIR, 0); if (!retval) inc_nlink(dir); return retval; } -static int ramfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) +static int ramfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { - return ramfs_mknod(dir, dentry, mode | S_IFREG, 0); + return ramfs_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0); } -static int ramfs_symlink(struct inode * dir, struct dentry *dentry, const char * symname) +static int ramfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { struct inode *inode; int error = -ENOSPC; diff --git a/fs/reiserfs/acl.h b/fs/reiserfs/acl.h index 0c1c847f992f..fd58618da360 100644 --- a/fs/reiserfs/acl.h +++ b/fs/reiserfs/acl.h @@ -49,7 +49,8 @@ static inline int reiserfs_acl_count(size_t size) #ifdef CONFIG_REISERFS_FS_POSIX_ACL struct posix_acl *reiserfs_get_acl(struct inode *inode, int type); -int reiserfs_set_acl(struct inode *inode, struct posix_acl *acl, int type); +int reiserfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); int reiserfs_acl_chmod(struct inode *inode); int reiserfs_inherit_default_acl(struct reiserfs_transaction_handle *th, struct inode *dir, struct dentry *dentry, diff --git a/fs/reiserfs/inode.c b/fs/reiserfs/inode.c index 944f2b487cf8..780bb90c1804 100644 --- a/fs/reiserfs/inode.c +++ b/fs/reiserfs/inode.c @@ -3282,7 +3282,8 @@ static ssize_t reiserfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) return ret; } -int reiserfs_setattr(struct dentry *dentry, struct iattr *attr) +int reiserfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); unsigned int ia_valid; diff --git a/fs/reiserfs/namei.c b/fs/reiserfs/namei.c index a67a7d371725..e6eb05e2b2f1 100644 --- a/fs/reiserfs/namei.c +++ b/fs/reiserfs/namei.c @@ -619,8 +619,8 @@ static int new_inode_init(struct inode *inode, struct inode *dir, umode_t mode) return dquot_initialize(inode); } -static int reiserfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int reiserfs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { int retval; struct inode *inode; @@ -698,8 +698,8 @@ out_failed: return retval; } -static int reiserfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, - dev_t rdev) +static int reiserfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { int retval; struct inode *inode; @@ -781,7 +781,8 @@ out_failed: return retval; } -static int reiserfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int reiserfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { int retval; struct inode *inode; @@ -1094,8 +1095,9 @@ out_unlink: return retval; } -static int reiserfs_symlink(struct inode *parent_dir, - struct dentry *dentry, const char *symname) +static int reiserfs_symlink(struct user_namespace *mnt_userns, + struct inode *parent_dir, struct dentry *dentry, + const char *symname) { int retval; struct inode *inode; @@ -1304,7 +1306,8 @@ static void set_ino_in_dir_entry(struct reiserfs_dir_entry *de, * one path. If it holds 2 or more, it can get into endless waiting in * get_empty_nodes or its clones */ -static int reiserfs_rename(struct inode *old_dir, struct dentry *old_dentry, +static int reiserfs_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { diff --git a/fs/reiserfs/reiserfs.h b/fs/reiserfs/reiserfs.h index f69871516167..0ca2ac62e534 100644 --- a/fs/reiserfs/reiserfs.h +++ b/fs/reiserfs/reiserfs.h @@ -3102,7 +3102,8 @@ static inline void reiserfs_update_sd(struct reiserfs_transaction_handle *th, } void sd_attrs_to_i_attrs(__u16 sd_attrs, struct inode *inode); -int reiserfs_setattr(struct dentry *dentry, struct iattr *attr); +int reiserfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr); int __reiserfs_write_begin(struct page *page, unsigned from, unsigned len); diff --git a/fs/reiserfs/xattr.c b/fs/reiserfs/xattr.c index ec440d1957a1..bd073836e141 100644 --- a/fs/reiserfs/xattr.c +++ b/fs/reiserfs/xattr.c @@ -66,14 +66,14 @@ static int xattr_create(struct inode *dir, struct dentry *dentry, int mode) { BUG_ON(!inode_is_locked(dir)); - return dir->i_op->create(dir, dentry, mode, true); + return dir->i_op->create(&init_user_ns, dir, dentry, mode, true); } #endif static int xattr_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) { BUG_ON(!inode_is_locked(dir)); - return dir->i_op->mkdir(dir, dentry, mode); + return dir->i_op->mkdir(&init_user_ns, dir, dentry, mode); } /* @@ -352,7 +352,7 @@ static int chown_one_xattr(struct dentry *dentry, void *data) * ATTR_MODE is set. */ attrs->ia_valid &= (ATTR_UID|ATTR_GID); - err = reiserfs_setattr(dentry, attrs); + err = reiserfs_setattr(&init_user_ns, dentry, attrs); attrs->ia_valid = ia_valid; return err; @@ -604,7 +604,7 @@ reiserfs_xattr_set_handle(struct reiserfs_transaction_handle *th, inode_lock_nested(d_inode(dentry), I_MUTEX_XATTR); inode_dio_wait(d_inode(dentry)); - err = reiserfs_setattr(dentry, &newattrs); + err = reiserfs_setattr(&init_user_ns, dentry, &newattrs); inode_unlock(d_inode(dentry)); } else update_ctime(inode); @@ -948,7 +948,8 @@ static int xattr_mount_check(struct super_block *s) return 0; } -int reiserfs_permission(struct inode *inode, int mask) +int reiserfs_permission(struct user_namespace *mnt_userns, struct inode *inode, + int mask) { /* * We don't do permission checks on the internal objects. diff --git a/fs/reiserfs/xattr.h b/fs/reiserfs/xattr.h index c764352447ba..9b3b06da568c 100644 --- a/fs/reiserfs/xattr.h +++ b/fs/reiserfs/xattr.h @@ -16,7 +16,8 @@ int reiserfs_xattr_init(struct super_block *sb, int mount_flags); int reiserfs_lookup_privroot(struct super_block *sb); int reiserfs_delete_xattrs(struct inode *inode); int reiserfs_chown_xattrs(struct inode *inode, struct iattr *attrs); -int reiserfs_permission(struct inode *inode, int mask); +int reiserfs_permission(struct user_namespace *mnt_userns, + struct inode *inode, int mask); #ifdef CONFIG_REISERFS_FS_XATTR #define has_xattr_dir(inode) (REISERFS_I(inode)->i_flags & i_has_xattr_dir) diff --git a/fs/reiserfs/xattr_acl.c b/fs/reiserfs/xattr_acl.c index 4bf976bc7bad..a9547144a099 100644 --- a/fs/reiserfs/xattr_acl.c +++ b/fs/reiserfs/xattr_acl.c @@ -18,7 +18,8 @@ static int __reiserfs_set_acl(struct reiserfs_transaction_handle *th, int -reiserfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) +reiserfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { int error, error2; struct reiserfs_transaction_handle th; diff --git a/fs/stat.c b/fs/stat.c index 2c471c2fd766..fbc171d038aa 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -75,6 +75,7 @@ EXPORT_SYMBOL(generic_fillattr); int vfs_getattr_nosec(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { + struct user_namespace *mnt_userns; struct inode *inode = d_backing_inode(path->dentry); memset(stat, 0, sizeof(*stat)); @@ -91,11 +92,12 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat, if (IS_DAX(inode)) stat->attributes |= STATX_ATTR_DAX; + mnt_userns = mnt_user_ns(path->mnt); if (inode->i_op->getattr) - return inode->i_op->getattr(path, stat, request_mask, - query_flags); + return inode->i_op->getattr(mnt_userns, path, stat, + request_mask, query_flags); - generic_fillattr(mnt_user_ns(path->mnt), inode, stat); + generic_fillattr(mnt_userns, inode, stat); return 0; } EXPORT_SYMBOL(vfs_getattr_nosec); diff --git a/fs/sysv/file.c b/fs/sysv/file.c index ca7e216b7b9e..90e00124ea07 100644 --- a/fs/sysv/file.c +++ b/fs/sysv/file.c @@ -29,7 +29,8 @@ const struct file_operations sysv_file_operations = { .splice_read = generic_file_splice_read, }; -static int sysv_setattr(struct dentry *dentry, struct iattr *attr) +static int sysv_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); int error; diff --git a/fs/sysv/itree.c b/fs/sysv/itree.c index 83cffab6955f..8b2e99b7bc9f 100644 --- a/fs/sysv/itree.c +++ b/fs/sysv/itree.c @@ -441,8 +441,8 @@ static unsigned sysv_nblocks(struct super_block *s, loff_t size) return blocks; } -int sysv_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +int sysv_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { struct super_block *s = path->dentry->d_sb; generic_fillattr(&init_user_ns, d_inode(path->dentry), stat); diff --git a/fs/sysv/namei.c b/fs/sysv/namei.c index ea2414b385ec..b2e6abc06a2d 100644 --- a/fs/sysv/namei.c +++ b/fs/sysv/namei.c @@ -41,7 +41,8 @@ static struct dentry *sysv_lookup(struct inode * dir, struct dentry * dentry, un return d_splice_alias(inode, dentry); } -static int sysv_mknod(struct inode * dir, struct dentry * dentry, umode_t mode, dev_t rdev) +static int sysv_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct inode * inode; int err; @@ -60,13 +61,14 @@ static int sysv_mknod(struct inode * dir, struct dentry * dentry, umode_t mode, return err; } -static int sysv_create(struct inode * dir, struct dentry * dentry, umode_t mode, bool excl) +static int sysv_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { - return sysv_mknod(dir, dentry, mode, 0); + return sysv_mknod(&init_user_ns, dir, dentry, mode, 0); } -static int sysv_symlink(struct inode * dir, struct dentry * dentry, - const char * symname) +static int sysv_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { int err = -ENAMETOOLONG; int l = strlen(symname)+1; @@ -108,7 +110,8 @@ static int sysv_link(struct dentry * old_dentry, struct inode * dir, return add_nondir(dentry, inode); } -static int sysv_mkdir(struct inode * dir, struct dentry *dentry, umode_t mode) +static int sysv_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode * inode; int err; @@ -186,9 +189,9 @@ static int sysv_rmdir(struct inode * dir, struct dentry * dentry) * Anybody can rename anything with this: the permission checks are left to the * higher-level routines. */ -static int sysv_rename(struct inode * old_dir, struct dentry * old_dentry, - struct inode * new_dir, struct dentry * new_dentry, - unsigned int flags) +static int sysv_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct inode * old_inode = d_inode(old_dentry); struct inode * new_inode = d_inode(new_dentry); diff --git a/fs/sysv/sysv.h b/fs/sysv/sysv.h index 1cff585526b1..99ddf033da4f 100644 --- a/fs/sysv/sysv.h +++ b/fs/sysv/sysv.h @@ -141,7 +141,8 @@ extern struct inode *sysv_iget(struct super_block *, unsigned int); extern int sysv_write_inode(struct inode *, struct writeback_control *wbc); extern int sysv_sync_inode(struct inode *); extern void sysv_set_inode(struct inode *, dev_t); -extern int sysv_getattr(const struct path *, struct kstat *, u32, unsigned int); +extern int sysv_getattr(struct user_namespace *, const struct path *, + struct kstat *, u32, unsigned int); extern int sysv_init_icache(void); extern void sysv_destroy_icache(void); diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index 0ee8c6dfb036..4b83cbded559 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -67,7 +67,9 @@ static char *get_dname(struct dentry *dentry) return name; } -static int tracefs_syscall_mkdir(struct inode *inode, struct dentry *dentry, umode_t mode) +static int tracefs_syscall_mkdir(struct user_namespace *mnt_userns, + struct inode *inode, struct dentry *dentry, + umode_t mode) { char *name; int ret; diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c index a8881ed61620..d9d8d7794eff 100644 --- a/fs/ubifs/dir.c +++ b/fs/ubifs/dir.c @@ -280,8 +280,8 @@ static int ubifs_prepare_create(struct inode *dir, struct dentry *dentry, return fscrypt_setup_filename(dir, &dentry->d_name, 0, nm); } -static int ubifs_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int ubifs_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct inode *inode; struct ubifs_info *c = dir->i_sb->s_fs_info; @@ -441,8 +441,8 @@ out_budg: return err; } -static int ubifs_tmpfile(struct inode *dir, struct dentry *dentry, - umode_t mode) +static int ubifs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { return do_tmpfile(dir, dentry, mode, NULL); } @@ -942,7 +942,8 @@ out_fname: return err; } -static int ubifs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int ubifs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode *inode; struct ubifs_inode *dir_ui = ubifs_inode(dir); @@ -1013,8 +1014,8 @@ out_budg: return err; } -static int ubifs_mknod(struct inode *dir, struct dentry *dentry, - umode_t mode, dev_t rdev) +static int ubifs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct inode *inode; struct ubifs_inode *ui; @@ -1102,8 +1103,8 @@ out_budg: return err; } -static int ubifs_symlink(struct inode *dir, struct dentry *dentry, - const char *symname) +static int ubifs_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { struct inode *inode; struct ubifs_inode *ui; @@ -1542,7 +1543,8 @@ static int ubifs_xrename(struct inode *old_dir, struct dentry *old_dentry, return err; } -static int ubifs_rename(struct inode *old_dir, struct dentry *old_dentry, +static int ubifs_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { @@ -1566,8 +1568,8 @@ static int ubifs_rename(struct inode *old_dir, struct dentry *old_dentry, return do_rename(old_dir, old_dentry, new_dir, new_dentry, flags); } -int ubifs_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +int ubifs_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) { loff_t size; struct inode *inode = d_inode(path->dentry); diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c index 76ef392b1e41..0e4b4be3aa26 100644 --- a/fs/ubifs/file.c +++ b/fs/ubifs/file.c @@ -1257,7 +1257,8 @@ static int do_setattr(struct ubifs_info *c, struct inode *inode, return err; } -int ubifs_setattr(struct dentry *dentry, struct iattr *attr) +int ubifs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { int err; struct inode *inode = d_inode(dentry); diff --git a/fs/ubifs/ubifs.h b/fs/ubifs/ubifs.h index fc2cdde3b549..7fdfdbda4b8a 100644 --- a/fs/ubifs/ubifs.h +++ b/fs/ubifs/ubifs.h @@ -1989,13 +1989,14 @@ int ubifs_calc_dark(const struct ubifs_info *c, int spc); /* file.c */ int ubifs_fsync(struct file *file, loff_t start, loff_t end, int datasync); -int ubifs_setattr(struct dentry *dentry, struct iattr *attr); +int ubifs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr); int ubifs_update_time(struct inode *inode, struct timespec64 *time, int flags); /* dir.c */ struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir, umode_t mode); -int ubifs_getattr(const struct path *path, struct kstat *stat, +int ubifs_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags); int ubifs_check_dir_empty(struct inode *dir); diff --git a/fs/udf/file.c b/fs/udf/file.c index 7c7d161315c2..2846dcd92197 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -253,7 +253,8 @@ const struct file_operations udf_file_operations = { .llseek = generic_file_llseek, }; -static int udf_setattr(struct dentry *dentry, struct iattr *attr) +static int udf_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); struct super_block *sb = inode->i_sb; diff --git a/fs/udf/namei.c b/fs/udf/namei.c index e169d8fe35b5..f146b3089f3d 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -604,8 +604,8 @@ static int udf_add_nondir(struct dentry *dentry, struct inode *inode) return 0; } -static int udf_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int udf_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { struct inode *inode = udf_new_inode(dir, mode); @@ -623,7 +623,8 @@ static int udf_create(struct inode *dir, struct dentry *dentry, umode_t mode, return udf_add_nondir(dentry, inode); } -static int udf_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) +static int udf_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode *inode = udf_new_inode(dir, mode); @@ -642,8 +643,8 @@ static int udf_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) return 0; } -static int udf_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, - dev_t rdev) +static int udf_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct inode *inode; @@ -658,7 +659,8 @@ static int udf_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, return udf_add_nondir(dentry, inode); } -static int udf_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int udf_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode *inode; struct udf_fileident_bh fibh; @@ -877,8 +879,8 @@ out: return retval; } -static int udf_symlink(struct inode *dir, struct dentry *dentry, - const char *symname) +static int udf_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { struct inode *inode = udf_new_inode(dir, S_IFLNK | 0777); struct pathComponent *pc; @@ -1065,9 +1067,9 @@ static int udf_link(struct dentry *old_dentry, struct inode *dir, /* Anybody can rename anything with this: the permission checks are left to the * higher-level routines. */ -static int udf_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int udf_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct inode *old_inode = d_inode(old_dentry); struct inode *new_inode = d_inode(new_dentry); diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index 54a44d1f023c..9b223421a3c5 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -152,8 +152,9 @@ out_unmap: return err; } -static int udf_symlink_getattr(const struct path *path, struct kstat *stat, - u32 request_mask, unsigned int flags) +static int udf_symlink_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags) { struct dentry *dentry = path->dentry; struct inode *inode = d_backing_inode(dentry); diff --git a/fs/ufs/inode.c b/fs/ufs/inode.c index 6b51f3b20143..debc282c1bb4 100644 --- a/fs/ufs/inode.c +++ b/fs/ufs/inode.c @@ -1211,7 +1211,8 @@ out: return err; } -int ufs_setattr(struct dentry *dentry, struct iattr *attr) +int ufs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); unsigned int ia_valid = attr->ia_valid; diff --git a/fs/ufs/namei.c b/fs/ufs/namei.c index 9ef40f100415..29d5a0e0c8f0 100644 --- a/fs/ufs/namei.c +++ b/fs/ufs/namei.c @@ -69,7 +69,8 @@ static struct dentry *ufs_lookup(struct inode * dir, struct dentry *dentry, unsi * If the create succeeds, we fill in the inode information * with d_instantiate(). */ -static int ufs_create (struct inode * dir, struct dentry * dentry, umode_t mode, +static int ufs_create (struct user_namespace * mnt_userns, + struct inode * dir, struct dentry * dentry, umode_t mode, bool excl) { struct inode *inode; @@ -85,7 +86,8 @@ static int ufs_create (struct inode * dir, struct dentry * dentry, umode_t mode, return ufs_add_nondir(dentry, inode); } -static int ufs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) +static int ufs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) { struct inode *inode; int err; @@ -104,8 +106,8 @@ static int ufs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev return err; } -static int ufs_symlink (struct inode * dir, struct dentry * dentry, - const char * symname) +static int ufs_symlink (struct user_namespace * mnt_userns, struct inode * dir, + struct dentry * dentry, const char * symname) { struct super_block * sb = dir->i_sb; int err; @@ -164,7 +166,8 @@ static int ufs_link (struct dentry * old_dentry, struct inode * dir, return error; } -static int ufs_mkdir(struct inode * dir, struct dentry * dentry, umode_t mode) +static int ufs_mkdir(struct user_namespace * mnt_userns, struct inode * dir, + struct dentry * dentry, umode_t mode) { struct inode * inode; int err; @@ -240,9 +243,9 @@ static int ufs_rmdir (struct inode * dir, struct dentry *dentry) return err; } -static int ufs_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int ufs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { struct inode *old_inode = d_inode(old_dentry); struct inode *new_inode = d_inode(new_dentry); diff --git a/fs/ufs/ufs.h b/fs/ufs/ufs.h index b49e0efdf3d7..550f7c5a3636 100644 --- a/fs/ufs/ufs.h +++ b/fs/ufs/ufs.h @@ -123,7 +123,8 @@ extern struct inode *ufs_iget(struct super_block *, unsigned long); extern int ufs_write_inode (struct inode *, struct writeback_control *); extern int ufs_sync_inode (struct inode *); extern void ufs_evict_inode (struct inode *); -extern int ufs_setattr(struct dentry *dentry, struct iattr *attr); +extern int ufs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr); /* namei.c */ extern const struct file_operations ufs_dir_operations; diff --git a/fs/vboxsf/dir.c b/fs/vboxsf/dir.c index 4d569f14a8d8..7aee0ec63ade 100644 --- a/fs/vboxsf/dir.c +++ b/fs/vboxsf/dir.c @@ -288,13 +288,15 @@ static int vboxsf_dir_create(struct inode *parent, struct dentry *dentry, return 0; } -static int vboxsf_dir_mkfile(struct inode *parent, struct dentry *dentry, +static int vboxsf_dir_mkfile(struct user_namespace *mnt_userns, + struct inode *parent, struct dentry *dentry, umode_t mode, bool excl) { return vboxsf_dir_create(parent, dentry, mode, 0); } -static int vboxsf_dir_mkdir(struct inode *parent, struct dentry *dentry, +static int vboxsf_dir_mkdir(struct user_namespace *mnt_userns, + struct inode *parent, struct dentry *dentry, umode_t mode) { return vboxsf_dir_create(parent, dentry, mode, 1); @@ -332,7 +334,8 @@ static int vboxsf_dir_unlink(struct inode *parent, struct dentry *dentry) return 0; } -static int vboxsf_dir_rename(struct inode *old_parent, +static int vboxsf_dir_rename(struct user_namespace *mnt_userns, + struct inode *old_parent, struct dentry *old_dentry, struct inode *new_parent, struct dentry *new_dentry, @@ -374,7 +377,8 @@ err_put_old_path: return err; } -static int vboxsf_dir_symlink(struct inode *parent, struct dentry *dentry, +static int vboxsf_dir_symlink(struct user_namespace *mnt_userns, + struct inode *parent, struct dentry *dentry, const char *symname) { struct vboxsf_inode *sf_parent_i = VBOXSF_I(parent); diff --git a/fs/vboxsf/utils.c b/fs/vboxsf/utils.c index d2cd1c99f48e..3b847e3fba24 100644 --- a/fs/vboxsf/utils.c +++ b/fs/vboxsf/utils.c @@ -212,8 +212,8 @@ int vboxsf_inode_revalidate(struct dentry *dentry) return 0; } -int vboxsf_getattr(const struct path *path, struct kstat *kstat, - u32 request_mask, unsigned int flags) +int vboxsf_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *kstat, u32 request_mask, unsigned int flags) { int err; struct dentry *dentry = path->dentry; @@ -237,7 +237,8 @@ int vboxsf_getattr(const struct path *path, struct kstat *kstat, return 0; } -int vboxsf_setattr(struct dentry *dentry, struct iattr *iattr) +int vboxsf_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr) { struct vboxsf_inode *sf_i = VBOXSF_I(d_inode(dentry)); struct vboxsf_sbi *sbi = VBOXSF_SBI(dentry->d_sb); diff --git a/fs/vboxsf/vfsmod.h b/fs/vboxsf/vfsmod.h index 18f95b00fc33..760524e78c88 100644 --- a/fs/vboxsf/vfsmod.h +++ b/fs/vboxsf/vfsmod.h @@ -90,9 +90,11 @@ int vboxsf_stat(struct vboxsf_sbi *sbi, struct shfl_string *path, struct shfl_fsobjinfo *info); int vboxsf_stat_dentry(struct dentry *dentry, struct shfl_fsobjinfo *info); int vboxsf_inode_revalidate(struct dentry *dentry); -int vboxsf_getattr(const struct path *path, struct kstat *kstat, - u32 request_mask, unsigned int query_flags); -int vboxsf_setattr(struct dentry *dentry, struct iattr *iattr); +int vboxsf_getattr(struct user_namespace *mnt_userns, const struct path *path, + struct kstat *kstat, u32 request_mask, + unsigned int query_flags); +int vboxsf_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *iattr); struct shfl_string *vboxsf_path_from_dentry(struct vboxsf_sbi *sbi, struct dentry *dentry); int vboxsf_nlscpy(struct vboxsf_sbi *sbi, char *name, size_t name_bound_len, diff --git a/fs/xfs/xfs_acl.c b/fs/xfs/xfs_acl.c index 368351298bd5..332e87153c6c 100644 --- a/fs/xfs/xfs_acl.c +++ b/fs/xfs/xfs_acl.c @@ -238,7 +238,8 @@ xfs_acl_set_mode( } int -xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) +xfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type) { umode_t mode; bool set_mode = false; diff --git a/fs/xfs/xfs_acl.h b/fs/xfs/xfs_acl.h index c042c0868016..7bdb3a4ed798 100644 --- a/fs/xfs/xfs_acl.h +++ b/fs/xfs/xfs_acl.h @@ -11,7 +11,8 @@ struct posix_acl; #ifdef CONFIG_XFS_POSIX_ACL extern struct posix_acl *xfs_get_acl(struct inode *inode, int type); -extern int xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type); +extern int xfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, + struct posix_acl *acl, int type); extern int __xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type); void xfs_forget_acl(struct inode *inode, const char *name); #else diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 26d22edef741..f5dfa128af64 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -220,29 +220,32 @@ xfs_generic_create( STATIC int xfs_vn_mknod( - struct inode *dir, - struct dentry *dentry, - umode_t mode, - dev_t rdev) + struct user_namespace *mnt_userns, + struct inode *dir, + struct dentry *dentry, + umode_t mode, + dev_t rdev) { return xfs_generic_create(dir, dentry, mode, rdev, false); } STATIC int xfs_vn_create( - struct inode *dir, - struct dentry *dentry, - umode_t mode, - bool flags) + struct user_namespace *mnt_userns, + struct inode *dir, + struct dentry *dentry, + umode_t mode, + bool flags) { return xfs_generic_create(dir, dentry, mode, 0, false); } STATIC int xfs_vn_mkdir( - struct inode *dir, - struct dentry *dentry, - umode_t mode) + struct user_namespace *mnt_userns, + struct inode *dir, + struct dentry *dentry, + umode_t mode) { return xfs_generic_create(dir, dentry, mode | S_IFDIR, 0, false); } @@ -361,9 +364,10 @@ xfs_vn_unlink( STATIC int xfs_vn_symlink( - struct inode *dir, - struct dentry *dentry, - const char *symname) + struct user_namespace *mnt_userns, + struct inode *dir, + struct dentry *dentry, + const char *symname) { struct inode *inode; struct xfs_inode *cip = NULL; @@ -403,11 +407,12 @@ xfs_vn_symlink( STATIC int xfs_vn_rename( - struct inode *odir, - struct dentry *odentry, - struct inode *ndir, - struct dentry *ndentry, - unsigned int flags) + struct user_namespace *mnt_userns, + struct inode *odir, + struct dentry *odentry, + struct inode *ndir, + struct dentry *ndentry, + unsigned int flags) { struct inode *new_inode = d_inode(ndentry); int omode = 0; @@ -529,6 +534,7 @@ xfs_stat_blksize( STATIC int xfs_vn_getattr( + struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 request_mask, @@ -1047,6 +1053,7 @@ xfs_vn_setattr_size( STATIC int xfs_vn_setattr( + struct user_namespace *mnt_userns, struct dentry *dentry, struct iattr *iattr) { @@ -1144,9 +1151,10 @@ xfs_vn_fiemap( STATIC int xfs_vn_tmpfile( - struct inode *dir, - struct dentry *dentry, - umode_t mode) + struct user_namespace *mnt_userns, + struct inode *dir, + struct dentry *dentry, + umode_t mode) { return xfs_generic_create(dir, dentry, mode, 0, true); } diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 8a1f69677784..76e45d66d4ce 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -480,7 +480,8 @@ unlock: return ret; } -static int zonefs_inode_setattr(struct dentry *dentry, struct iattr *iattr) +static int zonefs_inode_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *iattr) { struct inode *inode = d_inode(dentry); int ret; diff --git a/include/linux/fs.h b/include/linux/fs.h index f0601cca1930..7762d3d75230 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1930,22 +1930,28 @@ struct file_operations { struct inode_operations { struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int); const char * (*get_link) (struct dentry *, struct inode *, struct delayed_call *); - int (*permission) (struct inode *, int); + int (*permission) (struct user_namespace *, struct inode *, int); struct posix_acl * (*get_acl)(struct inode *, int); int (*readlink) (struct dentry *, char __user *,int); - int (*create) (struct inode *,struct dentry *, umode_t, bool); + int (*create) (struct user_namespace *, struct inode *,struct dentry *, + umode_t, bool); int (*link) (struct dentry *,struct inode *,struct dentry *); int (*unlink) (struct inode *,struct dentry *); - int (*symlink) (struct inode *,struct dentry *,const char *); - int (*mkdir) (struct inode *,struct dentry *,umode_t); + int (*symlink) (struct user_namespace *, struct inode *,struct dentry *, + const char *); + int (*mkdir) (struct user_namespace *, struct inode *,struct dentry *, + umode_t); int (*rmdir) (struct inode *,struct dentry *); - int (*mknod) (struct inode *,struct dentry *,umode_t,dev_t); - int (*rename) (struct inode *, struct dentry *, + int (*mknod) (struct user_namespace *, struct inode *,struct dentry *, + umode_t,dev_t); + int (*rename) (struct user_namespace *, struct inode *, struct dentry *, struct inode *, struct dentry *, unsigned int); - int (*setattr) (struct dentry *, struct iattr *); - int (*getattr) (const struct path *, struct kstat *, u32, unsigned int); + int (*setattr) (struct user_namespace *, struct dentry *, + struct iattr *); + int (*getattr) (struct user_namespace *, const struct path *, + struct kstat *, u32, unsigned int); ssize_t (*listxattr) (struct dentry *, char *, size_t); int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start, u64 len); @@ -1953,8 +1959,10 @@ struct inode_operations { int (*atomic_open)(struct inode *, struct dentry *, struct file *, unsigned open_flag, umode_t create_mode); - int (*tmpfile) (struct inode *, struct dentry *, umode_t); - int (*set_acl)(struct inode *, struct posix_acl *, int); + int (*tmpfile) (struct user_namespace *, struct inode *, + struct dentry *, umode_t); + int (*set_acl)(struct user_namespace *, struct inode *, + struct posix_acl *, int); } ____cacheline_aligned; static inline ssize_t call_read_iter(struct file *file, struct kiocb *kio, @@ -3227,15 +3235,18 @@ extern int dcache_dir_open(struct inode *, struct file *); extern int dcache_dir_close(struct inode *, struct file *); extern loff_t dcache_dir_lseek(struct file *, loff_t, int); extern int dcache_readdir(struct file *, struct dir_context *); -extern int simple_setattr(struct dentry *, struct iattr *); -extern int simple_getattr(const struct path *, struct kstat *, u32, unsigned int); +extern int simple_setattr(struct user_namespace *, struct dentry *, + struct iattr *); +extern int simple_getattr(struct user_namespace *, const struct path *, + struct kstat *, u32, unsigned int); extern int simple_statfs(struct dentry *, struct kstatfs *); extern int simple_open(struct inode *inode, struct file *file); extern int simple_link(struct dentry *, struct inode *, struct dentry *); extern int simple_unlink(struct inode *, struct dentry *); extern int simple_rmdir(struct inode *, struct dentry *); -extern int simple_rename(struct inode *, struct dentry *, - struct inode *, struct dentry *, unsigned int); +extern int simple_rename(struct user_namespace *, struct inode *, + struct dentry *, struct inode *, struct dentry *, + unsigned int); extern void simple_recursive_removal(struct dentry *, void (*callback)(struct dentry *)); extern int noop_fsync(struct file *, loff_t, loff_t, int); diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 681ed98e4ba8..8c6c4e32fc2f 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -379,10 +379,11 @@ extern int nfs_refresh_inode(struct inode *, struct nfs_fattr *); extern int nfs_post_op_update_inode(struct inode *inode, struct nfs_fattr *fattr); extern int nfs_post_op_update_inode_force_wcc(struct inode *inode, struct nfs_fattr *fattr); extern int nfs_post_op_update_inode_force_wcc_locked(struct inode *inode, struct nfs_fattr *fattr); -extern int nfs_getattr(const struct path *, struct kstat *, u32, unsigned int); +extern int nfs_getattr(struct user_namespace *, const struct path *, + struct kstat *, u32, unsigned int); extern void nfs_access_add_cache(struct inode *, struct nfs_access_entry *); extern void nfs_access_set_mask(struct nfs_access_entry *, u32); -extern int nfs_permission(struct inode *, int); +extern int nfs_permission(struct user_namespace *, struct inode *, int); extern int nfs_open(struct inode *, struct file *); extern int nfs_attribute_cache_expired(struct inode *inode); extern int nfs_revalidate_inode(struct nfs_server *server, struct inode *inode); @@ -390,7 +391,7 @@ extern int __nfs_revalidate_inode(struct nfs_server *, struct inode *); extern bool nfs_mapping_need_revalidate_inode(struct inode *inode); extern int nfs_revalidate_mapping(struct inode *inode, struct address_space *mapping); extern int nfs_revalidate_mapping_rcu(struct inode *inode); -extern int nfs_setattr(struct dentry *, struct iattr *); +extern int nfs_setattr(struct user_namespace *, struct dentry *, struct iattr *); extern void nfs_setattr_update_inode(struct inode *inode, struct iattr *attr, struct nfs_fattr *); extern void nfs_setsecurity(struct inode *inode, struct nfs_fattr *fattr, struct nfs4_label *label); diff --git a/include/linux/posix_acl.h b/include/linux/posix_acl.h index 6dcd8b8f6ab5..307094ebb88c 100644 --- a/include/linux/posix_acl.h +++ b/include/linux/posix_acl.h @@ -79,7 +79,8 @@ extern int posix_acl_create(struct inode *, umode_t *, struct posix_acl **, int posix_acl_update_mode(struct user_namespace *, struct inode *, umode_t *, struct posix_acl **); -extern int simple_set_acl(struct inode *, struct posix_acl *, int); +extern int simple_set_acl(struct user_namespace *, struct inode *, + struct posix_acl *, int); extern int simple_acl_create(struct inode *, struct inode *); struct posix_acl *get_cached_acl(struct inode *inode, int type); diff --git a/ipc/mqueue.c b/ipc/mqueue.c index fcd56e077733..8031464ed4ae 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -594,8 +594,8 @@ out_unlock: return error; } -static int mqueue_create(struct inode *dir, struct dentry *dentry, - umode_t mode, bool excl) +static int mqueue_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { return mqueue_create_attr(dentry, mode, NULL); } diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 05b1f51d15e0..1576ff331ee4 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -152,7 +152,8 @@ static void bpf_dentry_finalize(struct dentry *dentry, struct inode *inode, dir->i_ctime = dir->i_mtime; } -static int bpf_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int bpf_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode *inode; @@ -381,8 +382,8 @@ bpf_lookup(struct inode *dir, struct dentry *dentry, unsigned flags) return simple_lookup(dir, dentry, flags); } -static int bpf_symlink(struct inode *dir, struct dentry *dentry, - const char *target) +static int bpf_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *target) { char *link = kstrdup(target, GFP_USER | __GFP_NOWARN); struct inode *inode; diff --git a/mm/shmem.c b/mm/shmem.c index 339d5530d3a9..facdd1a9c524 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1060,7 +1060,8 @@ void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) } EXPORT_SYMBOL_GPL(shmem_truncate_range); -static int shmem_getattr(const struct path *path, struct kstat *stat, +static int shmem_getattr(struct user_namespace *mnt_userns, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = path->dentry->d_inode; @@ -1080,7 +1081,8 @@ static int shmem_getattr(const struct path *path, struct kstat *stat, return 0; } -static int shmem_setattr(struct dentry *dentry, struct iattr *attr) +static int shmem_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *attr) { struct inode *inode = d_inode(dentry); struct shmem_inode_info *info = SHMEM_I(inode); @@ -2917,7 +2919,8 @@ static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) * File creation. Allocate an inode, and we're done.. */ static int -shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) { struct inode *inode; int error = -ENOSPC; @@ -2946,7 +2949,8 @@ out_iput: } static int -shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) +shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode *inode; int error = -ENOSPC; @@ -2969,20 +2973,22 @@ out_iput: return error; } -static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +static int shmem_mkdir(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { int error; - if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) + if ((error = shmem_mknod(&init_user_ns, dir, dentry, + mode | S_IFDIR, 0))) return error; inc_nlink(dir); return 0; } -static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode, - bool excl) +static int shmem_create(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) { - return shmem_mknod(dir, dentry, mode | S_IFREG, 0); + return shmem_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0); } /* @@ -3062,7 +3068,8 @@ static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, stru return 0; } -static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry) +static int shmem_whiteout(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry) { struct dentry *whiteout; int error; @@ -3071,7 +3078,7 @@ static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry) if (!whiteout) return -ENOMEM; - error = shmem_mknod(old_dir, whiteout, + error = shmem_mknod(&init_user_ns, old_dir, whiteout, S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); dput(whiteout); if (error) @@ -3094,7 +3101,10 @@ static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry) * it exists so that the VFS layer correctly free's it when it * gets overwritten. */ -static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) +static int shmem_rename2(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) { struct inode *inode = d_inode(old_dentry); int they_are_dirs = S_ISDIR(inode->i_mode); @@ -3111,7 +3121,7 @@ static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struc if (flags & RENAME_WHITEOUT) { int error; - error = shmem_whiteout(old_dir, old_dentry); + error = shmem_whiteout(&init_user_ns, old_dir, old_dentry); if (error) return error; } @@ -3135,7 +3145,8 @@ static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struc return 0; } -static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) +static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, const char *symname) { int error; int len; diff --git a/net/socket.c b/net/socket.c index c76703c6f480..2826698ff97c 100644 --- a/net/socket.c +++ b/net/socket.c @@ -538,9 +538,10 @@ static ssize_t sockfs_listxattr(struct dentry *dentry, char *buffer, return used; } -static int sockfs_setattr(struct dentry *dentry, struct iattr *iattr) +static int sockfs_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *iattr) { - int err = simple_setattr(dentry, iattr); + int err = simple_setattr(&init_user_ns, dentry, iattr); if (!err && (iattr->ia_valid & ATTR_UID)) { struct socket *sock = SOCKET_I(d_inode(dentry)); diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index f95c6bfa8b8e..2ee3b3d29f10 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -1773,7 +1773,8 @@ fail2: return error; } -static int ns_mkdir_op(struct inode *dir, struct dentry *dentry, umode_t mode) +static int ns_mkdir_op(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct aa_ns *ns, *parent; /* TODO: improve permission check */ diff --git a/security/integrity/evm/evm_secfs.c b/security/integrity/evm/evm_secfs.c index cfc3075769bb..bbc85637e18b 100644 --- a/security/integrity/evm/evm_secfs.c +++ b/security/integrity/evm/evm_secfs.c @@ -219,7 +219,7 @@ static ssize_t evm_write_xattrs(struct file *file, const char __user *buf, newattrs.ia_valid = ATTR_MODE; inode = evm_xattrs->d_inode; inode_lock(inode); - err = simple_setattr(evm_xattrs, &newattrs); + err = simple_setattr(&init_user_ns, evm_xattrs, &newattrs); inode_unlock(inode); if (!err) err = count; -- cgit v1.2.3 From a2d2329e30e224ea68d575d2525b866df9805ea0 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:45 +0100 Subject: ima: handle idmapped mounts IMA does sometimes access the inode's i_uid and compares it against the rules' fowner. Enable IMA to handle idmapped mounts by passing down the mount's user namespace. We simply make use of the helpers we introduced before. If the initial user namespace is passed nothing changes so non-idmapped mounts will see identical behavior as before. Link: https://lore.kernel.org/r/20210121131959.646623-27-christian.brauner@ubuntu.com Signed-off-by: Christian Brauner --- fs/attr.c | 2 +- fs/namei.c | 4 +-- include/linux/ima.h | 18 +++++++++----- security/integrity/ima/ima.h | 19 ++++++++------ security/integrity/ima/ima_api.c | 10 +++++--- security/integrity/ima/ima_appraise.c | 15 ++++++----- security/integrity/ima/ima_asymmetric_keys.c | 3 ++- security/integrity/ima/ima_main.c | 37 ++++++++++++++++++---------- security/integrity/ima/ima_policy.c | 20 +++++++++------ security/integrity/ima/ima_queue_keys.c | 4 ++- 10 files changed, 83 insertions(+), 49 deletions(-) (limited to 'fs') diff --git a/fs/attr.c b/fs/attr.c index 41abd0d973d8..87ef39db1c34 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -401,7 +401,7 @@ int notify_change(struct user_namespace *mnt_userns, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); - ima_inode_post_setattr(dentry); + ima_inode_post_setattr(mnt_userns, dentry); evm_inode_post_setattr(dentry, ia_valid); } diff --git a/fs/namei.c b/fs/namei.c index d9ceb75ac169..dbf53b325ac9 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -3422,7 +3422,7 @@ struct dentry *vfs_tmpfile(struct user_namespace *mnt_userns, inode->i_state |= I_LINKABLE; spin_unlock(&inode->i_lock); } - ima_post_create_tmpfile(inode); + ima_post_create_tmpfile(mnt_userns, inode); return child; out_err: @@ -3750,7 +3750,7 @@ retry: error = vfs_create(mnt_userns, path.dentry->d_inode, dentry, mode, true); if (!error) - ima_post_path_mknod(dentry); + ima_post_path_mknod(mnt_userns, dentry); break; case S_IFCHR: case S_IFBLK: error = vfs_mknod(mnt_userns, path.dentry->d_inode, diff --git a/include/linux/ima.h b/include/linux/ima.h index 7db9cca1af34..5486312d8ee6 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -16,7 +16,8 @@ struct linux_binprm; #ifdef CONFIG_IMA extern int ima_bprm_check(struct linux_binprm *bprm); extern int ima_file_check(struct file *file, int mask); -extern void ima_post_create_tmpfile(struct inode *inode); +extern void ima_post_create_tmpfile(struct user_namespace *mnt_userns, + struct inode *inode); extern void ima_file_free(struct file *file); extern int ima_file_mmap(struct file *file, unsigned long prot); extern int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot); @@ -27,7 +28,8 @@ extern int ima_read_file(struct file *file, enum kernel_read_file_id id, bool contents); extern int ima_post_read_file(struct file *file, void *buf, loff_t size, enum kernel_read_file_id id); -extern void ima_post_path_mknod(struct dentry *dentry); +extern void ima_post_path_mknod(struct user_namespace *mnt_userns, + struct dentry *dentry); extern int ima_file_hash(struct file *file, char *buf, size_t buf_size); extern int ima_inode_hash(struct inode *inode, char *buf, size_t buf_size); extern void ima_kexec_cmdline(int kernel_fd, const void *buf, int size); @@ -68,7 +70,8 @@ static inline int ima_file_check(struct file *file, int mask) return 0; } -static inline void ima_post_create_tmpfile(struct inode *inode) +static inline void ima_post_create_tmpfile(struct user_namespace *mnt_userns, + struct inode *inode) { } @@ -112,7 +115,8 @@ static inline int ima_post_read_file(struct file *file, void *buf, loff_t size, return 0; } -static inline void ima_post_path_mknod(struct dentry *dentry) +static inline void ima_post_path_mknod(struct user_namespace *mnt_userns, + struct dentry *dentry) { return; } @@ -153,7 +157,8 @@ static inline void ima_post_key_create_or_update(struct key *keyring, #ifdef CONFIG_IMA_APPRAISE extern bool is_ima_appraise_enabled(void); -extern void ima_inode_post_setattr(struct dentry *dentry); +extern void ima_inode_post_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry); extern int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, size_t xattr_value_len); extern int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name); @@ -163,7 +168,8 @@ static inline bool is_ima_appraise_enabled(void) return 0; } -static inline void ima_inode_post_setattr(struct dentry *dentry) +static inline void ima_inode_post_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry) { return; } diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h index 8e8b1e3cb847..b87c9006d577 100644 --- a/security/integrity/ima/ima.h +++ b/security/integrity/ima/ima.h @@ -253,8 +253,9 @@ static inline void ima_process_queued_keys(void) {} #endif /* CONFIG_IMA_QUEUE_EARLY_BOOT_KEYS */ /* LIM API function definitions */ -int ima_get_action(struct inode *inode, const struct cred *cred, u32 secid, - int mask, enum ima_hooks func, int *pcr, +int ima_get_action(struct user_namespace *mnt_userns, struct inode *inode, + const struct cred *cred, u32 secid, int mask, + enum ima_hooks func, int *pcr, struct ima_template_desc **template_desc, const char *keyring); int ima_must_measure(struct inode *inode, int mask, enum ima_hooks func); @@ -266,7 +267,8 @@ void ima_store_measurement(struct integrity_iint_cache *iint, struct file *file, struct evm_ima_xattr_data *xattr_value, int xattr_len, const struct modsig *modsig, int pcr, struct ima_template_desc *template_desc); -void process_buffer_measurement(struct inode *inode, const void *buf, int size, +void process_buffer_measurement(struct user_namespace *mnt_userns, + struct inode *inode, const void *buf, int size, const char *eventname, enum ima_hooks func, int pcr, const char *keyring); void ima_audit_measurement(struct integrity_iint_cache *iint, @@ -281,8 +283,9 @@ void ima_free_template_entry(struct ima_template_entry *entry); const char *ima_d_path(const struct path *path, char **pathbuf, char *filename); /* IMA policy related functions */ -int ima_match_policy(struct inode *inode, const struct cred *cred, u32 secid, - enum ima_hooks func, int mask, int flags, int *pcr, +int ima_match_policy(struct user_namespace *mnt_userns, struct inode *inode, + const struct cred *cred, u32 secid, enum ima_hooks func, + int mask, int flags, int *pcr, struct ima_template_desc **template_desc, const char *keyring); void ima_init_policy(void); @@ -313,7 +316,8 @@ int ima_appraise_measurement(enum ima_hooks func, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, const struct modsig *modsig); -int ima_must_appraise(struct inode *inode, int mask, enum ima_hooks func); +int ima_must_appraise(struct user_namespace *mnt_userns, struct inode *inode, + int mask, enum ima_hooks func); void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file); enum integrity_status ima_get_cache_status(struct integrity_iint_cache *iint, enum ima_hooks func); @@ -340,7 +344,8 @@ static inline int ima_appraise_measurement(enum ima_hooks func, return INTEGRITY_UNKNOWN; } -static inline int ima_must_appraise(struct inode *inode, int mask, +static inline int ima_must_appraise(struct user_namespace *mnt_userns, + struct inode *inode, int mask, enum ima_hooks func) { return 0; diff --git a/security/integrity/ima/ima_api.c b/security/integrity/ima/ima_api.c index 4f39fb93f278..ed410efb3597 100644 --- a/security/integrity/ima/ima_api.c +++ b/security/integrity/ima/ima_api.c @@ -162,6 +162,7 @@ err_out: /** * ima_get_action - appraise & measure decision based on policy. + * @mnt_userns: user namespace of the mount the inode was found from * @inode: pointer to the inode associated with the object being validated * @cred: pointer to credentials structure to validate * @secid: secid of the task being validated @@ -183,8 +184,9 @@ err_out: * Returns IMA_MEASURE, IMA_APPRAISE mask. * */ -int ima_get_action(struct inode *inode, const struct cred *cred, u32 secid, - int mask, enum ima_hooks func, int *pcr, +int ima_get_action(struct user_namespace *mnt_userns, struct inode *inode, + const struct cred *cred, u32 secid, int mask, + enum ima_hooks func, int *pcr, struct ima_template_desc **template_desc, const char *keyring) { @@ -192,8 +194,8 @@ int ima_get_action(struct inode *inode, const struct cred *cred, u32 secid, flags &= ima_policy_flag; - return ima_match_policy(inode, cred, secid, func, mask, flags, pcr, - template_desc, keyring); + return ima_match_policy(mnt_userns, inode, cred, secid, func, mask, + flags, pcr, template_desc, keyring); } /* diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 70b643c41c6b..2e64b9f281cc 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -68,7 +68,8 @@ bool is_ima_appraise_enabled(void) * * Return 1 to appraise or hash */ -int ima_must_appraise(struct inode *inode, int mask, enum ima_hooks func) +int ima_must_appraise(struct user_namespace *mnt_userns, struct inode *inode, + int mask, enum ima_hooks func) { u32 secid; @@ -76,8 +77,8 @@ int ima_must_appraise(struct inode *inode, int mask, enum ima_hooks func) return 0; security_task_getsecid(current, &secid); - return ima_match_policy(inode, current_cred(), secid, func, mask, - IMA_APPRAISE | IMA_HASH, NULL, NULL, NULL); + return ima_match_policy(mnt_userns, inode, current_cred(), secid, func, + mask, IMA_APPRAISE | IMA_HASH, NULL, NULL, NULL); } static int ima_fix_xattr(struct dentry *dentry, @@ -350,7 +351,7 @@ int ima_check_blacklist(struct integrity_iint_cache *iint, rc = is_binary_blacklisted(digest, digestsize); if ((rc == -EPERM) && (iint->flags & IMA_MEASURE)) - process_buffer_measurement(NULL, digest, digestsize, + process_buffer_measurement(&init_user_ns, NULL, digest, digestsize, "blacklisted-hash", NONE, pcr, NULL); } @@ -501,6 +502,7 @@ void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) /** * ima_inode_post_setattr - reflect file metadata changes + * @mnt_userns: user namespace of the mount the inode was found from * @dentry: pointer to the affected dentry * * Changes to a dentry's metadata might result in needing to appraise. @@ -508,7 +510,8 @@ void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) * This function is called from notify_change(), which expects the caller * to lock the inode's i_mutex. */ -void ima_inode_post_setattr(struct dentry *dentry) +void ima_inode_post_setattr(struct user_namespace *mnt_userns, + struct dentry *dentry) { struct inode *inode = d_backing_inode(dentry); struct integrity_iint_cache *iint; @@ -518,7 +521,7 @@ void ima_inode_post_setattr(struct dentry *dentry) || !(inode->i_opflags & IOP_XATTR)) return; - action = ima_must_appraise(inode, MAY_ACCESS, POST_SETATTR); + action = ima_must_appraise(mnt_userns, inode, MAY_ACCESS, POST_SETATTR); if (!action) __vfs_removexattr(&init_user_ns, dentry, XATTR_NAME_IMA); iint = integrity_iint_find(inode); diff --git a/security/integrity/ima/ima_asymmetric_keys.c b/security/integrity/ima/ima_asymmetric_keys.c index 1c68c500c26f..c4ef69100789 100644 --- a/security/integrity/ima/ima_asymmetric_keys.c +++ b/security/integrity/ima/ima_asymmetric_keys.c @@ -10,6 +10,7 @@ */ #include +#include #include "ima.h" /** @@ -58,7 +59,7 @@ void ima_post_key_create_or_update(struct key *keyring, struct key *key, * if the IMA policy is configured to measure a key linked * to the given keyring. */ - process_buffer_measurement(NULL, payload, payload_len, + process_buffer_measurement(&init_user_ns, NULL, payload, payload_len, keyring->description, KEY_CHECK, 0, keyring->description); } diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index f87cb29329e9..cb1c56eccd6d 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -218,8 +218,8 @@ static int process_measurement(struct file *file, const struct cred *cred, * bitmask based on the appraise/audit/measurement policy. * Included is the appraise submask. */ - action = ima_get_action(inode, cred, secid, mask, func, &pcr, - &template_desc, NULL); + action = ima_get_action(file_mnt_user_ns(file), inode, cred, secid, + mask, func, &pcr, &template_desc, NULL); violation_check = ((func == FILE_CHECK || func == MMAP_CHECK) && (ima_policy_flag & IMA_MEASURE)); if (!action && !violation_check) @@ -431,8 +431,9 @@ int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot) security_task_getsecid(current, &secid); inode = file_inode(vma->vm_file); - action = ima_get_action(inode, current_cred(), secid, MAY_EXEC, - MMAP_CHECK, &pcr, &template, 0); + action = ima_get_action(file_mnt_user_ns(vma->vm_file), inode, + current_cred(), secid, MAY_EXEC, MMAP_CHECK, + &pcr, &template, 0); /* Is the mmap'ed file in policy? */ if (!(action & (IMA_MEASURE | IMA_APPRAISE_SUBMASK))) @@ -592,18 +593,21 @@ EXPORT_SYMBOL_GPL(ima_inode_hash); /** * ima_post_create_tmpfile - mark newly created tmpfile as new + * @mnt_userns: user namespace of the mount the inode was found from * @file : newly created tmpfile * * No measuring, appraising or auditing of newly created tmpfiles is needed. * Skip calling process_measurement(), but indicate which newly, created * tmpfiles are in policy. */ -void ima_post_create_tmpfile(struct inode *inode) +void ima_post_create_tmpfile(struct user_namespace *mnt_userns, + struct inode *inode) { struct integrity_iint_cache *iint; int must_appraise; - must_appraise = ima_must_appraise(inode, MAY_ACCESS, FILE_CHECK); + must_appraise = ima_must_appraise(mnt_userns, inode, MAY_ACCESS, + FILE_CHECK); if (!must_appraise) return; @@ -619,18 +623,21 @@ void ima_post_create_tmpfile(struct inode *inode) /** * ima_post_path_mknod - mark as a new inode + * @mnt_userns: user namespace of the mount the inode was found from * @dentry: newly created dentry * * Mark files created via the mknodat syscall as new, so that the * file data can be written later. */ -void ima_post_path_mknod(struct dentry *dentry) +void ima_post_path_mknod(struct user_namespace *mnt_userns, + struct dentry *dentry) { struct integrity_iint_cache *iint; struct inode *inode = dentry->d_inode; int must_appraise; - must_appraise = ima_must_appraise(inode, MAY_ACCESS, FILE_CHECK); + must_appraise = ima_must_appraise(mnt_userns, inode, MAY_ACCESS, + FILE_CHECK); if (!must_appraise) return; @@ -810,6 +817,7 @@ int ima_post_load_data(char *buf, loff_t size, /* * process_buffer_measurement - Measure the buffer to ima log. + * @mnt_userns: user namespace of the mount the inode was found from * @inode: inode associated with the object being measured (NULL for KEY_CHECK) * @buf: pointer to the buffer that needs to be added to the log. * @size: size of buffer(in bytes). @@ -820,7 +828,8 @@ int ima_post_load_data(char *buf, loff_t size, * * Based on policy, the buffer is measured into the ima log. */ -void process_buffer_measurement(struct inode *inode, const void *buf, int size, +void process_buffer_measurement(struct user_namespace *mnt_userns, + struct inode *inode, const void *buf, int size, const char *eventname, enum ima_hooks func, int pcr, const char *keyring) { @@ -860,8 +869,9 @@ void process_buffer_measurement(struct inode *inode, const void *buf, int size, */ if (func) { security_task_getsecid(current, &secid); - action = ima_get_action(inode, current_cred(), secid, 0, func, - &pcr, &template, keyring); + action = ima_get_action(mnt_userns, inode, current_cred(), + secid, 0, func, &pcr, &template, + keyring); if (!(action & IMA_MEASURE)) return; } @@ -919,8 +929,9 @@ void ima_kexec_cmdline(int kernel_fd, const void *buf, int size) if (!f.file) return; - process_buffer_measurement(file_inode(f.file), buf, size, - "kexec-cmdline", KEXEC_CMDLINE, 0, NULL); + process_buffer_measurement(file_mnt_user_ns(f.file), file_inode(f.file), + buf, size, "kexec-cmdline", KEXEC_CMDLINE, 0, + NULL); fdput(f); } diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c index 823a0c1379cb..e14426c24a19 100644 --- a/security/integrity/ima/ima_policy.c +++ b/security/integrity/ima/ima_policy.c @@ -488,6 +488,7 @@ static bool ima_match_keyring(struct ima_rule_entry *rule, /** * ima_match_rules - determine whether an inode matches the policy rule. * @rule: a pointer to a rule + * @mnt_userns: user namespace of the mount the inode was found from * @inode: a pointer to an inode * @cred: a pointer to a credentials structure for user validation * @secid: the secid of the task to be validated @@ -497,9 +498,10 @@ static bool ima_match_keyring(struct ima_rule_entry *rule, * * Returns true on rule match, false on failure. */ -static bool ima_match_rules(struct ima_rule_entry *rule, struct inode *inode, - const struct cred *cred, u32 secid, - enum ima_hooks func, int mask, +static bool ima_match_rules(struct ima_rule_entry *rule, + struct user_namespace *mnt_userns, + struct inode *inode, const struct cred *cred, + u32 secid, enum ima_hooks func, int mask, const char *keyring) { int i; @@ -539,7 +541,7 @@ static bool ima_match_rules(struct ima_rule_entry *rule, struct inode *inode, } if ((rule->flags & IMA_FOWNER) && - !rule->fowner_op(inode->i_uid, rule->fowner)) + !rule->fowner_op(i_uid_into_mnt(mnt_userns, inode), rule->fowner)) return false; for (i = 0; i < MAX_LSM_RULES; i++) { int rc = 0; @@ -602,6 +604,7 @@ static int get_subaction(struct ima_rule_entry *rule, enum ima_hooks func) /** * ima_match_policy - decision based on LSM and other conditions + * @mnt_userns: user namespace of the mount the inode was found from * @inode: pointer to an inode for which the policy decision is being made * @cred: pointer to a credentials structure for which the policy decision is * being made @@ -620,8 +623,9 @@ static int get_subaction(struct ima_rule_entry *rule, enum ima_hooks func) * list when walking it. Reads are many orders of magnitude more numerous * than writes so ima_match_policy() is classical RCU candidate. */ -int ima_match_policy(struct inode *inode, const struct cred *cred, u32 secid, - enum ima_hooks func, int mask, int flags, int *pcr, +int ima_match_policy(struct user_namespace *mnt_userns, struct inode *inode, + const struct cred *cred, u32 secid, enum ima_hooks func, + int mask, int flags, int *pcr, struct ima_template_desc **template_desc, const char *keyring) { @@ -637,8 +641,8 @@ int ima_match_policy(struct inode *inode, const struct cred *cred, u32 secid, if (!(entry->action & actmask)) continue; - if (!ima_match_rules(entry, inode, cred, secid, func, mask, - keyring)) + if (!ima_match_rules(entry, mnt_userns, inode, cred, secid, + func, mask, keyring)) continue; action |= entry->flags & IMA_ACTION_FLAGS; diff --git a/security/integrity/ima/ima_queue_keys.c b/security/integrity/ima/ima_queue_keys.c index 69a8626a35c0..ca3dea19da18 100644 --- a/security/integrity/ima/ima_queue_keys.c +++ b/security/integrity/ima/ima_queue_keys.c @@ -8,6 +8,7 @@ * Enables deferred processing of keys */ +#include #include #include #include "ima.h" @@ -158,7 +159,8 @@ void ima_process_queued_keys(void) list_for_each_entry_safe(entry, tmp, &ima_keys, list) { if (!timer_expired) - process_buffer_measurement(NULL, entry->payload, + process_buffer_measurement(&init_user_ns, NULL, + entry->payload, entry->payload_len, entry->keyring_name, KEY_CHECK, 0, -- cgit v1.2.3 From 0f16ff0f545d5ec90bbd548afd2983785b546780 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:46 +0100 Subject: ecryptfs: do not mount on top of idmapped mounts Prevent ecryptfs from being mounted on top of idmapped mounts. Stacking filesystems need to be prevented from being mounted on top of idmapped mounts until they have have been converted to handle this. Link: https://lore.kernel.org/r/20210121131959.646623-28-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: James Morris Signed-off-by: Christian Brauner --- fs/ecryptfs/main.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'fs') diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c index e63259fdef28..cdf40a54a35d 100644 --- a/fs/ecryptfs/main.c +++ b/fs/ecryptfs/main.c @@ -531,6 +531,12 @@ static struct dentry *ecryptfs_mount(struct file_system_type *fs_type, int flags goto out_free; } + if (mnt_user_ns(path.mnt) != &init_user_ns) { + rc = -EINVAL; + printk(KERN_ERR "Mounting on idmapped mounts currently disallowed\n"); + goto out_free; + } + if (check_ruid && !uid_eq(d_inode(path.dentry)->i_uid, current_uid())) { rc = -EPERM; printk(KERN_ERR "Mount of device (uid: %d) not owned by " -- cgit v1.2.3 From 029a52ada6a774cac457f3be34b79cfc85563ced Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:47 +0100 Subject: overlayfs: do not mount on top of idmapped mounts Prevent overlayfs from being mounted on top of idmapped mounts. Stacking filesystems need to be prevented from being mounted on top of idmapped mounts until they have have been converted to handle this. Link: https://lore.kernel.org/r/20210121131959.646623-29-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: James Morris Signed-off-by: Christian Brauner --- fs/overlayfs/super.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'fs') diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index c04612b19054..b702c576e783 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -858,6 +858,10 @@ static int ovl_mount_dir_noesc(const char *name, struct path *path) pr_err("filesystem on '%s' not supported\n", name); goto out_put; } + if (mnt_user_ns(path->mnt) != &init_user_ns) { + pr_err("idmapped layers are currently not supported\n"); + goto out_put; + } if (!d_is_dir(path->dentry)) { pr_err("'%s' not a directory\n", name); goto out_put; -- cgit v1.2.3 From 899bf2ceb3963b04fa412d7967480578311b1bc7 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Sat, 23 Jan 2021 12:00:02 +0100 Subject: nfs: do not export idmapped mounts Prevent nfs from exporting idmapped mounts until we have ported it to support exporting idmapped mounts. Link: https://lore.kernel.org/linux-api/20210123130958.3t6kvgkl634njpsm@wittgenstein Cc: Christoph Hellwig Cc: David Howells Cc: "J. Bruce Fields" Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Signed-off-by: Christian Brauner --- fs/nfsd/export.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 81e7bb12aca6..e456421f68b4 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -369,8 +369,9 @@ static struct svc_export *svc_export_update(struct svc_export *new, struct svc_export *old); static struct svc_export *svc_export_lookup(struct svc_export *); -static int check_export(struct inode *inode, int *flags, unsigned char *uuid) +static int check_export(struct path *path, int *flags, unsigned char *uuid) { + struct inode *inode = d_inode(path->dentry); /* * We currently export only dirs, regular files, and (for v4 @@ -394,6 +395,7 @@ static int check_export(struct inode *inode, int *flags, unsigned char *uuid) * or an FSID number (so NFSEXP_FSID or ->uuid is needed). * 2: We must be able to find an inode from a filehandle. * This means that s_export_op must be set. + * 3: We must not currently be on an idmapped mount. */ if (!(inode->i_sb->s_type->fs_flags & FS_REQUIRES_DEV) && !(*flags & NFSEXP_FSID) && @@ -408,6 +410,11 @@ static int check_export(struct inode *inode, int *flags, unsigned char *uuid) return -EINVAL; } + if (mnt_user_ns(path->mnt) != &init_user_ns) { + dprintk("exp_export: export of idmapped mounts not yet supported.\n"); + return -EINVAL; + } + if (inode->i_sb->s_export_op->flags & EXPORT_OP_NOSUBTREECHK && !(*flags & NFSEXP_NOSUBTREECHECK)) { dprintk("%s: %s does not support subtree checking!\n", @@ -636,8 +643,7 @@ static int svc_export_parse(struct cache_detail *cd, char *mesg, int mlen) goto out4; } - err = check_export(d_inode(exp.ex_path.dentry), &exp.ex_flags, - exp.ex_uuid); + err = check_export(&exp.ex_path, &exp.ex_flags, exp.ex_uuid); if (err) goto out4; /* -- cgit v1.2.3 From 68847c941700475575ced191108971d26e82ae29 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:48 +0100 Subject: namespace: take lock_mount_hash() directly when changing flags Changing mount options always ends up taking lock_mount_hash() but when MNT_READONLY is requested and neither the mount nor the superblock are MNT_READONLY we end up taking the lock, dropping it, and retaking it to change the other mount attributes. Instead, let's acquire the lock once when changing the mount attributes. This simplifies the locking in these codepath, makes them easier to reason about and avoids having to reacquire the lock right after dropping it. Link: https://lore.kernel.org/r/20210121131959.646623-30-christian.brauner@ubuntu.com Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/namespace.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/namespace.c b/fs/namespace.c index ecdc63ef881c..bdfb130f2c3c 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -464,7 +464,6 @@ static int mnt_make_readonly(struct mount *mnt) { int ret = 0; - lock_mount_hash(); mnt->mnt.mnt_flags |= MNT_WRITE_HOLD; /* * After storing MNT_WRITE_HOLD, we'll read the counters. This store @@ -498,18 +497,9 @@ static int mnt_make_readonly(struct mount *mnt) */ smp_wmb(); mnt->mnt.mnt_flags &= ~MNT_WRITE_HOLD; - unlock_mount_hash(); return ret; } -static int __mnt_unmake_readonly(struct mount *mnt) -{ - lock_mount_hash(); - mnt->mnt.mnt_flags &= ~MNT_READONLY; - unlock_mount_hash(); - return 0; -} - int sb_prepare_remount_readonly(struct super_block *sb) { struct mount *mnt; @@ -2523,7 +2513,8 @@ static int change_mount_ro_state(struct mount *mnt, unsigned int mnt_flags) if (readonly_request) return mnt_make_readonly(mnt); - return __mnt_unmake_readonly(mnt); + mnt->mnt.mnt_flags &= ~MNT_READONLY; + return 0; } /* @@ -2532,11 +2523,9 @@ static int change_mount_ro_state(struct mount *mnt, unsigned int mnt_flags) */ static void set_mount_attributes(struct mount *mnt, unsigned int mnt_flags) { - lock_mount_hash(); mnt_flags |= mnt->mnt.mnt_flags & ~MNT_USER_SETTABLE_MASK; mnt->mnt.mnt_flags = mnt_flags; touch_mnt_namespace(mnt->mnt_ns); - unlock_mount_hash(); } static void mnt_warn_timestamp_expiry(struct path *mountpoint, struct vfsmount *mnt) @@ -2582,9 +2571,11 @@ static int do_reconfigure_mnt(struct path *path, unsigned int mnt_flags) return -EPERM; down_write(&sb->s_umount); + lock_mount_hash(); ret = change_mount_ro_state(mnt, mnt_flags); if (ret == 0) set_mount_attributes(mnt, mnt_flags); + unlock_mount_hash(); up_write(&sb->s_umount); mnt_warn_timestamp_expiry(path, &mnt->mnt); @@ -2625,8 +2616,11 @@ static int do_remount(struct path *path, int ms_flags, int sb_flags, err = -EPERM; if (ns_capable(sb->s_user_ns, CAP_SYS_ADMIN)) { err = reconfigure_super(fc); - if (!err) + if (!err) { + lock_mount_hash(); set_mount_attributes(mnt, mnt_flags); + unlock_mount_hash(); + } } up_write(&sb->s_umount); } -- cgit v1.2.3 From d033cb6784c4f3a19a593cfe11f850e476197388 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:49 +0100 Subject: mount: make {lock,unlock}_mount_hash() static The lock_mount_hash() and unlock_mount_hash() helpers are never called outside a single file. Remove them from the header and make them static to reflect this fact. There's no need to have them callable from other places right now, as Christoph observed. Link: https://lore.kernel.org/r/20210121131959.646623-31-christian.brauner@ubuntu.com Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/mount.h | 10 ---------- fs/namespace.c | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/mount.h b/fs/mount.h index ce6c376e0bc2..0b6e08cf8afb 100644 --- a/fs/mount.h +++ b/fs/mount.h @@ -124,16 +124,6 @@ static inline void get_mnt_ns(struct mnt_namespace *ns) extern seqlock_t mount_lock; -static inline void lock_mount_hash(void) -{ - write_seqlock(&mount_lock); -} - -static inline void unlock_mount_hash(void) -{ - write_sequnlock(&mount_lock); -} - struct proc_mounts { struct mnt_namespace *ns; struct path root; diff --git a/fs/namespace.c b/fs/namespace.c index bdfb130f2c3c..5fceb2854395 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -87,6 +87,16 @@ EXPORT_SYMBOL_GPL(fs_kobj); */ __cacheline_aligned_in_smp DEFINE_SEQLOCK(mount_lock); +static inline void lock_mount_hash(void) +{ + write_seqlock(&mount_lock); +} + +static inline void unlock_mount_hash(void) +{ + write_sequnlock(&mount_lock); +} + static inline struct hlist_head *m_hash(struct vfsmount *mnt, struct dentry *dentry) { unsigned long tmp = ((unsigned long)mnt / L1_CACHE_BYTES); -- cgit v1.2.3 From e58ace1a0fa9d578f85f556b4b88c5fe9b871d08 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:50 +0100 Subject: namespace: only take read lock in do_reconfigure_mnt() do_reconfigure_mnt() used to take the down_write(&sb->s_umount) lock which seems unnecessary since we're not changing the superblock. We're only checking whether it is already read-only. Setting other mount attributes is protected by lock_mount_hash() afaict and not by s_umount. The history of down_write(&sb->s_umount) lock being taken when setting mount attributes dates back to the introduction of MNT_READONLY in [2]. This introduced the concept of having read-only mounts in contrast to just having a read-only superblock. When it got introduced it was simply plumbed into do_remount() which already took down_write(&sb->s_umount) because it was only used to actually change the superblock before [2]. Afaict, it would've already been possible back then to only use down_read(&sb->s_umount) for MS_BIND | MS_REMOUNT since actual mount options were protected by the vfsmount lock already. But that would've meant special casing the locking for MS_BIND | MS_REMOUNT in do_remount() which people might not have considered worth it. Then in [1] MS_BIND | MS_REMOUNT mount option changes were split out of do_remount() into do_reconfigure_mnt() but the down_write(&sb->s_umount) lock was simply copied over. Now that we have this be a separate helper only take the down_read(&sb->s_umount) lock since we're only interested in checking whether the super block is currently read-only and blocking any writers from changing it. Essentially, checking that the super block is read-only has the advantage that we can avoid having to go into the slowpath and through MNT_WRITE_HOLD and can simply set the read-only flag on the mount in set_mount_attributes(). [1]: commit 43f5e655eff7 ("vfs: Separate changing mount flags full remount") [2]: commit 2e4b7fcd9260 ("[PATCH] r/o bind mounts: honor mount writer counts at remount") Link: https://lore.kernel.org/r/20210121131959.646623-32-christian.brauner@ubuntu.com Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/namespace.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/namespace.c b/fs/namespace.c index 5fceb2854395..367f1c7cb6db 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -2527,10 +2527,6 @@ static int change_mount_ro_state(struct mount *mnt, unsigned int mnt_flags) return 0; } -/* - * Update the user-settable attributes on a mount. The caller must hold - * sb->s_umount for writing. - */ static void set_mount_attributes(struct mount *mnt, unsigned int mnt_flags) { mnt_flags |= mnt->mnt.mnt_flags & ~MNT_USER_SETTABLE_MASK; @@ -2580,13 +2576,17 @@ static int do_reconfigure_mnt(struct path *path, unsigned int mnt_flags) if (!can_change_locked_flags(mnt, mnt_flags)) return -EPERM; - down_write(&sb->s_umount); + /* + * We're only checking whether the superblock is read-only not + * changing it, so only take down_read(&sb->s_umount). + */ + down_read(&sb->s_umount); lock_mount_hash(); ret = change_mount_ro_state(mnt, mnt_flags); if (ret == 0) set_mount_attributes(mnt, mnt_flags); unlock_mount_hash(); - up_write(&sb->s_umount); + up_read(&sb->s_umount); mnt_warn_timestamp_expiry(path, &mnt->mnt); -- cgit v1.2.3 From fbdc2f6c40f6528fa0db79c73e844451234f3e26 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:51 +0100 Subject: fs: split out functions to hold writers When a mount is marked read-only we set MNT_WRITE_HOLD on it if there aren't currently any active writers. Split this logic out into simple helpers that we can use in follow-up patches. Link: https://lore.kernel.org/r/20210121131959.646623-33-christian.brauner@ubuntu.com Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/namespace.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/namespace.c b/fs/namespace.c index 367f1c7cb6db..774ae5f74716 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -470,10 +470,8 @@ void mnt_drop_write_file(struct file *file) } EXPORT_SYMBOL(mnt_drop_write_file); -static int mnt_make_readonly(struct mount *mnt) +static inline int mnt_hold_writers(struct mount *mnt) { - int ret = 0; - mnt->mnt.mnt_flags |= MNT_WRITE_HOLD; /* * After storing MNT_WRITE_HOLD, we'll read the counters. This store @@ -498,15 +496,29 @@ static int mnt_make_readonly(struct mount *mnt) * we're counting up here. */ if (mnt_get_writers(mnt) > 0) - ret = -EBUSY; - else - mnt->mnt.mnt_flags |= MNT_READONLY; + return -EBUSY; + + return 0; +} + +static inline void mnt_unhold_writers(struct mount *mnt) +{ /* * MNT_READONLY must become visible before ~MNT_WRITE_HOLD, so writers * that become unheld will see MNT_READONLY. */ smp_wmb(); mnt->mnt.mnt_flags &= ~MNT_WRITE_HOLD; +} + +static int mnt_make_readonly(struct mount *mnt) +{ + int ret; + + ret = mnt_hold_writers(mnt); + if (!ret) + mnt->mnt.mnt_flags |= MNT_READONLY; + mnt_unhold_writers(mnt); return ret; } -- cgit v1.2.3 From 5b490500f91b212d862560e7568aa5cdc141f9d0 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:52 +0100 Subject: fs: add attr_flags_to_mnt_flags helper Add a simple helper to translate uapi MOUNT_ATTR_* flags to MNT_* flags which we will use in follow-up patches too. Link: https://lore.kernel.org/r/20210121131959.646623-34-christian.brauner@ubuntu.com Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/namespace.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/namespace.c b/fs/namespace.c index 774ae5f74716..00ed0d6cb2ee 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -3465,6 +3465,28 @@ out_type: return ret; } +#define FSMOUNT_VALID_FLAGS \ + (MOUNT_ATTR_RDONLY | MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV | \ + MOUNT_ATTR_NOEXEC | MOUNT_ATTR__ATIME | MOUNT_ATTR_NODIRATIME) + +static unsigned int attr_flags_to_mnt_flags(u64 attr_flags) +{ + unsigned int mnt_flags = 0; + + if (attr_flags & MOUNT_ATTR_RDONLY) + mnt_flags |= MNT_READONLY; + if (attr_flags & MOUNT_ATTR_NOSUID) + mnt_flags |= MNT_NOSUID; + if (attr_flags & MOUNT_ATTR_NODEV) + mnt_flags |= MNT_NODEV; + if (attr_flags & MOUNT_ATTR_NOEXEC) + mnt_flags |= MNT_NOEXEC; + if (attr_flags & MOUNT_ATTR_NODIRATIME) + mnt_flags |= MNT_NODIRATIME; + + return mnt_flags; +} + /* * Create a kernel mount representation for a new, prepared superblock * (specified by fs_fd) and attach to an open_tree-like file descriptor. @@ -3487,24 +3509,10 @@ SYSCALL_DEFINE3(fsmount, int, fs_fd, unsigned int, flags, if ((flags & ~(FSMOUNT_CLOEXEC)) != 0) return -EINVAL; - if (attr_flags & ~(MOUNT_ATTR_RDONLY | - MOUNT_ATTR_NOSUID | - MOUNT_ATTR_NODEV | - MOUNT_ATTR_NOEXEC | - MOUNT_ATTR__ATIME | - MOUNT_ATTR_NODIRATIME)) + if (attr_flags & ~FSMOUNT_VALID_FLAGS) return -EINVAL; - if (attr_flags & MOUNT_ATTR_RDONLY) - mnt_flags |= MNT_READONLY; - if (attr_flags & MOUNT_ATTR_NOSUID) - mnt_flags |= MNT_NOSUID; - if (attr_flags & MOUNT_ATTR_NODEV) - mnt_flags |= MNT_NODEV; - if (attr_flags & MOUNT_ATTR_NOEXEC) - mnt_flags |= MNT_NOEXEC; - if (attr_flags & MOUNT_ATTR_NODIRATIME) - mnt_flags |= MNT_NODIRATIME; + mnt_flags = attr_flags_to_mnt_flags(attr_flags); switch (attr_flags & MOUNT_ATTR__ATIME) { case MOUNT_ATTR_STRICTATIME: -- cgit v1.2.3 From 2a1867219c7b27f928e2545782b86daaf9ad50bd Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:53 +0100 Subject: fs: add mount_setattr() This implements the missing mount_setattr() syscall. While the new mount api allows to change the properties of a superblock there is currently no way to change the properties of a mount or a mount tree using file descriptors which the new mount api is based on. In addition the old mount api has the restriction that mount options cannot be applied recursively. This hasn't changed since changing mount options on a per-mount basis was implemented in [1] and has been a frequent request not just for convenience but also for security reasons. The legacy mount syscall is unable to accommodate this behavior without introducing a whole new set of flags because MS_REC | MS_REMOUNT | MS_BIND | MS_RDONLY | MS_NOEXEC | [...] only apply the mount option to the topmost mount. Changing MS_REC to apply to the whole mount tree would mean introducing a significant uapi change and would likely cause significant regressions. The new mount_setattr() syscall allows to recursively clear and set mount options in one shot. Multiple calls to change mount options requesting the same changes are idempotent: int mount_setattr(int dfd, const char *path, unsigned flags, struct mount_attr *uattr, size_t usize); Flags to modify path resolution behavior are specified in the @flags argument. Currently, AT_EMPTY_PATH, AT_RECURSIVE, AT_SYMLINK_NOFOLLOW, and AT_NO_AUTOMOUNT are supported. If useful, additional lookup flags to restrict path resolution as introduced with openat2() might be supported in the future. The mount_setattr() syscall can be expected to grow over time and is designed with extensibility in mind. It follows the extensible syscall pattern we have used with other syscalls such as openat2(), clone3(), sched_{set,get}attr(), and others. The set of mount options is passed in the uapi struct mount_attr which currently has the following layout: struct mount_attr { __u64 attr_set; __u64 attr_clr; __u64 propagation; __u64 userns_fd; }; The @attr_set and @attr_clr members are used to clear and set mount options. This way a user can e.g. request that a set of flags is to be raised such as turning mounts readonly by raising MOUNT_ATTR_RDONLY in @attr_set while at the same time requesting that another set of flags is to be lowered such as removing noexec from a mount tree by specifying MOUNT_ATTR_NOEXEC in @attr_clr. Note, since the MOUNT_ATTR_ values are an enum starting from 0, not a bitmap, users wanting to transition to a different atime setting cannot simply specify the atime setting in @attr_set, but must also specify MOUNT_ATTR__ATIME in the @attr_clr field. So we ensure that MOUNT_ATTR__ATIME can't be partially set in @attr_clr and that @attr_set can't have any atime bits set if MOUNT_ATTR__ATIME isn't set in @attr_clr. The @propagation field lets callers specify the propagation type of a mount tree. Propagation is a single property that has four different settings and as such is not really a flag argument but an enum. Specifically, it would be unclear what setting and clearing propagation settings in combination would amount to. The legacy mount() syscall thus forbids the combination of multiple propagation settings too. The goal is to keep the semantics of mount propagation somewhat simple as they are overly complex as it is. The @userns_fd field lets user specify a user namespace whose idmapping becomes the idmapping of the mount. This is implemented and explained in detail in the next patch. [1]: commit 2e4b7fcd9260 ("[PATCH] r/o bind mounts: honor mount writer counts at remount") Link: https://lore.kernel.org/r/20210121131959.646623-35-christian.brauner@ubuntu.com Cc: David Howells Cc: Aleksa Sarai Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Cc: linux-api@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- arch/alpha/kernel/syscalls/syscall.tbl | 1 + arch/arm/tools/syscall.tbl | 1 + arch/arm64/include/asm/unistd.h | 2 +- arch/arm64/include/asm/unistd32.h | 2 + arch/ia64/kernel/syscalls/syscall.tbl | 1 + arch/m68k/kernel/syscalls/syscall.tbl | 1 + arch/microblaze/kernel/syscalls/syscall.tbl | 1 + arch/mips/kernel/syscalls/syscall_n32.tbl | 1 + arch/mips/kernel/syscalls/syscall_n64.tbl | 1 + arch/mips/kernel/syscalls/syscall_o32.tbl | 1 + arch/parisc/kernel/syscalls/syscall.tbl | 1 + arch/powerpc/kernel/syscalls/syscall.tbl | 1 + arch/s390/kernel/syscalls/syscall.tbl | 1 + arch/sh/kernel/syscalls/syscall.tbl | 1 + arch/sparc/kernel/syscalls/syscall.tbl | 1 + arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + arch/xtensa/kernel/syscalls/syscall.tbl | 1 + fs/namespace.c | 263 ++++++++++++++++++++++++++++ include/linux/syscalls.h | 4 + include/uapi/asm-generic/unistd.h | 4 +- include/uapi/linux/mount.h | 15 ++ tools/include/uapi/asm-generic/unistd.h | 4 +- 23 files changed, 307 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl index a6617067dbe6..02f0244e005c 100644 --- a/arch/alpha/kernel/syscalls/syscall.tbl +++ b/arch/alpha/kernel/syscalls/syscall.tbl @@ -481,3 +481,4 @@ 549 common faccessat2 sys_faccessat2 550 common process_madvise sys_process_madvise 551 common epoll_pwait2 sys_epoll_pwait2 +552 common mount_setattr sys_mount_setattr diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl index 20e1170e2e0a..dcc1191291a2 100644 --- a/arch/arm/tools/syscall.tbl +++ b/arch/arm/tools/syscall.tbl @@ -455,3 +455,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 86a9d7b3eabe..949788f5ba40 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -38,7 +38,7 @@ #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5) #define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800) -#define __NR_compat_syscalls 442 +#define __NR_compat_syscalls 443 #endif #define __ARCH_WANT_SYS_CLONE diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index cccfbbefbf95..3d874f624056 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -891,6 +891,8 @@ __SYSCALL(__NR_faccessat2, sys_faccessat2) __SYSCALL(__NR_process_madvise, sys_process_madvise) #define __NR_epoll_pwait2 441 __SYSCALL(__NR_epoll_pwait2, compat_sys_epoll_pwait2) +#define __NR_mount_setattr 442 +__SYSCALL(__NR_mount_setattr, sys_mount_setattr) /* * Please add new compat syscalls above this comment and update diff --git a/arch/ia64/kernel/syscalls/syscall.tbl b/arch/ia64/kernel/syscalls/syscall.tbl index bfc00f2bd437..d89231166e19 100644 --- a/arch/ia64/kernel/syscalls/syscall.tbl +++ b/arch/ia64/kernel/syscalls/syscall.tbl @@ -362,3 +362,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl index 7fe4e45c864c..72bde6707dd3 100644 --- a/arch/m68k/kernel/syscalls/syscall.tbl +++ b/arch/m68k/kernel/syscalls/syscall.tbl @@ -441,3 +441,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl index a522adf194ab..d603a5ec9338 100644 --- a/arch/microblaze/kernel/syscalls/syscall.tbl +++ b/arch/microblaze/kernel/syscalls/syscall.tbl @@ -447,3 +447,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl index 0f03ad223f33..8fd8c1790941 100644 --- a/arch/mips/kernel/syscalls/syscall_n32.tbl +++ b/arch/mips/kernel/syscalls/syscall_n32.tbl @@ -380,3 +380,4 @@ 439 n32 faccessat2 sys_faccessat2 440 n32 process_madvise sys_process_madvise 441 n32 epoll_pwait2 compat_sys_epoll_pwait2 +442 n32 mount_setattr sys_mount_setattr diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl index 91649690b52f..169f21438065 100644 --- a/arch/mips/kernel/syscalls/syscall_n64.tbl +++ b/arch/mips/kernel/syscalls/syscall_n64.tbl @@ -356,3 +356,4 @@ 439 n64 faccessat2 sys_faccessat2 440 n64 process_madvise sys_process_madvise 441 n64 epoll_pwait2 sys_epoll_pwait2 +442 n64 mount_setattr sys_mount_setattr diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl index 4bad0c40aed6..090d29ca80ff 100644 --- a/arch/mips/kernel/syscalls/syscall_o32.tbl +++ b/arch/mips/kernel/syscalls/syscall_o32.tbl @@ -429,3 +429,4 @@ 439 o32 faccessat2 sys_faccessat2 440 o32 process_madvise sys_process_madvise 441 o32 epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 o32 mount_setattr sys_mount_setattr diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl index 6bcc31966b44..271a92519683 100644 --- a/arch/parisc/kernel/syscalls/syscall.tbl +++ b/arch/parisc/kernel/syscalls/syscall.tbl @@ -439,3 +439,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl index f744eb5cba88..72e5aa67ab8a 100644 --- a/arch/powerpc/kernel/syscalls/syscall.tbl +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -531,3 +531,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl index d443423495e5..3abef2144dac 100644 --- a/arch/s390/kernel/syscalls/syscall.tbl +++ b/arch/s390/kernel/syscalls/syscall.tbl @@ -444,3 +444,4 @@ 439 common faccessat2 sys_faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr sys_mount_setattr diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl index 9df40ac0ebc0..d08eebad6b7f 100644 --- a/arch/sh/kernel/syscalls/syscall.tbl +++ b/arch/sh/kernel/syscalls/syscall.tbl @@ -444,3 +444,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl index 40d8c7cd8298..84403a99039c 100644 --- a/arch/sparc/kernel/syscalls/syscall.tbl +++ b/arch/sparc/kernel/syscalls/syscall.tbl @@ -487,3 +487,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 874aeacde2dd..a1c9f496fca6 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -446,3 +446,4 @@ 439 i386 faccessat2 sys_faccessat2 440 i386 process_madvise sys_process_madvise 441 i386 epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 i386 mount_setattr sys_mount_setattr diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 78672124d28b..7bf01cbe582f 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -363,6 +363,7 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr # # Due to a historical design error, certain syscalls are numbered differently diff --git a/arch/xtensa/kernel/syscalls/syscall.tbl b/arch/xtensa/kernel/syscalls/syscall.tbl index 46116a28eeed..365a9b849224 100644 --- a/arch/xtensa/kernel/syscalls/syscall.tbl +++ b/arch/xtensa/kernel/syscalls/syscall.tbl @@ -412,3 +412,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/fs/namespace.c b/fs/namespace.c index 00ed0d6cb2ee..726a51c8c46e 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -73,6 +73,14 @@ static DECLARE_RWSEM(namespace_sem); static HLIST_HEAD(unmounted); /* protected by namespace_sem */ static LIST_HEAD(ex_mountpoints); /* protected by namespace_sem */ +struct mount_kattr { + unsigned int attr_set; + unsigned int attr_clr; + unsigned int propagation; + unsigned int lookup_flags; + bool recurse; +}; + /* /sys/fs */ struct kobject *fs_kobj; EXPORT_SYMBOL_GPL(fs_kobj); @@ -3469,6 +3477,11 @@ out_type: (MOUNT_ATTR_RDONLY | MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV | \ MOUNT_ATTR_NOEXEC | MOUNT_ATTR__ATIME | MOUNT_ATTR_NODIRATIME) +#define MOUNT_SETATTR_VALID_FLAGS FSMOUNT_VALID_FLAGS + +#define MOUNT_SETATTR_PROPAGATION_FLAGS \ + (MS_UNBINDABLE | MS_PRIVATE | MS_SLAVE | MS_SHARED) + static unsigned int attr_flags_to_mnt_flags(u64 attr_flags) { unsigned int mnt_flags = 0; @@ -3820,6 +3833,256 @@ out0: return error; } +static unsigned int recalc_flags(struct mount_kattr *kattr, struct mount *mnt) +{ + unsigned int flags = mnt->mnt.mnt_flags; + + /* flags to clear */ + flags &= ~kattr->attr_clr; + /* flags to raise */ + flags |= kattr->attr_set; + + return flags; +} + +static struct mount *mount_setattr_prepare(struct mount_kattr *kattr, + struct mount *mnt, int *err) +{ + struct mount *m = mnt, *last = NULL; + + if (!is_mounted(&m->mnt)) { + *err = -EINVAL; + goto out; + } + + if (!(mnt_has_parent(m) ? check_mnt(m) : is_anon_ns(m->mnt_ns))) { + *err = -EINVAL; + goto out; + } + + do { + unsigned int flags; + + flags = recalc_flags(kattr, m); + if (!can_change_locked_flags(m, flags)) { + *err = -EPERM; + goto out; + } + + last = m; + + if ((kattr->attr_set & MNT_READONLY) && + !(m->mnt.mnt_flags & MNT_READONLY)) { + *err = mnt_hold_writers(m); + if (*err) + goto out; + } + } while (kattr->recurse && (m = next_mnt(m, mnt))); + +out: + return last; +} + +static void mount_setattr_commit(struct mount_kattr *kattr, + struct mount *mnt, struct mount *last, + int err) +{ + struct mount *m = mnt; + + do { + if (!err) { + unsigned int flags; + + flags = recalc_flags(kattr, m); + WRITE_ONCE(m->mnt.mnt_flags, flags); + } + + /* + * We either set MNT_READONLY above so make it visible + * before ~MNT_WRITE_HOLD or we failed to recursively + * apply mount options. + */ + if ((kattr->attr_set & MNT_READONLY) && + (m->mnt.mnt_flags & MNT_WRITE_HOLD)) + mnt_unhold_writers(m); + + if (!err && kattr->propagation) + change_mnt_propagation(m, kattr->propagation); + + /* + * On failure, only cleanup until we found the first mount + * we failed to handle. + */ + if (err && m == last) + break; + } while (kattr->recurse && (m = next_mnt(m, mnt))); + + if (!err) + touch_mnt_namespace(mnt->mnt_ns); +} + +static int do_mount_setattr(struct path *path, struct mount_kattr *kattr) +{ + struct mount *mnt = real_mount(path->mnt), *last = NULL; + int err = 0; + + if (path->dentry != mnt->mnt.mnt_root) + return -EINVAL; + + if (kattr->propagation) { + /* + * Only take namespace_lock() if we're actually changing + * propagation. + */ + namespace_lock(); + if (kattr->propagation == MS_SHARED) { + err = invent_group_ids(mnt, kattr->recurse); + if (err) { + namespace_unlock(); + return err; + } + } + } + + lock_mount_hash(); + + /* + * Get the mount tree in a shape where we can change mount + * properties without failure. + */ + last = mount_setattr_prepare(kattr, mnt, &err); + if (last) /* Commit all changes or revert to the old state. */ + mount_setattr_commit(kattr, mnt, last, err); + + unlock_mount_hash(); + + if (kattr->propagation) { + namespace_unlock(); + if (err) + cleanup_group_ids(mnt, NULL); + } + + return err; +} + +static int build_mount_kattr(const struct mount_attr *attr, + struct mount_kattr *kattr, unsigned int flags) +{ + unsigned int lookup_flags = LOOKUP_AUTOMOUNT | LOOKUP_FOLLOW; + + if (flags & AT_NO_AUTOMOUNT) + lookup_flags &= ~LOOKUP_AUTOMOUNT; + if (flags & AT_SYMLINK_NOFOLLOW) + lookup_flags &= ~LOOKUP_FOLLOW; + if (flags & AT_EMPTY_PATH) + lookup_flags |= LOOKUP_EMPTY; + + *kattr = (struct mount_kattr) { + .lookup_flags = lookup_flags, + .recurse = !!(flags & AT_RECURSIVE), + }; + + if (attr->propagation & ~MOUNT_SETATTR_PROPAGATION_FLAGS) + return -EINVAL; + if (hweight32(attr->propagation & MOUNT_SETATTR_PROPAGATION_FLAGS) > 1) + return -EINVAL; + kattr->propagation = attr->propagation; + + if ((attr->attr_set | attr->attr_clr) & ~MOUNT_SETATTR_VALID_FLAGS) + return -EINVAL; + + if (attr->userns_fd) + return -EINVAL; + + kattr->attr_set = attr_flags_to_mnt_flags(attr->attr_set); + kattr->attr_clr = attr_flags_to_mnt_flags(attr->attr_clr); + + /* + * Since the MOUNT_ATTR_ values are an enum, not a bitmap, + * users wanting to transition to a different atime setting cannot + * simply specify the atime setting in @attr_set, but must also + * specify MOUNT_ATTR__ATIME in the @attr_clr field. + * So ensure that MOUNT_ATTR__ATIME can't be partially set in + * @attr_clr and that @attr_set can't have any atime bits set if + * MOUNT_ATTR__ATIME isn't set in @attr_clr. + */ + if (attr->attr_clr & MOUNT_ATTR__ATIME) { + if ((attr->attr_clr & MOUNT_ATTR__ATIME) != MOUNT_ATTR__ATIME) + return -EINVAL; + + /* + * Clear all previous time settings as they are mutually + * exclusive. + */ + kattr->attr_clr |= MNT_RELATIME | MNT_NOATIME; + switch (attr->attr_set & MOUNT_ATTR__ATIME) { + case MOUNT_ATTR_RELATIME: + kattr->attr_set |= MNT_RELATIME; + break; + case MOUNT_ATTR_NOATIME: + kattr->attr_set |= MNT_NOATIME; + break; + case MOUNT_ATTR_STRICTATIME: + break; + default: + return -EINVAL; + } + } else { + if (attr->attr_set & MOUNT_ATTR__ATIME) + return -EINVAL; + } + + return 0; +} + +SYSCALL_DEFINE5(mount_setattr, int, dfd, const char __user *, path, + unsigned int, flags, struct mount_attr __user *, uattr, + size_t, usize) +{ + int err; + struct path target; + struct mount_attr attr; + struct mount_kattr kattr; + + BUILD_BUG_ON(sizeof(struct mount_attr) != MOUNT_ATTR_SIZE_VER0); + + if (flags & ~(AT_EMPTY_PATH | + AT_RECURSIVE | + AT_SYMLINK_NOFOLLOW | + AT_NO_AUTOMOUNT)) + return -EINVAL; + + if (unlikely(usize > PAGE_SIZE)) + return -E2BIG; + if (unlikely(usize < MOUNT_ATTR_SIZE_VER0)) + return -EINVAL; + + if (!may_mount()) + return -EPERM; + + err = copy_struct_from_user(&attr, sizeof(attr), uattr, usize); + if (err) + return err; + + /* Don't bother walking through the mounts if this is a nop. */ + if (attr.attr_set == 0 && + attr.attr_clr == 0 && + attr.propagation == 0) + return 0; + + err = build_mount_kattr(&attr, &kattr, flags); + if (err) + return err; + + err = user_path_at(dfd, path, kattr.lookup_flags, &target); + if (err) + return err; + + err = do_mount_setattr(&target, &kattr); + path_put(&target); + return err; +} + static void __init init_mount_tree(void) { struct vfsmount *mnt; diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 7688bc983de5..cd7b5c817ba2 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -68,6 +68,7 @@ union bpf_attr; struct io_uring_params; struct clone_args; struct open_how; +struct mount_attr; #include #include @@ -1028,6 +1029,9 @@ asmlinkage long sys_open_tree(int dfd, const char __user *path, unsigned flags); asmlinkage long sys_move_mount(int from_dfd, const char __user *from_path, int to_dfd, const char __user *to_path, unsigned int ms_flags); +asmlinkage long sys_mount_setattr(int dfd, const char __user *path, + unsigned int flags, + struct mount_attr __user *uattr, size_t usize); asmlinkage long sys_fsopen(const char __user *fs_name, unsigned int flags); asmlinkage long sys_fsconfig(int fs_fd, unsigned int cmd, const char __user *key, const void __user *value, int aux); diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 728752917785..ce58cff99b66 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -861,9 +861,11 @@ __SYSCALL(__NR_faccessat2, sys_faccessat2) __SYSCALL(__NR_process_madvise, sys_process_madvise) #define __NR_epoll_pwait2 441 __SC_COMP(__NR_epoll_pwait2, sys_epoll_pwait2, compat_sys_epoll_pwait2) +#define __NR_mount_setattr 442 +__SYSCALL(__NR_mount_setattr, sys_mount_setattr) #undef __NR_syscalls -#define __NR_syscalls 442 +#define __NR_syscalls 443 /* * 32 bit systems traditionally used different diff --git a/include/uapi/linux/mount.h b/include/uapi/linux/mount.h index dd8306ea336c..d3bcefdfa73d 100644 --- a/include/uapi/linux/mount.h +++ b/include/uapi/linux/mount.h @@ -1,6 +1,8 @@ #ifndef _UAPI_LINUX_MOUNT_H #define _UAPI_LINUX_MOUNT_H +#include + /* * These are the fs-independent mount-flags: up to 32 flags are supported * @@ -118,4 +120,17 @@ enum fsconfig_command { #define MOUNT_ATTR_STRICTATIME 0x00000020 /* - Always perform atime updates */ #define MOUNT_ATTR_NODIRATIME 0x00000080 /* Do not update directory access times */ +/* + * mount_setattr() + */ +struct mount_attr { + __u64 attr_set; + __u64 attr_clr; + __u64 propagation; + __u64 userns_fd; +}; + +/* List of all mount_attr versions. */ +#define MOUNT_ATTR_SIZE_VER0 32 /* sizeof first published struct */ + #endif /* _UAPI_LINUX_MOUNT_H */ diff --git a/tools/include/uapi/asm-generic/unistd.h b/tools/include/uapi/asm-generic/unistd.h index 728752917785..ce58cff99b66 100644 --- a/tools/include/uapi/asm-generic/unistd.h +++ b/tools/include/uapi/asm-generic/unistd.h @@ -861,9 +861,11 @@ __SYSCALL(__NR_faccessat2, sys_faccessat2) __SYSCALL(__NR_process_madvise, sys_process_madvise) #define __NR_epoll_pwait2 441 __SC_COMP(__NR_epoll_pwait2, sys_epoll_pwait2, compat_sys_epoll_pwait2) +#define __NR_mount_setattr 442 +__SYSCALL(__NR_mount_setattr, sys_mount_setattr) #undef __NR_syscalls -#define __NR_syscalls 442 +#define __NR_syscalls 443 /* * 32 bit systems traditionally used different -- cgit v1.2.3 From 9caccd41541a6f7d6279928d9f971f6642c361af Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:54 +0100 Subject: fs: introduce MOUNT_ATTR_IDMAP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a new mount bind mount property to allow idmapping mounts. The MOUNT_ATTR_IDMAP flag can be set via the new mount_setattr() syscall together with a file descriptor referring to a user namespace. The user namespace referenced by the namespace file descriptor will be attached to the bind mount. All interactions with the filesystem going through that mount will be mapped according to the mapping specified in the user namespace attached to it. Using user namespaces to mark mounts means we can reuse all the existing infrastructure in the kernel that already exists to handle idmappings and can also use this for permission checking to allow unprivileged user to create idmapped mounts in the future. Idmapping a mount is decoupled from the caller's user and mount namespace. This means idmapped mounts can be created in the initial user namespace which is an important use-case for systemd-homed, portable usb-sticks between systems, sharing data between the initial user namespace and unprivileged containers, and other use-cases that have been brought up. For example, assume a home directory where all files are owned by uid and gid 1000 and the home directory is brought to a new laptop where the user has id 12345. The system administrator can simply create a mount of this home directory with a mapping of 1000:12345:1 and other mappings to indicate the ids should be kept. (With this it is e.g. also possible to create idmapped mounts on the host with an identity mapping 1:1:100000 where the root user is not mapped. A user with root access that e.g. has been pivot rooted into such a mount on the host will be not be able to execute, read, write, or create files as root.) Given that mapping a mount is decoupled from the caller's user namespace a sufficiently privileged process such as a container manager can set up an idmapped mount for the container and the container can simply pivot root to it. There's no need for the container to do anything. The mount will appear correctly mapped independent of the user namespace the container uses. This means we don't need to mark a mount as idmappable. In order to create an idmapped mount the caller must currently be privileged in the user namespace of the superblock the mount belongs to. Once a mount has been idmapped we don't allow it to change its mapping. This keeps permission checking and life-cycle management simple. Users wanting to change the idmapped can always create a new detached mount with a different idmapping. Link: https://lore.kernel.org/r/20210121131959.646623-36-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Mauricio Vásquez Bernal Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/namespace.c | 122 ++++++++++++++++++++++++++++++++++++++++++--- fs/proc_namespace.c | 3 ++ include/linux/mount.h | 3 +- include/uapi/linux/mount.h | 1 + 4 files changed, 121 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/namespace.c b/fs/namespace.c index 726a51c8c46e..062fe984a697 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -79,6 +80,7 @@ struct mount_kattr { unsigned int propagation; unsigned int lookup_flags; bool recurse; + struct user_namespace *mnt_userns; }; /* /sys/fs */ @@ -3477,7 +3479,7 @@ out_type: (MOUNT_ATTR_RDONLY | MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV | \ MOUNT_ATTR_NOEXEC | MOUNT_ATTR__ATIME | MOUNT_ATTR_NODIRATIME) -#define MOUNT_SETATTR_VALID_FLAGS FSMOUNT_VALID_FLAGS +#define MOUNT_SETATTR_VALID_FLAGS (FSMOUNT_VALID_FLAGS | MOUNT_ATTR_IDMAP) #define MOUNT_SETATTR_PROPAGATION_FLAGS \ (MS_UNBINDABLE | MS_PRIVATE | MS_SLAVE | MS_SHARED) @@ -3845,6 +3847,36 @@ static unsigned int recalc_flags(struct mount_kattr *kattr, struct mount *mnt) return flags; } +static int can_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt) +{ + struct vfsmount *m = &mnt->mnt; + + if (!kattr->mnt_userns) + return 0; + + /* + * Once a mount has been idmapped we don't allow it to change its + * mapping. It makes things simpler and callers can just create + * another bind-mount they can idmap if they want to. + */ + if (mnt_user_ns(m) != &init_user_ns) + return -EPERM; + + /* The underlying filesystem doesn't support idmapped mounts yet. */ + if (!(m->mnt_sb->s_type->fs_flags & FS_ALLOW_IDMAP)) + return -EINVAL; + + /* We're not controlling the superblock. */ + if (!ns_capable(m->mnt_sb->s_user_ns, CAP_SYS_ADMIN)) + return -EPERM; + + /* Mount has already been visible in the filesystem hierarchy. */ + if (!is_anon_ns(mnt->mnt_ns)) + return -EINVAL; + + return 0; +} + static struct mount *mount_setattr_prepare(struct mount_kattr *kattr, struct mount *mnt, int *err) { @@ -3869,6 +3901,10 @@ static struct mount *mount_setattr_prepare(struct mount_kattr *kattr, goto out; } + *err = can_idmap_mount(kattr, m); + if (*err) + goto out; + last = m; if ((kattr->attr_set & MNT_READONLY) && @@ -3883,6 +3919,18 @@ out: return last; } +static void do_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt) +{ + struct user_namespace *mnt_userns; + + if (!kattr->mnt_userns) + return; + + mnt_userns = get_user_ns(kattr->mnt_userns); + /* Pairs with smp_load_acquire() in mnt_user_ns(). */ + smp_store_release(&mnt->mnt.mnt_userns, mnt_userns); +} + static void mount_setattr_commit(struct mount_kattr *kattr, struct mount *mnt, struct mount *last, int err) @@ -3893,6 +3941,7 @@ static void mount_setattr_commit(struct mount_kattr *kattr, if (!err) { unsigned int flags; + do_idmap_mount(kattr, m); flags = recalc_flags(kattr, m); WRITE_ONCE(m->mnt.mnt_flags, flags); } @@ -3965,7 +4014,62 @@ static int do_mount_setattr(struct path *path, struct mount_kattr *kattr) return err; } -static int build_mount_kattr(const struct mount_attr *attr, +static int build_mount_idmapped(const struct mount_attr *attr, size_t usize, + struct mount_kattr *kattr, unsigned int flags) +{ + int err = 0; + struct ns_common *ns; + struct user_namespace *mnt_userns; + struct file *file; + + if (!((attr->attr_set | attr->attr_clr) & MOUNT_ATTR_IDMAP)) + return 0; + + /* + * We currently do not support clearing an idmapped mount. If this ever + * is a use-case we can revisit this but for now let's keep it simple + * and not allow it. + */ + if (attr->attr_clr & MOUNT_ATTR_IDMAP) + return -EINVAL; + + if (attr->userns_fd > INT_MAX) + return -EINVAL; + + file = fget(attr->userns_fd); + if (!file) + return -EBADF; + + if (!proc_ns_file(file)) { + err = -EINVAL; + goto out_fput; + } + + ns = get_proc_ns(file_inode(file)); + if (ns->ops->type != CLONE_NEWUSER) { + err = -EINVAL; + goto out_fput; + } + + /* + * The init_user_ns is used to indicate that a vfsmount is not idmapped. + * This is simpler than just having to treat NULL as unmapped. Users + * wanting to idmap a mount to init_user_ns can just use a namespace + * with an identity mapping. + */ + mnt_userns = container_of(ns, struct user_namespace, ns); + if (mnt_userns == &init_user_ns) { + err = -EPERM; + goto out_fput; + } + kattr->mnt_userns = get_user_ns(mnt_userns); + +out_fput: + fput(file); + return err; +} + +static int build_mount_kattr(const struct mount_attr *attr, size_t usize, struct mount_kattr *kattr, unsigned int flags) { unsigned int lookup_flags = LOOKUP_AUTOMOUNT | LOOKUP_FOLLOW; @@ -3991,9 +4095,6 @@ static int build_mount_kattr(const struct mount_attr *attr, if ((attr->attr_set | attr->attr_clr) & ~MOUNT_SETATTR_VALID_FLAGS) return -EINVAL; - if (attr->userns_fd) - return -EINVAL; - kattr->attr_set = attr_flags_to_mnt_flags(attr->attr_set); kattr->attr_clr = attr_flags_to_mnt_flags(attr->attr_clr); @@ -4032,7 +4133,13 @@ static int build_mount_kattr(const struct mount_attr *attr, return -EINVAL; } - return 0; + return build_mount_idmapped(attr, usize, kattr, flags); +} + +static void finish_mount_kattr(struct mount_kattr *kattr) +{ + put_user_ns(kattr->mnt_userns); + kattr->mnt_userns = NULL; } SYSCALL_DEFINE5(mount_setattr, int, dfd, const char __user *, path, @@ -4070,7 +4177,7 @@ SYSCALL_DEFINE5(mount_setattr, int, dfd, const char __user *, path, attr.propagation == 0) return 0; - err = build_mount_kattr(&attr, &kattr, flags); + err = build_mount_kattr(&attr, usize, &kattr, flags); if (err) return err; @@ -4079,6 +4186,7 @@ SYSCALL_DEFINE5(mount_setattr, int, dfd, const char __user *, path, return err; err = do_mount_setattr(&target, &kattr); + finish_mount_kattr(&kattr); path_put(&target); return err; } diff --git a/fs/proc_namespace.c b/fs/proc_namespace.c index eafb75755fa3..392ef5162655 100644 --- a/fs/proc_namespace.c +++ b/fs/proc_namespace.c @@ -79,6 +79,9 @@ static void show_mnt_opts(struct seq_file *m, struct vfsmount *mnt) if (mnt->mnt_flags & fs_infop->flag) seq_puts(m, fs_infop->str); } + + if (mnt_user_ns(mnt) != &init_user_ns) + seq_puts(m, ",idmapped"); } static inline void mangle(struct seq_file *m, const char *s) diff --git a/include/linux/mount.h b/include/linux/mount.h index 52de25e08319..161f4419db6c 100644 --- a/include/linux/mount.h +++ b/include/linux/mount.h @@ -77,7 +77,8 @@ struct vfsmount { static inline struct user_namespace *mnt_user_ns(const struct vfsmount *mnt) { - return mnt->mnt_userns; + /* Pairs with smp_store_release() in do_idmap_mount(). */ + return smp_load_acquire(&mnt->mnt_userns); } struct file; /* forward dec */ diff --git a/include/uapi/linux/mount.h b/include/uapi/linux/mount.h index d3bcefdfa73d..e6524ead2b7b 100644 --- a/include/uapi/linux/mount.h +++ b/include/uapi/linux/mount.h @@ -119,6 +119,7 @@ enum fsconfig_command { #define MOUNT_ATTR_NOATIME 0x00000010 /* - Do not update access times. */ #define MOUNT_ATTR_STRICTATIME 0x00000020 /* - Always perform atime updates */ #define MOUNT_ATTR_NODIRATIME 0x00000080 /* Do not update directory access times */ +#define MOUNT_ATTR_IDMAP 0x00100000 /* Idmap mount to @userns_fd in struct mount_attr. */ /* * mount_setattr() -- cgit v1.2.3 From 4b78993681083361f19f66b4e9db298945647a3d Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:56 +0100 Subject: fat: handle idmapped mounts Let fat handle idmapped mounts. This allows to have the same fat mount appear in multiple locations with different id mappings. This allows to expose a vfat formatted USB stick to multiple user with different ids on the host or in user namespaces allowing for dac permissions: mount -o uid=1000,gid=1000 /dev/sdb /mnt u1001@f2-vm:/lower1$ ls -ln /mnt/ total 4 -rwxr-xr-x 1 1000 1000 4 Oct 28 03:44 aaa -rwxr-xr-x 1 1000 1000 0 Oct 28 01:09 bbb -rwxr-xr-x 1 1000 1000 0 Oct 28 01:10 ccc -rwxr-xr-x 1 1000 1000 0 Oct 28 03:46 ddd -rwxr-xr-x 1 1000 1000 0 Oct 28 04:01 eee mount-idmapped --map-mount b:1000:1001:1 u1001@f2-vm:/lower1$ ls -ln /lower1/ total 4 -rwxr-xr-x 1 1001 1001 4 Oct 28 03:44 aaa -rwxr-xr-x 1 1001 1001 0 Oct 28 01:09 bbb -rwxr-xr-x 1 1001 1001 0 Oct 28 01:10 ccc -rwxr-xr-x 1 1001 1001 0 Oct 28 03:46 ddd -rwxr-xr-x 1 1001 1001 0 Oct 28 04:01 eee u1001@f2-vm:/lower1$ touch /lower1/fff u1001@f2-vm:/lower1$ ls -ln /lower1/fff -rwxr-xr-x 1 1001 1001 0 Oct 28 04:03 /lower1/fff u1001@f2-vm:/lower1$ ls -ln /mnt/fff -rwxr-xr-x 1 1000 1000 0 Oct 28 04:03 /mnt/fff Link: https://lore.kernel.org/r/20210121131959.646623-38-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Signed-off-by: Christian Brauner --- fs/fat/file.c | 15 ++++++++------- fs/fat/namei_msdos.c | 2 +- fs/fat/namei_vfat.c | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/fat/file.c b/fs/fat/file.c index dd73d1b70c55..da7c56234189 100644 --- a/fs/fat/file.c +++ b/fs/fat/file.c @@ -398,7 +398,7 @@ int fat_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { struct inode *inode = d_inode(path->dentry); - generic_fillattr(&init_user_ns, inode, stat); + generic_fillattr(mnt_userns, inode, stat); stat->blksize = MSDOS_SB(inode->i_sb)->cluster_size; if (MSDOS_SB(inode->i_sb)->options.nfs == FAT_NFS_NOSTALE_RO) { @@ -447,12 +447,13 @@ static int fat_sanitize_mode(const struct msdos_sb_info *sbi, return 0; } -static int fat_allow_set_time(struct msdos_sb_info *sbi, struct inode *inode) +static int fat_allow_set_time(struct user_namespace *mnt_userns, + struct msdos_sb_info *sbi, struct inode *inode) { umode_t allow_utime = sbi->options.allow_utime; - if (!uid_eq(current_fsuid(), inode->i_uid)) { - if (in_group_p(inode->i_gid)) + if (!uid_eq(current_fsuid(), i_uid_into_mnt(mnt_userns, inode))) { + if (in_group_p(i_gid_into_mnt(mnt_userns, inode))) allow_utime >>= 3; if (allow_utime & MAY_WRITE) return 1; @@ -477,11 +478,11 @@ int fat_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, /* Check for setting the inode time. */ ia_valid = attr->ia_valid; if (ia_valid & TIMES_SET_FLAGS) { - if (fat_allow_set_time(sbi, inode)) + if (fat_allow_set_time(mnt_userns, sbi, inode)) attr->ia_valid &= ~TIMES_SET_FLAGS; } - error = setattr_prepare(&init_user_ns, dentry, attr); + error = setattr_prepare(mnt_userns, dentry, attr); attr->ia_valid = ia_valid; if (error) { if (sbi->options.quiet) @@ -551,7 +552,7 @@ int fat_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, fat_truncate_time(inode, &attr->ia_mtime, S_MTIME); attr->ia_valid &= ~(ATTR_ATIME|ATTR_CTIME|ATTR_MTIME); - setattr_copy(&init_user_ns, inode, attr); + setattr_copy(mnt_userns, inode, attr); mark_inode_dirty(inode); out: return error; diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c index a8f3375d9d10..efba301d68ae 100644 --- a/fs/fat/namei_msdos.c +++ b/fs/fat/namei_msdos.c @@ -667,7 +667,7 @@ static struct file_system_type msdos_fs_type = { .name = "msdos", .mount = msdos_mount, .kill_sb = kill_block_super, - .fs_flags = FS_REQUIRES_DEV, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, }; MODULE_ALIAS_FS("msdos"); diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c index 23936ecf79a5..5369d82e0bfb 100644 --- a/fs/fat/namei_vfat.c +++ b/fs/fat/namei_vfat.c @@ -1063,7 +1063,7 @@ static struct file_system_type vfat_fs_type = { .name = "vfat", .mount = vfat_mount, .kill_sb = kill_block_super, - .fs_flags = FS_REQUIRES_DEV, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, }; MODULE_ALIAS_FS("vfat"); -- cgit v1.2.3 From 14f3db5542e62bcf6fe088a09760ac52d55306c5 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Thu, 21 Jan 2021 14:19:57 +0100 Subject: ext4: support idmapped mounts Enable idmapped mounts for ext4. All dedicated helpers we need for this exist. So this basically just means we're passing down the user_namespace argument from the VFS methods to the relevant helpers. Let's create simple example where we idmap an ext4 filesystem: root@f2-vm:~# truncate -s 5G ext4.img root@f2-vm:~# mkfs.ext4 ./ext4.img mke2fs 1.45.5 (07-Jan-2020) Discarding device blocks: done Creating filesystem with 1310720 4k blocks and 327680 inodes Filesystem UUID: 3fd91794-c6ca-4b0f-9964-289a000919cf Superblock backups stored on blocks: 32768, 98304, 163840, 229376, 294912, 819200, 884736 Allocating group tables: done Writing inode tables: done Creating journal (16384 blocks): done Writing superblocks and filesystem accounting information: done root@f2-vm:~# losetup -f --show ./ext4.img /dev/loop0 root@f2-vm:~# mount /dev/loop0 /mnt root@f2-vm:~# ls -al /mnt/ total 24 drwxr-xr-x 3 root root 4096 Oct 28 13:34 . drwxr-xr-x 30 root root 4096 Oct 28 13:22 .. drwx------ 2 root root 16384 Oct 28 13:34 lost+found # Let's create an idmapped mount at /idmapped1 where we map uid and gid # 0 to uid and gid 1000 root@f2-vm:/# ./mount-idmapped --map-mount b:0:1000:1 /mnt/ /idmapped1/ root@f2-vm:/# ls -al /idmapped1/ total 24 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 28 13:34 . drwxr-xr-x 30 root root 4096 Oct 28 13:22 .. drwx------ 2 ubuntu ubuntu 16384 Oct 28 13:34 lost+found # Let's create an idmapped mount at /idmapped2 where we map uid and gid # 0 to uid and gid 2000 root@f2-vm:/# ./mount-idmapped --map-mount b:0:2000:1 /mnt/ /idmapped2/ root@f2-vm:/# ls -al /idmapped2/ total 24 drwxr-xr-x 3 2000 2000 4096 Oct 28 13:34 . drwxr-xr-x 31 root root 4096 Oct 28 13:39 .. drwx------ 2 2000 2000 16384 Oct 28 13:34 lost+found Let's create another example where we idmap the rootfs filesystem without a mapping for uid 0 and gid 0: # Create an idmapped mount of for a full POSIX range of rootfs under # /mnt but without a mapping for uid 0 to reduce attack surface root@f2-vm:/# ./mount-idmapped --map-mount b:1:1:65536 / /mnt/ # Since we don't have a mapping for uid and gid 0 all files owned by # uid and gid 0 should show up as uid and gid 65534: root@f2-vm:/# ls -al /mnt/ total 664 drwxr-xr-x 31 nobody nogroup 4096 Oct 28 13:39 . drwxr-xr-x 31 root root 4096 Oct 28 13:39 .. lrwxrwxrwx 1 nobody nogroup 7 Aug 25 07:44 bin -> usr/bin drwxr-xr-x 4 nobody nogroup 4096 Oct 28 13:17 boot drwxr-xr-x 2 nobody nogroup 4096 Aug 25 07:48 dev drwxr-xr-x 81 nobody nogroup 4096 Oct 28 04:00 etc drwxr-xr-x 4 nobody nogroup 4096 Oct 28 04:00 home lrwxrwxrwx 1 nobody nogroup 7 Aug 25 07:44 lib -> usr/lib lrwxrwxrwx 1 nobody nogroup 9 Aug 25 07:44 lib32 -> usr/lib32 lrwxrwxrwx 1 nobody nogroup 9 Aug 25 07:44 lib64 -> usr/lib64 lrwxrwxrwx 1 nobody nogroup 10 Aug 25 07:44 libx32 -> usr/libx32 drwx------ 2 nobody nogroup 16384 Aug 25 07:47 lost+found drwxr-xr-x 2 nobody nogroup 4096 Aug 25 07:44 media drwxr-xr-x 31 nobody nogroup 4096 Oct 28 13:39 mnt drwxr-xr-x 2 nobody nogroup 4096 Aug 25 07:44 opt drwxr-xr-x 2 nobody nogroup 4096 Apr 15 2020 proc drwx--x--x 6 nobody nogroup 4096 Oct 28 13:34 root drwxr-xr-x 2 nobody nogroup 4096 Aug 25 07:46 run lrwxrwxrwx 1 nobody nogroup 8 Aug 25 07:44 sbin -> usr/sbin drwxr-xr-x 2 nobody nogroup 4096 Aug 25 07:44 srv drwxr-xr-x 2 nobody nogroup 4096 Apr 15 2020 sys drwxrwxrwt 10 nobody nogroup 4096 Oct 28 13:19 tmp drwxr-xr-x 14 nobody nogroup 4096 Oct 20 13:00 usr drwxr-xr-x 12 nobody nogroup 4096 Aug 25 07:45 var # Since we do have a mapping for uid and gid 1000 all files owned by # uid and gid 1000 should simply show up as uid and gid 1000: root@f2-vm:/# ls -al /mnt/home/ubuntu/ total 40 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 28 00:43 . drwxr-xr-x 4 nobody nogroup 4096 Oct 28 04:00 .. -rw------- 1 ubuntu ubuntu 2936 Oct 28 12:26 .bash_history -rw-r--r-- 1 ubuntu ubuntu 220 Feb 25 2020 .bash_logout -rw-r--r-- 1 ubuntu ubuntu 3771 Feb 25 2020 .bashrc -rw-r--r-- 1 ubuntu ubuntu 807 Feb 25 2020 .profile -rw-r--r-- 1 ubuntu ubuntu 0 Oct 16 16:11 .sudo_as_admin_successful -rw------- 1 ubuntu ubuntu 1144 Oct 28 00:43 .viminfo Link: https://lore.kernel.org/r/20210121131959.646623-39-christian.brauner@ubuntu.com Cc: Christoph Hellwig Cc: David Howells Cc: Al Viro Cc: linux-ext4@vger.kernel.org Cc: linux-fsdevel@vger.kernel.org Signed-off-by: Christian Brauner --- fs/ext4/acl.c | 3 +-- fs/ext4/ext4.h | 13 +++++++------ fs/ext4/ialloc.c | 7 ++++--- fs/ext4/inode.c | 11 ++++++----- fs/ext4/ioctl.c | 19 +++++++++++-------- fs/ext4/namei.c | 30 ++++++++++++++++-------------- fs/ext4/super.c | 2 +- 7 files changed, 46 insertions(+), 39 deletions(-) (limited to 'fs') diff --git a/fs/ext4/acl.c b/fs/ext4/acl.c index 059434e0f36c..c5eaffccecc3 100644 --- a/fs/ext4/acl.c +++ b/fs/ext4/acl.c @@ -246,8 +246,7 @@ retry: ext4_fc_start_update(inode); if ((type == ACL_TYPE_ACCESS) && acl) { - error = posix_acl_update_mode(&init_user_ns, inode, &mode, - &acl); + error = posix_acl_update_mode(mnt_userns, inode, &mode, &acl); if (error) goto out_stop; if (mode != inode->i_mode) diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 3c750f5e8ebd..644fd69185d3 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -2755,18 +2755,19 @@ extern int ext4fs_dirhash(const struct inode *dir, const char *name, int len, /* ialloc.c */ extern int ext4_mark_inode_used(struct super_block *sb, int ino); -extern struct inode *__ext4_new_inode(handle_t *, struct inode *, umode_t, +extern struct inode *__ext4_new_inode(struct user_namespace *, handle_t *, + struct inode *, umode_t, const struct qstr *qstr, __u32 goal, uid_t *owner, __u32 i_flags, int handle_type, unsigned int line_no, int nblocks); -#define ext4_new_inode(handle, dir, mode, qstr, goal, owner, i_flags) \ - __ext4_new_inode((handle), (dir), (mode), (qstr), (goal), (owner), \ - i_flags, 0, 0, 0) -#define ext4_new_inode_start_handle(dir, mode, qstr, goal, owner, \ +#define ext4_new_inode(handle, dir, mode, qstr, goal, owner, i_flags) \ + __ext4_new_inode(&init_user_ns, (handle), (dir), (mode), (qstr), \ + (goal), (owner), i_flags, 0, 0, 0) +#define ext4_new_inode_start_handle(mnt_userns, dir, mode, qstr, goal, owner, \ type, nblocks) \ - __ext4_new_inode(NULL, (dir), (mode), (qstr), (goal), (owner), \ + __ext4_new_inode((mnt_userns), NULL, (dir), (mode), (qstr), (goal), (owner), \ 0, (type), __LINE__, (nblocks)) diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index 00c1ec6eee16..bf9028950a51 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -919,7 +919,8 @@ static int ext4_xattr_credits_for_new_inode(struct inode *dir, mode_t mode, * For other inodes, search forward from the parent directory's block * group to find a free inode. */ -struct inode *__ext4_new_inode(handle_t *handle, struct inode *dir, +struct inode *__ext4_new_inode(struct user_namespace *mnt_userns, + handle_t *handle, struct inode *dir, umode_t mode, const struct qstr *qstr, __u32 goal, uid_t *owner, __u32 i_flags, int handle_type, unsigned int line_no, @@ -969,10 +970,10 @@ struct inode *__ext4_new_inode(handle_t *handle, struct inode *dir, i_gid_write(inode, owner[1]); } else if (test_opt(sb, GRPID)) { inode->i_mode = mode; - inode->i_uid = current_fsuid(); + inode->i_uid = fsuid_into_mnt(mnt_userns); inode->i_gid = dir->i_gid; } else - inode_init_owner(&init_user_ns, inode, dir, mode); + inode_init_owner(mnt_userns, inode, dir, mode); if (ext4_has_feature_project(sb) && ext4_test_inode_flag(dir, EXT4_INODE_PROJINHERIT)) diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index ce45535336fa..8fbf85b3547e 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -20,6 +20,7 @@ */ #include +#include #include #include #include @@ -5338,7 +5339,7 @@ int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, ATTR_GID | ATTR_TIMES_SET)))) return -EPERM; - error = setattr_prepare(&init_user_ns, dentry, attr); + error = setattr_prepare(mnt_userns, dentry, attr); if (error) return error; @@ -5513,7 +5514,7 @@ out_mmap_sem: } if (!error) { - setattr_copy(&init_user_ns, inode, attr); + setattr_copy(mnt_userns, inode, attr); mark_inode_dirty(inode); } @@ -5525,7 +5526,7 @@ out_mmap_sem: ext4_orphan_del(NULL, inode); if (!error && (ia_valid & ATTR_MODE)) - rc = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); + rc = posix_acl_chmod(mnt_userns, inode, inode->i_mode); err_out: if (error) @@ -5572,7 +5573,7 @@ int ext4_getattr(struct user_namespace *mnt_userns, const struct path *path, STATX_ATTR_NODUMP | STATX_ATTR_VERITY); - generic_fillattr(&init_user_ns, inode, stat); + generic_fillattr(mnt_userns, inode, stat); return 0; } @@ -5583,7 +5584,7 @@ int ext4_file_getattr(struct user_namespace *mnt_userns, struct inode *inode = d_inode(path->dentry); u64 delalloc_blocks; - ext4_getattr(&init_user_ns, path, stat, request_mask, query_flags); + ext4_getattr(mnt_userns, path, stat, request_mask, query_flags); /* * If there is inline data in the inode, the inode will normally not diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index ab80e2493fdc..56ad9c4b6350 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -107,10 +107,12 @@ void ext4_reset_inode_seed(struct inode *inode) * important fields of the inodes. * * @sb: the super block of the filesystem + * @mnt_userns: user namespace of the mount the inode was found from * @inode: the inode to swap with EXT4_BOOT_LOADER_INO * */ static long swap_inode_boot_loader(struct super_block *sb, + struct user_namespace *mnt_userns, struct inode *inode) { handle_t *handle; @@ -139,7 +141,7 @@ static long swap_inode_boot_loader(struct super_block *sb, } if (IS_RDONLY(inode) || IS_APPEND(inode) || IS_IMMUTABLE(inode) || - !inode_owner_or_capable(&init_user_ns, inode) || + !inode_owner_or_capable(mnt_userns, inode) || !capable(CAP_SYS_ADMIN)) { err = -EPERM; goto journal_err_out; @@ -815,6 +817,7 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) struct inode *inode = file_inode(filp); struct super_block *sb = inode->i_sb; struct ext4_inode_info *ei = EXT4_I(inode); + struct user_namespace *mnt_userns = file_mnt_user_ns(filp); unsigned int flags; ext4_debug("cmd = %u, arg = %lu\n", cmd, arg); @@ -830,7 +833,7 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) case FS_IOC_SETFLAGS: { int err; - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(mnt_userns, inode)) return -EACCES; if (get_user(flags, (int __user *) arg)) @@ -872,7 +875,7 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) __u32 generation; int err; - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(mnt_userns, inode)) return -EPERM; if (ext4_has_metadata_csum(inode->i_sb)) { @@ -1011,7 +1014,7 @@ mext_out: case EXT4_IOC_MIGRATE: { int err; - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(mnt_userns, inode)) return -EACCES; err = mnt_want_write_file(filp); @@ -1033,7 +1036,7 @@ mext_out: case EXT4_IOC_ALLOC_DA_BLKS: { int err; - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(mnt_userns, inode)) return -EACCES; err = mnt_want_write_file(filp); @@ -1052,7 +1055,7 @@ mext_out: err = mnt_want_write_file(filp); if (err) return err; - err = swap_inode_boot_loader(sb, inode); + err = swap_inode_boot_loader(sb, mnt_userns, inode); mnt_drop_write_file(filp); return err; } @@ -1218,7 +1221,7 @@ resizefs_out: case EXT4_IOC_CLEAR_ES_CACHE: { - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(mnt_userns, inode)) return -EACCES; ext4_clear_inode_es(inode); return 0; @@ -1264,7 +1267,7 @@ resizefs_out: return -EFAULT; /* Make sure caller has proper permission */ - if (!inode_owner_or_capable(&init_user_ns, inode)) + if (!inode_owner_or_capable(mnt_userns, inode)) return -EACCES; if (fa.fsx_xflags & ~EXT4_SUPPORTED_FS_XFLAGS) diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 13dff80aedcb..877c602ae063 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -2610,8 +2610,8 @@ static int ext4_create(struct user_namespace *mnt_userns, struct inode *dir, credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) + EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3); retry: - inode = ext4_new_inode_start_handle(dir, mode, &dentry->d_name, 0, - NULL, EXT4_HT_DIR, credits); + inode = ext4_new_inode_start_handle(mnt_userns, dir, mode, &dentry->d_name, + 0, NULL, EXT4_HT_DIR, credits); handle = ext4_journal_current_handle(); err = PTR_ERR(inode); if (!IS_ERR(inode)) { @@ -2645,8 +2645,8 @@ static int ext4_mknod(struct user_namespace *mnt_userns, struct inode *dir, credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) + EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3); retry: - inode = ext4_new_inode_start_handle(dir, mode, &dentry->d_name, 0, - NULL, EXT4_HT_DIR, credits); + inode = ext4_new_inode_start_handle(mnt_userns, dir, mode, &dentry->d_name, + 0, NULL, EXT4_HT_DIR, credits); handle = ext4_journal_current_handle(); err = PTR_ERR(inode); if (!IS_ERR(inode)) { @@ -2677,7 +2677,7 @@ static int ext4_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, return err; retry: - inode = ext4_new_inode_start_handle(dir, mode, + inode = ext4_new_inode_start_handle(mnt_userns, dir, mode, NULL, 0, NULL, EXT4_HT_DIR, EXT4_MAXQUOTAS_INIT_BLOCKS(dir->i_sb) + @@ -2792,7 +2792,7 @@ static int ext4_mkdir(struct user_namespace *mnt_userns, struct inode *dir, credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) + EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3); retry: - inode = ext4_new_inode_start_handle(dir, S_IFDIR | mode, + inode = ext4_new_inode_start_handle(mnt_userns, dir, S_IFDIR | mode, &dentry->d_name, 0, NULL, EXT4_HT_DIR, credits); handle = ext4_journal_current_handle(); @@ -3335,7 +3335,7 @@ static int ext4_symlink(struct user_namespace *mnt_userns, struct inode *dir, EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3; } - inode = ext4_new_inode_start_handle(dir, S_IFLNK|S_IRWXUGO, + inode = ext4_new_inode_start_handle(mnt_userns, dir, S_IFLNK|S_IRWXUGO, &dentry->d_name, 0, NULL, EXT4_HT_DIR, credits); handle = ext4_journal_current_handle(); @@ -3664,7 +3664,8 @@ static void ext4_update_dir_count(handle_t *handle, struct ext4_renament *ent) } } -static struct inode *ext4_whiteout_for_rename(struct ext4_renament *ent, +static struct inode *ext4_whiteout_for_rename(struct user_namespace *mnt_userns, + struct ext4_renament *ent, int credits, handle_t **h) { struct inode *wh; @@ -3678,7 +3679,8 @@ static struct inode *ext4_whiteout_for_rename(struct ext4_renament *ent, credits += (EXT4_MAXQUOTAS_TRANS_BLOCKS(ent->dir->i_sb) + EXT4_XATTR_TRANS_BLOCKS + 4); retry: - wh = ext4_new_inode_start_handle(ent->dir, S_IFCHR | WHITEOUT_MODE, + wh = ext4_new_inode_start_handle(mnt_userns, ent->dir, + S_IFCHR | WHITEOUT_MODE, &ent->dentry->d_name, 0, NULL, EXT4_HT_DIR, credits); @@ -3705,9 +3707,9 @@ retry: * while new_{dentry,inode) refers to the destination dentry/inode * This comes from rename(const char *oldpath, const char *newpath) */ -static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) +static int ext4_rename(struct user_namespace *mnt_userns, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) { handle_t *handle = NULL; struct ext4_renament old = { @@ -3791,7 +3793,7 @@ static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry, goto end_rename; } } else { - whiteout = ext4_whiteout_for_rename(&old, credits, &handle); + whiteout = ext4_whiteout_for_rename(mnt_userns, &old, credits, &handle); if (IS_ERR(whiteout)) { retval = PTR_ERR(whiteout); whiteout = NULL; @@ -4110,7 +4112,7 @@ static int ext4_rename2(struct user_namespace *mnt_userns, new_dir, new_dentry); } - return ext4_rename(old_dir, old_dentry, new_dir, new_dentry, flags); + return ext4_rename(mnt_userns, old_dir, old_dentry, new_dir, new_dentry, flags); } /* diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 9a6f9875aa34..a77fbb79e813 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -6654,7 +6654,7 @@ static struct file_system_type ext4_fs_type = { .name = "ext4", .mount = ext4_mount, .kill_sb = kill_block_super, - .fs_flags = FS_REQUIRES_DEV, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, }; MODULE_ALIAS_FS("ext4"); -- cgit v1.2.3 From f736d93d76d3e97d6986c6d26c8eaa32536ccc5c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 21 Jan 2021 14:19:58 +0100 Subject: xfs: support idmapped mounts Enable idmapped mounts for xfs. This basically just means passing down the user_namespace argument from the VFS methods down to where it is passed to the relevant helpers. Note that full-filesystem bulkstat is not supported from inside idmapped mounts as it is an administrative operation that acts on the whole file system. The limitation is not applied to the bulkstat single operation that just operates on a single inode. Link: https://lore.kernel.org/r/20210121131959.646623-40-christian.brauner@ubuntu.com Signed-off-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/xfs/xfs_acl.c | 3 +-- fs/xfs/xfs_file.c | 4 +++- fs/xfs/xfs_inode.c | 26 ++++++++++++++++++-------- fs/xfs/xfs_inode.h | 16 ++++++++++------ fs/xfs/xfs_ioctl.c | 35 +++++++++++++++++++++-------------- fs/xfs/xfs_ioctl32.c | 6 ++++-- fs/xfs/xfs_iops.c | 49 ++++++++++++++++++++++++++++--------------------- fs/xfs/xfs_iops.h | 3 ++- fs/xfs/xfs_itable.c | 17 +++++++++++++---- fs/xfs/xfs_itable.h | 1 + fs/xfs/xfs_qm.c | 3 ++- fs/xfs/xfs_super.c | 2 +- fs/xfs/xfs_symlink.c | 5 +++-- fs/xfs/xfs_symlink.h | 5 +++-- 14 files changed, 110 insertions(+), 65 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_acl.c b/fs/xfs/xfs_acl.c index 332e87153c6c..d02bef24b32b 100644 --- a/fs/xfs/xfs_acl.c +++ b/fs/xfs/xfs_acl.c @@ -253,8 +253,7 @@ xfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, return error; if (type == ACL_TYPE_ACCESS) { - error = posix_acl_update_mode(&init_user_ns, inode, &mode, - &acl); + error = posix_acl_update_mode(mnt_userns, inode, &mode, &acl); if (error) return error; set_mode = true; diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 5b0f93f73837..1bdc3560aed9 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -29,6 +29,7 @@ #include #include #include +#include static const struct vm_operations_struct xfs_file_vm_ops; @@ -994,7 +995,8 @@ xfs_file_fallocate( iattr.ia_valid = ATTR_SIZE; iattr.ia_size = new_size; - error = xfs_vn_setattr_size(file_dentry(file), &iattr); + error = xfs_vn_setattr_size(file_mnt_user_ns(file), + file_dentry(file), &iattr); if (error) goto out_unlock; } diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index b7352bc4c815..95b7f2ba4e06 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -766,6 +766,7 @@ xfs_inode_inherit_flags2( */ static int xfs_init_new_inode( + struct user_namespace *mnt_userns, struct xfs_trans *tp, struct xfs_inode *pip, xfs_ino_t ino, @@ -806,7 +807,7 @@ xfs_init_new_inode( inode = VFS_I(ip); inode->i_mode = mode; set_nlink(inode, nlink); - inode->i_uid = current_fsuid(); + inode->i_uid = fsuid_into_mnt(mnt_userns); inode->i_rdev = rdev; ip->i_d.di_projid = prid; @@ -815,7 +816,7 @@ xfs_init_new_inode( if ((VFS_I(pip)->i_mode & S_ISGID) && S_ISDIR(mode)) inode->i_mode |= S_ISGID; } else { - inode->i_gid = current_fsgid(); + inode->i_gid = fsgid_into_mnt(mnt_userns); } /* @@ -824,7 +825,8 @@ xfs_init_new_inode( * (and only if the irix_sgid_inherit compatibility variable is set). */ if (irix_sgid_inherit && - (inode->i_mode & S_ISGID) && !in_group_p(inode->i_gid)) + (inode->i_mode & S_ISGID) && + !in_group_p(i_gid_into_mnt(mnt_userns, inode))) inode->i_mode &= ~S_ISGID; ip->i_d.di_size = 0; @@ -901,6 +903,7 @@ xfs_init_new_inode( */ int xfs_dir_ialloc( + struct user_namespace *mnt_userns, struct xfs_trans **tpp, struct xfs_inode *dp, umode_t mode, @@ -933,7 +936,8 @@ xfs_dir_ialloc( return error; ASSERT(ino != NULLFSINO); - return xfs_init_new_inode(*tpp, dp, ino, mode, nlink, rdev, prid, ipp); + return xfs_init_new_inode(mnt_userns, *tpp, dp, ino, mode, nlink, rdev, + prid, ipp); } /* @@ -973,6 +977,7 @@ xfs_bumplink( int xfs_create( + struct user_namespace *mnt_userns, xfs_inode_t *dp, struct xfs_name *name, umode_t mode, @@ -1047,7 +1052,8 @@ xfs_create( * entry pointing to them, but a directory also the "." entry * pointing to itself. */ - error = xfs_dir_ialloc(&tp, dp, mode, is_dir ? 2 : 1, rdev, prid, &ip); + error = xfs_dir_ialloc(mnt_userns, &tp, dp, mode, is_dir ? 2 : 1, rdev, + prid, &ip); if (error) goto out_trans_cancel; @@ -1128,6 +1134,7 @@ xfs_create( int xfs_create_tmpfile( + struct user_namespace *mnt_userns, struct xfs_inode *dp, umode_t mode, struct xfs_inode **ipp) @@ -1169,7 +1176,7 @@ xfs_create_tmpfile( if (error) goto out_trans_cancel; - error = xfs_dir_ialloc(&tp, dp, mode, 0, 0, prid, &ip); + error = xfs_dir_ialloc(mnt_userns, &tp, dp, mode, 0, 0, prid, &ip); if (error) goto out_trans_cancel; @@ -2977,13 +2984,15 @@ out_trans_abort: */ static int xfs_rename_alloc_whiteout( + struct user_namespace *mnt_userns, struct xfs_inode *dp, struct xfs_inode **wip) { struct xfs_inode *tmpfile; int error; - error = xfs_create_tmpfile(dp, S_IFCHR | WHITEOUT_MODE, &tmpfile); + error = xfs_create_tmpfile(mnt_userns, dp, S_IFCHR | WHITEOUT_MODE, + &tmpfile); if (error) return error; @@ -3005,6 +3014,7 @@ xfs_rename_alloc_whiteout( */ int xfs_rename( + struct user_namespace *mnt_userns, struct xfs_inode *src_dp, struct xfs_name *src_name, struct xfs_inode *src_ip, @@ -3036,7 +3046,7 @@ xfs_rename( */ if (flags & RENAME_WHITEOUT) { ASSERT(!(flags & (RENAME_NOREPLACE | RENAME_EXCHANGE))); - error = xfs_rename_alloc_whiteout(target_dp, &wip); + error = xfs_rename_alloc_whiteout(mnt_userns, target_dp, &wip); if (error) return error; diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index eca333f5f715..88ee4c3930ae 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -369,15 +369,18 @@ int xfs_release(struct xfs_inode *ip); void xfs_inactive(struct xfs_inode *ip); int xfs_lookup(struct xfs_inode *dp, struct xfs_name *name, struct xfs_inode **ipp, struct xfs_name *ci_name); -int xfs_create(struct xfs_inode *dp, struct xfs_name *name, +int xfs_create(struct user_namespace *mnt_userns, + struct xfs_inode *dp, struct xfs_name *name, umode_t mode, dev_t rdev, struct xfs_inode **ipp); -int xfs_create_tmpfile(struct xfs_inode *dp, umode_t mode, +int xfs_create_tmpfile(struct user_namespace *mnt_userns, + struct xfs_inode *dp, umode_t mode, struct xfs_inode **ipp); int xfs_remove(struct xfs_inode *dp, struct xfs_name *name, struct xfs_inode *ip); int xfs_link(struct xfs_inode *tdp, struct xfs_inode *sip, struct xfs_name *target_name); -int xfs_rename(struct xfs_inode *src_dp, struct xfs_name *src_name, +int xfs_rename(struct user_namespace *mnt_userns, + struct xfs_inode *src_dp, struct xfs_name *src_name, struct xfs_inode *src_ip, struct xfs_inode *target_dp, struct xfs_name *target_name, struct xfs_inode *target_ip, unsigned int flags); @@ -407,9 +410,10 @@ void xfs_lock_two_inodes(struct xfs_inode *ip0, uint ip0_mode, xfs_extlen_t xfs_get_extsz_hint(struct xfs_inode *ip); xfs_extlen_t xfs_get_cowextsz_hint(struct xfs_inode *ip); -int xfs_dir_ialloc(struct xfs_trans **tpp, struct xfs_inode *dp, umode_t mode, - xfs_nlink_t nlink, dev_t dev, prid_t prid, - struct xfs_inode **ipp); +int xfs_dir_ialloc(struct user_namespace *mnt_userns, + struct xfs_trans **tpp, struct xfs_inode *dp, + umode_t mode, xfs_nlink_t nlink, dev_t dev, + prid_t prid, struct xfs_inode **ipp); static inline int xfs_itruncate_extents( diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 218e80afc859..3d4c7ca080fb 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -693,7 +693,8 @@ xfs_ioc_space( iattr.ia_valid = ATTR_SIZE; iattr.ia_size = bf->l_start; - error = xfs_vn_setattr_size(file_dentry(filp), &iattr); + error = xfs_vn_setattr_size(file_mnt_user_ns(filp), file_dentry(filp), + &iattr); if (error) goto out_unlock; @@ -734,13 +735,15 @@ xfs_fsinumbers_fmt( STATIC int xfs_ioc_fsbulkstat( - xfs_mount_t *mp, + struct file *file, unsigned int cmd, void __user *arg) { + struct xfs_mount *mp = XFS_I(file_inode(file))->i_mount; struct xfs_fsop_bulkreq bulkreq; struct xfs_ibulk breq = { .mp = mp, + .mnt_userns = file_mnt_user_ns(file), .ocount = 0, }; xfs_ino_t lastino; @@ -908,13 +911,15 @@ xfs_bulk_ireq_teardown( /* Handle the v5 bulkstat ioctl. */ STATIC int xfs_ioc_bulkstat( - struct xfs_mount *mp, + struct file *file, unsigned int cmd, struct xfs_bulkstat_req __user *arg) { + struct xfs_mount *mp = XFS_I(file_inode(file))->i_mount; struct xfs_bulk_ireq hdr; struct xfs_ibulk breq = { .mp = mp, + .mnt_userns = file_mnt_user_ns(file), }; int error; @@ -1275,8 +1280,9 @@ xfs_ioctl_setattr_prepare_dax( */ static struct xfs_trans * xfs_ioctl_setattr_get_trans( - struct xfs_inode *ip) + struct file *file) { + struct xfs_inode *ip = XFS_I(file_inode(file)); struct xfs_mount *mp = ip->i_mount; struct xfs_trans *tp; int error = -EROFS; @@ -1300,7 +1306,7 @@ xfs_ioctl_setattr_get_trans( * The user ID of the calling process must be equal to the file owner * ID, except in cases where the CAP_FSETID capability is applicable. */ - if (!inode_owner_or_capable(&init_user_ns, VFS_I(ip))) { + if (!inode_owner_or_capable(file_mnt_user_ns(file), VFS_I(ip))) { error = -EPERM; goto out_cancel; } @@ -1428,9 +1434,11 @@ xfs_ioctl_setattr_check_projid( STATIC int xfs_ioctl_setattr( - xfs_inode_t *ip, + struct file *file, struct fsxattr *fa) { + struct user_namespace *mnt_userns = file_mnt_user_ns(file); + struct xfs_inode *ip = XFS_I(file_inode(file)); struct fsxattr old_fa; struct xfs_mount *mp = ip->i_mount; struct xfs_trans *tp; @@ -1462,7 +1470,7 @@ xfs_ioctl_setattr( xfs_ioctl_setattr_prepare_dax(ip, fa); - tp = xfs_ioctl_setattr_get_trans(ip); + tp = xfs_ioctl_setattr_get_trans(file); if (IS_ERR(tp)) { code = PTR_ERR(tp); goto error_free_dquots; @@ -1502,7 +1510,7 @@ xfs_ioctl_setattr( */ if ((VFS_I(ip)->i_mode & (S_ISUID|S_ISGID)) && - !capable_wrt_inode_uidgid(&init_user_ns, VFS_I(ip), CAP_FSETID)) + !capable_wrt_inode_uidgid(mnt_userns, VFS_I(ip), CAP_FSETID)) VFS_I(ip)->i_mode &= ~(S_ISUID|S_ISGID); /* Change the ownerships and register project quota modifications */ @@ -1549,7 +1557,6 @@ error_free_dquots: STATIC int xfs_ioc_fssetxattr( - xfs_inode_t *ip, struct file *filp, void __user *arg) { @@ -1562,7 +1569,7 @@ xfs_ioc_fssetxattr( error = mnt_want_write_file(filp); if (error) return error; - error = xfs_ioctl_setattr(ip, &fa); + error = xfs_ioctl_setattr(filp, &fa); mnt_drop_write_file(filp); return error; } @@ -1608,7 +1615,7 @@ xfs_ioc_setxflags( xfs_ioctl_setattr_prepare_dax(ip, &fa); - tp = xfs_ioctl_setattr_get_trans(ip); + tp = xfs_ioctl_setattr_get_trans(filp); if (IS_ERR(tp)) { error = PTR_ERR(tp); goto out_drop_write; @@ -2119,10 +2126,10 @@ xfs_file_ioctl( case XFS_IOC_FSBULKSTAT_SINGLE: case XFS_IOC_FSBULKSTAT: case XFS_IOC_FSINUMBERS: - return xfs_ioc_fsbulkstat(mp, cmd, arg); + return xfs_ioc_fsbulkstat(filp, cmd, arg); case XFS_IOC_BULKSTAT: - return xfs_ioc_bulkstat(mp, cmd, arg); + return xfs_ioc_bulkstat(filp, cmd, arg); case XFS_IOC_INUMBERS: return xfs_ioc_inumbers(mp, cmd, arg); @@ -2144,7 +2151,7 @@ xfs_file_ioctl( case XFS_IOC_FSGETXATTRA: return xfs_ioc_fsgetxattr(ip, 1, arg); case XFS_IOC_FSSETXATTR: - return xfs_ioc_fssetxattr(ip, filp, arg); + return xfs_ioc_fssetxattr(filp, arg); case XFS_IOC_GETXFLAGS: return xfs_ioc_getxflags(ip, arg); case XFS_IOC_SETXFLAGS: diff --git a/fs/xfs/xfs_ioctl32.c b/fs/xfs/xfs_ioctl32.c index c1771e728117..926427b19573 100644 --- a/fs/xfs/xfs_ioctl32.c +++ b/fs/xfs/xfs_ioctl32.c @@ -209,14 +209,16 @@ xfs_fsbulkstat_one_fmt_compat( /* copied from xfs_ioctl.c */ STATIC int xfs_compat_ioc_fsbulkstat( - xfs_mount_t *mp, + struct file *file, unsigned int cmd, struct compat_xfs_fsop_bulkreq __user *p32) { + struct xfs_mount *mp = XFS_I(file_inode(file))->i_mount; u32 addr; struct xfs_fsop_bulkreq bulkreq; struct xfs_ibulk breq = { .mp = mp, + .mnt_userns = file_mnt_user_ns(file), .ocount = 0, }; xfs_ino_t lastino; @@ -507,7 +509,7 @@ xfs_file_compat_ioctl( case XFS_IOC_FSBULKSTAT_32: case XFS_IOC_FSBULKSTAT_SINGLE_32: case XFS_IOC_FSINUMBERS_32: - return xfs_compat_ioc_fsbulkstat(mp, cmd, arg); + return xfs_compat_ioc_fsbulkstat(filp, cmd, arg); case XFS_IOC_FD_TO_HANDLE_32: case XFS_IOC_PATH_TO_HANDLE_32: case XFS_IOC_PATH_TO_FSHANDLE_32: { diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index f5dfa128af64..816a0f77a39f 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -128,6 +128,7 @@ xfs_cleanup_inode( STATIC int xfs_generic_create( + struct user_namespace *mnt_userns, struct inode *dir, struct dentry *dentry, umode_t mode, @@ -161,9 +162,10 @@ xfs_generic_create( goto out_free_acl; if (!tmpfile) { - error = xfs_create(XFS_I(dir), &name, mode, rdev, &ip); + error = xfs_create(mnt_userns, XFS_I(dir), &name, mode, rdev, + &ip); } else { - error = xfs_create_tmpfile(XFS_I(dir), mode, &ip); + error = xfs_create_tmpfile(mnt_userns, XFS_I(dir), mode, &ip); } if (unlikely(error)) goto out_free_acl; @@ -226,7 +228,7 @@ xfs_vn_mknod( umode_t mode, dev_t rdev) { - return xfs_generic_create(dir, dentry, mode, rdev, false); + return xfs_generic_create(mnt_userns, dir, dentry, mode, rdev, false); } STATIC int @@ -237,7 +239,7 @@ xfs_vn_create( umode_t mode, bool flags) { - return xfs_generic_create(dir, dentry, mode, 0, false); + return xfs_generic_create(mnt_userns, dir, dentry, mode, 0, false); } STATIC int @@ -247,7 +249,8 @@ xfs_vn_mkdir( struct dentry *dentry, umode_t mode) { - return xfs_generic_create(dir, dentry, mode | S_IFDIR, 0, false); + return xfs_generic_create(mnt_userns, dir, dentry, mode | S_IFDIR, 0, + false); } STATIC struct dentry * @@ -381,7 +384,7 @@ xfs_vn_symlink( if (unlikely(error)) goto out; - error = xfs_symlink(XFS_I(dir), &name, symname, mode, &cip); + error = xfs_symlink(mnt_userns, XFS_I(dir), &name, symname, mode, &cip); if (unlikely(error)) goto out; @@ -436,8 +439,8 @@ xfs_vn_rename( if (unlikely(error)) return error; - return xfs_rename(XFS_I(odir), &oname, XFS_I(d_inode(odentry)), - XFS_I(ndir), &nname, + return xfs_rename(mnt_userns, XFS_I(odir), &oname, + XFS_I(d_inode(odentry)), XFS_I(ndir), &nname, new_inode ? XFS_I(new_inode) : NULL, flags); } @@ -553,8 +556,8 @@ xfs_vn_getattr( stat->dev = inode->i_sb->s_dev; stat->mode = inode->i_mode; stat->nlink = inode->i_nlink; - stat->uid = inode->i_uid; - stat->gid = inode->i_gid; + stat->uid = i_uid_into_mnt(mnt_userns, inode); + stat->gid = i_gid_into_mnt(mnt_userns, inode); stat->ino = ip->i_ino; stat->atime = inode->i_atime; stat->mtime = inode->i_mtime; @@ -632,8 +635,9 @@ xfs_setattr_time( static int xfs_vn_change_ok( - struct dentry *dentry, - struct iattr *iattr) + struct user_namespace *mnt_userns, + struct dentry *dentry, + struct iattr *iattr) { struct xfs_mount *mp = XFS_I(d_inode(dentry))->i_mount; @@ -643,7 +647,7 @@ xfs_vn_change_ok( if (XFS_FORCED_SHUTDOWN(mp)) return -EIO; - return setattr_prepare(&init_user_ns, dentry, iattr); + return setattr_prepare(mnt_userns, dentry, iattr); } /* @@ -654,6 +658,7 @@ xfs_vn_change_ok( */ static int xfs_setattr_nonsize( + struct user_namespace *mnt_userns, struct xfs_inode *ip, struct iattr *iattr) { @@ -813,7 +818,7 @@ xfs_setattr_nonsize( * Posix ACL code seems to care about this issue either. */ if (mask & ATTR_MODE) { - error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); + error = posix_acl_chmod(mnt_userns, inode, inode->i_mode); if (error) return error; } @@ -837,6 +842,7 @@ out_dqrele: */ STATIC int xfs_setattr_size( + struct user_namespace *mnt_userns, struct xfs_inode *ip, struct iattr *iattr) { @@ -868,7 +874,7 @@ xfs_setattr_size( * Use the regular setattr path to update the timestamps. */ iattr->ia_valid &= ~ATTR_SIZE; - return xfs_setattr_nonsize(ip, iattr); + return xfs_setattr_nonsize(mnt_userns, ip, iattr); } /* @@ -1037,6 +1043,7 @@ out_trans_cancel: int xfs_vn_setattr_size( + struct user_namespace *mnt_userns, struct dentry *dentry, struct iattr *iattr) { @@ -1045,10 +1052,10 @@ xfs_vn_setattr_size( trace_xfs_setattr(ip); - error = xfs_vn_change_ok(dentry, iattr); + error = xfs_vn_change_ok(mnt_userns, dentry, iattr); if (error) return error; - return xfs_setattr_size(ip, iattr); + return xfs_setattr_size(mnt_userns, ip, iattr); } STATIC int @@ -1073,14 +1080,14 @@ xfs_vn_setattr( return error; } - error = xfs_vn_setattr_size(dentry, iattr); + error = xfs_vn_setattr_size(mnt_userns, dentry, iattr); xfs_iunlock(ip, XFS_MMAPLOCK_EXCL); } else { trace_xfs_setattr(ip); - error = xfs_vn_change_ok(dentry, iattr); + error = xfs_vn_change_ok(mnt_userns, dentry, iattr); if (!error) - error = xfs_setattr_nonsize(ip, iattr); + error = xfs_setattr_nonsize(mnt_userns, ip, iattr); } return error; @@ -1156,7 +1163,7 @@ xfs_vn_tmpfile( struct dentry *dentry, umode_t mode) { - return xfs_generic_create(dir, dentry, mode, 0, true); + return xfs_generic_create(mnt_userns, dir, dentry, mode, 0, true); } static const struct inode_operations xfs_inode_operations = { diff --git a/fs/xfs/xfs_iops.h b/fs/xfs/xfs_iops.h index 99ca745c1071..278949056048 100644 --- a/fs/xfs/xfs_iops.h +++ b/fs/xfs/xfs_iops.h @@ -14,6 +14,7 @@ extern const struct file_operations xfs_dir_file_operations; extern ssize_t xfs_vn_listxattr(struct dentry *, char *data, size_t size); extern void xfs_setattr_time(struct xfs_inode *ip, struct iattr *iattr); -extern int xfs_vn_setattr_size(struct dentry *dentry, struct iattr *vap); +int xfs_vn_setattr_size(struct user_namespace *mnt_userns, + struct dentry *dentry, struct iattr *vap); #endif /* __XFS_IOPS_H__ */ diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c index 16ca97a7ff00..ca310a125d1e 100644 --- a/fs/xfs/xfs_itable.c +++ b/fs/xfs/xfs_itable.c @@ -54,10 +54,12 @@ struct xfs_bstat_chunk { STATIC int xfs_bulkstat_one_int( struct xfs_mount *mp, + struct user_namespace *mnt_userns, struct xfs_trans *tp, xfs_ino_t ino, struct xfs_bstat_chunk *bc) { + struct user_namespace *sb_userns = mp->m_super->s_user_ns; struct xfs_icdinode *dic; /* dinode core info pointer */ struct xfs_inode *ip; /* incore inode pointer */ struct inode *inode; @@ -86,8 +88,8 @@ xfs_bulkstat_one_int( */ buf->bs_projectid = ip->i_d.di_projid; buf->bs_ino = ino; - buf->bs_uid = i_uid_read(inode); - buf->bs_gid = i_gid_read(inode); + buf->bs_uid = from_kuid(sb_userns, i_uid_into_mnt(mnt_userns, inode)); + buf->bs_gid = from_kgid(sb_userns, i_gid_into_mnt(mnt_userns, inode)); buf->bs_size = dic->di_size; buf->bs_nlink = inode->i_nlink; @@ -173,7 +175,8 @@ xfs_bulkstat_one( if (!bc.buf) return -ENOMEM; - error = xfs_bulkstat_one_int(breq->mp, NULL, breq->startino, &bc); + error = xfs_bulkstat_one_int(breq->mp, breq->mnt_userns, NULL, + breq->startino, &bc); kmem_free(bc.buf); @@ -194,9 +197,10 @@ xfs_bulkstat_iwalk( xfs_ino_t ino, void *data) { + struct xfs_bstat_chunk *bc = data; int error; - error = xfs_bulkstat_one_int(mp, tp, ino, data); + error = xfs_bulkstat_one_int(mp, bc->breq->mnt_userns, tp, ino, data); /* bulkstat just skips over missing inodes */ if (error == -ENOENT || error == -EINVAL) return 0; @@ -239,6 +243,11 @@ xfs_bulkstat( }; int error; + if (breq->mnt_userns != &init_user_ns) { + xfs_warn_ratelimited(breq->mp, + "bulkstat not supported inside of idmapped mounts."); + return -EINVAL; + } if (xfs_bulkstat_already_done(breq->mp, breq->startino)) return 0; diff --git a/fs/xfs/xfs_itable.h b/fs/xfs/xfs_itable.h index 96a1e2a9be3f..7078d10c9b12 100644 --- a/fs/xfs/xfs_itable.h +++ b/fs/xfs/xfs_itable.h @@ -8,6 +8,7 @@ /* In-memory representation of a userspace request for batch inode data. */ struct xfs_ibulk { struct xfs_mount *mp; + struct user_namespace *mnt_userns; void __user *ubuffer; /* user output buffer */ xfs_ino_t startino; /* start with this inode */ unsigned int icount; /* number of elements in ubuffer */ diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index c134eb4aeaa8..1b7b1393cab2 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -787,7 +787,8 @@ xfs_qm_qino_alloc( return error; if (need_alloc) { - error = xfs_dir_ialloc(&tp, NULL, S_IFREG, 1, 0, 0, ipp); + error = xfs_dir_ialloc(&init_user_ns, &tp, NULL, S_IFREG, 1, 0, + 0, ipp); if (error) { xfs_trans_cancel(tp); return error; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 813be879a5e5..e95c1eff95e0 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -1912,7 +1912,7 @@ static struct file_system_type xfs_fs_type = { .init_fs_context = xfs_init_fs_context, .parameters = xfs_fs_parameters, .kill_sb = kill_block_super, - .fs_flags = FS_REQUIRES_DEV, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, }; MODULE_ALIAS_FS("xfs"); diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 1f43fd7f3209..77c8ea3229f1 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -134,6 +134,7 @@ xfs_readlink( int xfs_symlink( + struct user_namespace *mnt_userns, struct xfs_inode *dp, struct xfs_name *link_name, const char *target_path, @@ -223,8 +224,8 @@ xfs_symlink( /* * Allocate an inode for the symlink. */ - error = xfs_dir_ialloc(&tp, dp, S_IFLNK | (mode & ~S_IFMT), 1, 0, - prid, &ip); + error = xfs_dir_ialloc(mnt_userns, &tp, dp, S_IFLNK | (mode & ~S_IFMT), + 1, 0, prid, &ip); if (error) goto out_trans_cancel; diff --git a/fs/xfs/xfs_symlink.h b/fs/xfs/xfs_symlink.h index b1fa091427e6..2586b7e393f3 100644 --- a/fs/xfs/xfs_symlink.h +++ b/fs/xfs/xfs_symlink.h @@ -7,8 +7,9 @@ /* Kernel only symlink definitions */ -int xfs_symlink(struct xfs_inode *dp, struct xfs_name *link_name, - const char *target_path, umode_t mode, struct xfs_inode **ipp); +int xfs_symlink(struct user_namespace *mnt_userns, struct xfs_inode *dp, + struct xfs_name *link_name, const char *target_path, + umode_t mode, struct xfs_inode **ipp); int xfs_readlink_bmap_ilocked(struct xfs_inode *ip, char *link); int xfs_readlink(struct xfs_inode *ip, char *link); int xfs_inactive_symlink(struct xfs_inode *ip); -- cgit v1.2.3 From 84965ff8a84f0368b154c9b367b62e59c1193f30 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 23 Jan 2021 15:51:11 -0700 Subject: io_uring: if we see flush on exit, cancel related tasks Ensure we match tasks that belong to a dead or dying task as well, as we need to reap those in addition to those belonging to the exiting task. Cc: stable@vger.kernel.org # 5.9+ Reported-by: Josef Grieb Signed-off-by: Jens Axboe --- fs/io_uring.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index c07913ec0cca..695fe00bafdc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1069,8 +1069,12 @@ static bool io_match_task(struct io_kiocb *head, { struct io_kiocb *req; - if (task && head->task != task) + if (task && head->task != task) { + /* in terms of cancelation, always match if req task is dead */ + if (head->task->flags & PF_EXITING) + return true; return false; + } if (!files) return true; @@ -9136,6 +9140,9 @@ static int io_uring_flush(struct file *file, void *data) struct io_uring_task *tctx = current->io_uring; struct io_ring_ctx *ctx = file->private_data; + if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) + io_uring_cancel_task_requests(ctx, NULL); + if (!tctx) return 0; -- cgit v1.2.3 From b18032bb0a883cd7edd22a7fe6c57e1059b81ed0 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 24 Jan 2021 16:58:56 -0700 Subject: io_uring: only call io_cqring_ev_posted() if events were posted This normally doesn't cause any extra harm, but it does mean that we'll increment the eventfd notification count, if one has been registered with the ring. This can confuse applications, when they see more notifications on the eventfd side than are available in the ring. Do the nice thing and only increment this count, if we actually posted (or even overflowed) events. Reported-and-tested-by: Dan Melnic Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 695fe00bafdc..2166c469789d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1779,12 +1779,13 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, struct io_kiocb *req, *tmp; struct io_uring_cqe *cqe; unsigned long flags; - bool all_flushed; + bool all_flushed, posted; LIST_HEAD(list); if (!force && __io_cqring_events(ctx) == rings->cq_ring_entries) return false; + posted = false; spin_lock_irqsave(&ctx->completion_lock, flags); list_for_each_entry_safe(req, tmp, &ctx->cq_overflow_list, compl.list) { if (!io_match_task(req, tsk, files)) @@ -1804,6 +1805,7 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, WRITE_ONCE(ctx->rings->cq_overflow, ctx->cached_cq_overflow); } + posted = true; } all_flushed = list_empty(&ctx->cq_overflow_list); @@ -1813,9 +1815,11 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW; } - io_commit_cqring(ctx); + if (posted) + io_commit_cqring(ctx); spin_unlock_irqrestore(&ctx->completion_lock, flags); - io_cqring_ev_posted(ctx); + if (posted) + io_cqring_ev_posted(ctx); while (!list_empty(&list)) { req = list_first_entry(&list, struct io_kiocb, compl.list); -- cgit v1.2.3 From 6f0d9689b670bc9f9640ff87b3f9226b7806dea2 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 9 Jan 2021 11:42:50 +0100 Subject: block: remove the NULL bdev check in bdev_read_only Only a single caller can end up in bdev_read_only, so move the check there. Signed-off-by: Christoph Hellwig Reviewed-by: Ming Lei Reviewed-by: Martin K. Petersen Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/genhd.c | 3 --- fs/super.c | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/block/genhd.c b/block/genhd.c index 419548e92d82..484a474648d5 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -1657,11 +1657,8 @@ EXPORT_SYMBOL(set_disk_ro); int bdev_read_only(struct block_device *bdev) { - if (!bdev) - return 0; return bdev->bd_read_only; } - EXPORT_SYMBOL(bdev_read_only); /* diff --git a/fs/super.c b/fs/super.c index 2c6cdea2ab2d..5a1f384ffc74 100644 --- a/fs/super.c +++ b/fs/super.c @@ -865,7 +865,8 @@ int reconfigure_super(struct fs_context *fc) if (fc->sb_flags_mask & SB_RDONLY) { #ifdef CONFIG_BLOCK - if (!(fc->sb_flags & SB_RDONLY) && bdev_read_only(sb->s_bdev)) + if (!(fc->sb_flags & SB_RDONLY) && sb->s_bdev && + bdev_read_only(sb->s_bdev)) return -EACCES; #endif -- cgit v1.2.3 From 309dca309fc39a9e3c31b916393b74bd174fd74e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Jan 2021 11:02:34 +0100 Subject: block: store a block_device pointer in struct bio Replace the gendisk pointer in struct bio with a pointer to the newly improved struct block device. From that the gendisk can be trivially accessed with an extra indirection, but it also allows to directly look up all information related to partition remapping. Signed-off-by: Christoph Hellwig Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- arch/m68k/emu/nfblock.c | 2 +- arch/xtensa/platforms/iss/simdisk.c | 2 +- block/bio-integrity.c | 18 +++++++++--------- block/bio.c | 31 +++++++++++-------------------- block/blk-cgroup.c | 7 ++++--- block/blk-core.c | 37 +++++++++++++++++-------------------- block/blk-crypto-fallback.c | 2 +- block/blk-crypto.c | 2 +- block/blk-merge.c | 17 ++++++++--------- block/blk-mq.c | 2 +- block/blk-throttle.c | 2 +- block/blk.h | 2 -- block/bounce.c | 2 +- block/genhd.c | 2 +- drivers/block/brd.c | 2 +- drivers/block/drbd/drbd_int.h | 4 ++-- drivers/block/drbd/drbd_req.c | 2 +- drivers/block/null_blk/main.c | 2 +- drivers/block/pktcdvd.c | 4 ++-- drivers/block/ps3vram.c | 2 +- drivers/block/rsxx/dev.c | 2 +- drivers/block/umem.c | 2 +- drivers/block/zram/zram_drv.c | 2 +- drivers/lightnvm/pblk-init.c | 2 +- drivers/md/bcache/debug.c | 2 +- drivers/md/bcache/request.c | 7 ++++--- drivers/md/dm-bio-record.h | 9 +++------ drivers/md/dm-raid1.c | 10 +++++----- drivers/md/dm.c | 14 +++++++------- drivers/md/md-linear.c | 2 +- drivers/md/md.c | 2 +- drivers/md/md.h | 6 +++--- drivers/md/raid1.c | 6 +++--- drivers/md/raid10.c | 12 ++++++------ drivers/md/raid5.c | 2 +- drivers/nvdimm/blk.c | 4 ++-- drivers/nvdimm/btt.c | 4 ++-- drivers/nvdimm/pmem.c | 4 ++-- drivers/nvme/host/core.c | 6 +++--- drivers/nvme/host/lightnvm.c | 3 +-- drivers/nvme/host/multipath.c | 6 +++--- drivers/nvme/host/rdma.c | 2 +- drivers/s390/block/dcssblk.c | 2 +- drivers/s390/block/xpram.c | 2 +- fs/btrfs/check-integrity.c | 10 +++++----- fs/btrfs/raid56.c | 7 ++----- fs/btrfs/scrub.c | 2 +- fs/direct-io.c | 2 +- fs/f2fs/data.c | 12 +----------- include/linux/bio.h | 18 ++++++++---------- include/linux/blk-mq.h | 4 ++-- include/linux/blk_types.h | 3 +-- include/linux/blkdev.h | 5 +++-- kernel/trace/blktrace.c | 16 +++++++++------- mm/page_io.c | 2 +- 55 files changed, 154 insertions(+), 184 deletions(-) (limited to 'fs') diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index 92d26c812441..ba808543161a 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c @@ -61,7 +61,7 @@ struct nfhd_device { static blk_qc_t nfhd_submit_bio(struct bio *bio) { - struct nfhd_device *dev = bio->bi_disk->private_data; + struct nfhd_device *dev = bio->bi_bdev->bd_disk->private_data; struct bio_vec bvec; struct bvec_iter iter; int dir, len, shift; diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c index 3447556d276d..fc09be7b1347 100644 --- a/arch/xtensa/platforms/iss/simdisk.c +++ b/arch/xtensa/platforms/iss/simdisk.c @@ -103,7 +103,7 @@ static void simdisk_transfer(struct simdisk *dev, unsigned long sector, static blk_qc_t simdisk_submit_bio(struct bio *bio) { - struct simdisk *dev = bio->bi_disk->private_data; + struct simdisk *dev = bio->bi_bdev->bd_disk->private_data; struct bio_vec bvec; struct bvec_iter iter; sector_t sector = bio->bi_iter.bi_sector; diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 9ffd7e289554..c3e5abcfdc98 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -140,7 +140,7 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, iv = bip->bip_vec + bip->bip_vcnt; if (bip->bip_vcnt && - bvec_gap_to_prev(bio->bi_disk->queue, + bvec_gap_to_prev(bio->bi_bdev->bd_disk->queue, &bip->bip_vec[bip->bip_vcnt - 1], offset)) return 0; @@ -162,7 +162,7 @@ EXPORT_SYMBOL(bio_integrity_add_page); static blk_status_t bio_integrity_process(struct bio *bio, struct bvec_iter *proc_iter, integrity_processing_fn *proc_fn) { - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); struct blk_integrity_iter iter; struct bvec_iter bviter; struct bio_vec bv; @@ -171,7 +171,7 @@ static blk_status_t bio_integrity_process(struct bio *bio, void *prot_buf = page_address(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset; - iter.disk_name = bio->bi_disk->disk_name; + iter.disk_name = bio->bi_bdev->bd_disk->disk_name; iter.interval = 1 << bi->interval_exp; iter.seed = proc_iter->bi_sector; iter.prot_buf = prot_buf; @@ -208,8 +208,8 @@ static blk_status_t bio_integrity_process(struct bio *bio, bool bio_integrity_prep(struct bio *bio) { struct bio_integrity_payload *bip; - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); - struct request_queue *q = bio->bi_disk->queue; + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); + struct request_queue *q = bio->bi_bdev->bd_disk->queue; void *buf; unsigned long start, end; unsigned int len, nr_pages; @@ -329,7 +329,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) struct bio_integrity_payload *bip = container_of(work, struct bio_integrity_payload, bip_work); struct bio *bio = bip->bip_bio; - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); /* * At the moment verify is called bio's iterator was advanced @@ -355,7 +355,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) */ bool __bio_integrity_endio(struct bio *bio) { - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); struct bio_integrity_payload *bip = bio_integrity(bio); if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && @@ -381,7 +381,7 @@ bool __bio_integrity_endio(struct bio *bio) void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) { struct bio_integrity_payload *bip = bio_integrity(bio); - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); bip->bip_iter.bi_sector += bytes_done >> 9; @@ -397,7 +397,7 @@ void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) void bio_integrity_trim(struct bio *bio) { struct bio_integrity_payload *bip = bio_integrity(bio); - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio)); } diff --git a/block/bio.c b/block/bio.c index 1f2cc1fbe283..0b70ade17da6 100644 --- a/block/bio.c +++ b/block/bio.c @@ -607,16 +607,7 @@ void bio_truncate(struct bio *bio, unsigned new_size) */ void guard_bio_eod(struct bio *bio) { - sector_t maxsector; - struct block_device *part; - - rcu_read_lock(); - part = __disk_get_part(bio->bi_disk, bio->bi_partno); - if (part) - maxsector = bdev_nr_sectors(part); - else - maxsector = get_capacity(bio->bi_disk); - rcu_read_unlock(); + sector_t maxsector = bdev_nr_sectors(bio->bi_bdev); if (!maxsector) return; @@ -676,11 +667,10 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio)); /* - * most users will be overriding ->bi_disk with a new target, + * most users will be overriding ->bi_bdev with a new target, * so we don't set nor calculate new physical/hw segment counts here */ - bio->bi_disk = bio_src->bi_disk; - bio->bi_partno = bio_src->bi_partno; + bio->bi_bdev = bio_src->bi_bdev; bio_set_flag(bio, BIO_CLONED); if (bio_flagged(bio_src, BIO_THROTTLED)) bio_set_flag(bio, BIO_THROTTLED); @@ -730,7 +720,7 @@ EXPORT_SYMBOL(bio_clone_fast); const char *bio_devname(struct bio *bio, char *buf) { - return disk_name(bio->bi_disk, bio->bi_partno, buf); + return bdevname(bio->bi_bdev, buf); } EXPORT_SYMBOL(bio_devname); @@ -1037,7 +1027,7 @@ static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter) { unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; unsigned int max_append_sectors = queue_max_zone_append_sectors(q); struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; struct page **pages = (struct page **)bv; @@ -1145,7 +1135,8 @@ static void submit_bio_wait_endio(struct bio *bio) */ int submit_bio_wait(struct bio *bio) { - DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_disk->lockdep_map); + DECLARE_COMPLETION_ONSTACK_MAP(done, + bio->bi_bdev->bd_disk->lockdep_map); unsigned long hang_check; bio->bi_private = &done; @@ -1422,8 +1413,8 @@ again: if (!bio_integrity_endio(bio)) return; - if (bio->bi_disk) - rq_qos_done_bio(bio->bi_disk->queue, bio); + if (bio->bi_bdev) + rq_qos_done_bio(bio->bi_bdev->bd_disk->queue, bio); /* * Need to have a real endio function for chained bios, otherwise @@ -1438,8 +1429,8 @@ again: goto again; } - if (bio->bi_disk && bio_flagged(bio, BIO_TRACE_COMPLETION)) { - trace_block_bio_complete(bio->bi_disk->queue, bio); + if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) { + trace_block_bio_complete(bio->bi_bdev->bd_disk->queue, bio); bio_clear_flag(bio, BIO_TRACE_COMPLETION); } diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 031114d454a6..3465d6ee708e 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1800,7 +1800,8 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, struct blkcg_gq *blkg, *ret_blkg = NULL; rcu_read_lock(); - blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_disk->queue); + blkg = blkg_lookup_create(css_to_blkcg(css), + bio->bi_bdev->bd_disk->queue); while (blkg) { if (blkg_tryget(blkg)) { ret_blkg = blkg; @@ -1836,8 +1837,8 @@ void bio_associate_blkg_from_css(struct bio *bio, if (css && css->parent) { bio->bi_blkg = blkg_tryget_closest(bio, css); } else { - blkg_get(bio->bi_disk->queue->root_blkg); - bio->bi_blkg = bio->bi_disk->queue->root_blkg; + blkg_get(bio->bi_bdev->bd_disk->queue->root_blkg); + bio->bi_blkg = bio->bi_bdev->bd_disk->queue->root_blkg; } } EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css); diff --git a/block/blk-core.c b/block/blk-core.c index 08ff8ca32529..a3a54cd86c9c 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -476,7 +476,7 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) static inline int bio_queue_enter(struct bio *bio) { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; bool nowait = bio->bi_opf & REQ_NOWAIT; int ret; @@ -712,7 +712,7 @@ static inline bool bio_check_ro(struct bio *bio, struct block_device *part) static noinline int should_fail_bio(struct bio *bio) { - if (should_fail_request(bio->bi_disk->part0, bio->bi_iter.bi_size)) + if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size)) return -EIO; return 0; } @@ -741,13 +741,9 @@ static inline int bio_check_eod(struct bio *bio, sector_t maxsector) */ static inline int blk_partition_remap(struct bio *bio) { - struct block_device *p; + struct block_device *p = bio->bi_bdev; int ret = -EIO; - rcu_read_lock(); - p = __disk_get_part(bio->bi_disk, bio->bi_partno); - if (unlikely(!p)) - goto out; if (unlikely(should_fail_request(p, bio->bi_iter.bi_size))) goto out; if (unlikely(bio_check_ro(bio, p))) @@ -761,10 +757,9 @@ static inline int blk_partition_remap(struct bio *bio) bio->bi_iter.bi_sector - p->bd_start_sect); } - bio->bi_partno = 0; + bio->bi_bdev = bdev_whole(p); ret = 0; out: - rcu_read_unlock(); return ret; } @@ -805,7 +800,8 @@ static inline blk_status_t blk_check_zone_append(struct request_queue *q, static noinline_for_stack bool submit_bio_checks(struct bio *bio) { - struct request_queue *q = bio->bi_disk->queue; + struct block_device *bdev = bio->bi_bdev; + struct request_queue *q = bdev->bd_disk->queue; blk_status_t status = BLK_STS_IOERR; struct blk_plug *plug; @@ -825,13 +821,13 @@ static noinline_for_stack bool submit_bio_checks(struct bio *bio) if (should_fail_bio(bio)) goto end_io; - if (bio->bi_partno) { + if (bio->bi_bdev->bd_partno) { if (unlikely(blk_partition_remap(bio))) goto end_io; } else { - if (unlikely(bio_check_ro(bio, bio->bi_disk->part0))) + if (unlikely(bio_check_ro(bio, bdev_whole(bdev)))) goto end_io; - if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk)))) + if (unlikely(bio_check_eod(bio, get_capacity(bdev->bd_disk)))) goto end_io; } @@ -924,7 +920,7 @@ end_io: static blk_qc_t __submit_bio(struct bio *bio) { - struct gendisk *disk = bio->bi_disk; + struct gendisk *disk = bio->bi_bdev->bd_disk; blk_qc_t ret = BLK_QC_T_NONE; if (blk_crypto_bio_prep(&bio)) { @@ -966,7 +962,7 @@ static blk_qc_t __submit_bio_noacct(struct bio *bio) current->bio_list = bio_list_on_stack; do { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; struct bio_list lower, same; if (unlikely(bio_queue_enter(bio) != 0)) @@ -987,7 +983,7 @@ static blk_qc_t __submit_bio_noacct(struct bio *bio) bio_list_init(&lower); bio_list_init(&same); while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL) - if (q == bio->bi_disk->queue) + if (q == bio->bi_bdev->bd_disk->queue) bio_list_add(&same, bio); else bio_list_add(&lower, bio); @@ -1012,7 +1008,7 @@ static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) current->bio_list = bio_list; do { - struct gendisk *disk = bio->bi_disk; + struct gendisk *disk = bio->bi_bdev->bd_disk; if (unlikely(bio_queue_enter(bio) != 0)) continue; @@ -1055,7 +1051,7 @@ blk_qc_t submit_bio_noacct(struct bio *bio) return BLK_QC_T_NONE; } - if (!bio->bi_disk->fops->submit_bio) + if (!bio->bi_bdev->bd_disk->fops->submit_bio) return __submit_bio_noacct_mq(bio); return __submit_bio_noacct(bio); } @@ -1067,7 +1063,7 @@ EXPORT_SYMBOL(submit_bio_noacct); * * submit_bio() is used to submit I/O requests to block devices. It is passed a * fully set up &struct bio that describes the I/O that needs to be done. The - * bio will be send to the device described by the bi_disk and bi_partno fields. + * bio will be send to the device described by the bi_bdev field. * * The success/failure status of the request, along with notification of * completion, is delivered asynchronously through the ->bi_end_io() callback @@ -1087,7 +1083,8 @@ blk_qc_t submit_bio(struct bio *bio) unsigned int count; if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) - count = queue_logical_block_size(bio->bi_disk->queue) >> 9; + count = queue_logical_block_size( + bio->bi_bdev->bd_disk->queue) >> 9; else count = bio_sectors(bio); diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index c162b754efbd..8f1e18176731 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -167,7 +167,7 @@ static struct bio *blk_crypto_clone_bio(struct bio *bio_src) bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL); if (!bio) return NULL; - bio->bi_disk = bio_src->bi_disk; + bio->bi_bdev = bio_src->bi_bdev; bio->bi_opf = bio_src->bi_opf; bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 5da43f0973b4..09fcb18fa778 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -280,7 +280,7 @@ bool __blk_crypto_bio_prep(struct bio **bio_ptr) * Success if device supports the encryption context, or if we succeeded * in falling back to the crypto API. */ - if (blk_ksm_crypto_cfg_supported(bio->bi_disk->queue->ksm, + if (blk_ksm_crypto_cfg_supported(bio->bi_bdev->bd_disk->queue->ksm, &bc_key->crypto_cfg)) return true; diff --git a/block/blk-merge.c b/block/blk-merge.c index 808768f6b174..ffb4aa0ea68b 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -298,14 +298,13 @@ split: * Split a bio into two bios, chain the two bios, submit the second half and * store a pointer to the first half in *@bio. If the second bio is still too * big it will be split by a recursive call to this function. Since this - * function may allocate a new bio from @bio->bi_disk->queue->bio_split, it is - * the responsibility of the caller to ensure that - * @bio->bi_disk->queue->bio_split is only released after processing of the - * split bio has finished. + * function may allocate a new bio from q->bio_split, it is the responsibility + * of the caller to ensure that q->bio_split is only released after processing + * of the split bio has finished. */ void __blk_queue_split(struct bio **bio, unsigned int *nr_segs) { - struct request_queue *q = (*bio)->bi_disk->queue; + struct request_queue *q = (*bio)->bi_bdev->bd_disk->queue; struct bio *split = NULL; switch (bio_op(*bio)) { @@ -358,9 +357,9 @@ void __blk_queue_split(struct bio **bio, unsigned int *nr_segs) * * Split a bio into two bios, chains the two bios, submit the second half and * store a pointer to the first half in *@bio. Since this function may allocate - * a new bio from @bio->bi_disk->queue->bio_split, it is the responsibility of - * the caller to ensure that @bio->bi_disk->queue->bio_split is only released - * after processing of the split bio has finished. + * a new bio from q->bio_split, it is the responsibility of the caller to ensure + * that q->bio_split is only released after processing of the split bio has + * finished. */ void blk_queue_split(struct bio **bio) { @@ -866,7 +865,7 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) return false; /* must be same device */ - if (rq->rq_disk != bio->bi_disk) + if (rq->rq_disk != bio->bi_bdev->bd_disk) return false; /* only merge integrity protected bio into ditto rq */ diff --git a/block/blk-mq.c b/block/blk-mq.c index f285a9123a8b..74b17b396f4c 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2128,7 +2128,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) */ blk_qc_t blk_mq_submit_bio(struct bio *bio) { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; const int is_sync = op_is_sync(bio->bi_opf); const int is_flush_fua = op_is_flush(bio->bi_opf); struct blk_mq_alloc_data data = { diff --git a/block/blk-throttle.c b/block/blk-throttle.c index d52cac9f3a7c..b1b22d863bdf 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -2178,7 +2178,7 @@ static inline void throtl_update_latency_buckets(struct throtl_data *td) bool blk_throtl_bio(struct bio *bio) { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; struct blkcg_gq *blkg = bio->bi_blkg; struct throtl_qnode *qn = NULL; struct throtl_grp *tg = blkg_to_tg(blkg); diff --git a/block/blk.h b/block/blk.h index 7550364c326c..10ab7c0d0766 100644 --- a/block/blk.h +++ b/block/blk.h @@ -202,8 +202,6 @@ static inline void elevator_exit(struct request_queue *q, __elevator_exit(q, e); } -struct block_device *__disk_get_part(struct gendisk *disk, int partno); - ssize_t part_size_show(struct device *dev, struct device_attribute *attr, char *buf); ssize_t part_stat_show(struct device *dev, struct device_attribute *attr, diff --git a/block/bounce.c b/block/bounce.c index d3f51acd6e3b..a22a8a1942b2 100644 --- a/block/bounce.c +++ b/block/bounce.c @@ -246,7 +246,7 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs); if (!bio) return NULL; - bio->bi_disk = bio_src->bi_disk; + bio->bi_bdev = bio_src->bi_bdev; bio->bi_opf = bio_src->bi_opf; bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; diff --git a/block/genhd.c b/block/genhd.c index ca5d880af512..e536d0b4bbae 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -161,7 +161,7 @@ static void part_in_flight_rw(struct block_device *part, inflight[1] = 0; } -struct block_device *__disk_get_part(struct gendisk *disk, int partno) +static struct block_device *__disk_get_part(struct gendisk *disk, int partno) { struct disk_part_tbl *ptbl = rcu_dereference(disk->part_tbl); diff --git a/drivers/block/brd.c b/drivers/block/brd.c index c7c821419079..18bf99906662 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -284,7 +284,7 @@ out: static blk_qc_t brd_submit_bio(struct bio *bio) { - struct brd_device *brd = bio->bi_disk->private_data; + struct brd_device *brd = bio->bi_bdev->bd_disk->private_data; sector_t sector = bio->bi_iter.bi_sector; struct bio_vec bvec; struct bvec_iter iter; diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index 8f879e5c2f67..b2c93a29c251 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -1579,8 +1579,8 @@ static inline void drbd_submit_bio_noacct(struct drbd_device *device, int fault_type, struct bio *bio) { __release(local); - if (!bio->bi_disk) { - drbd_err(device, "drbd_submit_bio_noacct: bio->bi_disk == NULL\n"); + if (!bio->bi_bdev) { + drbd_err(device, "drbd_submit_bio_noacct: bio->bi_bdev == NULL\n"); bio->bi_status = BLK_STS_IOERR; bio_endio(bio); return; diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index 330f851cb8f0..ea0f31ab3343 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -1595,7 +1595,7 @@ void do_submit(struct work_struct *ws) blk_qc_t drbd_submit_bio(struct bio *bio) { - struct drbd_device *device = bio->bi_disk->private_data; + struct drbd_device *device = bio->bi_bdev->bd_disk->private_data; unsigned long start_jif; blk_queue_split(&bio); diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 5357c3a4a36f..d6c821d48090 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -1420,7 +1420,7 @@ static blk_qc_t null_submit_bio(struct bio *bio) { sector_t sector = bio->bi_iter.bi_sector; sector_t nr_sectors = bio_sectors(bio); - struct nullb *nullb = bio->bi_disk->private_data; + struct nullb *nullb = bio->bi_bdev->bd_disk->private_data; struct nullb_queue *nq = nullb_to_queue(nullb); struct nullb_cmd *cmd; diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index b8bb8ec7538d..658a0981cb54 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -2374,7 +2374,7 @@ static blk_qc_t pkt_submit_bio(struct bio *bio) blk_queue_split(&bio); - pd = bio->bi_disk->queue->queuedata; + pd = bio->bi_bdev->bd_disk->queue->queuedata; if (!pd) { pr_err("%s incorrect request queue\n", bio_devname(bio, b)); goto end_io; @@ -2418,7 +2418,7 @@ static blk_qc_t pkt_submit_bio(struct bio *bio) split = bio; } - pkt_make_request_write(bio->bi_disk->queue, split); + pkt_make_request_write(bio->bi_bdev->bd_disk->queue, split); } while (split != bio); return BLK_QC_T_NONE; diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c index b71d28372ef3..1d738999fb69 100644 --- a/drivers/block/ps3vram.c +++ b/drivers/block/ps3vram.c @@ -581,7 +581,7 @@ out: static blk_qc_t ps3vram_submit_bio(struct bio *bio) { - struct ps3_system_bus_device *dev = bio->bi_disk->private_data; + struct ps3_system_bus_device *dev = bio->bi_bdev->bd_disk->private_data; struct ps3vram_priv *priv = ps3_system_bus_get_drvdata(dev); int busy; diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c index edacefff6e35..9a28322a8cd8 100644 --- a/drivers/block/rsxx/dev.c +++ b/drivers/block/rsxx/dev.c @@ -122,7 +122,7 @@ static void bio_dma_done_cb(struct rsxx_cardinfo *card, static blk_qc_t rsxx_submit_bio(struct bio *bio) { - struct rsxx_cardinfo *card = bio->bi_disk->private_data; + struct rsxx_cardinfo *card = bio->bi_bdev->bd_disk->private_data; struct rsxx_bio_meta *bio_meta; blk_status_t st = BLK_STS_IOERR; diff --git a/drivers/block/umem.c b/drivers/block/umem.c index 2b95d7b33b91..982732dbe82e 100644 --- a/drivers/block/umem.c +++ b/drivers/block/umem.c @@ -521,7 +521,7 @@ static int mm_check_plugged(struct cardinfo *card) static blk_qc_t mm_submit_bio(struct bio *bio) { - struct cardinfo *card = bio->bi_disk->private_data; + struct cardinfo *card = bio->bi_bdev->bd_disk->private_data; pr_debug("mm_make_request %llu %u\n", (unsigned long long)bio->bi_iter.bi_sector, diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index e2933cb7a82a..d6243dbc53cc 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1596,7 +1596,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio) */ static blk_qc_t zram_submit_bio(struct bio *bio) { - struct zram *zram = bio->bi_disk->private_data; + struct zram *zram = bio->bi_bdev->bd_disk->private_data; if (!valid_io_request(zram, bio->bi_iter.bi_sector, bio->bi_iter.bi_size)) { diff --git a/drivers/lightnvm/pblk-init.c b/drivers/lightnvm/pblk-init.c index b6246f73895c..5924f09c217b 100644 --- a/drivers/lightnvm/pblk-init.c +++ b/drivers/lightnvm/pblk-init.c @@ -49,7 +49,7 @@ struct bio_set pblk_bio_set; static blk_qc_t pblk_submit_bio(struct bio *bio) { - struct pblk *pblk = bio->bi_disk->queue->queuedata; + struct pblk *pblk = bio->bi_bdev->bd_disk->queue->queuedata; if (bio_op(bio) == REQ_OP_DISCARD) { pblk_discard(pblk, bio); diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c index b00fd08d696b..058dd8014428 100644 --- a/drivers/md/bcache/debug.c +++ b/drivers/md/bcache/debug.c @@ -114,7 +114,7 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio) check = bio_kmalloc(GFP_NOIO, bio_segments(bio)); if (!check) return; - check->bi_disk = bio->bi_disk; + check->bi_bdev = bio->bi_bdev; check->bi_opf = REQ_OP_READ; check->bi_iter.bi_sector = bio->bi_iter.bi_sector; check->bi_iter.bi_size = bio->bi_iter.bi_size; diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index 85b1f2a9b72d..dfc35d6d05ed 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -894,7 +894,8 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s, !(bio->bi_opf & (REQ_META|REQ_PRIO)) && s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA) reada = min_t(sector_t, dc->readahead >> 9, - get_capacity(bio->bi_disk) - bio_end_sector(bio)); + get_capacity(bio->bi_bdev->bd_disk) - + bio_end_sector(bio)); s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada); @@ -1167,7 +1168,7 @@ static void quit_max_writeback_rate(struct cache_set *c, blk_qc_t cached_dev_submit_bio(struct bio *bio) { struct search *s; - struct bcache_device *d = bio->bi_disk->private_data; + struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; struct cached_dev *dc = container_of(d, struct cached_dev, disk); int rw = bio_data_dir(bio); @@ -1274,7 +1275,7 @@ blk_qc_t flash_dev_submit_bio(struct bio *bio) { struct search *s; struct closure *cl; - struct bcache_device *d = bio->bi_disk->private_data; + struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; if (unlikely(d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags))) { bio->bi_status = BLK_STS_IOERR; diff --git a/drivers/md/dm-bio-record.h b/drivers/md/dm-bio-record.h index 2ea0360108e1..a3b71350eec8 100644 --- a/drivers/md/dm-bio-record.h +++ b/drivers/md/dm-bio-record.h @@ -18,8 +18,7 @@ */ struct dm_bio_details { - struct gendisk *bi_disk; - u8 bi_partno; + struct block_device *bi_bdev; int __bi_remaining; unsigned long bi_flags; struct bvec_iter bi_iter; @@ -31,8 +30,7 @@ struct dm_bio_details { static inline void dm_bio_record(struct dm_bio_details *bd, struct bio *bio) { - bd->bi_disk = bio->bi_disk; - bd->bi_partno = bio->bi_partno; + bd->bi_bdev = bio->bi_bdev; bd->bi_flags = bio->bi_flags; bd->bi_iter = bio->bi_iter; bd->__bi_remaining = atomic_read(&bio->__bi_remaining); @@ -44,8 +42,7 @@ static inline void dm_bio_record(struct dm_bio_details *bd, struct bio *bio) static inline void dm_bio_restore(struct dm_bio_details *bd, struct bio *bio) { - bio->bi_disk = bd->bi_disk; - bio->bi_partno = bd->bi_partno; + bio->bi_bdev = bd->bi_bdev; bio->bi_flags = bd->bi_flags; bio->bi_iter = bd->bi_iter; atomic_set(&bio->__bi_remaining, bd->__bi_remaining); diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index fa09bc4e4c54..b0a82f29a2e4 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -145,7 +145,7 @@ static void dispatch_bios(void *context, struct bio_list *bio_list) struct dm_raid1_bio_record { struct mirror *m; - /* if details->bi_disk == NULL, details were not saved */ + /* if details->bi_bdev == NULL, details were not saved */ struct dm_bio_details details; region_t write_region; }; @@ -1190,7 +1190,7 @@ static int mirror_map(struct dm_target *ti, struct bio *bio) struct dm_raid1_bio_record *bio_record = dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record)); - bio_record->details.bi_disk = NULL; + bio_record->details.bi_bdev = NULL; if (rw == WRITE) { /* Save region for mirror_end_io() handler */ @@ -1257,7 +1257,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, goto out; if (unlikely(*error)) { - if (!bio_record->details.bi_disk) { + if (!bio_record->details.bi_bdev) { /* * There wasn't enough memory to record necessary * information for a retry or there was no other @@ -1282,7 +1282,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, bd = &bio_record->details; dm_bio_restore(bd, bio); - bio_record->details.bi_disk = NULL; + bio_record->details.bi_bdev = NULL; bio->bi_status = 0; queue_bio(ms, bio, rw); @@ -1292,7 +1292,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, } out: - bio_record->details.bi_disk = NULL; + bio_record->details.bi_bdev = NULL; return DM_ENDIO_DONE; } diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 7bac564f3faa..479ec5bea09e 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -977,16 +977,17 @@ static void clone_endio(struct bio *bio) struct mapped_device *md = tio->io->md; dm_endio_fn endio = tio->ti->type->end_io; struct bio *orig_bio = io->orig_bio; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; if (unlikely(error == BLK_STS_TARGET)) { if (bio_op(bio) == REQ_OP_DISCARD && - !bio->bi_disk->queue->limits.max_discard_sectors) + !q->limits.max_discard_sectors) disable_discard(md); else if (bio_op(bio) == REQ_OP_WRITE_SAME && - !bio->bi_disk->queue->limits.max_write_same_sectors) + !q->limits.max_write_same_sectors) disable_write_same(md); else if (bio_op(bio) == REQ_OP_WRITE_ZEROES && - !bio->bi_disk->queue->limits.max_write_zeroes_sectors) + !q->limits.max_write_zeroes_sectors) disable_write_zeroes(md); } @@ -996,7 +997,7 @@ static void clone_endio(struct bio *bio) */ if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) { sector_t written_sector = bio->bi_iter.bi_sector; - struct request_queue *q = orig_bio->bi_disk->queue; + struct request_queue *q = orig_bio->bi_bdev->bd_disk->queue; u64 mask = (u64)blk_queue_zone_sectors(q) - 1; orig_bio->bi_iter.bi_sector += written_sector & mask; @@ -1422,8 +1423,7 @@ static int __send_empty_flush(struct clone_info *ci) */ bio_init(&flush_bio, NULL, 0); flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC; - flush_bio.bi_disk = ci->io->md->disk; - bio_associate_blkg(&flush_bio); + bio_set_dev(&flush_bio, ci->io->md->disk->part0); ci->bio = &flush_bio; ci->sector_count = 0; @@ -1626,7 +1626,7 @@ static blk_qc_t __split_and_process_bio(struct mapped_device *md, static blk_qc_t dm_submit_bio(struct bio *bio) { - struct mapped_device *md = bio->bi_disk->private_data; + struct mapped_device *md = bio->bi_bdev->bd_disk->private_data; blk_qc_t ret = BLK_QC_T_NONE; int srcu_idx; struct dm_table *map; diff --git a/drivers/md/md-linear.c b/drivers/md/md-linear.c index 68cac7d19278..63ed8329a98d 100644 --- a/drivers/md/md-linear.c +++ b/drivers/md/md-linear.c @@ -252,7 +252,7 @@ static bool linear_make_request(struct mddev *mddev, struct bio *bio) start_sector + data_offset; if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && - !blk_queue_discard(bio->bi_disk->queue))) { + !blk_queue_discard(bio->bi_bdev->bd_disk->queue))) { /* Just ignore it */ bio_endio(bio); } else { diff --git a/drivers/md/md.c b/drivers/md/md.c index 04384452a7ab..cf06dbb1aa53 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -486,7 +486,7 @@ static void md_end_io(struct bio *bio) static blk_qc_t md_submit_bio(struct bio *bio) { const int rw = bio_data_dir(bio); - struct mddev *mddev = bio->bi_disk->private_data; + struct mddev *mddev = bio->bi_bdev->bd_disk->private_data; if (mddev == NULL || mddev->pers == NULL) { bio_io_error(bio); diff --git a/drivers/md/md.h b/drivers/md/md.h index 34070ab30a8a..f13290ccc1c2 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -556,7 +556,7 @@ static inline void md_sync_acct(struct block_device *bdev, unsigned long nr_sect static inline void md_sync_acct_bio(struct bio *bio, unsigned long nr_sectors) { - atomic_add(nr_sectors, &bio->bi_disk->sync_io); + md_sync_acct(bio->bi_bdev, nr_sectors); } struct md_personality @@ -793,14 +793,14 @@ static inline void mddev_clear_unsupported_flags(struct mddev *mddev, static inline void mddev_check_writesame(struct mddev *mddev, struct bio *bio) { if (bio_op(bio) == REQ_OP_WRITE_SAME && - !bio->bi_disk->queue->limits.max_write_same_sectors) + !bio->bi_bdev->bd_disk->queue->limits.max_write_same_sectors) mddev->queue->limits.max_write_same_sectors = 0; } static inline void mddev_check_write_zeroes(struct mddev *mddev, struct bio *bio) { if (bio_op(bio) == REQ_OP_WRITE_ZEROES && - !bio->bi_disk->queue->limits.max_write_zeroes_sectors) + !bio->bi_bdev->bd_disk->queue->limits.max_write_zeroes_sectors) mddev->queue->limits.max_write_zeroes_sectors = 0; } diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index c0347997f6ff..3b19141cdb4b 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -794,13 +794,13 @@ static void flush_bio_list(struct r1conf *conf, struct bio *bio) while (bio) { /* submit pending writes */ struct bio *next = bio->bi_next; - struct md_rdev *rdev = (void *)bio->bi_disk; + struct md_rdev *rdev = (void *)bio->bi_bdev; bio->bi_next = NULL; bio_set_dev(bio, rdev->bdev); if (test_bit(Faulty, &rdev->flags)) { bio_io_error(bio); } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && - !blk_queue_discard(bio->bi_disk->queue))) + !blk_queue_discard(bio->bi_bdev->bd_disk->queue))) /* Just ignore it */ bio_endio(bio); else @@ -1520,7 +1520,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, trace_block_bio_remap(mbio, disk_devt(mddev->gendisk), r1_bio->sector); /* flush_pending_writes() needs access to the rdev so...*/ - mbio->bi_disk = (void *)conf->mirrors[i].rdev; + mbio->bi_bdev = (void *)conf->mirrors[i].rdev; cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug)); if (cb) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index c5d88ef6a45c..be8f14afb6d1 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -882,13 +882,13 @@ static void flush_pending_writes(struct r10conf *conf) while (bio) { /* submit pending writes */ struct bio *next = bio->bi_next; - struct md_rdev *rdev = (void*)bio->bi_disk; + struct md_rdev *rdev = (void*)bio->bi_bdev; bio->bi_next = NULL; bio_set_dev(bio, rdev->bdev); if (test_bit(Faulty, &rdev->flags)) { bio_io_error(bio); } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && - !blk_queue_discard(bio->bi_disk->queue))) + !blk_queue_discard(bio->bi_bdev->bd_disk->queue))) /* Just ignore it */ bio_endio(bio); else @@ -1075,13 +1075,13 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) while (bio) { /* submit pending writes */ struct bio *next = bio->bi_next; - struct md_rdev *rdev = (void*)bio->bi_disk; + struct md_rdev *rdev = (void*)bio->bi_bdev; bio->bi_next = NULL; bio_set_dev(bio, rdev->bdev); if (test_bit(Faulty, &rdev->flags)) { bio_io_error(bio); } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && - !blk_queue_discard(bio->bi_disk->queue))) + !blk_queue_discard(bio->bi_bdev->bd_disk->queue))) /* Just ignore it */ bio_endio(bio); else @@ -1253,7 +1253,7 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, trace_block_bio_remap(mbio, disk_devt(conf->mddev->gendisk), r10_bio->sector); /* flush_pending_writes() needs access to the rdev so...*/ - mbio->bi_disk = (void *)rdev; + mbio->bi_bdev = (void *)rdev; atomic_inc(&r10_bio->remaining); @@ -3003,7 +3003,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, /* Again, very different code for resync and recovery. * Both must result in an r10bio with a list of bios that - * have bi_end_io, bi_sector, bi_disk set, + * have bi_end_io, bi_sector, bi_bdev set, * and bi_private set to the r10bio. * For recovery, we may actually create several r10bios * with 2 bios in each, that correspond to the bios in the main one. diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 3a90cc0e43ca..f411b9e5c332 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -5310,7 +5310,7 @@ static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) unsigned int chunk_sectors; unsigned int bio_sectors = bio_sectors(bio); - WARN_ON_ONCE(bio->bi_partno); + WARN_ON_ONCE(bio->bi_bdev->bd_partno); chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors); return chunk_sectors >= diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c index 22e5617b2cea..e03a1f38d750 100644 --- a/drivers/nvdimm/blk.c +++ b/drivers/nvdimm/blk.c @@ -165,7 +165,7 @@ static int nsblk_do_bvec(struct nd_namespace_blk *nsblk, static blk_qc_t nd_blk_submit_bio(struct bio *bio) { struct bio_integrity_payload *bip; - struct nd_namespace_blk *nsblk = bio->bi_disk->private_data; + struct nd_namespace_blk *nsblk = bio->bi_bdev->bd_disk->private_data; struct bvec_iter iter; unsigned long start; struct bio_vec bvec; @@ -177,7 +177,7 @@ static blk_qc_t nd_blk_submit_bio(struct bio *bio) bip = bio_integrity(bio); rw = bio_data_dir(bio); - do_acct = blk_queue_io_stat(bio->bi_disk->queue); + do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); if (do_acct) start = bio_start_io_acct(bio); bio_for_each_segment(bvec, bio, iter) { diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 12ff6f8784ac..41aa1f01fc07 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -1442,7 +1442,7 @@ static int btt_do_bvec(struct btt *btt, struct bio_integrity_payload *bip, static blk_qc_t btt_submit_bio(struct bio *bio) { struct bio_integrity_payload *bip = bio_integrity(bio); - struct btt *btt = bio->bi_disk->private_data; + struct btt *btt = bio->bi_bdev->bd_disk->private_data; struct bvec_iter iter; unsigned long start; struct bio_vec bvec; @@ -1452,7 +1452,7 @@ static blk_qc_t btt_submit_bio(struct bio *bio) if (!bio_integrity_prep(bio)) return BLK_QC_T_NONE; - do_acct = blk_queue_io_stat(bio->bi_disk->queue); + do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); if (do_acct) start = bio_start_io_acct(bio); bio_for_each_segment(bvec, bio, iter) { diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 875076b0ea6c..72740835c85c 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -197,13 +197,13 @@ static blk_qc_t pmem_submit_bio(struct bio *bio) unsigned long start; struct bio_vec bvec; struct bvec_iter iter; - struct pmem_device *pmem = bio->bi_disk->private_data; + struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data; struct nd_region *nd_region = to_region(pmem); if (bio->bi_opf & REQ_PREFLUSH) ret = nvdimm_flush(nd_region, bio); - do_acct = blk_queue_io_stat(bio->bi_disk->queue); + do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); if (do_acct) start = bio_start_io_acct(bio); bio_for_each_segment(bvec, bio, iter) { diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 566788ba4e7d..a39befb4deba 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1113,7 +1113,7 @@ static int nvme_submit_user_cmd(struct request_queue *q, { bool write = nvme_is_write(cmd); struct nvme_ns *ns = q->queuedata; - struct gendisk *disk = ns ? ns->disk : NULL; + struct block_device *bdev = ns ? ns->disk->part0 : NULL; struct request *req; struct bio *bio = NULL; void *meta = NULL; @@ -1133,8 +1133,8 @@ static int nvme_submit_user_cmd(struct request_queue *q, if (ret) goto out; bio = req->bio; - bio->bi_disk = disk; - if (disk && meta_buffer && meta_len) { + bio->bi_bdev = bdev; + if (bdev && meta_buffer && meta_len) { meta = nvme_add_user_metadata(bio, meta_buffer, meta_len, meta_seed, write); if (IS_ERR(meta)) { diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c index 470cef3abec3..6c8eab8de288 100644 --- a/drivers/nvme/host/lightnvm.c +++ b/drivers/nvme/host/lightnvm.c @@ -757,7 +757,6 @@ static int nvme_nvm_submit_user_cmd(struct request_queue *q, { bool write = nvme_is_write((struct nvme_command *)vcmd); struct nvm_dev *dev = ns->ndev; - struct gendisk *disk = ns->disk; struct request *rq; struct bio *bio = NULL; __le64 *ppa_list = NULL; @@ -817,7 +816,7 @@ static int nvme_nvm_submit_user_cmd(struct request_queue *q, vcmd->ph_rw.metadata = cpu_to_le64(metadata_dma); } - bio->bi_disk = disk; + bio->bi_bdev = ns->disk->part0; } blk_execute_rq(q, NULL, rq, 0); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 9ac762b28811..a6d44e7a775f 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -296,7 +296,7 @@ static bool nvme_available_path(struct nvme_ns_head *head) blk_qc_t nvme_ns_head_submit_bio(struct bio *bio) { - struct nvme_ns_head *head = bio->bi_disk->private_data; + struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data; struct device *dev = disk_to_dev(head->disk); struct nvme_ns *ns; blk_qc_t ret = BLK_QC_T_NONE; @@ -312,7 +312,7 @@ blk_qc_t nvme_ns_head_submit_bio(struct bio *bio) srcu_idx = srcu_read_lock(&head->srcu); ns = nvme_find_path(head); if (likely(ns)) { - bio->bi_disk = ns->disk; + bio->bi_bdev = ns->disk->part0; bio->bi_opf |= REQ_NVME_MPATH; trace_block_bio_remap(bio, disk_devt(ns->head->disk), bio->bi_iter.bi_sector); @@ -352,7 +352,7 @@ static void nvme_requeue_work(struct work_struct *work) * Reset disk to the mpath node and resubmit to select a new * path. */ - bio->bi_disk = head->disk; + bio->bi_bdev = head->disk->part0; submit_bio_noacct(bio); } } diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index b7ce4f221d99..f5ef3edeb2fd 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -1468,7 +1468,7 @@ static int nvme_rdma_map_sg_pi(struct nvme_rdma_queue *queue, if (unlikely(nr)) goto mr_put; - nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_disk), c, + nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_bdev->bd_disk), c, req->mr->sig_attrs, ns->pi_type); nvme_rdma_set_prot_checks(c, &req->mr->sig_attrs->check_mask); diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index 5c5cff3f2374..da33cb4cba28 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -879,7 +879,7 @@ dcssblk_submit_bio(struct bio *bio) blk_queue_split(&bio); bytes_done = 0; - dev_info = bio->bi_disk->private_data; + dev_info = bio->bi_bdev->bd_disk->private_data; if (dev_info == NULL) goto fail; if ((bio->bi_iter.bi_sector & 7) != 0 || diff --git a/drivers/s390/block/xpram.c b/drivers/s390/block/xpram.c index c2536f7767b3..d1ed39162943 100644 --- a/drivers/s390/block/xpram.c +++ b/drivers/s390/block/xpram.c @@ -184,7 +184,7 @@ static unsigned long xpram_highest_page_index(void) */ static blk_qc_t xpram_submit_bio(struct bio *bio) { - xpram_device_t *xdev = bio->bi_disk->private_data; + xpram_device_t *xdev = bio->bi_bdev->bd_disk->private_data; struct bio_vec bvec; struct bvec_iter iter; unsigned int index; diff --git a/fs/btrfs/check-integrity.c b/fs/btrfs/check-integrity.c index 6ff44e53814c..113cb85c1fd4 100644 --- a/fs/btrfs/check-integrity.c +++ b/fs/btrfs/check-integrity.c @@ -2674,7 +2674,7 @@ static void __btrfsic_submit_bio(struct bio *bio) mutex_lock(&btrfsic_mutex); /* since btrfsic_submit_bio() is also called before * btrfsic_mount(), this might return NULL */ - dev_state = btrfsic_dev_state_lookup(bio_dev(bio) + bio->bi_partno); + dev_state = btrfsic_dev_state_lookup(bio->bi_bdev->bd_dev); if (NULL != dev_state && (bio_op(bio) == REQ_OP_WRITE) && bio_has_data(bio)) { unsigned int i = 0; @@ -2690,9 +2690,9 @@ static void __btrfsic_submit_bio(struct bio *bio) bio_is_patched = 0; if (dev_state->state->print_mask & BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH) - pr_info("submit_bio(rw=%d,0x%x, bi_vcnt=%u, bi_sector=%llu (bytenr %llu), bi_disk=%p)\n", + pr_info("submit_bio(rw=%d,0x%x, bi_vcnt=%u, bi_sector=%llu (bytenr %llu), bi_bdev=%p)\n", bio_op(bio), bio->bi_opf, segs, - bio->bi_iter.bi_sector, dev_bytenr, bio->bi_disk); + bio->bi_iter.bi_sector, dev_bytenr, bio->bi_bdev); mapped_datav = kmalloc_array(segs, sizeof(*mapped_datav), GFP_NOFS); @@ -2721,8 +2721,8 @@ static void __btrfsic_submit_bio(struct bio *bio) } else if (NULL != dev_state && (bio->bi_opf & REQ_PREFLUSH)) { if (dev_state->state->print_mask & BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH) - pr_info("submit_bio(rw=%d,0x%x FLUSH, disk=%p)\n", - bio_op(bio), bio->bi_opf, bio->bi_disk); + pr_info("submit_bio(rw=%d,0x%x FLUSH, bdev=%p)\n", + bio_op(bio), bio->bi_opf, bio->bi_bdev); if (!dev_state->dummy_block_for_bio_bh_flush.is_iodone) { if ((dev_state->state->print_mask & (BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH | diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 93fbf87bdc8d..b2204a2942cb 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1105,8 +1105,7 @@ static int rbio_add_io_page(struct btrfs_raid_bio *rbio, * devices or if they are not contiguous */ if (last_end == disk_start && !last->bi_status && - last->bi_disk == stripe->dev->bdev->bd_disk && - last->bi_partno == stripe->dev->bdev->bd_partno) { + last->bi_bdev == stripe->dev->bdev) { ret = bio_add_page(last, page, PAGE_SIZE, 0); if (ret == PAGE_SIZE) return 0; @@ -1357,9 +1356,7 @@ static int find_bio_stripe(struct btrfs_raid_bio *rbio, for (i = 0; i < rbio->bbio->num_stripes; i++) { stripe = &rbio->bbio->stripes[i]; if (in_range(physical, stripe->physical, rbio->stripe_len) && - stripe->dev->bdev && - bio->bi_disk == stripe->dev->bdev->bd_disk && - bio->bi_partno == stripe->dev->bdev->bd_partno) { + stripe->dev->bdev && bio->bi_bdev == stripe->dev->bdev) { return i; } } diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 5f4f88a4d2c8..33f8f0f108bf 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -1695,7 +1695,7 @@ static void scrub_wr_submit(struct scrub_ctx *sctx) sbio = sctx->wr_curr_bio; sctx->wr_curr_bio = NULL; - WARN_ON(!sbio->bio->bi_disk); + WARN_ON(!sbio->bio->bi_bdev); scrub_pending_bio_inc(sctx); /* process all writes in a single worker thread. Then the block layer * orders the requests before sending them to the driver which diff --git a/fs/direct-io.c b/fs/direct-io.c index d53fa92a1ab6..2660e744da2d 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -434,7 +434,7 @@ static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) bio_set_pages_dirty(bio); - dio->bio_disk = bio->bi_disk; + dio->bio_disk = bio->bi_bdev->bd_disk; if (sdio->submit_io) { sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index aa34d620bec9..8cbf03159752 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -427,16 +427,6 @@ int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) return 0; } -/* - * Return true, if pre_bio's bdev is same as its target device. - */ -static bool __same_bdev(struct f2fs_sb_info *sbi, - block_t blk_addr, struct bio *bio) -{ - struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL); - return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno; -} - static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) { struct f2fs_sb_info *sbi = fio->sbi; @@ -741,7 +731,7 @@ static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, return false; if (last_blkaddr + 1 != cur_blkaddr) return false; - return __same_bdev(sbi, cur_blkaddr, bio); + return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL); } static bool io_type_is_mergeable(struct f2fs_bio_info *io, diff --git a/include/linux/bio.h b/include/linux/bio.h index 1edda614f7ce..12af7aa5db37 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -483,24 +483,22 @@ extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int); extern unsigned int bvec_nr_vecs(unsigned short idx); extern const char *bio_devname(struct bio *bio, char *buffer); -#define bio_set_dev(bio, bdev) \ -do { \ - if ((bio)->bi_disk != (bdev)->bd_disk) \ - bio_clear_flag(bio, BIO_THROTTLED);\ - (bio)->bi_disk = (bdev)->bd_disk; \ - (bio)->bi_partno = (bdev)->bd_partno; \ - bio_associate_blkg(bio); \ +#define bio_set_dev(bio, bdev) \ +do { \ + if ((bio)->bi_bdev != (bdev)) \ + bio_clear_flag(bio, BIO_THROTTLED); \ + (bio)->bi_bdev = (bdev); \ + bio_associate_blkg(bio); \ } while (0) #define bio_copy_dev(dst, src) \ do { \ - (dst)->bi_disk = (src)->bi_disk; \ - (dst)->bi_partno = (src)->bi_partno; \ + (dst)->bi_bdev = (src)->bi_bdev; \ bio_clone_blkg_association(dst, src); \ } while (0) #define bio_dev(bio) \ - disk_devt((bio)->bi_disk) + disk_devt((bio)->bi_bdev->bd_disk) #ifdef CONFIG_BLK_CGROUP void bio_associate_blkg(struct bio *bio); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index d705b174d346..6b410dab48ee 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -602,8 +602,8 @@ static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio, rq->bio = rq->biotail = bio; rq->ioprio = bio_prio(bio); - if (bio->bi_disk) - rq->rq_disk = bio->bi_disk; + if (bio->bi_bdev) + rq->rq_disk = bio->bi_bdev->bd_disk; } blk_qc_t blk_mq_submit_bio(struct bio *bio); diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 866f74261b3b..8ebd8be3e050 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -222,7 +222,7 @@ static inline void bio_issue_init(struct bio_issue *issue, */ struct bio { struct bio *bi_next; /* request queue link */ - struct gendisk *bi_disk; + struct block_device *bi_bdev; unsigned int bi_opf; /* bottom bits req flags, * top bits REQ_OP. Use * accessors. @@ -231,7 +231,6 @@ struct bio { unsigned short bi_ioprio; unsigned short bi_write_hint; blk_status_t bi_status; - u8 bi_partno; atomic_t __bi_remaining; struct bvec_iter bi_iter; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index f94ee3089e01..b55bd534b2e1 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1967,7 +1967,8 @@ void part_end_io_acct(struct block_device *part, struct bio *bio, */ static inline unsigned long bio_start_io_acct(struct bio *bio) { - return disk_start_io_acct(bio->bi_disk, bio_sectors(bio), bio_op(bio)); + return disk_start_io_acct(bio->bi_bdev->bd_disk, bio_sectors(bio), + bio_op(bio)); } /** @@ -1977,7 +1978,7 @@ static inline unsigned long bio_start_io_acct(struct bio *bio) */ static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time) { - return disk_end_io_acct(bio->bi_disk, bio_op(bio), start_time); + return disk_end_io_acct(bio->bi_bdev->bd_disk, bio_op(bio), start_time); } int bdev_read_only(struct block_device *bdev); diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index fb0fe4c66b84..9e9ee4945043 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -903,7 +903,7 @@ static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio) { - blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BOUNCE, 0); + blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BOUNCE, 0); } static void blk_add_trace_bio_complete(void *ignore, @@ -915,22 +915,24 @@ static void blk_add_trace_bio_complete(void *ignore, static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio) { - blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BACKMERGE, 0); + blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BACKMERGE, + 0); } static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio) { - blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_FRONTMERGE, 0); + blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_FRONTMERGE, + 0); } static void blk_add_trace_bio_queue(void *ignore, struct bio *bio) { - blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_QUEUE, 0); + blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_QUEUE, 0); } static void blk_add_trace_getrq(void *ignore, struct bio *bio) { - blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_GETRQ, 0); + blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_GETRQ, 0); } static void blk_add_trace_plug(void *ignore, struct request_queue *q) @@ -967,7 +969,7 @@ static void blk_add_trace_unplug(void *ignore, struct request_queue *q, static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; struct blk_trace *bt; rcu_read_lock(); @@ -997,7 +999,7 @@ static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev, sector_t from) { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; struct blk_trace *bt; struct blk_io_trace_remap r; diff --git a/mm/page_io.c b/mm/page_io.c index 9bca17ecc4df..a75f35464a4e 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -433,7 +433,7 @@ int swap_readpage(struct page *page, bool synchronous) ret = -ENOMEM; goto out; } - disk = bio->bi_disk; + disk = bio->bi_bdev->bd_disk; /* * Keep this task valid during swap readpage because the oom killer may * attempt to access it in the page fault retry time check. -- cgit v1.2.3 From 814b84971388cd5fb182f2e914265b3827758455 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 21 Jan 2021 16:34:37 -0500 Subject: pNFS/NFSv4: Fix a layout segment leak in pnfs_layout_process() If the server returns a new stateid that does not match the one in our cache, then pnfs_layout_process() will leak the layout segments returned by pnfs_mark_layout_stateid_invalid(). Fixes: 9888d837f3cf ("pNFS: Force a retry of LAYOUTGET if the stateid doesn't match our cache") Signed-off-by: Trond Myklebust --- fs/nfs/pnfs.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 4f274f21c4ab..e68e6f8cb407 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -2417,6 +2417,7 @@ out_forget: spin_unlock(&ino->i_lock); lseg->pls_layout = lo; NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); + pnfs_free_lseg_list(&free_me); return ERR_PTR(-EAGAIN); } -- cgit v1.2.3 From 08bd8dbe88825760e953759d7ec212903a026c75 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 21 Jan 2021 17:11:42 -0500 Subject: pNFS/NFSv4: Try to return invalid layout in pnfs_layout_process() If the server returns a new stateid that does not match the one in our cache, then try to return the one we hold instead of just invalidating it on the client side. This ensures that both client and server will agree that the stateid is invalid. Signed-off-by: Trond Myklebust --- fs/nfs/pnfs.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index e68e6f8cb407..d6262289cf4a 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -2398,7 +2398,13 @@ pnfs_layout_process(struct nfs4_layoutget *lgp) * We got an entirely new state ID. Mark all segments for the * inode invalid, and retry the layoutget */ - pnfs_mark_layout_stateid_invalid(lo, &free_me); + struct pnfs_layout_range range = { + .iomode = IOMODE_ANY, + .length = NFS4_MAX_UINT64, + }; + pnfs_set_plh_return_info(lo, IOMODE_ANY, 0); + pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, + &range, 0); goto out_forget; } @@ -2417,7 +2423,6 @@ out_forget: spin_unlock(&ino->i_lock); lseg->pls_layout = lo; NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); - pnfs_free_lseg_list(&free_me); return ERR_PTR(-EAGAIN); } -- cgit v1.2.3 From 1bcf34fdac5f8c2fcd16796495db75744612ca27 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 21 Jan 2021 13:51:50 -0500 Subject: pNFS/NFSv4: Update the layout barrier when we schedule a layoutreturn When we're scheduling a layoutreturn, we need to ignore any further incoming layouts with sequence ids that are going to be affected by the layout return. Fixes: 44ea8dfce021 ("NFS/pnfs: Reference the layout cred in pnfs_prepare_layoutreturn()") Signed-off-by: Trond Myklebust --- fs/nfs/pnfs.c | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'fs') diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index d6262289cf4a..acb63ec00053 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -324,6 +324,21 @@ pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo) return NULL; } +/* + * Compare 2 layout stateid sequence ids, to see which is newer, + * taking into account wraparound issues. + */ +static bool pnfs_seqid_is_newer(u32 s1, u32 s2) +{ + return (s32)(s1 - s2) > 0; +} + +static void pnfs_barrier_update(struct pnfs_layout_hdr *lo, u32 newseq) +{ + if (pnfs_seqid_is_newer(newseq, lo->plh_barrier)) + lo->plh_barrier = newseq; +} + static void pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, u32 seq) @@ -335,6 +350,7 @@ pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, if (seq != 0) { WARN_ON_ONCE(lo->plh_return_seq != 0 && lo->plh_return_seq != seq); lo->plh_return_seq = seq; + pnfs_barrier_update(lo, seq); } } @@ -639,15 +655,6 @@ static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, return rv; } -/* - * Compare 2 layout stateid sequence ids, to see which is newer, - * taking into account wraparound issues. - */ -static bool pnfs_seqid_is_newer(u32 s1, u32 s2) -{ - return (s32)(s1 - s2) > 0; -} - static bool pnfs_should_free_range(const struct pnfs_layout_range *lseg_range, const struct pnfs_layout_range *recall_range) @@ -984,8 +991,7 @@ pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, new_barrier = be32_to_cpu(new->seqid); else if (new_barrier == 0) return; - if (pnfs_seqid_is_newer(new_barrier, lo->plh_barrier)) - lo->plh_barrier = new_barrier; + pnfs_barrier_update(lo, new_barrier); } static bool @@ -1183,20 +1189,17 @@ pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, return false; set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); pnfs_get_layout_hdr(lo); + nfs4_stateid_copy(stateid, &lo->plh_stateid); + *cred = get_cred(lo->plh_lc_cred); if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { - nfs4_stateid_copy(stateid, &lo->plh_stateid); - *cred = get_cred(lo->plh_lc_cred); if (lo->plh_return_seq != 0) stateid->seqid = cpu_to_be32(lo->plh_return_seq); if (iomode != NULL) *iomode = lo->plh_return_iomode; pnfs_clear_layoutreturn_info(lo); - return true; - } - nfs4_stateid_copy(stateid, &lo->plh_stateid); - *cred = get_cred(lo->plh_lc_cred); - if (iomode != NULL) + } else if (iomode != NULL) *iomode = IOMODE_ANY; + pnfs_barrier_update(lo, be32_to_cpu(stateid->seqid)); return true; } -- cgit v1.2.3 From d29b468da4f940bd2bff2628ba8d2d652671d244 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 22 Jan 2021 10:05:51 -0500 Subject: pNFS/NFSv4: Improve rejection of out-of-order layouts If a layoutget ends up being reordered w.r.t. a layoutreturn, e.g. due to a layoutget-on-open not knowing a priori which file to lock, then we must assume the layout is no longer being considered valid state by the server. Incrementally improve our ability to reject such states by using the cached old stateid in conjunction with the plh_barrier to try to identify them. Signed-off-by: Trond Myklebust --- fs/nfs/pnfs.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index acb63ec00053..af64b4e6fd1f 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1000,7 +1000,7 @@ pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, { u32 seqid = be32_to_cpu(stateid->seqid); - return !pnfs_seqid_is_newer(seqid, lo->plh_barrier); + return !pnfs_seqid_is_newer(seqid, lo->plh_barrier) && lo->plh_barrier; } /* lget is set to 1 if called from inside send_layoutget call chain */ @@ -1912,6 +1912,11 @@ static void nfs_layoutget_end(struct pnfs_layout_hdr *lo) wake_up_var(&lo->plh_outstanding); } +static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) +{ + return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); +} + static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) { unsigned long *bitlock = &lo->plh_flags; @@ -2386,17 +2391,17 @@ pnfs_layout_process(struct nfs4_layoutget *lgp) goto out_forget; } - if (!pnfs_layout_is_valid(lo)) { - /* We have a completely new layout */ - pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); - } else if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { + if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { /* existing state ID, make sure the sequence number matches. */ if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { + if (!pnfs_layout_is_valid(lo) && + pnfs_is_first_layoutget(lo)) + lo->plh_barrier = 0; dprintk("%s forget reply due to sequence\n", __func__); goto out_forget; } pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false); - } else { + } else if (pnfs_layout_is_valid(lo)) { /* * We got an entirely new state ID. Mark all segments for the * inode invalid, and retry the layoutget @@ -2409,6 +2414,11 @@ pnfs_layout_process(struct nfs4_layoutget *lgp) pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, &range, 0); goto out_forget; + } else { + /* We have a completely new layout */ + if (!pnfs_is_first_layoutget(lo)) + goto out_forget; + pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); } pnfs_get_lseg(lseg); -- cgit v1.2.3 From 684da7628d93bbdcfba9081b917d99f29ad04c23 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Mon, 25 Jan 2021 05:49:58 +0100 Subject: block: remove unnecessary argument from blk_execute_rq We can remove 'q' from blk_execute_rq as well after the previous change in blk_execute_rq_nowait. And more importantly it never really was needed to start with given that we can trivial derive it from struct request. Cc: linux-scsi@vger.kernel.org Cc: virtualization@lists.linux-foundation.org Cc: linux-ide@vger.kernel.org Cc: linux-mmc@vger.kernel.org Cc: linux-nvme@lists.infradead.org Cc: linux-nfs@vger.kernel.org Acked-by: Ulf Hansson # for mmc Signed-off-by: Guoqing Jiang Signed-off-by: Jens Axboe --- block/blk-exec.c | 3 +-- block/bsg.c | 2 +- block/scsi_ioctl.c | 6 +++--- drivers/block/mtip32xx/mtip32xx.c | 2 +- drivers/block/paride/pd.c | 2 +- drivers/block/pktcdvd.c | 2 +- drivers/block/virtio_blk.c | 2 +- drivers/cdrom/cdrom.c | 2 +- drivers/ide/ide-atapi.c | 2 +- drivers/ide/ide-cd.c | 2 +- drivers/ide/ide-cd_ioctl.c | 2 +- drivers/ide/ide-devsets.c | 2 +- drivers/ide/ide-disk.c | 2 +- drivers/ide/ide-ioctls.c | 4 ++-- drivers/ide/ide-park.c | 2 +- drivers/ide/ide-pm.c | 4 ++-- drivers/ide/ide-tape.c | 2 +- drivers/ide/ide-taskfile.c | 2 +- drivers/mmc/core/block.c | 10 +++++----- drivers/nvme/host/core.c | 4 ++-- drivers/nvme/host/lightnvm.c | 2 +- drivers/scsi/scsi_lib.c | 2 +- fs/nfsd/blocklayout.c | 2 +- include/linux/blkdev.h | 3 +-- 24 files changed, 33 insertions(+), 35 deletions(-) (limited to 'fs') diff --git a/block/blk-exec.c b/block/blk-exec.c index 2e37e85456fb..0ab873f10133 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c @@ -74,8 +74,7 @@ EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); * Insert a fully prepared request at the back of the I/O scheduler queue * for execution and wait for completion. */ -void blk_execute_rq(struct request_queue *q, struct gendisk *bd_disk, - struct request *rq, int at_head) +void blk_execute_rq(struct gendisk *bd_disk, struct request *rq, int at_head) { DECLARE_COMPLETION_ONSTACK(wait); unsigned long hang_check; diff --git a/block/bsg.c b/block/bsg.c index 3d78e843a83f..bd10922d5cbb 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -183,7 +183,7 @@ static int bsg_sg_io(struct request_queue *q, fmode_t mode, void __user *uarg) bio = rq->bio; - blk_execute_rq(q, NULL, rq, !(hdr.flags & BSG_FLAG_Q_AT_TAIL)); + blk_execute_rq(NULL, rq, !(hdr.flags & BSG_FLAG_Q_AT_TAIL)); ret = rq->q->bsg_dev.ops->complete_rq(rq, &hdr); blk_rq_unmap_user(bio); diff --git a/block/scsi_ioctl.c b/block/scsi_ioctl.c index c9f009cc0446..6599bac0a78c 100644 --- a/block/scsi_ioctl.c +++ b/block/scsi_ioctl.c @@ -357,7 +357,7 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk, * (if he doesn't check that is his problem). * N.B. a non-zero SCSI status is _not_ necessarily an error. */ - blk_execute_rq(q, bd_disk, rq, at_head); + blk_execute_rq(bd_disk, rq, at_head); hdr->duration = jiffies_to_msecs(jiffies - start_time); @@ -493,7 +493,7 @@ int sg_scsi_ioctl(struct request_queue *q, struct gendisk *disk, fmode_t mode, goto error; } - blk_execute_rq(q, disk, rq, 0); + blk_execute_rq(disk, rq, 0); err = req->result & 0xff; /* only 8 bit SCSI status */ if (err) { @@ -532,7 +532,7 @@ static int __blk_send_generic(struct request_queue *q, struct gendisk *bd_disk, scsi_req(rq)->cmd[0] = cmd; scsi_req(rq)->cmd[4] = data; scsi_req(rq)->cmd_len = 6; - blk_execute_rq(q, bd_disk, rq, 0); + blk_execute_rq(bd_disk, rq, 0); err = scsi_req(rq)->result ? -EIO : 0; blk_put_request(rq); diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 53ac59d19ae5..3fd99836bb1c 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -1015,7 +1015,7 @@ static int mtip_exec_internal_command(struct mtip_port *port, rq->timeout = timeout; /* insert request and run queue */ - blk_execute_rq(rq->q, NULL, rq, true); + blk_execute_rq(NULL, rq, true); if (int_cmd->status) { dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n", diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index a7af4f27b7c3..897acda20ac8 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -781,7 +781,7 @@ static int pd_special_command(struct pd_unit *disk, req = blk_mq_rq_to_pdu(rq); req->func = func; - blk_execute_rq(disk->gd->queue, disk->gd, rq, 0); + blk_execute_rq(disk->gd, rq, 0); blk_put_request(rq); return 0; } diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 658a0981cb54..fc4b0f1aa86d 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -722,7 +722,7 @@ static int pkt_generic_packet(struct pktcdvd_device *pd, struct packet_command * if (cgc->quiet) rq->rq_flags |= RQF_QUIET; - blk_execute_rq(rq->q, pd->bdev->bd_disk, rq, 0); + blk_execute_rq(pd->bdev->bd_disk, rq, 0); if (scsi_req(rq)->result) ret = -EIO; out: diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 145606dc52db..b0285db7cf4f 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -320,7 +320,7 @@ static int virtblk_get_id(struct gendisk *disk, char *id_str) if (err) goto out; - blk_execute_rq(vblk->disk->queue, vblk->disk, req, false); + blk_execute_rq(vblk->disk, req, false); err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req))); out: blk_put_request(req); diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c index 8f0e52a71493..90ad34c6ef8e 100644 --- a/drivers/cdrom/cdrom.c +++ b/drivers/cdrom/cdrom.c @@ -2214,7 +2214,7 @@ static int cdrom_read_cdda_bpc(struct cdrom_device_info *cdi, __u8 __user *ubuf, rq->timeout = 60 * HZ; bio = rq->bio; - blk_execute_rq(q, cdi->disk, rq, 0); + blk_execute_rq(cdi->disk, rq, 0); if (scsi_req(rq)->result) { struct scsi_sense_hdr sshdr; diff --git a/drivers/ide/ide-atapi.c b/drivers/ide/ide-atapi.c index 013ad33fbbc8..a1ce9f5ac3aa 100644 --- a/drivers/ide/ide-atapi.c +++ b/drivers/ide/ide-atapi.c @@ -107,7 +107,7 @@ int ide_queue_pc_tail(ide_drive_t *drive, struct gendisk *disk, memcpy(scsi_req(rq)->cmd, pc->c, 12); if (drive->media == ide_tape) scsi_req(rq)->cmd[13] = REQ_IDETAPE_PC1; - blk_execute_rq(drive->queue, disk, rq, 0); + blk_execute_rq(disk, rq, 0); error = scsi_req(rq)->result ? -EIO : 0; put_req: blk_put_request(rq); diff --git a/drivers/ide/ide-cd.c b/drivers/ide/ide-cd.c index 25d2d88e82ad..cffbcc27a34c 100644 --- a/drivers/ide/ide-cd.c +++ b/drivers/ide/ide-cd.c @@ -467,7 +467,7 @@ int ide_cd_queue_pc(ide_drive_t *drive, const unsigned char *cmd, } } - blk_execute_rq(drive->queue, info->disk, rq, 0); + blk_execute_rq(info->disk, rq, 0); error = scsi_req(rq)->result ? -EIO : 0; if (buffer) diff --git a/drivers/ide/ide-cd_ioctl.c b/drivers/ide/ide-cd_ioctl.c index 46f2df288c6a..011eab9c69b7 100644 --- a/drivers/ide/ide-cd_ioctl.c +++ b/drivers/ide/ide-cd_ioctl.c @@ -299,7 +299,7 @@ int ide_cdrom_reset(struct cdrom_device_info *cdi) rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0); ide_req(rq)->type = ATA_PRIV_MISC; rq->rq_flags = RQF_QUIET; - blk_execute_rq(drive->queue, cd->disk, rq, 0); + blk_execute_rq(cd->disk, rq, 0); ret = scsi_req(rq)->result ? -EIO : 0; blk_put_request(rq); /* diff --git a/drivers/ide/ide-devsets.c b/drivers/ide/ide-devsets.c index f2f93ed40356..ca1d4b3d3878 100644 --- a/drivers/ide/ide-devsets.c +++ b/drivers/ide/ide-devsets.c @@ -173,7 +173,7 @@ int ide_devset_execute(ide_drive_t *drive, const struct ide_devset *setting, *(int *)&scsi_req(rq)->cmd[1] = arg; ide_req(rq)->special = setting->set; - blk_execute_rq(q, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); ret = scsi_req(rq)->result; blk_put_request(rq); diff --git a/drivers/ide/ide-disk.c b/drivers/ide/ide-disk.c index 34b9441084f8..8413731c6259 100644 --- a/drivers/ide/ide-disk.c +++ b/drivers/ide/ide-disk.c @@ -482,7 +482,7 @@ static int set_multcount(ide_drive_t *drive, int arg) drive->mult_req = arg; drive->special_flags |= IDE_SFLAG_SET_MULTMODE; - blk_execute_rq(drive->queue, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); blk_put_request(rq); return (drive->mult_count == arg) ? 0 : -EIO; diff --git a/drivers/ide/ide-ioctls.c b/drivers/ide/ide-ioctls.c index 58994da10c06..43fbc37d85c3 100644 --- a/drivers/ide/ide-ioctls.c +++ b/drivers/ide/ide-ioctls.c @@ -137,7 +137,7 @@ static int ide_cmd_ioctl(ide_drive_t *drive, void __user *argp) rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0); ide_req(rq)->type = ATA_PRIV_TASKFILE; - blk_execute_rq(drive->queue, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); err = scsi_req(rq)->result ? -EIO : 0; blk_put_request(rq); @@ -235,7 +235,7 @@ static int generic_drive_reset(ide_drive_t *drive) ide_req(rq)->type = ATA_PRIV_MISC; scsi_req(rq)->cmd_len = 1; scsi_req(rq)->cmd[0] = REQ_DRIVE_RESET; - blk_execute_rq(drive->queue, NULL, rq, 1); + blk_execute_rq(NULL, rq, 1); ret = scsi_req(rq)->result; blk_put_request(rq); return ret; diff --git a/drivers/ide/ide-park.c b/drivers/ide/ide-park.c index 8af7af6001eb..a80a0f28f7b9 100644 --- a/drivers/ide/ide-park.c +++ b/drivers/ide/ide-park.c @@ -37,7 +37,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout) scsi_req(rq)->cmd_len = 1; ide_req(rq)->type = ATA_PRIV_MISC; ide_req(rq)->special = &timeout; - blk_execute_rq(q, NULL, rq, 1); + blk_execute_rq(NULL, rq, 1); rc = scsi_req(rq)->result ? -EIO : 0; blk_put_request(rq); if (rc) diff --git a/drivers/ide/ide-pm.c b/drivers/ide/ide-pm.c index 82ab308f1aaf..d680b3e3295f 100644 --- a/drivers/ide/ide-pm.c +++ b/drivers/ide/ide-pm.c @@ -27,7 +27,7 @@ int generic_ide_suspend(struct device *dev, pm_message_t mesg) mesg.event = PM_EVENT_FREEZE; rqpm.pm_state = mesg.event; - blk_execute_rq(drive->queue, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); ret = scsi_req(rq)->result ? -EIO : 0; blk_put_request(rq); @@ -50,7 +50,7 @@ static int ide_pm_execute_rq(struct request *rq) blk_mq_end_request(rq, BLK_STS_OK); return -ENXIO; } - blk_execute_rq(q, NULL, rq, true); + blk_execute_rq(NULL, rq, true); return scsi_req(rq)->result ? -EIO : 0; } diff --git a/drivers/ide/ide-tape.c b/drivers/ide/ide-tape.c index 88b96437b22e..fa05e7e7d609 100644 --- a/drivers/ide/ide-tape.c +++ b/drivers/ide/ide-tape.c @@ -868,7 +868,7 @@ static int idetape_queue_rw_tail(ide_drive_t *drive, int cmd, int size) goto out_put; } - blk_execute_rq(drive->queue, tape->disk, rq, 0); + blk_execute_rq(tape->disk, rq, 0); /* calculate the number of transferred bytes and update buffer state */ size -= scsi_req(rq)->resid_len; diff --git a/drivers/ide/ide-taskfile.c b/drivers/ide/ide-taskfile.c index d016cbe68cba..6665fc4724b9 100644 --- a/drivers/ide/ide-taskfile.c +++ b/drivers/ide/ide-taskfile.c @@ -443,7 +443,7 @@ int ide_raw_taskfile(ide_drive_t *drive, struct ide_cmd *cmd, u8 *buf, ide_req(rq)->special = cmd; cmd->rq = rq; - blk_execute_rq(drive->queue, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); error = scsi_req(rq)->result ? -EIO : 0; put_req: blk_put_request(rq); diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 42e27a298218..a1d6b68320ae 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -253,7 +253,7 @@ static ssize_t power_ro_lock_store(struct device *dev, goto out_put; } req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP; - blk_execute_rq(mq->queue, NULL, req, 0); + blk_execute_rq(NULL, req, 0); ret = req_to_mmc_queue_req(req)->drv_op_result; blk_put_request(req); @@ -629,7 +629,7 @@ static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md, rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL; req_to_mmc_queue_req(req)->drv_op_data = idatas; req_to_mmc_queue_req(req)->ioc_count = 1; - blk_execute_rq(mq->queue, NULL, req, 0); + blk_execute_rq(NULL, req, 0); ioc_err = req_to_mmc_queue_req(req)->drv_op_result; err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata); blk_put_request(req); @@ -698,7 +698,7 @@ static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md, rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL; req_to_mmc_queue_req(req)->drv_op_data = idata; req_to_mmc_queue_req(req)->ioc_count = num_of_cmds; - blk_execute_rq(mq->queue, NULL, req, 0); + blk_execute_rq(NULL, req, 0); ioc_err = req_to_mmc_queue_req(req)->drv_op_result; /* copy to user if data and response */ @@ -2722,7 +2722,7 @@ static int mmc_dbg_card_status_get(void *data, u64 *val) if (IS_ERR(req)) return PTR_ERR(req); req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS; - blk_execute_rq(mq->queue, NULL, req, 0); + blk_execute_rq(NULL, req, 0); ret = req_to_mmc_queue_req(req)->drv_op_result; if (ret >= 0) { *val = ret; @@ -2761,7 +2761,7 @@ static int mmc_ext_csd_open(struct inode *inode, struct file *filp) } req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD; req_to_mmc_queue_req(req)->drv_op_data = &ext_csd; - blk_execute_rq(mq->queue, NULL, req, 0); + blk_execute_rq(NULL, req, 0); err = req_to_mmc_queue_req(req)->drv_op_result; blk_put_request(req); if (err) { diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 0bea9ae03092..eb7963fb167b 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -964,7 +964,7 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, if (poll) nvme_execute_rq_polled(req->q, NULL, req, at_head); else - blk_execute_rq(req->q, NULL, req, at_head); + blk_execute_rq(NULL, req, at_head); if (result) *result = nvme_req(req)->result; if (nvme_req(req)->flags & NVME_REQ_CANCELLED) @@ -1101,7 +1101,7 @@ void nvme_execute_passthru_rq(struct request *rq) u32 effects; effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode); - blk_execute_rq(rq->q, disk, rq, 0); + blk_execute_rq(disk, rq, 0); nvme_passthru_end(ctrl, effects); } EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU); diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c index 0e5a55075e35..ec38128f51e9 100644 --- a/drivers/nvme/host/lightnvm.c +++ b/drivers/nvme/host/lightnvm.c @@ -819,7 +819,7 @@ static int nvme_nvm_submit_user_cmd(struct request_queue *q, bio->bi_bdev = ns->disk->part0; } - blk_execute_rq(q, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); if (nvme_req(rq)->flags & NVME_REQ_CANCELLED) ret = -EINTR; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index b3f14f05340a..4d2280658559 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -269,7 +269,7 @@ int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, /* * head injection *required* here otherwise quiesce won't work */ - blk_execute_rq(req->q, NULL, req, 1); + blk_execute_rq(NULL, req, 1); /* * Some devices (USB mass-storage in particular) may transfer diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c index a07c39c94bbd..1058659a8d31 100644 --- a/fs/nfsd/blocklayout.c +++ b/fs/nfsd/blocklayout.c @@ -254,7 +254,7 @@ again: req->cmd[4] = bufflen & 0xff; req->cmd_len = COMMAND_SIZE(INQUIRY); - blk_execute_rq(rq->q, NULL, rq, 1); + blk_execute_rq(NULL, rq, 1); if (req->result) { pr_err("pNFS: INQUIRY 0x83 failed with: %x\n", req->result); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 623a61239429..20f3706b6b2e 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -948,8 +948,7 @@ extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, uns extern int blk_rq_map_user_iov(struct request_queue *, struct request *, struct rq_map_data *, const struct iov_iter *, gfp_t); -extern void blk_execute_rq(struct request_queue *, struct gendisk *, - struct request *, int); +extern void blk_execute_rq(struct gendisk *, struct request *, int); extern void blk_execute_rq_nowait(struct gendisk *, struct request *, int, rq_end_io_fn *); -- cgit v1.2.3 From bff2e532d433a1de3ec11aa0a525ceb08ba8f960 Mon Sep 17 00:00:00 2001 From: Andrew Price Date: Tue, 12 Jan 2021 11:46:56 +0000 Subject: gfs2: Fix invalid block size message Signed-off-by: Andrew Price Signed-off-by: Andreas Gruenbacher Signed-off-by: Bob Peterson --- fs/gfs2/ops_fstype.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index f2c6bbe5cdb8..6118f975030c 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -179,7 +179,7 @@ static int gfs2_check_sb(struct gfs2_sbd *sdp, int silent) if (sb->sb_bsize < 512 || sb->sb_bsize > PAGE_SIZE || (sb->sb_bsize & (sb->sb_bsize - 1))) { - pr_warn("Invalid superblock size\n"); + pr_warn("Invalid block size\n"); return -EINVAL; } -- cgit v1.2.3 From 4a011849f7595c95a221551ced9179aa3ffb4890 Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Wed, 20 Jan 2021 10:12:59 -0500 Subject: Revert "GFS2: Re-add a call to log_flush_wait when flushing the journal" This reverts commit 428fd95d859b24fea448380fa21ad6d841b34241. Patch 428fd95d85b2 added a call to log_flush_wait to function gfs2_log_flush. Then gfs2_log_flush calls log_write_header which submits a write request with the REQ_PREFLUSH flag which also forces it to wait. This patch removes the unnecessary call to log_flush_wait. Signed-off-by: Bob Peterson --- fs/gfs2/log.c | 1 - 1 file changed, 1 deletion(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 2e9314091c81..41d13f19d1b4 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -1004,7 +1004,6 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) goto out_withdraw; if (sdp->sd_log_head != sdp->sd_log_flush_head) { - log_flush_wait(sdp); log_write_header(sdp, flags); } else if (sdp->sd_log_tail != current_tail(sdp) && !sdp->sd_log_idle){ atomic_dec(&sdp->sd_log_blks_free); /* Adjust for unreserved buffer */ -- cgit v1.2.3 From f5f02fde9f52b2d769c1c2ddfd3d9c4a1fe739a7 Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Mon, 18 Jan 2021 15:18:59 -0500 Subject: gfs2: fix glock confusion in function signal_our_withdraw If go_free is defined, function signal_our_withdraw is supposed to synchronize on the GLF_FREEING flag of the inode glock, but it accidentally does that on the live glock. Fix that and disambiguate the glock variables. Fixes: 601ef0d52e96 ("gfs2: Force withdraw to replay journals and wait for it to finish") Cc: stable@vger.kernel.org # v5.7+ Signed-off-by: Bob Peterson --- fs/gfs2/util.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/util.c b/fs/gfs2/util.c index 97b1bdc76871..8d3c670c990f 100644 --- a/fs/gfs2/util.c +++ b/fs/gfs2/util.c @@ -118,9 +118,10 @@ void gfs2_freeze_unlock(struct gfs2_holder *freeze_gh) static void signal_our_withdraw(struct gfs2_sbd *sdp) { - struct gfs2_glock *gl = sdp->sd_live_gh.gh_gl; + struct gfs2_glock *live_gl = sdp->sd_live_gh.gh_gl; struct inode *inode = sdp->sd_jdesc->jd_inode; struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_glock *i_gl = ip->i_gl; u64 no_formal_ino = ip->i_no_formal_ino; int log_write_allowed = test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); int ret = 0; @@ -180,7 +181,8 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) atomic_set(&sdp->sd_freeze_state, SFS_FROZEN); thaw_super(sdp->sd_vfs); } else { - wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE); + wait_on_bit(&i_gl->gl_flags, GLF_DEMOTE, + TASK_UNINTERRUPTIBLE); } /* @@ -200,15 +202,15 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) * on other nodes to be successful, otherwise we remain the owner of * the glock as far as dlm is concerned. */ - if (gl->gl_ops->go_free) { - set_bit(GLF_FREEING, &gl->gl_flags); - wait_on_bit(&gl->gl_flags, GLF_FREEING, TASK_UNINTERRUPTIBLE); + if (i_gl->gl_ops->go_free) { + set_bit(GLF_FREEING, &i_gl->gl_flags); + wait_on_bit(&i_gl->gl_flags, GLF_FREEING, TASK_UNINTERRUPTIBLE); } /* * Dequeue the "live" glock, but keep a reference so it's never freed. */ - gfs2_glock_hold(gl); + gfs2_glock_hold(live_gl); gfs2_glock_dq_wait(&sdp->sd_live_gh); /* * We enqueue the "live" glock in EX so that all other nodes @@ -247,7 +249,7 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) gfs2_glock_nq(&sdp->sd_live_gh); } - gfs2_glock_queue_put(gl); /* drop the extra reference we acquired */ + gfs2_glock_queue_put(live_gl); /* drop extra reference we acquired */ clear_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags); /* -- cgit v1.2.3 From 82218943058d5e3fe692a38b5a549479738dab33 Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Thu, 21 Jan 2021 10:10:26 -0500 Subject: gfs2: keep bios separate for each journal The recovery func can recover multiple journals, but they were all using the same bio. This resulted in use-after-free related to sdp->sd_log_bio. This patch moves the variable to the journal descriptor, jd, so that every recovery can operate on its own bio. And hopefully we never run out. Signed-off-by: Bob Peterson --- fs/gfs2/incore.h | 2 +- fs/gfs2/log.c | 6 +++--- fs/gfs2/lops.c | 14 ++++++++------ fs/gfs2/lops.h | 5 +++-- fs/gfs2/recovery.c | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index 8e1ab8ed4abc..9933af070121 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -531,6 +531,7 @@ struct gfs2_jdesc { unsigned int nr_extents; struct work_struct jd_work; struct inode *jd_inode; + struct bio *jd_log_bio; unsigned long jd_flags; #define JDF_RECOVERY 1 unsigned int jd_jid; @@ -844,7 +845,6 @@ struct gfs2_sbd { struct rw_semaphore sd_log_flush_lock; atomic_t sd_log_in_flight; - struct bio *sd_log_bio; wait_queue_head_t sd_log_flush_wait; int sd_log_error; /* First log error */ wait_queue_head_t sd_withdraw_wait; diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 41d13f19d1b4..d3fa14ec2c44 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -822,8 +822,8 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, sb->s_blocksize - LH_V1_SIZE - 4); lh->lh_crc = cpu_to_be32(crc); - gfs2_log_write(sdp, page, sb->s_blocksize, 0, dblock); - gfs2_log_submit_bio(&sdp->sd_log_bio, REQ_OP_WRITE | op_flags); + gfs2_log_write(sdp, jd, page, sb->s_blocksize, 0, dblock); + gfs2_log_submit_bio(&jd->jd_log_bio, REQ_OP_WRITE | op_flags); out: log_flush_wait(sdp); } @@ -999,7 +999,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) lops_before_commit(sdp, tr); if (gfs2_withdrawn(sdp)) goto out_withdraw; - gfs2_log_submit_bio(&sdp->sd_log_bio, REQ_OP_WRITE); + gfs2_log_submit_bio(&sdp->sd_jdesc->jd_log_bio, REQ_OP_WRITE); if (gfs2_withdrawn(sdp)) goto out_withdraw; diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index 9e9dd486bed9..921e156d3181 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c @@ -322,17 +322,18 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno, * then add the page segment to that. */ -void gfs2_log_write(struct gfs2_sbd *sdp, struct page *page, - unsigned size, unsigned offset, u64 blkno) +void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, + struct page *page, unsigned size, unsigned offset, + u64 blkno) { struct bio *bio; int ret; - bio = gfs2_log_get_bio(sdp, blkno, &sdp->sd_log_bio, REQ_OP_WRITE, + bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, REQ_OP_WRITE, gfs2_end_log_write, false); ret = bio_add_page(bio, page, size, offset); if (ret == 0) { - bio = gfs2_log_get_bio(sdp, blkno, &sdp->sd_log_bio, + bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, REQ_OP_WRITE, gfs2_end_log_write, true); ret = bio_add_page(bio, page, size, offset); WARN_ON(ret == 0); @@ -355,7 +356,8 @@ static void gfs2_log_write_bh(struct gfs2_sbd *sdp, struct buffer_head *bh) dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head); gfs2_log_incr_head(sdp); - gfs2_log_write(sdp, bh->b_page, bh->b_size, bh_offset(bh), dblock); + gfs2_log_write(sdp, sdp->sd_jdesc, bh->b_page, bh->b_size, + bh_offset(bh), dblock); } /** @@ -376,7 +378,7 @@ static void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page) dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head); gfs2_log_incr_head(sdp); - gfs2_log_write(sdp, page, sb->s_blocksize, 0, dblock); + gfs2_log_write(sdp, sdp->sd_jdesc, page, sb->s_blocksize, 0, dblock); } /** diff --git a/fs/gfs2/lops.h b/fs/gfs2/lops.h index 2280f68862de..9a85b9d7ad60 100644 --- a/fs/gfs2/lops.h +++ b/fs/gfs2/lops.h @@ -20,8 +20,9 @@ extern const struct gfs2_log_operations *gfs2_log_ops[]; extern void gfs2_log_incr_head(struct gfs2_sbd *sdp); extern u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lbn); -extern void gfs2_log_write(struct gfs2_sbd *sdp, struct page *page, - unsigned size, unsigned offset, u64 blkno); +extern void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, + struct page *page, unsigned size, unsigned offset, + u64 blkno); extern void gfs2_log_submit_bio(struct bio **biop, int opf); extern void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh); extern int gfs2_find_jhead(struct gfs2_jdesc *jd, diff --git a/fs/gfs2/recovery.c b/fs/gfs2/recovery.c index 74ab1fcaf558..cff8b9a6612c 100644 --- a/fs/gfs2/recovery.c +++ b/fs/gfs2/recovery.c @@ -505,7 +505,7 @@ void gfs2_recover_func(struct work_struct *work) /* We take the sd_log_flush_lock here primarily to prevent log * flushes and simultaneous journal replays from stomping on - * each other wrt sd_log_bio. */ + * each other wrt jd_log_bio. */ down_read(&sdp->sd_log_flush_lock); for (pass = 0; pass < 2; pass++) { lops_before_scan(jd, &head, pass); -- cgit v1.2.3 From 2289e87b5951f97783f07fc895e6c5e804b53668 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 17 Sep 2020 17:22:49 -0400 Subject: SUNRPC: Make trace_svc_process() display the RPC procedure symbolically The next few patches will employ these strings to help make server- side trace logs more human-readable. A similar technique is already in use in kernel RPC client code. Signed-off-by: Chuck Lever --- fs/lockd/svc4proc.c | 24 ++++++++++++++++++++++++ fs/lockd/svcproc.c | 24 ++++++++++++++++++++++++ fs/nfs/callback_xdr.c | 2 ++ fs/nfsd/nfs2acl.c | 5 +++++ fs/nfsd/nfs3acl.c | 3 +++ fs/nfsd/nfs3proc.c | 22 ++++++++++++++++++++++ fs/nfsd/nfs4proc.c | 2 ++ fs/nfsd/nfsproc.c | 18 ++++++++++++++++++ include/linux/sunrpc/svc.h | 1 + 9 files changed, 101 insertions(+) (limited to 'fs') diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c index fa41dda39925..4c10fb5138f1 100644 --- a/fs/lockd/svc4proc.c +++ b/fs/lockd/svc4proc.c @@ -512,6 +512,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "NULL", }, [NLMPROC_TEST] = { .pc_func = nlm4svc_proc_test, @@ -520,6 +521,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+2+No+Rg, + .pc_name = "TEST", }, [NLMPROC_LOCK] = { .pc_func = nlm4svc_proc_lock, @@ -528,6 +530,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "LOCK", }, [NLMPROC_CANCEL] = { .pc_func = nlm4svc_proc_cancel, @@ -536,6 +539,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "CANCEL", }, [NLMPROC_UNLOCK] = { .pc_func = nlm4svc_proc_unlock, @@ -544,6 +548,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "UNLOCK", }, [NLMPROC_GRANTED] = { .pc_func = nlm4svc_proc_granted, @@ -552,6 +557,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "GRANTED", }, [NLMPROC_TEST_MSG] = { .pc_func = nlm4svc_proc_test_msg, @@ -560,6 +566,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "TEST_MSG", }, [NLMPROC_LOCK_MSG] = { .pc_func = nlm4svc_proc_lock_msg, @@ -568,6 +575,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "LOCK_MSG", }, [NLMPROC_CANCEL_MSG] = { .pc_func = nlm4svc_proc_cancel_msg, @@ -576,6 +584,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "CANCEL_MSG", }, [NLMPROC_UNLOCK_MSG] = { .pc_func = nlm4svc_proc_unlock_msg, @@ -584,6 +593,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNLOCK_MSG", }, [NLMPROC_GRANTED_MSG] = { .pc_func = nlm4svc_proc_granted_msg, @@ -592,6 +602,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "GRANTED_MSG", }, [NLMPROC_TEST_RES] = { .pc_func = nlm4svc_proc_null, @@ -600,6 +611,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "TEST_RES", }, [NLMPROC_LOCK_RES] = { .pc_func = nlm4svc_proc_null, @@ -608,6 +620,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "LOCK_RES", }, [NLMPROC_CANCEL_RES] = { .pc_func = nlm4svc_proc_null, @@ -616,6 +629,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "CANCEL_RES", }, [NLMPROC_UNLOCK_RES] = { .pc_func = nlm4svc_proc_null, @@ -624,6 +638,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNLOCK_RES", }, [NLMPROC_GRANTED_RES] = { .pc_func = nlm4svc_proc_granted_res, @@ -632,6 +647,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "GRANTED_RES", }, [NLMPROC_NSM_NOTIFY] = { .pc_func = nlm4svc_proc_sm_notify, @@ -640,6 +656,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_reboot), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "SM_NOTIFY", }, [17] = { .pc_func = nlm4svc_proc_unused, @@ -648,6 +665,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, + .pc_name = "UNUSED", }, [18] = { .pc_func = nlm4svc_proc_unused, @@ -656,6 +674,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, + .pc_name = "UNUSED", }, [19] = { .pc_func = nlm4svc_proc_unused, @@ -664,6 +683,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, + .pc_name = "UNUSED", }, [NLMPROC_SHARE] = { .pc_func = nlm4svc_proc_share, @@ -672,6 +692,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, + .pc_name = "SHARE", }, [NLMPROC_UNSHARE] = { .pc_func = nlm4svc_proc_unshare, @@ -680,6 +701,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, + .pc_name = "UNSHARE", }, [NLMPROC_NM_LOCK] = { .pc_func = nlm4svc_proc_nm_lock, @@ -688,6 +710,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "NM_LOCK", }, [NLMPROC_FREE_ALL] = { .pc_func = nlm4svc_proc_free_all, @@ -696,5 +719,6 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "FREE_ALL", }, }; diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c index 50855f2c1f4b..4ae4b63b5392 100644 --- a/fs/lockd/svcproc.c +++ b/fs/lockd/svcproc.c @@ -554,6 +554,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "NULL", }, [NLMPROC_TEST] = { .pc_func = nlmsvc_proc_test, @@ -562,6 +563,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+2+No+Rg, + .pc_name = "TEST", }, [NLMPROC_LOCK] = { .pc_func = nlmsvc_proc_lock, @@ -570,6 +572,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "LOCK", }, [NLMPROC_CANCEL] = { .pc_func = nlmsvc_proc_cancel, @@ -578,6 +581,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "CANCEL", }, [NLMPROC_UNLOCK] = { .pc_func = nlmsvc_proc_unlock, @@ -586,6 +590,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "UNLOCK", }, [NLMPROC_GRANTED] = { .pc_func = nlmsvc_proc_granted, @@ -594,6 +599,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "GRANTED", }, [NLMPROC_TEST_MSG] = { .pc_func = nlmsvc_proc_test_msg, @@ -602,6 +608,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "TEST_MSG", }, [NLMPROC_LOCK_MSG] = { .pc_func = nlmsvc_proc_lock_msg, @@ -610,6 +617,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "LOCK_MSG", }, [NLMPROC_CANCEL_MSG] = { .pc_func = nlmsvc_proc_cancel_msg, @@ -618,6 +626,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "CANCEL_MSG", }, [NLMPROC_UNLOCK_MSG] = { .pc_func = nlmsvc_proc_unlock_msg, @@ -626,6 +635,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNLOCK_MSG", }, [NLMPROC_GRANTED_MSG] = { .pc_func = nlmsvc_proc_granted_msg, @@ -634,6 +644,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "GRANTED_MSG", }, [NLMPROC_TEST_RES] = { .pc_func = nlmsvc_proc_null, @@ -642,6 +653,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "TEST_RES", }, [NLMPROC_LOCK_RES] = { .pc_func = nlmsvc_proc_null, @@ -650,6 +662,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "LOCK_RES", }, [NLMPROC_CANCEL_RES] = { .pc_func = nlmsvc_proc_null, @@ -658,6 +671,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "CANCEL_RES", }, [NLMPROC_UNLOCK_RES] = { .pc_func = nlmsvc_proc_null, @@ -666,6 +680,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNLOCK_RES", }, [NLMPROC_GRANTED_RES] = { .pc_func = nlmsvc_proc_granted_res, @@ -674,6 +689,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "GRANTED_RES", }, [NLMPROC_NSM_NOTIFY] = { .pc_func = nlmsvc_proc_sm_notify, @@ -682,6 +698,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_reboot), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "SM_NOTIFY", }, [17] = { .pc_func = nlmsvc_proc_unused, @@ -690,6 +707,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNUSED", }, [18] = { .pc_func = nlmsvc_proc_unused, @@ -698,6 +716,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNUSED", }, [19] = { .pc_func = nlmsvc_proc_unused, @@ -706,6 +725,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNUSED", }, [NLMPROC_SHARE] = { .pc_func = nlmsvc_proc_share, @@ -714,6 +734,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, + .pc_name = "SHARE", }, [NLMPROC_UNSHARE] = { .pc_func = nlmsvc_proc_unshare, @@ -722,6 +743,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, + .pc_name = "UNSHARE", }, [NLMPROC_NM_LOCK] = { .pc_func = nlmsvc_proc_nm_lock, @@ -730,6 +752,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "NM_LOCK", }, [NLMPROC_FREE_ALL] = { .pc_func = nlmsvc_proc_free_all, @@ -738,5 +761,6 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, + .pc_name = "FREE_ALL", }, }; diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c index 79ff172eb1c8..c5348ba81129 100644 --- a/fs/nfs/callback_xdr.c +++ b/fs/nfs/callback_xdr.c @@ -1060,6 +1060,7 @@ static const struct svc_procedure nfs4_callback_procedures1[] = { .pc_decode = nfs4_decode_void, .pc_encode = nfs4_encode_void, .pc_xdrressize = 1, + .pc_name = "NULL", }, [CB_COMPOUND] = { .pc_func = nfs4_callback_compound, @@ -1067,6 +1068,7 @@ static const struct svc_procedure nfs4_callback_procedures1[] = { .pc_argsize = 256, .pc_ressize = 256, .pc_xdrressize = NFS4_CALLBACK_BUFSIZE, + .pc_name = "COMPOUND", } }; diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index b0f66604532a..899762da23c9 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -371,6 +371,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST, + .pc_name = "NULL", }, [ACLPROC2_GETACL] = { .pc_func = nfsacld_proc_getacl, @@ -381,6 +382,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_ressize = sizeof(struct nfsd3_getaclres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+1+2*(1+ACL), + .pc_name = "GETACL", }, [ACLPROC2_SETACL] = { .pc_func = nfsacld_proc_setacl, @@ -391,6 +393,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, + .pc_name = "SETACL", }, [ACLPROC2_GETATTR] = { .pc_func = nfsacld_proc_getattr, @@ -401,6 +404,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, + .pc_name = "GETATTR", }, [ACLPROC2_ACCESS] = { .pc_func = nfsacld_proc_access, @@ -411,6 +415,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_ressize = sizeof(struct nfsd3_accessres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT+1, + .pc_name = "SETATTR", }, }; diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index 7c30876a31a1..9e1a92fb9771 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -251,6 +251,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST, + .pc_name = "NULL", }, [ACLPROC3_GETACL] = { .pc_func = nfsd3_proc_getacl, @@ -261,6 +262,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_ressize = sizeof(struct nfsd3_getaclres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+1+2*(1+ACL), + .pc_name = "GETACL", }, [ACLPROC3_SETACL] = { .pc_func = nfsd3_proc_setacl, @@ -271,6 +273,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_ressize = sizeof(struct nfsd3_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT, + .pc_name = "SETACL", }, }; diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index 76931f4f57c3..c9c64471c568 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -708,6 +708,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST, + .pc_name = "NULL", }, [NFS3PROC_GETATTR] = { .pc_func = nfsd3_proc_getattr, @@ -718,6 +719,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_attrstatres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, + .pc_name = "GETATTR", }, [NFS3PROC_SETATTR] = { .pc_func = nfsd3_proc_setattr, @@ -728,6 +730,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_wccstatres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC, + .pc_name = "SETATTR", }, [NFS3PROC_LOOKUP] = { .pc_func = nfsd3_proc_lookup, @@ -738,6 +741,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_diropres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+FH+pAT+pAT, + .pc_name = "LOOKUP", }, [NFS3PROC_ACCESS] = { .pc_func = nfsd3_proc_access, @@ -748,6 +752,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_accessres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+1, + .pc_name = "ACCESS", }, [NFS3PROC_READLINK] = { .pc_func = nfsd3_proc_readlink, @@ -758,6 +763,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_readlinkres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+1+NFS3_MAXPATHLEN/4, + .pc_name = "READLINK", }, [NFS3PROC_READ] = { .pc_func = nfsd3_proc_read, @@ -768,6 +774,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_readres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+4+NFSSVC_MAXBLKSIZE/4, + .pc_name = "READ", }, [NFS3PROC_WRITE] = { .pc_func = nfsd3_proc_write, @@ -778,6 +785,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_writeres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC+4, + .pc_name = "WRITE", }, [NFS3PROC_CREATE] = { .pc_func = nfsd3_proc_create, @@ -788,6 +796,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "CREATE", }, [NFS3PROC_MKDIR] = { .pc_func = nfsd3_proc_mkdir, @@ -798,6 +807,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "MKDIR", }, [NFS3PROC_SYMLINK] = { .pc_func = nfsd3_proc_symlink, @@ -808,6 +818,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "SYMLINK", }, [NFS3PROC_MKNOD] = { .pc_func = nfsd3_proc_mknod, @@ -818,6 +829,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "MKNOD", }, [NFS3PROC_REMOVE] = { .pc_func = nfsd3_proc_remove, @@ -828,6 +840,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_wccstatres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC, + .pc_name = "REMOVE", }, [NFS3PROC_RMDIR] = { .pc_func = nfsd3_proc_rmdir, @@ -838,6 +851,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_wccstatres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC, + .pc_name = "RMDIR", }, [NFS3PROC_RENAME] = { .pc_func = nfsd3_proc_rename, @@ -848,6 +862,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_renameres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC+WC, + .pc_name = "RENAME", }, [NFS3PROC_LINK] = { .pc_func = nfsd3_proc_link, @@ -858,6 +873,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_linkres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+pAT+WC, + .pc_name = "LINK", }, [NFS3PROC_READDIR] = { .pc_func = nfsd3_proc_readdir, @@ -867,6 +883,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_argsize = sizeof(struct nfsd3_readdirargs), .pc_ressize = sizeof(struct nfsd3_readdirres), .pc_cachetype = RC_NOCACHE, + .pc_name = "READDIR", }, [NFS3PROC_READDIRPLUS] = { .pc_func = nfsd3_proc_readdirplus, @@ -876,6 +893,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_argsize = sizeof(struct nfsd3_readdirplusargs), .pc_ressize = sizeof(struct nfsd3_readdirres), .pc_cachetype = RC_NOCACHE, + .pc_name = "READDIRPLUS", }, [NFS3PROC_FSSTAT] = { .pc_func = nfsd3_proc_fsstat, @@ -885,6 +903,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_fsstatres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+2*6+1, + .pc_name = "FSSTAT", }, [NFS3PROC_FSINFO] = { .pc_func = nfsd3_proc_fsinfo, @@ -894,6 +913,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_fsinfores), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+12, + .pc_name = "FSINFO", }, [NFS3PROC_PATHCONF] = { .pc_func = nfsd3_proc_pathconf, @@ -903,6 +923,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_pathconfres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+6, + .pc_name = "PATHCONF", }, [NFS3PROC_COMMIT] = { .pc_func = nfsd3_proc_commit, @@ -913,6 +934,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_commitres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+WC+2, + .pc_name = "COMMIT", }, }; diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 8d6d2678abad..f567592692ee 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -3305,6 +3305,7 @@ static const struct svc_procedure nfsd_procedures4[2] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 1, + .pc_name = "NULL", }, [NFSPROC4_COMPOUND] = { .pc_func = nfsd4_proc_compound, @@ -3315,6 +3316,7 @@ static const struct svc_procedure nfsd_procedures4[2] = { .pc_release = nfsd4_release_compoundargs, .pc_cachetype = RC_NOCACHE, .pc_xdrressize = NFSD_BUFSIZE/4, + .pc_name = "COMPOUND", }, }; diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 9473d048efec..1f85a4dc9d1b 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -623,6 +623,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 0, + .pc_name = "NULL", }, [NFSPROC_GETATTR] = { .pc_func = nfsd_proc_getattr, @@ -633,6 +634,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, + .pc_name = "GETATTR", }, [NFSPROC_SETATTR] = { .pc_func = nfsd_proc_setattr, @@ -643,6 +645,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+AT, + .pc_name = "SETATTR", }, [NFSPROC_ROOT] = { .pc_func = nfsd_proc_root, @@ -652,6 +655,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 0, + .pc_name = "ROOT", }, [NFSPROC_LOOKUP] = { .pc_func = nfsd_proc_lookup, @@ -662,6 +666,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_diropres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+FH+AT, + .pc_name = "LOOKUP", }, [NFSPROC_READLINK] = { .pc_func = nfsd_proc_readlink, @@ -671,6 +676,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_readlinkres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+1+NFS_MAXPATHLEN/4, + .pc_name = "READLINK", }, [NFSPROC_READ] = { .pc_func = nfsd_proc_read, @@ -681,6 +687,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_readres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT+1+NFSSVC_MAXBLKSIZE_V2/4, + .pc_name = "READ", }, [NFSPROC_WRITECACHE] = { .pc_func = nfsd_proc_writecache, @@ -690,6 +697,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 0, + .pc_name = "WRITECACHE", }, [NFSPROC_WRITE] = { .pc_func = nfsd_proc_write, @@ -700,6 +708,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+AT, + .pc_name = "WRITE", }, [NFSPROC_CREATE] = { .pc_func = nfsd_proc_create, @@ -710,6 +719,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_diropres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+FH+AT, + .pc_name = "CREATE", }, [NFSPROC_REMOVE] = { .pc_func = nfsd_proc_remove, @@ -719,6 +729,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, + .pc_name = "REMOVE", }, [NFSPROC_RENAME] = { .pc_func = nfsd_proc_rename, @@ -728,6 +739,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, + .pc_name = "RENAME", }, [NFSPROC_LINK] = { .pc_func = nfsd_proc_link, @@ -737,6 +749,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, + .pc_name = "LINK", }, [NFSPROC_SYMLINK] = { .pc_func = nfsd_proc_symlink, @@ -746,6 +759,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, + .pc_name = "SYMLINK", }, [NFSPROC_MKDIR] = { .pc_func = nfsd_proc_mkdir, @@ -756,6 +770,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_diropres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+FH+AT, + .pc_name = "MKDIR", }, [NFSPROC_RMDIR] = { .pc_func = nfsd_proc_rmdir, @@ -765,6 +780,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, + .pc_name = "RMDIR", }, [NFSPROC_READDIR] = { .pc_func = nfsd_proc_readdir, @@ -773,6 +789,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_argsize = sizeof(struct nfsd_readdirargs), .pc_ressize = sizeof(struct nfsd_readdirres), .pc_cachetype = RC_NOCACHE, + .pc_name = "READDIR", }, [NFSPROC_STATFS] = { .pc_func = nfsd_proc_statfs, @@ -782,6 +799,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_statfsres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+5, + .pc_name = "STATFS", }, }; diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 34c2a69820e9..31ee3b6047c3 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -463,6 +463,7 @@ struct svc_procedure { unsigned int pc_ressize; /* result struct size */ unsigned int pc_cachetype; /* cache info (NFS) */ unsigned int pc_xdrressize; /* maximum size of XDR reply */ + const char * pc_name; /* for display */ }; /* -- cgit v1.2.3 From 9575363a9e4c8d7e2f9ba5e79884d623fff0be6f Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 14:30:02 -0400 Subject: NFSD: Update GETATTR3args decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3proc.c | 3 +-- fs/nfsd/nfs3xdr.c | 31 +++++++++++++++++++++++++------ fs/nfsd/xdr3.h | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index c9c64471c568..4b66f055141b 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -683,7 +683,6 @@ out: * NFSv3 Server procedures. * Only the results of non-idempotent operations are cached. */ -#define nfs3svc_decode_fhandleargs nfs3svc_decode_fhandle #define nfs3svc_encode_attrstatres nfs3svc_encode_attrstat #define nfs3svc_encode_wccstatres nfs3svc_encode_wccstat #define nfsd3_mkdirargs nfsd3_createargs @@ -715,7 +714,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_decode = nfs3svc_decode_fhandleargs, .pc_encode = nfs3svc_encode_attrstatres, .pc_release = nfs3svc_release_fhandle, - .pc_argsize = sizeof(struct nfsd3_fhandleargs), + .pc_argsize = sizeof(struct nfsd_fhandle), .pc_ressize = sizeof(struct nfsd3_attrstatres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 34b880211e5e..3a2b4abea1a4 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -29,8 +29,9 @@ static u32 nfs3_ftypes[] = { /* - * XDR functions for basic NFS types + * Basic NFSv3 data types (RFC 1813 Sections 2.5 and 2.6) */ + static __be32 * encode_time3(__be32 *p, struct timespec64 *time) { @@ -46,6 +47,26 @@ decode_time3(__be32 *p, struct timespec64 *time) return p; } +static bool +svcxdr_decode_nfs_fh3(struct xdr_stream *xdr, struct svc_fh *fhp) +{ + __be32 *p; + u32 size; + + if (xdr_stream_decode_u32(xdr, &size) < 0) + return false; + if (size == 0 || size > NFS3_FHSIZE) + return false; + p = xdr_inline_decode(xdr, size); + if (!p) + return false; + fh_init(fhp, NFS3_FHSIZE); + fhp->fh_handle.fh_size = size; + memcpy(&fhp->fh_handle.fh_base, p, size); + + return true; +} + static __be32 * decode_fh(__be32 *p, struct svc_fh *fhp) { @@ -312,14 +333,12 @@ void fill_post_wcc(struct svc_fh *fhp) */ int -nfs3svc_decode_fhandle(struct svc_rqst *rqstp, __be32 *p) +nfs3svc_decode_fhandleargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_fhandle *args = rqstp->rq_argp; - p = decode_fh(p, &args->fh); - if (!p) - return 0; - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_nfs_fh3(xdr, &args->fh); } int diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index 456fcd7a1038..62ea669768cf 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -273,7 +273,7 @@ union nfsd3_xdrstore { #define NFS3_SVC_XDRSIZE sizeof(union nfsd3_xdrstore) -int nfs3svc_decode_fhandle(struct svc_rqst *, __be32 *); +int nfs3svc_decode_fhandleargs(struct svc_rqst *, __be32 *); int nfs3svc_decode_sattrargs(struct svc_rqst *, __be32 *); int nfs3svc_decode_diropargs(struct svc_rqst *, __be32 *); int nfs3svc_decode_accessargs(struct svc_rqst *, __be32 *); -- cgit v1.2.3 From 3b921a2b14251e9e203f1e8af76e8ade79f50e50 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 14:32:04 -0400 Subject: NFSD: Update ACCESS3arg decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 9 +++++---- fs/nfsd/xdr3.h | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 3a2b4abea1a4..e07cebd80ef7 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -375,14 +375,15 @@ nfs3svc_decode_diropargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_accessargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_accessargs *args = rqstp->rq_argp; - p = decode_fh(p, &args->fh); - if (!p) + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return 0; + if (xdr_stream_decode_u32(xdr, &args->access) < 0) return 0; - args->access = ntohl(*p++); - return xdr_argsize_check(rqstp, p); + return 1; } int diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index 62ea669768cf..a4dce4baec7c 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -25,7 +25,7 @@ struct nfsd3_diropargs { struct nfsd3_accessargs { struct svc_fh fh; - unsigned int access; + __u32 access; }; struct nfsd3_readargs { -- cgit v1.2.3 From be63bd2ac6bbf8c065a0ef6dfbea76934326c352 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 14:34:40 -0400 Subject: NFSD: Update READ3arg decoder to use struct xdr_stream The code that sets up rq_vec is refactored so that it is now adjacent to the nfsd_read() call site where it is used. Signed-off-by: Chuck Lever --- fs/nfsd/nfs3proc.c | 23 ++++++++++++++++++----- fs/nfsd/nfs3xdr.c | 28 +++++++--------------------- fs/nfsd/xdr3.h | 1 - 3 files changed, 25 insertions(+), 27 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index 4b66f055141b..acdf47179a38 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -144,25 +144,38 @@ nfsd3_proc_read(struct svc_rqst *rqstp) { struct nfsd3_readargs *argp = rqstp->rq_argp; struct nfsd3_readres *resp = rqstp->rq_resp; - u32 max_blocksize = svc_max_payload(rqstp); - unsigned long cnt = min(argp->count, max_blocksize); + u32 max_blocksize = svc_max_payload(rqstp); + unsigned int len; + int v; + + argp->count = min_t(u32, argp->count, max_blocksize); dprintk("nfsd: READ(3) %s %lu bytes at %Lu\n", SVCFH_fmt(&argp->fh), (unsigned long) argp->count, (unsigned long long) argp->offset); + v = 0; + len = argp->count; + while (len > 0) { + struct page *page = *(rqstp->rq_next_page++); + + rqstp->rq_vec[v].iov_base = page_address(page); + rqstp->rq_vec[v].iov_len = min_t(unsigned int, len, PAGE_SIZE); + len -= rqstp->rq_vec[v].iov_len; + v++; + } + /* Obtain buffer pointer for payload. * 1 (status) + 22 (post_op_attr) + 1 (count) + 1 (eof) * + 1 (xdr opaque byte count) = 26 */ - resp->count = cnt; + resp->count = argp->count; svc_reserve_auth(rqstp, ((1 + NFS3_POST_OP_ATTR_WORDS + 3)<<2) + resp->count +4); fh_copy(&resp->fh, &argp->fh); resp->status = nfsd_read(rqstp, &resp->fh, argp->offset, - rqstp->rq_vec, argp->vlen, &resp->count, - &resp->eof); + rqstp->rq_vec, v, &resp->count, &resp->eof); return rpc_success; } diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index e07cebd80ef7..2f32df15a7e8 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -389,31 +389,17 @@ nfs3svc_decode_accessargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_readargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_readargs *args = rqstp->rq_argp; - unsigned int len; - int v; - u32 max_blocksize = svc_max_payload(rqstp); - p = decode_fh(p, &args->fh); - if (!p) + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return 0; + if (xdr_stream_decode_u64(xdr, &args->offset) < 0) + return 0; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) return 0; - p = xdr_decode_hyper(p, &args->offset); - - args->count = ntohl(*p++); - len = min(args->count, max_blocksize); - - /* set up the kvec */ - v=0; - while (len > 0) { - struct page *p = *(rqstp->rq_next_page++); - rqstp->rq_vec[v].iov_base = page_address(p); - rqstp->rq_vec[v].iov_len = min_t(unsigned int, len, PAGE_SIZE); - len -= rqstp->rq_vec[v].iov_len; - v++; - } - args->vlen = v; - return xdr_argsize_check(rqstp, p); + return 1; } int diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index a4dce4baec7c..7dfeeaa4e1df 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -32,7 +32,6 @@ struct nfsd3_readargs { struct svc_fh fh; __u64 offset; __u32 count; - int vlen; }; struct nfsd3_writeargs { -- cgit v1.2.3 From c43b2f229a01969a7ccf94b033c5085e0ec2040c Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 22 Oct 2020 11:14:55 -0400 Subject: NFSD: Update WRITE3arg decoder to use struct xdr_stream As part of the update, open code that sanity-checks the size of the data payload against the length of the RPC Call message has to be re-implemented to use xdr_stream infrastructure. Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 51 ++++++++++++++++++++------------------------------- 1 file changed, 20 insertions(+), 31 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 2f32df15a7e8..c06467e8ac82 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -405,52 +405,41 @@ nfs3svc_decode_readargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_writeargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_writeargs *args = rqstp->rq_argp; - unsigned int len, hdr, dlen; u32 max_blocksize = svc_max_payload(rqstp); struct kvec *head = rqstp->rq_arg.head; struct kvec *tail = rqstp->rq_arg.tail; + size_t remaining; - p = decode_fh(p, &args->fh); - if (!p) + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) return 0; - p = xdr_decode_hyper(p, &args->offset); - - args->count = ntohl(*p++); - args->stable = ntohl(*p++); - len = args->len = ntohl(*p++); - if ((void *)p > head->iov_base + head->iov_len) + if (xdr_stream_decode_u64(xdr, &args->offset) < 0) return 0; - /* - * The count must equal the amount of data passed. - */ - if (args->count != args->len) + if (xdr_stream_decode_u32(xdr, &args->count) < 0) + return 0; + if (xdr_stream_decode_u32(xdr, &args->stable) < 0) return 0; - /* - * Check to make sure that we got the right number of - * bytes. - */ - hdr = (void*)p - head->iov_base; - dlen = head->iov_len + rqstp->rq_arg.page_len + tail->iov_len - hdr; - /* - * Round the length of the data which was specified up to - * the next multiple of XDR units and then compare that - * against the length which was actually received. - * Note that when RPCSEC/GSS (for example) is used, the - * data buffer can be padded so dlen might be larger - * than required. It must never be smaller. - */ - if (dlen < XDR_QUADLEN(len)*4) + /* opaque data */ + if (xdr_stream_decode_u32(xdr, &args->len) < 0) return 0; + /* request sanity */ + if (args->count != args->len) + return 0; + remaining = head->iov_len + rqstp->rq_arg.page_len + tail->iov_len; + remaining -= xdr_stream_pos(xdr); + if (remaining < xdr_align_size(args->len)) + return 0; if (args->count > max_blocksize) { args->count = max_blocksize; - len = args->len = max_blocksize; + args->len = max_blocksize; } - args->first.iov_base = (void *)p; - args->first.iov_len = head->iov_len - hdr; + args->first.iov_base = xdr->p; + args->first.iov_len = head->iov_len - xdr_stream_pos(xdr); + return 1; } -- cgit v1.2.3 From 224c1c894e48cd72e4dd9fb6311be80cbe1369b0 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sat, 24 Oct 2020 12:51:18 -0400 Subject: NFSD: Update READLINK3arg decoder to use struct xdr_stream The NFSv3 READLINK request takes a single filehandle, so it can re-use GETATTR's decoder. Signed-off-by: Chuck Lever --- fs/nfsd/nfs3proc.c | 9 +++++---- fs/nfsd/nfs3xdr.c | 13 ------------- fs/nfsd/xdr3.h | 6 ------ 3 files changed, 5 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index acdf47179a38..9e289e0f439b 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -124,15 +124,16 @@ nfsd3_proc_access(struct svc_rqst *rqstp) static __be32 nfsd3_proc_readlink(struct svc_rqst *rqstp) { - struct nfsd3_readlinkargs *argp = rqstp->rq_argp; + struct nfsd_fhandle *argp = rqstp->rq_argp; struct nfsd3_readlinkres *resp = rqstp->rq_resp; + char *buffer = page_address(*(rqstp->rq_next_page++)); dprintk("nfsd: READLINK(3) %s\n", SVCFH_fmt(&argp->fh)); /* Read the symlink. */ fh_copy(&resp->fh, &argp->fh); resp->len = NFS3_MAXPATHLEN; - resp->status = nfsd_readlink(rqstp, &resp->fh, argp->buffer, &resp->len); + resp->status = nfsd_readlink(rqstp, &resp->fh, buffer, &resp->len); return rpc_success; } @@ -768,10 +769,10 @@ static const struct svc_procedure nfsd_procedures3[22] = { }, [NFS3PROC_READLINK] = { .pc_func = nfsd3_proc_readlink, - .pc_decode = nfs3svc_decode_readlinkargs, + .pc_decode = nfs3svc_decode_fhandleargs, .pc_encode = nfs3svc_encode_readlinkres, .pc_release = nfs3svc_release_fhandle, - .pc_argsize = sizeof(struct nfsd3_readlinkargs), + .pc_argsize = sizeof(struct nfsd_fhandle), .pc_ressize = sizeof(struct nfsd3_readlinkres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+1+NFS3_MAXPATHLEN/4, diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index c06467e8ac82..6b6a839c1fc8 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -543,19 +543,6 @@ nfs3svc_decode_renameargs(struct svc_rqst *rqstp, __be32 *p) return xdr_argsize_check(rqstp, p); } -int -nfs3svc_decode_readlinkargs(struct svc_rqst *rqstp, __be32 *p) -{ - struct nfsd3_readlinkargs *args = rqstp->rq_argp; - - p = decode_fh(p, &args->fh); - if (!p) - return 0; - args->buffer = page_address(*(rqstp->rq_next_page++)); - - return xdr_argsize_check(rqstp, p); -} - int nfs3svc_decode_linkargs(struct svc_rqst *rqstp, __be32 *p) { diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index 7dfeeaa4e1df..08f909142ddf 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -70,11 +70,6 @@ struct nfsd3_renameargs { unsigned int tlen; }; -struct nfsd3_readlinkargs { - struct svc_fh fh; - char * buffer; -}; - struct nfsd3_linkargs { struct svc_fh ffh; struct svc_fh tfh; @@ -282,7 +277,6 @@ int nfs3svc_decode_createargs(struct svc_rqst *, __be32 *); int nfs3svc_decode_mkdirargs(struct svc_rqst *, __be32 *); int nfs3svc_decode_mknodargs(struct svc_rqst *, __be32 *); int nfs3svc_decode_renameargs(struct svc_rqst *, __be32 *); -int nfs3svc_decode_readlinkargs(struct svc_rqst *, __be32 *); int nfs3svc_decode_linkargs(struct svc_rqst *, __be32 *); int nfs3svc_decode_symlinkargs(struct svc_rqst *, __be32 *); int nfs3svc_decode_readdirargs(struct svc_rqst *, __be32 *); -- cgit v1.2.3 From 0a8f37fb34a96267c656f7254e69bb9a2fc89fe4 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 10 Nov 2020 10:24:39 -0500 Subject: NFSD: Fix returned READDIR offset cookie Code inspection shows that the server's NFSv3 READDIR implementation handles offset cookies slightly differently than the NFSv2 READDIR, NFSv3 READDIRPLUS, and NFSv4 READDIR implementations, and there doesn't seem to be any need for this difference. As a clean up, I copied the logic from nfsd3_proc_readdirplus(). Signed-off-by: Chuck Lever --- fs/nfsd/nfs3proc.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index 9e289e0f439b..7ea2fb127f6f 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -444,6 +444,7 @@ nfsd3_proc_readdir(struct svc_rqst *rqstp) struct nfsd3_readdirargs *argp = rqstp->rq_argp; struct nfsd3_readdirres *resp = rqstp->rq_resp; int count = 0; + loff_t offset; struct page **p; caddr_t page_addr = NULL; @@ -462,7 +463,9 @@ nfsd3_proc_readdir(struct svc_rqst *rqstp) resp->common.err = nfs_ok; resp->buffer = argp->buffer; resp->rqstp = rqstp; - resp->status = nfsd_readdir(rqstp, &resp->fh, (loff_t *)&argp->cookie, + offset = argp->cookie; + + resp->status = nfsd_readdir(rqstp, &resp->fh, &offset, &resp->common, nfs3svc_encode_entry); memcpy(resp->verf, argp->verf, 8); count = 0; @@ -478,8 +481,6 @@ nfsd3_proc_readdir(struct svc_rqst *rqstp) } resp->count = count >> 2; if (resp->offset) { - loff_t offset = argp->cookie; - if (unlikely(resp->offset1)) { /* we ended up with offset on a page boundary */ *resp->offset = htonl(offset >> 32); -- cgit v1.2.3 From 40116ebd0934cca7e46423bdb3397d3d27eb9fb9 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 17 Nov 2020 09:50:23 -0500 Subject: NFSD: Add helper to set up the pages where the dirlist is encoded De-duplicate some code that is used by both READDIR and READDIRPLUS to build the dirlist in the Reply. Because this code is not related to decoding READ arguments, it is moved to a more appropriate spot. Signed-off-by: Chuck Lever --- fs/nfsd/nfs3proc.c | 29 +++++++++++++++++++---------- fs/nfsd/nfs3xdr.c | 20 -------------------- fs/nfsd/xdr3.h | 1 - 3 files changed, 19 insertions(+), 31 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index 7ea2fb127f6f..8675851199f8 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -435,6 +435,23 @@ nfsd3_proc_link(struct svc_rqst *rqstp) return rpc_success; } +static void nfsd3_init_dirlist_pages(struct svc_rqst *rqstp, + struct nfsd3_readdirres *resp, + int count) +{ + count = min_t(u32, count, svc_max_payload(rqstp)); + + /* Convert byte count to number of words (i.e. >> 2), + * and reserve room for the NULL ptr & eof flag (-2 words) */ + resp->buflen = (count >> 2) - 2; + + resp->buffer = page_address(*rqstp->rq_next_page); + while (count > 0) { + rqstp->rq_next_page++; + count -= PAGE_SIZE; + } +} + /* * Read a portion of a directory. */ @@ -452,16 +469,12 @@ nfsd3_proc_readdir(struct svc_rqst *rqstp) SVCFH_fmt(&argp->fh), argp->count, (u32) argp->cookie); - /* Make sure we've room for the NULL ptr & eof flag, and shrink to - * client read size */ - count = (argp->count >> 2) - 2; + nfsd3_init_dirlist_pages(rqstp, resp, argp->count); /* Read directory and encode entries on the fly */ fh_copy(&resp->fh, &argp->fh); - resp->buflen = count; resp->common.err = nfs_ok; - resp->buffer = argp->buffer; resp->rqstp = rqstp; offset = argp->cookie; @@ -513,16 +526,12 @@ nfsd3_proc_readdirplus(struct svc_rqst *rqstp) SVCFH_fmt(&argp->fh), argp->count, (u32) argp->cookie); - /* Convert byte count to number of words (i.e. >> 2), - * and reserve room for the NULL ptr & eof flag (-2 words) */ - resp->count = (argp->count >> 2) - 2; + nfsd3_init_dirlist_pages(rqstp, resp, argp->count); /* Read directory and encode entries on the fly */ fh_copy(&resp->fh, &argp->fh); resp->common.err = nfs_ok; - resp->buffer = argp->buffer; - resp->buflen = resp->count; resp->rqstp = rqstp; offset = argp->cookie; diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 6b6a839c1fc8..8394aeb8381e 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -560,8 +560,6 @@ int nfs3svc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p) { struct nfsd3_readdirargs *args = rqstp->rq_argp; - int len; - u32 max_blocksize = svc_max_payload(rqstp); p = decode_fh(p, &args->fh); if (!p) @@ -570,14 +568,6 @@ nfs3svc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p) args->verf = p; p += 2; args->dircount = ~0; args->count = ntohl(*p++); - len = args->count = min_t(u32, args->count, max_blocksize); - - while (len > 0) { - struct page *p = *(rqstp->rq_next_page++); - if (!args->buffer) - args->buffer = page_address(p); - len -= PAGE_SIZE; - } return xdr_argsize_check(rqstp, p); } @@ -586,8 +576,6 @@ int nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, __be32 *p) { struct nfsd3_readdirargs *args = rqstp->rq_argp; - int len; - u32 max_blocksize = svc_max_payload(rqstp); p = decode_fh(p, &args->fh); if (!p) @@ -597,14 +585,6 @@ nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, __be32 *p) args->dircount = ntohl(*p++); args->count = ntohl(*p++); - len = args->count = min(args->count, max_blocksize); - while (len > 0) { - struct page *p = *(rqstp->rq_next_page++); - if (!args->buffer) - args->buffer = page_address(p); - len -= PAGE_SIZE; - } - return xdr_argsize_check(rqstp, p); } diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index 08f909142ddf..789a364d5e69 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -93,7 +93,6 @@ struct nfsd3_readdirargs { __u32 dircount; __u32 count; __be32 * verf; - __be32 * buffer; }; struct nfsd3_commitargs { -- cgit v1.2.3 From 9cedc2e64c296efb3bebe93a0ceeb5e71e8d722d Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 19 Oct 2020 13:23:52 -0400 Subject: NFSD: Update READDIR3args decoders to use struct xdr_stream As an additional clean up, neither nfsd3_proc_readdir() nor nfsd3_proc_readdirplus() make use of the dircount argument, so remove it from struct nfsd3_readdirargs. Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 38 ++++++++++++++++++++++++-------------- fs/nfsd/xdr3.h | 1 - 2 files changed, 24 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 8394aeb8381e..eb55be106a04 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -559,33 +559,43 @@ nfs3svc_decode_linkargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_readdirargs *args = rqstp->rq_argp; - p = decode_fh(p, &args->fh); - if (!p) + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return 0; + if (xdr_stream_decode_u64(xdr, &args->cookie) < 0) + return 0; + args->verf = xdr_inline_decode(xdr, NFS3_COOKIEVERFSIZE); + if (!args->verf) + return 0; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) return 0; - p = xdr_decode_hyper(p, &args->cookie); - args->verf = p; p += 2; - args->dircount = ~0; - args->count = ntohl(*p++); - return xdr_argsize_check(rqstp, p); + return 1; } int nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_readdirargs *args = rqstp->rq_argp; + u32 dircount; - p = decode_fh(p, &args->fh); - if (!p) + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return 0; + if (xdr_stream_decode_u64(xdr, &args->cookie) < 0) + return 0; + args->verf = xdr_inline_decode(xdr, NFS3_COOKIEVERFSIZE); + if (!args->verf) + return 0; + /* dircount is ignored */ + if (xdr_stream_decode_u32(xdr, &dircount) < 0) + return 0; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) return 0; - p = xdr_decode_hyper(p, &args->cookie); - args->verf = p; p += 2; - args->dircount = ntohl(*p++); - args->count = ntohl(*p++); - return xdr_argsize_check(rqstp, p); + return 1; } int diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index 789a364d5e69..64af5b01c5d7 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -90,7 +90,6 @@ struct nfsd3_symlinkargs { struct nfsd3_readdirargs { struct svc_fh fh; __u64 cookie; - __u32 dircount; __u32 count; __be32 * verf; }; -- cgit v1.2.3 From c8d26a0acfe77f0880e0acfe77e4209cf8f3a38b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 14:41:56 -0400 Subject: NFSD: Update COMMIT3arg decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index eb55be106a04..bafb84c97861 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -601,14 +601,17 @@ nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_commitargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_commitargs *args = rqstp->rq_argp; - p = decode_fh(p, &args->fh); - if (!p) + + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return 0; + if (xdr_stream_decode_u64(xdr, &args->offset) < 0) + return 0; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) return 0; - p = xdr_decode_hyper(p, &args->offset); - args->count = ntohl(*p++); - return xdr_argsize_check(rqstp, p); + return 1; } /* -- cgit v1.2.3 From 54d1d43dc709f58be38d278bfc38e9bfb38d35fc Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 15:42:33 -0400 Subject: NFSD: Update the NFSv3 DIROPargs decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index bafb84c97861..299ea8bbd685 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -117,6 +117,39 @@ decode_filename(__be32 *p, char **namp, unsigned int *lenp) return p; } +static bool +svcxdr_decode_filename3(struct xdr_stream *xdr, char **name, unsigned int *len) +{ + u32 size, i; + __be32 *p; + char *c; + + if (xdr_stream_decode_u32(xdr, &size) < 0) + return false; + if (size == 0 || size > NFS3_MAXNAMLEN) + return false; + p = xdr_inline_decode(xdr, size); + if (!p) + return false; + + *len = size; + *name = (char *)p; + for (i = 0, c = *name; i < size; i++, c++) { + if (*c == '\0' || *c == '/') + return false; + } + + return true; +} + +static bool +svcxdr_decode_diropargs3(struct xdr_stream *xdr, struct svc_fh *fhp, + char **name, unsigned int *len) +{ + return svcxdr_decode_nfs_fh3(xdr, fhp) && + svcxdr_decode_filename3(xdr, name, len); +} + static __be32 * decode_sattr3(__be32 *p, struct iattr *iap, struct user_namespace *userns) { @@ -363,13 +396,10 @@ nfs3svc_decode_sattrargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_diropargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_diropargs *args = rqstp->rq_argp; - if (!(p = decode_fh(p, &args->fh)) - || !(p = decode_filename(p, &args->name, &args->len))) - return 0; - - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_diropargs3(xdr, &args->fh, &args->name, &args->len); } int -- cgit v1.2.3 From d181e0a4bef36ee74d1338e5b5c2561d7463a5d0 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 15:44:12 -0400 Subject: NFSD: Update the RENAME3args decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 299ea8bbd685..f870a068aad8 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -562,15 +562,13 @@ nfs3svc_decode_mknodargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_renameargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_renameargs *args = rqstp->rq_argp; - if (!(p = decode_fh(p, &args->ffh)) - || !(p = decode_filename(p, &args->fname, &args->flen)) - || !(p = decode_fh(p, &args->tfh)) - || !(p = decode_filename(p, &args->tname, &args->tlen))) - return 0; - - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_diropargs3(xdr, &args->ffh, + &args->fname, &args->flen) && + svcxdr_decode_diropargs3(xdr, &args->tfh, + &args->tname, &args->tlen); } int -- cgit v1.2.3 From efaa1e7c2c7475f0a9bbeb904d9aba09b73dd52a Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 19 Oct 2020 13:26:32 -0400 Subject: NFSD: Update the LINK3args decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index f870a068aad8..9437dda2646f 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -574,14 +574,12 @@ nfs3svc_decode_renameargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_linkargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_linkargs *args = rqstp->rq_argp; - if (!(p = decode_fh(p, &args->ffh)) - || !(p = decode_fh(p, &args->tfh)) - || !(p = decode_filename(p, &args->tname, &args->tlen))) - return 0; - - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_nfs_fh3(xdr, &args->ffh) && + svcxdr_decode_diropargs3(xdr, &args->tfh, + &args->tname, &args->tlen); } int -- cgit v1.2.3 From 9cde9360d18d8b352b737d10f90f2aecccf93dbe Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 15:48:22 -0400 Subject: NFSD: Update the SETATTR3args decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 138 ++++++++++++++++++++++++++++++++++++++++------ include/uapi/linux/nfs3.h | 6 ++ 2 files changed, 127 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 9437dda2646f..6a6bf8e34d82 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -39,12 +39,18 @@ encode_time3(__be32 *p, struct timespec64 *time) return p; } -static __be32 * -decode_time3(__be32 *p, struct timespec64 *time) +static bool +svcxdr_decode_nfstime3(struct xdr_stream *xdr, struct timespec64 *timep) { - time->tv_sec = ntohl(*p++); - time->tv_nsec = ntohl(*p++); - return p; + __be32 *p; + + p = xdr_inline_decode(xdr, XDR_UNIT * 2); + if (!p) + return false; + timep->tv_sec = be32_to_cpup(p++); + timep->tv_nsec = be32_to_cpup(p); + + return true; } static bool @@ -150,6 +156,112 @@ svcxdr_decode_diropargs3(struct xdr_stream *xdr, struct svc_fh *fhp, svcxdr_decode_filename3(xdr, name, len); } +static bool +svcxdr_decode_sattr3(struct svc_rqst *rqstp, struct xdr_stream *xdr, + struct iattr *iap) +{ + u32 set_it; + + iap->ia_valid = 0; + + if (xdr_stream_decode_bool(xdr, &set_it) < 0) + return false; + if (set_it) { + u32 mode; + + if (xdr_stream_decode_u32(xdr, &mode) < 0) + return false; + iap->ia_valid |= ATTR_MODE; + iap->ia_mode = mode; + } + if (xdr_stream_decode_bool(xdr, &set_it) < 0) + return false; + if (set_it) { + u32 uid; + + if (xdr_stream_decode_u32(xdr, &uid) < 0) + return false; + iap->ia_uid = make_kuid(nfsd_user_namespace(rqstp), uid); + if (uid_valid(iap->ia_uid)) + iap->ia_valid |= ATTR_UID; + } + if (xdr_stream_decode_bool(xdr, &set_it) < 0) + return false; + if (set_it) { + u32 gid; + + if (xdr_stream_decode_u32(xdr, &gid) < 0) + return false; + iap->ia_gid = make_kgid(nfsd_user_namespace(rqstp), gid); + if (gid_valid(iap->ia_gid)) + iap->ia_valid |= ATTR_GID; + } + if (xdr_stream_decode_bool(xdr, &set_it) < 0) + return false; + if (set_it) { + u64 newsize; + + if (xdr_stream_decode_u64(xdr, &newsize) < 0) + return false; + iap->ia_valid |= ATTR_SIZE; + iap->ia_size = min_t(u64, newsize, NFS_OFFSET_MAX); + } + if (xdr_stream_decode_u32(xdr, &set_it) < 0) + return false; + switch (set_it) { + case DONT_CHANGE: + break; + case SET_TO_SERVER_TIME: + iap->ia_valid |= ATTR_ATIME; + break; + case SET_TO_CLIENT_TIME: + if (!svcxdr_decode_nfstime3(xdr, &iap->ia_atime)) + return false; + iap->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET; + break; + default: + return false; + } + if (xdr_stream_decode_u32(xdr, &set_it) < 0) + return false; + switch (set_it) { + case DONT_CHANGE: + break; + case SET_TO_SERVER_TIME: + iap->ia_valid |= ATTR_MTIME; + break; + case SET_TO_CLIENT_TIME: + if (!svcxdr_decode_nfstime3(xdr, &iap->ia_mtime)) + return false; + iap->ia_valid |= ATTR_MTIME | ATTR_MTIME_SET; + break; + default: + return false; + } + + return true; +} + +static bool +svcxdr_decode_sattrguard3(struct xdr_stream *xdr, struct nfsd3_sattrargs *args) +{ + __be32 *p; + u32 check; + + if (xdr_stream_decode_bool(xdr, &check) < 0) + return false; + if (check) { + p = xdr_inline_decode(xdr, XDR_UNIT * 2); + if (!p) + return false; + args->check_guard = 1; + args->guardtime = be32_to_cpup(p); + } else + args->check_guard = 0; + + return true; +} + static __be32 * decode_sattr3(__be32 *p, struct iattr *iap, struct user_namespace *userns) { @@ -377,20 +489,12 @@ nfs3svc_decode_fhandleargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_sattrargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_sattrargs *args = rqstp->rq_argp; - p = decode_fh(p, &args->fh); - if (!p) - return 0; - p = decode_sattr3(p, &args->attrs, nfsd_user_namespace(rqstp)); - - if ((args->check_guard = ntohl(*p++)) != 0) { - struct timespec64 time; - p = decode_time3(p, &time); - args->guardtime = time.tv_sec; - } - - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_nfs_fh3(xdr, &args->fh) && + svcxdr_decode_sattr3(rqstp, xdr, &args->attrs) && + svcxdr_decode_sattrguard3(xdr, args); } int diff --git a/include/uapi/linux/nfs3.h b/include/uapi/linux/nfs3.h index 37e4b34e6b43..c22ab77713bd 100644 --- a/include/uapi/linux/nfs3.h +++ b/include/uapi/linux/nfs3.h @@ -63,6 +63,12 @@ enum nfs3_ftype { NF3BAD = 8 }; +enum nfs3_time_how { + DONT_CHANGE = 0, + SET_TO_SERVER_TIME = 1, + SET_TO_CLIENT_TIME = 2, +}; + struct nfs3_fh { unsigned short size; unsigned char data[NFS3_FHSIZE]; -- cgit v1.2.3 From 6b3a11960d898b25a30103cc6a2ff0b24b90a83b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 15:56:11 -0400 Subject: NFSD: Update the CREATE3args decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 6a6bf8e34d82..24db3725a070 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -580,26 +580,26 @@ nfs3svc_decode_writeargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_createargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_createargs *args = rqstp->rq_argp; - if (!(p = decode_fh(p, &args->fh)) - || !(p = decode_filename(p, &args->name, &args->len))) + if (!svcxdr_decode_diropargs3(xdr, &args->fh, &args->name, &args->len)) return 0; - - switch (args->createmode = ntohl(*p++)) { + if (xdr_stream_decode_u32(xdr, &args->createmode) < 0) + return 0; + switch (args->createmode) { case NFS3_CREATE_UNCHECKED: case NFS3_CREATE_GUARDED: - p = decode_sattr3(p, &args->attrs, nfsd_user_namespace(rqstp)); - break; + return svcxdr_decode_sattr3(rqstp, xdr, &args->attrs); case NFS3_CREATE_EXCLUSIVE: - args->verf = p; - p += 2; + args->verf = xdr_inline_decode(xdr, NFS3_CREATEVERFSIZE); + if (!args->verf) + return 0; break; default: return 0; } - - return xdr_argsize_check(rqstp, p); + return 1; } int -- cgit v1.2.3 From 83374c278db193f3e8b2608b45da1132b867a760 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 17:02:16 -0400 Subject: NFSD: Update the MKDIR3args decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 24db3725a070..b4071cda1d65 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -605,14 +605,12 @@ nfs3svc_decode_createargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_mkdirargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_createargs *args = rqstp->rq_argp; - if (!(p = decode_fh(p, &args->fh)) || - !(p = decode_filename(p, &args->name, &args->len))) - return 0; - p = decode_sattr3(p, &args->attrs, nfsd_user_namespace(rqstp)); - - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_diropargs3(xdr, &args->fh, + &args->name, &args->len) && + svcxdr_decode_sattr3(rqstp, xdr, &args->attrs); } int -- cgit v1.2.3 From da39201637297460c13134c29286a00f3a1c92fe Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 16:01:16 -0400 Subject: NFSD: Update the SYMLINK3args decoder to use struct xdr_stream Similar to the WRITE decoder, code that checks the sanity of the payload size is re-wired to work with xdr_stream infrastructure. Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index b4071cda1d65..eb17231ab166 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -616,25 +616,28 @@ nfs3svc_decode_mkdirargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_symlinkargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_symlinkargs *args = rqstp->rq_argp; - char *base = (char *)p; - size_t dlen; + struct kvec *head = rqstp->rq_arg.head; + struct kvec *tail = rqstp->rq_arg.tail; + size_t remaining; - if (!(p = decode_fh(p, &args->ffh)) || - !(p = decode_filename(p, &args->fname, &args->flen))) + if (!svcxdr_decode_diropargs3(xdr, &args->ffh, &args->fname, &args->flen)) + return 0; + if (!svcxdr_decode_sattr3(rqstp, xdr, &args->attrs)) + return 0; + if (xdr_stream_decode_u32(xdr, &args->tlen) < 0) return 0; - p = decode_sattr3(p, &args->attrs, nfsd_user_namespace(rqstp)); - args->tlen = ntohl(*p++); + /* request sanity */ + remaining = head->iov_len + rqstp->rq_arg.page_len + tail->iov_len; + remaining -= xdr_stream_pos(xdr); + if (remaining < xdr_align_size(args->tlen)) + return 0; - args->first.iov_base = p; - args->first.iov_len = rqstp->rq_arg.head[0].iov_len; - args->first.iov_len -= (char *)p - base; + args->first.iov_base = xdr->p; + args->first.iov_len = head->iov_len - xdr_stream_pos(xdr); - dlen = args->first.iov_len + rqstp->rq_arg.page_len + - rqstp->rq_arg.tail[0].iov_len; - if (dlen < XDR_QUADLEN(args->tlen) << 2) - return 0; return 1; } -- cgit v1.2.3 From f8a38e2d6c885f9d7cd03febc515d36293de4a5b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 17:04:03 -0400 Subject: NFSD: Update the MKNOD3args decoder to use struct xdr_stream This commit removes the last usage of the original decode_sattr3(), so it is removed as a clean-up. Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 107 ++++++++++++++++++------------------------------------ 1 file changed, 35 insertions(+), 72 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index eb17231ab166..a30b418a5116 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -103,26 +103,6 @@ encode_fh(__be32 *p, struct svc_fh *fhp) return p + XDR_QUADLEN(size); } -/* - * Decode a file name and make sure that the path contains - * no slashes or null bytes. - */ -static __be32 * -decode_filename(__be32 *p, char **namp, unsigned int *lenp) -{ - char *name; - unsigned int i; - - if ((p = xdr_decode_string_inplace(p, namp, lenp, NFS3_MAXNAMLEN)) != NULL) { - for (i = 0, name = *namp; i < *lenp; i++, name++) { - if (*name == '\0' || *name == '/') - return NULL; - } - } - - return p; -} - static bool svcxdr_decode_filename3(struct xdr_stream *xdr, char **name, unsigned int *len) { @@ -262,49 +242,26 @@ svcxdr_decode_sattrguard3(struct xdr_stream *xdr, struct nfsd3_sattrargs *args) return true; } -static __be32 * -decode_sattr3(__be32 *p, struct iattr *iap, struct user_namespace *userns) +static bool +svcxdr_decode_specdata3(struct xdr_stream *xdr, struct nfsd3_mknodargs *args) { - u32 tmp; + __be32 *p; - iap->ia_valid = 0; + p = xdr_inline_decode(xdr, XDR_UNIT * 2); + if (!p) + return false; + args->major = be32_to_cpup(p++); + args->minor = be32_to_cpup(p); - if (*p++) { - iap->ia_valid |= ATTR_MODE; - iap->ia_mode = ntohl(*p++); - } - if (*p++) { - iap->ia_uid = make_kuid(userns, ntohl(*p++)); - if (uid_valid(iap->ia_uid)) - iap->ia_valid |= ATTR_UID; - } - if (*p++) { - iap->ia_gid = make_kgid(userns, ntohl(*p++)); - if (gid_valid(iap->ia_gid)) - iap->ia_valid |= ATTR_GID; - } - if (*p++) { - u64 newsize; + return true; +} - iap->ia_valid |= ATTR_SIZE; - p = xdr_decode_hyper(p, &newsize); - iap->ia_size = min_t(u64, newsize, NFS_OFFSET_MAX); - } - if ((tmp = ntohl(*p++)) == 1) { /* set to server time */ - iap->ia_valid |= ATTR_ATIME; - } else if (tmp == 2) { /* set to client time */ - iap->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET; - iap->ia_atime.tv_sec = ntohl(*p++); - iap->ia_atime.tv_nsec = ntohl(*p++); - } - if ((tmp = ntohl(*p++)) == 1) { /* set to server time */ - iap->ia_valid |= ATTR_MTIME; - } else if (tmp == 2) { /* set to client time */ - iap->ia_valid |= ATTR_MTIME | ATTR_MTIME_SET; - iap->ia_mtime.tv_sec = ntohl(*p++); - iap->ia_mtime.tv_nsec = ntohl(*p++); - } - return p; +static bool +svcxdr_decode_devicedata3(struct svc_rqst *rqstp, struct xdr_stream *xdr, + struct nfsd3_mknodargs *args) +{ + return svcxdr_decode_sattr3(rqstp, xdr, &args->attrs) && + svcxdr_decode_specdata3(xdr, args); } static __be32 *encode_fsid(__be32 *p, struct svc_fh *fhp) @@ -644,24 +601,30 @@ nfs3svc_decode_symlinkargs(struct svc_rqst *rqstp, __be32 *p) int nfs3svc_decode_mknodargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_mknodargs *args = rqstp->rq_argp; - if (!(p = decode_fh(p, &args->fh)) - || !(p = decode_filename(p, &args->name, &args->len))) + if (!svcxdr_decode_diropargs3(xdr, &args->fh, &args->name, &args->len)) + return 0; + if (xdr_stream_decode_u32(xdr, &args->ftype) < 0) + return 0; + switch (args->ftype) { + case NF3CHR: + case NF3BLK: + return svcxdr_decode_devicedata3(rqstp, xdr, args); + case NF3SOCK: + case NF3FIFO: + return svcxdr_decode_sattr3(rqstp, xdr, &args->attrs); + case NF3REG: + case NF3DIR: + case NF3LNK: + /* Valid XDR but illegal file types */ + break; + default: return 0; - - args->ftype = ntohl(*p++); - - if (args->ftype == NF3BLK || args->ftype == NF3CHR - || args->ftype == NF3SOCK || args->ftype == NF3FIFO) - p = decode_sattr3(p, &args->attrs, nfsd_user_namespace(rqstp)); - - if (args->ftype == NF3BLK || args->ftype == NF3CHR) { - args->major = ntohl(*p++); - args->minor = ntohl(*p++); } - return xdr_argsize_check(rqstp, p); + return 1; } int -- cgit v1.2.3 From ebcd8e8b28535b643a4c06685bd363b3b73a96af Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 21 Oct 2020 12:14:23 -0400 Subject: NFSD: Update the NFSv2 GETATTR argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfsproc.c | 4 ++-- fs/nfsd/nfsxdr.c | 26 ++++++++++++++++++++------ fs/nfsd/xdr.h | 2 +- 3 files changed, 23 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 1f85a4dc9d1b..b9bc162a5c77 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -627,7 +627,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { }, [NFSPROC_GETATTR] = { .pc_func = nfsd_proc_getattr, - .pc_decode = nfssvc_decode_fhandle, + .pc_decode = nfssvc_decode_fhandleargs, .pc_encode = nfssvc_encode_attrstat, .pc_release = nfssvc_release_attrstat, .pc_argsize = sizeof(struct nfsd_fhandle), @@ -793,7 +793,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { }, [NFSPROC_STATFS] = { .pc_func = nfsd_proc_statfs, - .pc_decode = nfssvc_decode_fhandle, + .pc_decode = nfssvc_decode_fhandleargs, .pc_encode = nfssvc_encode_statfsres, .pc_argsize = sizeof(struct nfsd_fhandle), .pc_ressize = sizeof(struct nfsd_statfsres), diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 7aa6e8aca2c1..f3189e1be20f 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -23,8 +23,9 @@ static u32 nfs_ftypes[] = { /* - * XDR functions for basic NFS types + * Basic NFSv2 data types (RFC 1094 Section 2.3) */ + static __be32 * decode_fh(__be32 *p, struct svc_fh *fhp) { @@ -37,6 +38,21 @@ decode_fh(__be32 *p, struct svc_fh *fhp) return p + (NFS_FHSIZE >> 2); } +static bool +svcxdr_decode_fhandle(struct xdr_stream *xdr, struct svc_fh *fhp) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS_FHSIZE); + if (!p) + return false; + fh_init(fhp, NFS_FHSIZE); + memcpy(&fhp->fh_handle.fh_base, p, NFS_FHSIZE); + fhp->fh_handle.fh_size = NFS_FHSIZE; + + return true; +} + /* Helper function for NFSv2 ACL code */ __be32 *nfs2svc_decode_fh(__be32 *p, struct svc_fh *fhp) { @@ -194,14 +210,12 @@ __be32 *nfs2svc_encode_fattr(struct svc_rqst *rqstp, __be32 *p, struct svc_fh *f */ int -nfssvc_decode_fhandle(struct svc_rqst *rqstp, __be32 *p) +nfssvc_decode_fhandleargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_fhandle *args = rqstp->rq_argp; - p = decode_fh(p, &args->fh); - if (!p) - return 0; - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_fhandle(xdr, &args->fh); } int diff --git a/fs/nfsd/xdr.h b/fs/nfsd/xdr.h index ad77387734cc..84256a6a1ba1 100644 --- a/fs/nfsd/xdr.h +++ b/fs/nfsd/xdr.h @@ -144,7 +144,7 @@ union nfsd_xdrstore { #define NFS2_SVC_XDRSIZE sizeof(union nfsd_xdrstore) -int nfssvc_decode_fhandle(struct svc_rqst *, __be32 *); +int nfssvc_decode_fhandleargs(struct svc_rqst *, __be32 *); int nfssvc_decode_sattrargs(struct svc_rqst *, __be32 *); int nfssvc_decode_diropargs(struct svc_rqst *, __be32 *); int nfssvc_decode_readargs(struct svc_rqst *, __be32 *); -- cgit v1.2.3 From 8c293ef993c8df0b1bea9ecb0de6eb96dec3ac9d Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 21 Oct 2020 12:15:51 -0400 Subject: NFSD: Update the NFSv2 READ argument decoder to use struct xdr_stream The code that sets up rq_vec is refactored so that it is now adjacent to the nfsd_read() call site where it is used. Signed-off-by: Chuck Lever --- fs/nfsd/nfsproc.c | 32 ++++++++++++++++++-------------- fs/nfsd/nfsxdr.c | 36 ++++++++++++------------------------ fs/nfsd/xdr.h | 1 - 3 files changed, 30 insertions(+), 39 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index b9bc162a5c77..814762793f9c 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -171,32 +171,36 @@ nfsd_proc_read(struct svc_rqst *rqstp) { struct nfsd_readargs *argp = rqstp->rq_argp; struct nfsd_readres *resp = rqstp->rq_resp; + unsigned int len; u32 eof; + int v; dprintk("nfsd: READ %s %d bytes at %d\n", SVCFH_fmt(&argp->fh), argp->count, argp->offset); + argp->count = min_t(u32, argp->count, NFSSVC_MAXBLKSIZE_V2); + + v = 0; + len = argp->count; + while (len > 0) { + struct page *page = *(rqstp->rq_next_page++); + + rqstp->rq_vec[v].iov_base = page_address(page); + rqstp->rq_vec[v].iov_len = min_t(unsigned int, len, PAGE_SIZE); + len -= rqstp->rq_vec[v].iov_len; + v++; + } + /* Obtain buffer pointer for payload. 19 is 1 word for * status, 17 words for fattr, and 1 word for the byte count. */ - - if (NFSSVC_MAXBLKSIZE_V2 < argp->count) { - char buf[RPC_MAX_ADDRBUFLEN]; - printk(KERN_NOTICE - "oversized read request from %s (%d bytes)\n", - svc_print_addr(rqstp, buf, sizeof(buf)), - argp->count); - argp->count = NFSSVC_MAXBLKSIZE_V2; - } svc_reserve_auth(rqstp, (19<<2) + argp->count + 4); resp->count = argp->count; - resp->status = nfsd_read(rqstp, fh_copy(&resp->fh, &argp->fh), - argp->offset, - rqstp->rq_vec, argp->vlen, - &resp->count, - &eof); + fh_copy(&resp->fh, &argp->fh); + resp->status = nfsd_read(rqstp, &resp->fh, argp->offset, + rqstp->rq_vec, v, &resp->count, &eof); if (resp->status == nfs_ok) resp->status = fh_getattr(&resp->fh, &resp->stat); else if (resp->status == nfserr_jukebox) diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index f3189e1be20f..1eacaa2c13a9 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -246,33 +246,21 @@ nfssvc_decode_diropargs(struct svc_rqst *rqstp, __be32 *p) int nfssvc_decode_readargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_readargs *args = rqstp->rq_argp; - unsigned int len; - int v; - p = decode_fh(p, &args->fh); - if (!p) - return 0; + u32 totalcount; - args->offset = ntohl(*p++); - len = args->count = ntohl(*p++); - p++; /* totalcount - unused */ - - len = min_t(unsigned int, len, NFSSVC_MAXBLKSIZE_V2); + if (!svcxdr_decode_fhandle(xdr, &args->fh)) + return 0; + if (xdr_stream_decode_u32(xdr, &args->offset) < 0) + return 0; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) + return 0; + /* totalcount is ignored */ + if (xdr_stream_decode_u32(xdr, &totalcount) < 0) + return 0; - /* set up somewhere to store response. - * We take pages, put them on reslist and include in iovec - */ - v=0; - while (len > 0) { - struct page *p = *(rqstp->rq_next_page++); - - rqstp->rq_vec[v].iov_base = page_address(p); - rqstp->rq_vec[v].iov_len = min_t(unsigned int, len, PAGE_SIZE); - len -= rqstp->rq_vec[v].iov_len; - v++; - } - args->vlen = v; - return xdr_argsize_check(rqstp, p); + return 1; } int diff --git a/fs/nfsd/xdr.h b/fs/nfsd/xdr.h index 84256a6a1ba1..d2ffda96975d 100644 --- a/fs/nfsd/xdr.h +++ b/fs/nfsd/xdr.h @@ -27,7 +27,6 @@ struct nfsd_readargs { struct svc_fh fh; __u32 offset; __u32 count; - int vlen; }; struct nfsd_writeargs { -- cgit v1.2.3 From a51b5b737a0be93fae6ea2a18df03ab2359a3f4b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 21 Oct 2020 12:18:36 -0400 Subject: NFSD: Update the NFSv2 WRITE argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfsxdr.c | 52 +++++++++++++++++++++------------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 1eacaa2c13a9..11d27b219cff 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -266,46 +266,36 @@ nfssvc_decode_readargs(struct svc_rqst *rqstp, __be32 *p) int nfssvc_decode_writeargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_writeargs *args = rqstp->rq_argp; - unsigned int len, hdr, dlen; struct kvec *head = rqstp->rq_arg.head; + struct kvec *tail = rqstp->rq_arg.tail; + u32 beginoffset, totalcount; + size_t remaining; - p = decode_fh(p, &args->fh); - if (!p) + if (!svcxdr_decode_fhandle(xdr, &args->fh)) return 0; - - p++; /* beginoffset */ - args->offset = ntohl(*p++); /* offset */ - p++; /* totalcount */ - len = args->len = ntohl(*p++); - /* - * The protocol specifies a maximum of 8192 bytes. - */ - if (len > NFSSVC_MAXBLKSIZE_V2) + /* beginoffset is ignored */ + if (xdr_stream_decode_u32(xdr, &beginoffset) < 0) return 0; - - /* - * Check to make sure that we got the right number of - * bytes. - */ - hdr = (void*)p - head->iov_base; - if (hdr > head->iov_len) + if (xdr_stream_decode_u32(xdr, &args->offset) < 0) + return 0; + /* totalcount is ignored */ + if (xdr_stream_decode_u32(xdr, &totalcount) < 0) return 0; - dlen = head->iov_len + rqstp->rq_arg.page_len - hdr; - /* - * Round the length of the data which was specified up to - * the next multiple of XDR units and then compare that - * against the length which was actually received. - * Note that when RPCSEC/GSS (for example) is used, the - * data buffer can be padded so dlen might be larger - * than required. It must never be smaller. - */ - if (dlen < XDR_QUADLEN(len)*4) + /* opaque data */ + if (xdr_stream_decode_u32(xdr, &args->len) < 0) + return 0; + if (args->len > NFSSVC_MAXBLKSIZE_V2) + return 0; + remaining = head->iov_len + rqstp->rq_arg.page_len + tail->iov_len; + remaining -= xdr_stream_pos(xdr); + if (remaining < xdr_align_size(args->len)) return 0; + args->first.iov_base = xdr->p; + args->first.iov_len = head->iov_len - xdr_stream_pos(xdr); - args->first.iov_base = (void *)p; - args->first.iov_len = head->iov_len - hdr; return 1; } -- cgit v1.2.3 From 1fcbd1c9456ba129d38420e345e91c4b6363db47 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 21 Oct 2020 12:21:25 -0400 Subject: NFSD: Update the NFSv2 READLINK argument decoder to use struct xdr_stream If the code that sets up the sink buffer for nfsd_readlink() is moved adjacent to the nfsd_readlink() call site that uses it, then the only argument is a file handle, and the fhandle decoder can be used instead. Signed-off-by: Chuck Lever --- fs/nfsd/nfsproc.c | 9 +++++---- fs/nfsd/nfsxdr.c | 13 ------------- fs/nfsd/xdr.h | 6 ------ 3 files changed, 5 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 814762793f9c..bdb47848f7fd 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -149,14 +149,15 @@ out: static __be32 nfsd_proc_readlink(struct svc_rqst *rqstp) { - struct nfsd_readlinkargs *argp = rqstp->rq_argp; + struct nfsd_fhandle *argp = rqstp->rq_argp; struct nfsd_readlinkres *resp = rqstp->rq_resp; + char *buffer = page_address(*(rqstp->rq_next_page++)); dprintk("nfsd: READLINK %s\n", SVCFH_fmt(&argp->fh)); /* Read the symlink. */ resp->len = NFS_MAXPATHLEN; - resp->status = nfsd_readlink(rqstp, &argp->fh, argp->buffer, &resp->len); + resp->status = nfsd_readlink(rqstp, &argp->fh, buffer, &resp->len); fh_put(&argp->fh); return rpc_success; @@ -674,9 +675,9 @@ static const struct svc_procedure nfsd_procedures2[18] = { }, [NFSPROC_READLINK] = { .pc_func = nfsd_proc_readlink, - .pc_decode = nfssvc_decode_readlinkargs, + .pc_decode = nfssvc_decode_fhandleargs, .pc_encode = nfssvc_encode_readlinkres, - .pc_argsize = sizeof(struct nfsd_readlinkargs), + .pc_argsize = sizeof(struct nfsd_fhandle), .pc_ressize = sizeof(struct nfsd_readlinkres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+1+NFS_MAXPATHLEN/4, diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 11d27b219cff..02dd9888d93b 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -326,19 +326,6 @@ nfssvc_decode_renameargs(struct svc_rqst *rqstp, __be32 *p) return xdr_argsize_check(rqstp, p); } -int -nfssvc_decode_readlinkargs(struct svc_rqst *rqstp, __be32 *p) -{ - struct nfsd_readlinkargs *args = rqstp->rq_argp; - - p = decode_fh(p, &args->fh); - if (!p) - return 0; - args->buffer = page_address(*(rqstp->rq_next_page++)); - - return xdr_argsize_check(rqstp, p); -} - int nfssvc_decode_linkargs(struct svc_rqst *rqstp, __be32 *p) { diff --git a/fs/nfsd/xdr.h b/fs/nfsd/xdr.h index d2ffda96975d..288c29a999db 100644 --- a/fs/nfsd/xdr.h +++ b/fs/nfsd/xdr.h @@ -52,11 +52,6 @@ struct nfsd_renameargs { unsigned int tlen; }; -struct nfsd_readlinkargs { - struct svc_fh fh; - char * buffer; -}; - struct nfsd_linkargs { struct svc_fh ffh; struct svc_fh tfh; @@ -150,7 +145,6 @@ int nfssvc_decode_readargs(struct svc_rqst *, __be32 *); int nfssvc_decode_writeargs(struct svc_rqst *, __be32 *); int nfssvc_decode_createargs(struct svc_rqst *, __be32 *); int nfssvc_decode_renameargs(struct svc_rqst *, __be32 *); -int nfssvc_decode_readlinkargs(struct svc_rqst *, __be32 *); int nfssvc_decode_linkargs(struct svc_rqst *, __be32 *); int nfssvc_decode_symlinkargs(struct svc_rqst *, __be32 *); int nfssvc_decode_readdirargs(struct svc_rqst *, __be32 *); -- cgit v1.2.3 From 788cd46ecf83ee2d561cb4e754e276dc8089b787 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 13 Nov 2020 17:03:49 -0500 Subject: NFSD: Add helper to set up the pages where the dirlist is encoded Add a helper similar to nfsd3_init_dirlist_pages(). Signed-off-by: Chuck Lever --- fs/nfsd/nfsproc.c | 29 ++++++++++++++++++----------- fs/nfsd/nfsxdr.c | 2 -- fs/nfsd/xdr.h | 1 - 3 files changed, 18 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index bdb47848f7fd..b2f8035f166b 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -553,6 +553,20 @@ nfsd_proc_rmdir(struct svc_rqst *rqstp) return rpc_success; } +static void nfsd_init_dirlist_pages(struct svc_rqst *rqstp, + struct nfsd_readdirres *resp, + int count) +{ + count = min_t(u32, count, PAGE_SIZE); + + /* Convert byte count to number of words (i.e. >> 2), + * and reserve room for the NULL ptr & eof flag (-2 words) */ + resp->buflen = (count >> 2) - 2; + + resp->buffer = page_address(*rqstp->rq_next_page); + rqstp->rq_next_page++; +} + /* * Read a portion of a directory. */ @@ -561,31 +575,24 @@ nfsd_proc_readdir(struct svc_rqst *rqstp) { struct nfsd_readdirargs *argp = rqstp->rq_argp; struct nfsd_readdirres *resp = rqstp->rq_resp; - int count; loff_t offset; + __be32 *buffer; dprintk("nfsd: READDIR %s %d bytes at %d\n", SVCFH_fmt(&argp->fh), argp->count, argp->cookie); - /* Shrink to the client read size */ - count = (argp->count >> 2) - 2; - - /* Make sure we've room for the NULL ptr & eof flag */ - count -= 2; - if (count < 0) - count = 0; + nfsd_init_dirlist_pages(rqstp, resp, argp->count); + buffer = resp->buffer; - resp->buffer = argp->buffer; resp->offset = NULL; - resp->buflen = count; resp->common.err = nfs_ok; /* Read directory and encode entries on the fly */ offset = argp->cookie; resp->status = nfsd_readdir(rqstp, &argp->fh, &offset, &resp->common, nfssvc_encode_entry); - resp->count = resp->buffer - argp->buffer; + resp->count = resp->buffer - buffer; if (resp->offset) *resp->offset = htonl(offset); diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 02dd9888d93b..3d72334e1673 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -388,8 +388,6 @@ nfssvc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p) return 0; args->cookie = ntohl(*p++); args->count = ntohl(*p++); - args->count = min_t(u32, args->count, PAGE_SIZE); - args->buffer = page_address(*(rqstp->rq_next_page++)); return xdr_argsize_check(rqstp, p); } diff --git a/fs/nfsd/xdr.h b/fs/nfsd/xdr.h index 288c29a999db..d700838f6512 100644 --- a/fs/nfsd/xdr.h +++ b/fs/nfsd/xdr.h @@ -73,7 +73,6 @@ struct nfsd_readdirargs { struct svc_fh fh; __u32 cookie; __u32 count; - __be32 * buffer; }; struct nfsd_stat { -- cgit v1.2.3 From 8688361ae2edb8f7e61d926dc5000c9a44f29370 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 19 Oct 2020 14:15:51 -0400 Subject: NFSD: Update the NFSv2 READDIR argument decoder to use struct xdr_stream As an additional clean up, move code not related to XDR decoding into readdir's .pc_func call out. Signed-off-by: Chuck Lever --- fs/nfsd/nfsxdr.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 3d72334e1673..7b33093f8d8b 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -381,15 +381,17 @@ nfssvc_decode_symlinkargs(struct svc_rqst *rqstp, __be32 *p) int nfssvc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_readdirargs *args = rqstp->rq_argp; - p = decode_fh(p, &args->fh); - if (!p) + if (!svcxdr_decode_fhandle(xdr, &args->fh)) + return 0; + if (xdr_stream_decode_u32(xdr, &args->cookie) < 0) + return 0; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) return 0; - args->cookie = ntohl(*p++); - args->count = ntohl(*p++); - return xdr_argsize_check(rqstp, p); + return 1; } /* -- cgit v1.2.3 From 6d742c1864c18f143ea2031f1ed66bcd8f4812de Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 19 Oct 2020 14:33:24 -0400 Subject: NFSD: Update NFSv2 diropargs decoding to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfsxdr.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 7b33093f8d8b..00a7db8548eb 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -86,6 +86,38 @@ decode_filename(__be32 *p, char **namp, unsigned int *lenp) return p; } +static bool +svcxdr_decode_filename(struct xdr_stream *xdr, char **name, unsigned int *len) +{ + u32 size, i; + __be32 *p; + char *c; + + if (xdr_stream_decode_u32(xdr, &size) < 0) + return false; + if (size == 0 || size > NFS_MAXNAMLEN) + return false; + p = xdr_inline_decode(xdr, size); + if (!p) + return false; + + *len = size; + *name = (char *)p; + for (i = 0, c = *name; i < size; i++, c++) + if (*c == '\0' || *c == '/') + return false; + + return true; +} + +static bool +svcxdr_decode_diropargs(struct xdr_stream *xdr, struct svc_fh *fhp, + char **name, unsigned int *len) +{ + return svcxdr_decode_fhandle(xdr, fhp) && + svcxdr_decode_filename(xdr, name, len); +} + static __be32 * decode_sattr(__be32 *p, struct iattr *iap, struct user_namespace *userns) { @@ -234,13 +266,10 @@ nfssvc_decode_sattrargs(struct svc_rqst *rqstp, __be32 *p) int nfssvc_decode_diropargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_diropargs *args = rqstp->rq_argp; - if (!(p = decode_fh(p, &args->fh)) - || !(p = decode_filename(p, &args->name, &args->len))) - return 0; - - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_diropargs(xdr, &args->fh, &args->name, &args->len); } int -- cgit v1.2.3 From 62aa557efb81ea3339fabe7f5b1a343e742bbbdf Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 21 Oct 2020 12:35:41 -0400 Subject: NFSD: Update the NFSv2 RENAME argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfsxdr.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 00a7db8548eb..d4f4729c7b1c 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -344,15 +344,13 @@ nfssvc_decode_createargs(struct svc_rqst *rqstp, __be32 *p) int nfssvc_decode_renameargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_renameargs *args = rqstp->rq_argp; - if (!(p = decode_fh(p, &args->ffh)) - || !(p = decode_filename(p, &args->fname, &args->flen)) - || !(p = decode_fh(p, &args->tfh)) - || !(p = decode_filename(p, &args->tname, &args->tlen))) - return 0; - - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_diropargs(xdr, &args->ffh, + &args->fname, &args->flen) && + svcxdr_decode_diropargs(xdr, &args->tfh, + &args->tname, &args->tlen); } int -- cgit v1.2.3 From 77edcdf91f6245a9881b84e4e101738148bd039a Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 21 Oct 2020 12:34:24 -0400 Subject: NFSD: Update the NFSv2 LINK argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfsxdr.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index d4f4729c7b1c..3d0fe03a3fb9 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -356,14 +356,12 @@ nfssvc_decode_renameargs(struct svc_rqst *rqstp, __be32 *p) int nfssvc_decode_linkargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_linkargs *args = rqstp->rq_argp; - if (!(p = decode_fh(p, &args->ffh)) - || !(p = decode_fh(p, &args->tfh)) - || !(p = decode_filename(p, &args->tname, &args->tlen))) - return 0; - - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_fhandle(xdr, &args->ffh) && + svcxdr_decode_diropargs(xdr, &args->tfh, + &args->tname, &args->tlen); } int -- cgit v1.2.3 From 2fdd6bd293b9e7dda61220538b2759fbf06f5af0 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 21 Oct 2020 12:39:06 -0400 Subject: NFSD: Update the NFSv2 SETATTR argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfsxdr.c | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 3d0fe03a3fb9..6c87ea8f3876 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -173,6 +173,79 @@ decode_sattr(__be32 *p, struct iattr *iap, struct user_namespace *userns) return p; } +static bool +svcxdr_decode_sattr(struct svc_rqst *rqstp, struct xdr_stream *xdr, + struct iattr *iap) +{ + u32 tmp1, tmp2; + __be32 *p; + + p = xdr_inline_decode(xdr, XDR_UNIT * 8); + if (!p) + return false; + + iap->ia_valid = 0; + + /* + * Some Sun clients put 0xffff in the mode field when they + * mean 0xffffffff. + */ + tmp1 = be32_to_cpup(p++); + if (tmp1 != (u32)-1 && tmp1 != 0xffff) { + iap->ia_valid |= ATTR_MODE; + iap->ia_mode = tmp1; + } + + tmp1 = be32_to_cpup(p++); + if (tmp1 != (u32)-1) { + iap->ia_uid = make_kuid(nfsd_user_namespace(rqstp), tmp1); + if (uid_valid(iap->ia_uid)) + iap->ia_valid |= ATTR_UID; + } + + tmp1 = be32_to_cpup(p++); + if (tmp1 != (u32)-1) { + iap->ia_gid = make_kgid(nfsd_user_namespace(rqstp), tmp1); + if (gid_valid(iap->ia_gid)) + iap->ia_valid |= ATTR_GID; + } + + tmp1 = be32_to_cpup(p++); + if (tmp1 != (u32)-1) { + iap->ia_valid |= ATTR_SIZE; + iap->ia_size = tmp1; + } + + tmp1 = be32_to_cpup(p++); + tmp2 = be32_to_cpup(p++); + if (tmp1 != (u32)-1 && tmp2 != (u32)-1) { + iap->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET; + iap->ia_atime.tv_sec = tmp1; + iap->ia_atime.tv_nsec = tmp2 * NSEC_PER_USEC; + } + + tmp1 = be32_to_cpup(p++); + tmp2 = be32_to_cpup(p++); + if (tmp1 != (u32)-1 && tmp2 != (u32)-1) { + iap->ia_valid |= ATTR_MTIME | ATTR_MTIME_SET; + iap->ia_mtime.tv_sec = tmp1; + iap->ia_mtime.tv_nsec = tmp2 * NSEC_PER_USEC; + /* + * Passing the invalid value useconds=1000000 for mtime + * is a Sun convention for "set both mtime and atime to + * current server time". It's needed to make permissions + * checks for the "touch" program across v2 mounts to + * Solaris and Irix boxes work correctly. See description of + * sattr in section 6.1 of "NFS Illustrated" by + * Brent Callaghan, Addison-Wesley, ISBN 0-201-32750-5 + */ + if (tmp2 == 1000000) + iap->ia_valid &= ~(ATTR_ATIME_SET|ATTR_MTIME_SET); + } + + return true; +} + static __be32 * encode_fattr(struct svc_rqst *rqstp, __be32 *p, struct svc_fh *fhp, struct kstat *stat) @@ -253,14 +326,11 @@ nfssvc_decode_fhandleargs(struct svc_rqst *rqstp, __be32 *p) int nfssvc_decode_sattrargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_sattrargs *args = rqstp->rq_argp; - p = decode_fh(p, &args->fh); - if (!p) - return 0; - p = decode_sattr(p, &args->attrs, nfsd_user_namespace(rqstp)); - - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_fhandle(xdr, &args->fh) && + svcxdr_decode_sattr(rqstp, xdr, &args->attrs); } int -- cgit v1.2.3 From 7dcf65b91ecaf60ce593e7859ae2b29b7c46ccbd Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 21 Oct 2020 12:43:58 -0400 Subject: NFSD: Update the NFSv2 CREATE argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfsxdr.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 6c87ea8f3876..2e2806cbe7b8 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -401,14 +401,12 @@ nfssvc_decode_writeargs(struct svc_rqst *rqstp, __be32 *p) int nfssvc_decode_createargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_createargs *args = rqstp->rq_argp; - if ( !(p = decode_fh(p, &args->fh)) - || !(p = decode_filename(p, &args->name, &args->len))) - return 0; - p = decode_sattr(p, &args->attrs, nfsd_user_namespace(rqstp)); - - return xdr_argsize_check(rqstp, p); + return svcxdr_decode_diropargs(xdr, &args->fh, + &args->name, &args->len) && + svcxdr_decode_sattr(rqstp, xdr, &args->attrs); } int -- cgit v1.2.3 From 09f75a5375ac61f4adb94da0accc1cfc60eb4f2b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 21 Oct 2020 12:46:03 -0400 Subject: NFSD: Update the NFSv2 SYMLINK argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfsxdr.c | 113 +++++-------------------------------------------------- 1 file changed, 10 insertions(+), 103 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 2e2806cbe7b8..f2cb4794aeaf 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -66,26 +66,6 @@ encode_fh(__be32 *p, struct svc_fh *fhp) return p + (NFS_FHSIZE>> 2); } -/* - * Decode a file name and make sure that the path contains - * no slashes or null bytes. - */ -static __be32 * -decode_filename(__be32 *p, char **namp, unsigned int *lenp) -{ - char *name; - unsigned int i; - - if ((p = xdr_decode_string_inplace(p, namp, lenp, NFS_MAXNAMLEN)) != NULL) { - for (i = 0, name = *namp; i < *lenp; i++, name++) { - if (*name == '\0' || *name == '/') - return NULL; - } - } - - return p; -} - static bool svcxdr_decode_filename(struct xdr_stream *xdr, char **name, unsigned int *len) { @@ -118,61 +98,6 @@ svcxdr_decode_diropargs(struct xdr_stream *xdr, struct svc_fh *fhp, svcxdr_decode_filename(xdr, name, len); } -static __be32 * -decode_sattr(__be32 *p, struct iattr *iap, struct user_namespace *userns) -{ - u32 tmp, tmp1; - - iap->ia_valid = 0; - - /* Sun client bug compatibility check: some sun clients seem to - * put 0xffff in the mode field when they mean 0xffffffff. - * Quoting the 4.4BSD nfs server code: Nah nah nah nah na nah. - */ - if ((tmp = ntohl(*p++)) != (u32)-1 && tmp != 0xffff) { - iap->ia_valid |= ATTR_MODE; - iap->ia_mode = tmp; - } - if ((tmp = ntohl(*p++)) != (u32)-1) { - iap->ia_uid = make_kuid(userns, tmp); - if (uid_valid(iap->ia_uid)) - iap->ia_valid |= ATTR_UID; - } - if ((tmp = ntohl(*p++)) != (u32)-1) { - iap->ia_gid = make_kgid(userns, tmp); - if (gid_valid(iap->ia_gid)) - iap->ia_valid |= ATTR_GID; - } - if ((tmp = ntohl(*p++)) != (u32)-1) { - iap->ia_valid |= ATTR_SIZE; - iap->ia_size = tmp; - } - tmp = ntohl(*p++); tmp1 = ntohl(*p++); - if (tmp != (u32)-1 && tmp1 != (u32)-1) { - iap->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET; - iap->ia_atime.tv_sec = tmp; - iap->ia_atime.tv_nsec = tmp1 * 1000; - } - tmp = ntohl(*p++); tmp1 = ntohl(*p++); - if (tmp != (u32)-1 && tmp1 != (u32)-1) { - iap->ia_valid |= ATTR_MTIME | ATTR_MTIME_SET; - iap->ia_mtime.tv_sec = tmp; - iap->ia_mtime.tv_nsec = tmp1 * 1000; - /* - * Passing the invalid value useconds=1000000 for mtime - * is a Sun convention for "set both mtime and atime to - * current server time". It's needed to make permissions - * checks for the "touch" program across v2 mounts to - * Solaris and Irix boxes work correctly. See description of - * sattr in section 6.1 of "NFS Illustrated" by - * Brent Callaghan, Addison-Wesley, ISBN 0-201-32750-5 - */ - if (tmp1 == 1000000) - iap->ia_valid &= ~(ATTR_ATIME_SET|ATTR_MTIME_SET); - } - return p; -} - static bool svcxdr_decode_sattr(struct svc_rqst *rqstp, struct xdr_stream *xdr, struct iattr *iap) @@ -435,40 +360,22 @@ nfssvc_decode_linkargs(struct svc_rqst *rqstp, __be32 *p) int nfssvc_decode_symlinkargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd_symlinkargs *args = rqstp->rq_argp; - char *base = (char *)p; - size_t xdrlen; + struct kvec *head = rqstp->rq_arg.head; - if ( !(p = decode_fh(p, &args->ffh)) - || !(p = decode_filename(p, &args->fname, &args->flen))) + if (!svcxdr_decode_diropargs(xdr, &args->ffh, &args->fname, &args->flen)) + return 0; + if (xdr_stream_decode_u32(xdr, &args->tlen) < 0) return 0; - - args->tlen = ntohl(*p++); if (args->tlen == 0) return 0; - args->first.iov_base = p; - args->first.iov_len = rqstp->rq_arg.head[0].iov_len; - args->first.iov_len -= (char *)p - base; - - /* This request is never larger than a page. Therefore, - * transport will deliver either: - * 1. pathname in the pagelist -> sattr is in the tail. - * 2. everything in the head buffer -> sattr is in the head. - */ - if (rqstp->rq_arg.page_len) { - if (args->tlen != rqstp->rq_arg.page_len) - return 0; - p = rqstp->rq_arg.tail[0].iov_base; - } else { - xdrlen = XDR_QUADLEN(args->tlen); - if (xdrlen > args->first.iov_len - (8 * sizeof(__be32))) - return 0; - p += xdrlen; - } - decode_sattr(p, &args->attrs, nfsd_user_namespace(rqstp)); - - return 1; + args->first.iov_len = head->iov_len - xdr_stream_pos(xdr); + args->first.iov_base = xdr_inline_decode(xdr, args->tlen); + if (!args->first.iov_base) + return 0; + return svcxdr_decode_sattr(rqstp, xdr, &args->attrs); } int -- cgit v1.2.3 From 5650682e16f41722f735b7beeb2dbc3411dfbeb6 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 10:08:19 -0400 Subject: NFSD: Remove argument length checking in nfsd_dispatch() Now that the argument decoders for NFSv2 and NFSv3 use the xdr_stream mechanism, the version-specific length checking logic in nfsd_dispatch() is no longer necessary. Signed-off-by: Chuck Lever --- fs/nfsd/nfssvc.c | 34 ---------------------------------- 1 file changed, 34 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index f9c9f4c63cc7..6de406322106 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -955,37 +955,6 @@ out: return 0; } -/* - * A write procedure can have a large argument, and a read procedure can - * have a large reply, but no NFSv2 or NFSv3 procedure has argument and - * reply that can both be larger than a page. The xdr code has taken - * advantage of this assumption to be a sloppy about bounds checking in - * some cases. Pending a rewrite of the NFSv2/v3 xdr code to fix that - * problem, we enforce these assumptions here: - */ -static bool nfs_request_too_big(struct svc_rqst *rqstp, - const struct svc_procedure *proc) -{ - /* - * The ACL code has more careful bounds-checking and is not - * susceptible to this problem: - */ - if (rqstp->rq_prog != NFS_PROGRAM) - return false; - /* - * Ditto NFSv4 (which can in theory have argument and reply both - * more than a page): - */ - if (rqstp->rq_vers >= 4) - return false; - /* The reply will be small, we're OK: */ - if (proc->pc_xdrressize > 0 && - proc->pc_xdrressize < XDR_QUADLEN(PAGE_SIZE)) - return false; - - return rqstp->rq_arg.len > PAGE_SIZE; -} - /** * nfsd_dispatch - Process an NFS or NFSACL Request * @rqstp: incoming request @@ -1004,9 +973,6 @@ int nfsd_dispatch(struct svc_rqst *rqstp, __be32 *statp) struct kvec *resv = &rqstp->rq_res.head[0]; __be32 *p; - if (nfs_request_too_big(rqstp, proc)) - goto out_decode_err; - /* * Give the xdr decoder a chance to change this if it wants * (necessary in the NFSv4.0 compound case) -- cgit v1.2.3 From 635a45d34706400c59c3b18ca9fccba195147bda Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 17 Nov 2020 11:32:04 -0500 Subject: NFSD: Update the NFSv2 GETACL argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs2acl.c | 10 +++++----- fs/nfsd/nfsxdr.c | 11 ++++++++++- fs/nfsd/xdr.h | 1 + fs/nfsd/xdr3.h | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 899762da23c9..df2e145cfab0 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -188,17 +188,17 @@ out: static int nfsaclsvc_decode_getaclargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_getaclargs *argp = rqstp->rq_argp; - p = nfs2svc_decode_fh(p, &argp->fh); - if (!p) + if (!svcxdr_decode_fhandle(xdr, &argp->fh)) + return 0; + if (xdr_stream_decode_u32(xdr, &argp->mask) < 0) return 0; - argp->mask = ntohl(*p); p++; - return xdr_argsize_check(rqstp, p); + return 1; } - static int nfsaclsvc_decode_setaclargs(struct svc_rqst *rqstp, __be32 *p) { struct nfsd3_setaclargs *argp = rqstp->rq_argp; diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index f2cb4794aeaf..5ab9fc14816c 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -38,7 +38,16 @@ decode_fh(__be32 *p, struct svc_fh *fhp) return p + (NFS_FHSIZE >> 2); } -static bool +/** + * svcxdr_decode_fhandle - Decode an NFSv2 file handle + * @xdr: XDR stream positioned at an encoded NFSv2 FH + * @fhp: OUT: filled-in server file handle + * + * Return values: + * %false: The encoded file handle was not valid + * %true: @fhp has been initialized + */ +bool svcxdr_decode_fhandle(struct xdr_stream *xdr, struct svc_fh *fhp) { __be32 *p; diff --git a/fs/nfsd/xdr.h b/fs/nfsd/xdr.h index d700838f6512..035c99c7b384 100644 --- a/fs/nfsd/xdr.h +++ b/fs/nfsd/xdr.h @@ -165,5 +165,6 @@ void nfssvc_release_readres(struct svc_rqst *rqstp); /* Helper functions for NFSv2 ACL code */ __be32 *nfs2svc_encode_fattr(struct svc_rqst *rqstp, __be32 *p, struct svc_fh *fhp, struct kstat *stat); __be32 *nfs2svc_decode_fh(__be32 *p, struct svc_fh *fhp); +bool svcxdr_decode_fhandle(struct xdr_stream *xdr, struct svc_fh *fhp); #endif /* LINUX_NFSD_H */ diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index 64af5b01c5d7..43db4206cd25 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -102,7 +102,7 @@ struct nfsd3_commitargs { struct nfsd3_getaclargs { struct svc_fh fh; - int mask; + __u32 mask; }; struct posix_acl; -- cgit v1.2.3 From 6bb844b4eb6e3b109a2fdaffb60e6da722dc4356 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 17 Nov 2020 10:38:46 -0500 Subject: NFSD: Add an xdr_stream-based decoder for NFSv2/3 ACLs Signed-off-by: Chuck Lever --- fs/nfs_common/nfsacl.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfsacl.h | 3 +++ 2 files changed, 55 insertions(+) (limited to 'fs') diff --git a/fs/nfs_common/nfsacl.c b/fs/nfs_common/nfsacl.c index d056ad2fdefd..79c563c1a5e8 100644 --- a/fs/nfs_common/nfsacl.c +++ b/fs/nfs_common/nfsacl.c @@ -295,3 +295,55 @@ int nfsacl_decode(struct xdr_buf *buf, unsigned int base, unsigned int *aclcnt, nfsacl_desc.desc.array_len; } EXPORT_SYMBOL_GPL(nfsacl_decode); + +/** + * nfs_stream_decode_acl - Decode an NFSv3 ACL + * + * @xdr: an xdr_stream positioned at an encoded ACL + * @aclcnt: OUT: count of ACEs in decoded posix_acl + * @pacl: OUT: a dynamically-allocated buffer containing the decoded posix_acl + * + * Return values: + * %false: The encoded ACL is not valid + * %true: @pacl contains a decoded ACL, and @xdr is advanced + * + * On a successful return, caller must release *pacl using posix_acl_release(). + */ +bool nfs_stream_decode_acl(struct xdr_stream *xdr, unsigned int *aclcnt, + struct posix_acl **pacl) +{ + const size_t elem_size = XDR_UNIT * 3; + struct nfsacl_decode_desc nfsacl_desc = { + .desc = { + .elem_size = elem_size, + .xcode = pacl ? xdr_nfsace_decode : NULL, + }, + }; + unsigned int base; + u32 entries; + + if (xdr_stream_decode_u32(xdr, &entries) < 0) + return false; + if (entries > NFS_ACL_MAX_ENTRIES) + return false; + + base = xdr_stream_pos(xdr); + if (!xdr_inline_decode(xdr, XDR_UNIT + elem_size * entries)) + return false; + nfsacl_desc.desc.array_maxlen = entries; + if (xdr_decode_array2(xdr->buf, base, &nfsacl_desc.desc)) + return false; + + if (pacl) { + if (entries != nfsacl_desc.desc.array_len || + posix_acl_from_nfsacl(nfsacl_desc.acl) != 0) { + posix_acl_release(nfsacl_desc.acl); + return false; + } + *pacl = nfsacl_desc.acl; + } + if (aclcnt) + *aclcnt = entries; + return true; +} +EXPORT_SYMBOL_GPL(nfs_stream_decode_acl); diff --git a/include/linux/nfsacl.h b/include/linux/nfsacl.h index 103d44695323..0ba99c513649 100644 --- a/include/linux/nfsacl.h +++ b/include/linux/nfsacl.h @@ -38,5 +38,8 @@ nfsacl_encode(struct xdr_buf *buf, unsigned int base, struct inode *inode, extern int nfsacl_decode(struct xdr_buf *buf, unsigned int base, unsigned int *aclcnt, struct posix_acl **pacl); +extern bool +nfs_stream_decode_acl(struct xdr_stream *xdr, unsigned int *aclcnt, + struct posix_acl **pacl); #endif /* __LINUX_NFSACL_H */ -- cgit v1.2.3 From 427eab3ba22891845265f9a3846de6ac152ec836 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 17 Nov 2020 11:37:35 -0500 Subject: NFSD: Update the NFSv2 SETACL argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs2acl.c | 29 ++++++++++++----------------- fs/nfsd/xdr3.h | 2 +- 2 files changed, 13 insertions(+), 18 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index df2e145cfab0..123820ec79d3 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -201,28 +201,23 @@ static int nfsaclsvc_decode_getaclargs(struct svc_rqst *rqstp, __be32 *p) static int nfsaclsvc_decode_setaclargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_setaclargs *argp = rqstp->rq_argp; - struct kvec *head = rqstp->rq_arg.head; - unsigned int base; - int n; - p = nfs2svc_decode_fh(p, &argp->fh); - if (!p) + if (!svcxdr_decode_fhandle(xdr, &argp->fh)) + return 0; + if (xdr_stream_decode_u32(xdr, &argp->mask) < 0) + return 0; + if (argp->mask & ~NFS_ACL_MASK) return 0; - argp->mask = ntohl(*p++); - if (argp->mask & ~NFS_ACL_MASK || - !xdr_argsize_check(rqstp, p)) + if (!nfs_stream_decode_acl(xdr, NULL, (argp->mask & NFS_ACL) ? + &argp->acl_access : NULL)) + return 0; + if (!nfs_stream_decode_acl(xdr, NULL, (argp->mask & NFS_DFACL) ? + &argp->acl_default : NULL)) return 0; - base = (char *)p - (char *)head->iov_base; - n = nfsacl_decode(&rqstp->rq_arg, base, NULL, - (argp->mask & NFS_ACL) ? - &argp->acl_access : NULL); - if (n > 0) - n = nfsacl_decode(&rqstp->rq_arg, base + n, NULL, - (argp->mask & NFS_DFACL) ? - &argp->acl_default : NULL); - return (n > 0); + return 1; } static int nfsaclsvc_decode_fhandleargs(struct svc_rqst *rqstp, __be32 *p) diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index 43db4206cd25..5afb3ce4f062 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -108,7 +108,7 @@ struct nfsd3_getaclargs { struct posix_acl; struct nfsd3_setaclargs { struct svc_fh fh; - int mask; + __u32 mask; struct posix_acl *acl_access; struct posix_acl *acl_default; }; -- cgit v1.2.3 From 571d31f37a57729c9d3463b5a692a84e619b408a Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 17 Nov 2020 11:46:50 -0500 Subject: NFSD: Update the NFSv2 ACL GETATTR argument decoder to use struct xdr_stream Since the ACL GETATTR procedure is the same as the normal GETATTR procedure, simply re-use nfssvc_decode_fhandleargs. Signed-off-by: Chuck Lever --- fs/nfsd/nfs2acl.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 123820ec79d3..0274348f6679 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -220,16 +220,6 @@ static int nfsaclsvc_decode_setaclargs(struct svc_rqst *rqstp, __be32 *p) return 1; } -static int nfsaclsvc_decode_fhandleargs(struct svc_rqst *rqstp, __be32 *p) -{ - struct nfsd_fhandle *argp = rqstp->rq_argp; - - p = nfs2svc_decode_fh(p, &argp->fh); - if (!p) - return 0; - return xdr_argsize_check(rqstp, p); -} - static int nfsaclsvc_decode_accessargs(struct svc_rqst *rqstp, __be32 *p) { struct nfsd3_accessargs *argp = rqstp->rq_argp; @@ -392,7 +382,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { }, [ACLPROC2_GETATTR] = { .pc_func = nfsacld_proc_getattr, - .pc_decode = nfsaclsvc_decode_fhandleargs, + .pc_decode = nfssvc_decode_fhandleargs, .pc_encode = nfsaclsvc_encode_attrstatres, .pc_release = nfsaclsvc_release_attrstat, .pc_argsize = sizeof(struct nfsd_fhandle), -- cgit v1.2.3 From 64063892efc1daa3a48882673811ff327ba75ed5 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 17 Nov 2020 11:49:29 -0500 Subject: NFSD: Update the NFSv2 ACL ACCESS argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs2acl.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 0274348f6679..7eeac5b81c20 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -222,14 +222,15 @@ static int nfsaclsvc_decode_setaclargs(struct svc_rqst *rqstp, __be32 *p) static int nfsaclsvc_decode_accessargs(struct svc_rqst *rqstp, __be32 *p) { - struct nfsd3_accessargs *argp = rqstp->rq_argp; + struct xdr_stream *xdr = &rqstp->rq_arg_stream; + struct nfsd3_accessargs *args = rqstp->rq_argp; - p = nfs2svc_decode_fh(p, &argp->fh); - if (!p) + if (!svcxdr_decode_fhandle(xdr, &args->fh)) + return 0; + if (xdr_stream_decode_u32(xdr, &args->access) < 0) return 0; - argp->access = ntohl(*p++); - return xdr_argsize_check(rqstp, p); + return 1; } /* -- cgit v1.2.3 From baadce65d6ee3032b921d9c043ba808bc69d6b13 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 19 Oct 2020 17:49:16 -0400 Subject: NFSD: Clean up after updating NFSv2 ACL decoders Signed-off-by: Chuck Lever --- fs/nfsd/nfsxdr.c | 18 ------------------ fs/nfsd/xdr.h | 1 - 2 files changed, 19 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index 5ab9fc14816c..5d79ef6a0c7f 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -26,18 +26,6 @@ static u32 nfs_ftypes[] = { * Basic NFSv2 data types (RFC 1094 Section 2.3) */ -static __be32 * -decode_fh(__be32 *p, struct svc_fh *fhp) -{ - fh_init(fhp, NFS_FHSIZE); - memcpy(&fhp->fh_handle.fh_base, p, NFS_FHSIZE); - fhp->fh_handle.fh_size = NFS_FHSIZE; - - /* FIXME: Look up export pointer here and verify - * Sun Secure RPC if requested */ - return p + (NFS_FHSIZE >> 2); -} - /** * svcxdr_decode_fhandle - Decode an NFSv2 file handle * @xdr: XDR stream positioned at an encoded NFSv2 FH @@ -62,12 +50,6 @@ svcxdr_decode_fhandle(struct xdr_stream *xdr, struct svc_fh *fhp) return true; } -/* Helper function for NFSv2 ACL code */ -__be32 *nfs2svc_decode_fh(__be32 *p, struct svc_fh *fhp) -{ - return decode_fh(p, fhp); -} - static __be32 * encode_fh(__be32 *p, struct svc_fh *fhp) { diff --git a/fs/nfsd/xdr.h b/fs/nfsd/xdr.h index 035c99c7b384..3018b52b6d5e 100644 --- a/fs/nfsd/xdr.h +++ b/fs/nfsd/xdr.h @@ -164,7 +164,6 @@ void nfssvc_release_readres(struct svc_rqst *rqstp); /* Helper functions for NFSv2 ACL code */ __be32 *nfs2svc_encode_fattr(struct svc_rqst *rqstp, __be32 *p, struct svc_fh *fhp, struct kstat *stat); -__be32 *nfs2svc_decode_fh(__be32 *p, struct svc_fh *fhp); bool svcxdr_decode_fhandle(struct xdr_stream *xdr, struct svc_fh *fhp); #endif /* LINUX_NFSD_H */ -- cgit v1.2.3 From 05027eafc266487c6e056d10ab352861df95b5d4 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 17 Nov 2020 11:52:04 -0500 Subject: NFSD: Update the NFSv3 GETACL argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3acl.c | 11 ++++++----- fs/nfsd/nfs3xdr.c | 11 ++++++++++- fs/nfsd/xdr3.h | 1 + 3 files changed, 17 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index 9e1a92fb9771..addb0d7d5500 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -124,19 +124,20 @@ out: /* * XDR decode functions */ + static int nfs3svc_decode_getaclargs(struct svc_rqst *rqstp, __be32 *p) { + struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct nfsd3_getaclargs *args = rqstp->rq_argp; - p = nfs3svc_decode_fh(p, &args->fh); - if (!p) + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return 0; + if (xdr_stream_decode_u32(xdr, &args->mask) < 0) return 0; - args->mask = ntohl(*p); p++; - return xdr_argsize_check(rqstp, p); + return 1; } - static int nfs3svc_decode_setaclargs(struct svc_rqst *rqstp, __be32 *p) { struct nfsd3_setaclargs *args = rqstp->rq_argp; diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index a30b418a5116..aa55d0ba2a54 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -53,7 +53,16 @@ svcxdr_decode_nfstime3(struct xdr_stream *xdr, struct timespec64 *timep) return true; } -static bool +/** + * svcxdr_decode_nfs_fh3 - Decode an NFSv3 file handle + * @xdr: XDR stream positioned at an undecoded NFSv3 FH + * @fhp: OUT: filled-in server file handle + * + * Return values: + * %false: The encoded file handle was not valid + * %true: @fhp has been initialized + */ +bool svcxdr_decode_nfs_fh3(struct xdr_stream *xdr, struct svc_fh *fhp) { __be32 *p; diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index 5afb3ce4f062..7456aee74f3d 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -308,6 +308,7 @@ int nfs3svc_encode_entry_plus(void *, const char *name, __be32 *nfs3svc_encode_post_op_attr(struct svc_rqst *rqstp, __be32 *p, struct svc_fh *fhp); __be32 *nfs3svc_decode_fh(__be32 *p, struct svc_fh *fhp); +bool svcxdr_decode_nfs_fh3(struct xdr_stream *xdr, struct svc_fh *fhp); #endif /* _LINUX_NFSD_XDR3_H */ -- cgit v1.2.3 From 68519ff2a1c72c67fcdc4b81671acda59f420af9 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 17 Nov 2020 11:56:26 -0500 Subject: NFSD: Update the NFSv2 SETACL argument decoder to use struct xdr_stream Signed-off-by: Chuck Lever --- fs/nfsd/nfs3acl.c | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index addb0d7d5500..a568b842e9eb 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -140,28 +140,23 @@ static int nfs3svc_decode_getaclargs(struct svc_rqst *rqstp, __be32 *p) static int nfs3svc_decode_setaclargs(struct svc_rqst *rqstp, __be32 *p) { - struct nfsd3_setaclargs *args = rqstp->rq_argp; - struct kvec *head = rqstp->rq_arg.head; - unsigned int base; - int n; + struct xdr_stream *xdr = &rqstp->rq_arg_stream; + struct nfsd3_setaclargs *argp = rqstp->rq_argp; - p = nfs3svc_decode_fh(p, &args->fh); - if (!p) + if (!svcxdr_decode_nfs_fh3(xdr, &argp->fh)) + return 0; + if (xdr_stream_decode_u32(xdr, &argp->mask) < 0) return 0; - args->mask = ntohl(*p++); - if (args->mask & ~NFS_ACL_MASK || - !xdr_argsize_check(rqstp, p)) + if (argp->mask & ~NFS_ACL_MASK) + return 0; + if (!nfs_stream_decode_acl(xdr, NULL, (argp->mask & NFS_ACL) ? + &argp->acl_access : NULL)) + return 0; + if (!nfs_stream_decode_acl(xdr, NULL, (argp->mask & NFS_DFACL) ? + &argp->acl_default : NULL)) return 0; - base = (char *)p - (char *)head->iov_base; - n = nfsacl_decode(&rqstp->rq_arg, base, NULL, - (args->mask & NFS_ACL) ? - &args->acl_access : NULL); - if (n > 0) - n = nfsacl_decode(&rqstp->rq_arg, base + n, NULL, - (args->mask & NFS_DFACL) ? - &args->acl_default : NULL); - return (n > 0); + return 1; } /* -- cgit v1.2.3 From 9cee763ee654ce8622d673b8e32687d738e24ace Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 20 Oct 2020 09:56:52 -0400 Subject: NFSD: Clean up after updating NFSv3 ACL decoders Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 20 -------------------- fs/nfsd/xdr3.h | 2 -- 2 files changed, 22 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index aa55d0ba2a54..00a96054280a 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -82,26 +82,6 @@ svcxdr_decode_nfs_fh3(struct xdr_stream *xdr, struct svc_fh *fhp) return true; } -static __be32 * -decode_fh(__be32 *p, struct svc_fh *fhp) -{ - unsigned int size; - fh_init(fhp, NFS3_FHSIZE); - size = ntohl(*p++); - if (size > NFS3_FHSIZE) - return NULL; - - memcpy(&fhp->fh_handle.fh_base, p, size); - fhp->fh_handle.fh_size = size; - return p + XDR_QUADLEN(size); -} - -/* Helper function for NFSv3 ACL code */ -__be32 *nfs3svc_decode_fh(__be32 *p, struct svc_fh *fhp) -{ - return decode_fh(p, fhp); -} - static __be32 * encode_fh(__be32 *p, struct svc_fh *fhp) { diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h index 7456aee74f3d..3e1578953f54 100644 --- a/fs/nfsd/xdr3.h +++ b/fs/nfsd/xdr3.h @@ -307,8 +307,6 @@ int nfs3svc_encode_entry_plus(void *, const char *name, /* Helper functions for NFSv3 ACL code */ __be32 *nfs3svc_encode_post_op_attr(struct svc_rqst *rqstp, __be32 *p, struct svc_fh *fhp); -__be32 *nfs3svc_decode_fh(__be32 *p, struct svc_fh *fhp); bool svcxdr_decode_nfs_fh3(struct xdr_stream *xdr, struct svc_fh *fhp); - #endif /* _LINUX_NFSD_XDR3_H */ -- cgit v1.2.3 From 1b76d1df1a3683b6b23cd1c813d13c5e6a9d35e5 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Wed, 6 Jan 2021 09:52:34 +0200 Subject: nfsd: remove unused stats counters Commit 501cb1849f86 ("nfsd: rip out the raparms cache") removed the code that updates read-ahead cache stats counters, commit 8bbfa9f3889b ("knfsd: remove the nfsd thread busy histogram") removed code that updates the thread busy stats counters back in 2009 and code that updated filehandle cache stats was removed back in 2002. Remove the unused stats counters from nfsd_stats struct and print hardcoded zeros in /proc/net/rpc/nfsd. Signed-off-by: Amir Goldstein Signed-off-by: Chuck Lever --- fs/nfsd/stats.c | 41 ++++++++++++++++------------------------- fs/nfsd/stats.h | 10 ---------- 2 files changed, 16 insertions(+), 35 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/stats.c b/fs/nfsd/stats.c index b1bc582b0493..e928e224205a 100644 --- a/fs/nfsd/stats.c +++ b/fs/nfsd/stats.c @@ -7,16 +7,14 @@ * Format: * rc * Statistsics for the reply cache - * fh + * fh * statistics for filehandle lookup * io * statistics for IO throughput - * th <10%-20%> <20%-30%> ... <90%-100%> <100%> - * time (seconds) when nfsd thread usage above thresholds - * and number of times that all threads were in use - * ra cache-size <10% <20% <30% ... <100% not-found - * number of times that read-ahead entry was found that deep in - * the cache. + * th + * number of threads + * ra + * * plus generic RPC stats (see net/sunrpc/stats.c) * * Copyright (C) 1995, 1996, 1997 Olaf Kirch @@ -38,31 +36,24 @@ static int nfsd_proc_show(struct seq_file *seq, void *v) { int i; - seq_printf(seq, "rc %u %u %u\nfh %u %u %u %u %u\nio %u %u\n", + seq_printf(seq, "rc %u %u %u\nfh %u 0 0 0 0\nio %u %u\n", nfsdstats.rchits, nfsdstats.rcmisses, nfsdstats.rcnocache, nfsdstats.fh_stale, - nfsdstats.fh_lookup, - nfsdstats.fh_anon, - nfsdstats.fh_nocache_dir, - nfsdstats.fh_nocache_nondir, nfsdstats.io_read, nfsdstats.io_write); + /* thread usage: */ - seq_printf(seq, "th %u %u", nfsdstats.th_cnt, nfsdstats.th_fullcnt); - for (i=0; i<10; i++) { - unsigned int jifs = nfsdstats.th_usage[i]; - unsigned int sec = jifs / HZ, msec = (jifs % HZ)*1000/HZ; - seq_printf(seq, " %u.%03u", sec, msec); - } - - /* newline and ra-cache */ - seq_printf(seq, "\nra %u", nfsdstats.ra_size); - for (i=0; i<11; i++) - seq_printf(seq, " %u", nfsdstats.ra_depth[i]); - seq_putc(seq, '\n'); - + seq_printf(seq, "th %u 0", nfsdstats.th_cnt); + + /* deprecated thread usage histogram stats */ + for (i = 0; i < 10; i++) + seq_puts(seq, " 0.000"); + + /* deprecated ra-cache stats */ + seq_puts(seq, "\nra 0 0 0 0 0 0 0 0 0 0 0 0\n"); + /* show my rpc info */ svc_seq_show(seq, &nfsd_svcstats); diff --git a/fs/nfsd/stats.h b/fs/nfsd/stats.h index b23fdac69820..5e3cdf21556a 100644 --- a/fs/nfsd/stats.h +++ b/fs/nfsd/stats.h @@ -15,19 +15,9 @@ struct nfsd_stats { unsigned int rcmisses; /* repcache hits */ unsigned int rcnocache; /* uncached reqs */ unsigned int fh_stale; /* FH stale error */ - unsigned int fh_lookup; /* dentry cached */ - unsigned int fh_anon; /* anon file dentry returned */ - unsigned int fh_nocache_dir; /* filehandle not found in dcache */ - unsigned int fh_nocache_nondir; /* filehandle not found in dcache */ unsigned int io_read; /* bytes returned to read requests */ unsigned int io_write; /* bytes passed in write requests */ unsigned int th_cnt; /* number of available threads */ - unsigned int th_usage[10]; /* number of ticks during which n perdeciles - * of available threads were in use */ - unsigned int th_fullcnt; /* number of times last free thread was used */ - unsigned int ra_size; /* size of ra cache */ - unsigned int ra_depth[11]; /* number of times ra entry was found that deep - * in the cache (10percentiles). [10] = not found */ #ifdef CONFIG_NFSD_V4 unsigned int nfs4_opcount[LAST_NFS4_OP + 1]; /* count of individual nfsv4 operations */ #endif -- cgit v1.2.3 From e567b98ce9a4b35b63c364d24828a9e5cd7a8179 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Wed, 6 Jan 2021 09:52:35 +0200 Subject: nfsd: protect concurrent access to nfsd stats counters nfsd stats counters can be updated by concurrent nfsd threads without any protection. Convert some nfsd_stats and nfsd_net struct members to use percpu counters. The longest_chain* members of struct nfsd_net remain unprotected. Signed-off-by: Amir Goldstein Signed-off-by: Chuck Lever --- fs/nfsd/netns.h | 23 ++++++++++------ fs/nfsd/nfs4proc.c | 2 +- fs/nfsd/nfscache.c | 52 +++++++++++++++++++++++++---------- fs/nfsd/nfsctl.c | 5 +++- fs/nfsd/nfsfh.c | 2 +- fs/nfsd/stats.c | 77 +++++++++++++++++++++++++++++++++++++++++++--------- fs/nfsd/stats.h | 80 ++++++++++++++++++++++++++++++++++++++++++++++-------- fs/nfsd/vfs.c | 4 +-- 8 files changed, 192 insertions(+), 53 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 7346acda9d76..c330f5bd0cf3 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -10,6 +10,7 @@ #include #include +#include /* Hash tables for nfs4_clientid state */ #define CLIENT_HASH_BITS 4 @@ -21,6 +22,14 @@ struct cld_net; struct nfsd4_client_tracking_ops; +enum { + /* cache misses due only to checksum comparison failures */ + NFSD_NET_PAYLOAD_MISSES, + /* amount of memory (in bytes) currently consumed by the DRC */ + NFSD_NET_DRC_MEM_USAGE, + NFSD_NET_COUNTERS_NUM +}; + /* * Represents a nfsd "container". With respect to nfsv4 state tracking, the * fields of interest are the *_id_hashtbls and the *_name_tree. These track @@ -149,20 +158,16 @@ struct nfsd_net { /* * Stats and other tracking of on the duplicate reply cache. - * These fields and the "rc" fields in nfsdstats are modified - * with only the per-bucket cache lock, which isn't really safe - * and should be fixed if we want the statistics to be - * completely accurate. + * The longest_chain* fields are modified with only the per-bucket + * cache lock, which isn't really safe and should be fixed if we want + * these statistics to be completely accurate. */ /* total number of entries */ atomic_t num_drc_entries; - /* cache misses due only to checksum comparison failures */ - unsigned int payload_misses; - - /* amount of memory (in bytes) currently consumed by the DRC */ - unsigned int drc_mem_usage; + /* Per-netns stats counters */ + struct percpu_counter counter[NFSD_NET_COUNTERS_NUM]; /* longest hash chain seen */ unsigned int longest_chain; diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index f567592692ee..8273a99359d1 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -2174,7 +2174,7 @@ nfsd4_proc_null(struct svc_rqst *rqstp) static inline void nfsd4_increment_op_stats(u32 opnum) { if (opnum >= FIRST_NFS4_OP && opnum <= LAST_NFS4_OP) - nfsdstats.nfs4_opcount[opnum]++; + percpu_counter_inc(&nfsdstats.counter[NFSD_STATS_NFS4_OP(opnum)]); } static const struct nfsd4_operation nfsd4_ops[]; diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c index 80c90fc231a5..96cdf77925f3 100644 --- a/fs/nfsd/nfscache.c +++ b/fs/nfsd/nfscache.c @@ -121,14 +121,14 @@ nfsd_reply_cache_free_locked(struct nfsd_drc_bucket *b, struct svc_cacherep *rp, struct nfsd_net *nn) { if (rp->c_type == RC_REPLBUFF && rp->c_replvec.iov_base) { - nn->drc_mem_usage -= rp->c_replvec.iov_len; + nfsd_stats_drc_mem_usage_sub(nn, rp->c_replvec.iov_len); kfree(rp->c_replvec.iov_base); } if (rp->c_state != RC_UNUSED) { rb_erase(&rp->c_node, &b->rb_head); list_del(&rp->c_lru); atomic_dec(&nn->num_drc_entries); - nn->drc_mem_usage -= sizeof(*rp); + nfsd_stats_drc_mem_usage_sub(nn, sizeof(*rp)); } kmem_cache_free(drc_slab, rp); } @@ -154,6 +154,16 @@ void nfsd_drc_slab_free(void) kmem_cache_destroy(drc_slab); } +static int nfsd_reply_cache_stats_init(struct nfsd_net *nn) +{ + return nfsd_percpu_counters_init(nn->counter, NFSD_NET_COUNTERS_NUM); +} + +static void nfsd_reply_cache_stats_destroy(struct nfsd_net *nn) +{ + nfsd_percpu_counters_destroy(nn->counter, NFSD_NET_COUNTERS_NUM); +} + int nfsd_reply_cache_init(struct nfsd_net *nn) { unsigned int hashsize; @@ -165,12 +175,16 @@ int nfsd_reply_cache_init(struct nfsd_net *nn) hashsize = nfsd_hashsize(nn->max_drc_entries); nn->maskbits = ilog2(hashsize); + status = nfsd_reply_cache_stats_init(nn); + if (status) + goto out_nomem; + nn->nfsd_reply_cache_shrinker.scan_objects = nfsd_reply_cache_scan; nn->nfsd_reply_cache_shrinker.count_objects = nfsd_reply_cache_count; nn->nfsd_reply_cache_shrinker.seeks = 1; status = register_shrinker(&nn->nfsd_reply_cache_shrinker); if (status) - goto out_nomem; + goto out_stats_destroy; nn->drc_hashtbl = kvzalloc(array_size(hashsize, sizeof(*nn->drc_hashtbl)), GFP_KERNEL); @@ -186,6 +200,8 @@ int nfsd_reply_cache_init(struct nfsd_net *nn) return 0; out_shrinker: unregister_shrinker(&nn->nfsd_reply_cache_shrinker); +out_stats_destroy: + nfsd_reply_cache_stats_destroy(nn); out_nomem: printk(KERN_ERR "nfsd: failed to allocate reply cache\n"); return -ENOMEM; @@ -196,6 +212,7 @@ void nfsd_reply_cache_shutdown(struct nfsd_net *nn) struct svc_cacherep *rp; unsigned int i; + nfsd_reply_cache_stats_destroy(nn); unregister_shrinker(&nn->nfsd_reply_cache_shrinker); for (i = 0; i < nn->drc_hashsize; i++) { @@ -324,7 +341,7 @@ nfsd_cache_key_cmp(const struct svc_cacherep *key, { if (key->c_key.k_xid == rp->c_key.k_xid && key->c_key.k_csum != rp->c_key.k_csum) { - ++nn->payload_misses; + nfsd_stats_payload_misses_inc(nn); trace_nfsd_drc_mismatch(nn, key, rp); } @@ -407,7 +424,7 @@ int nfsd_cache_lookup(struct svc_rqst *rqstp) rqstp->rq_cacherep = NULL; if (type == RC_NOCACHE) { - nfsdstats.rcnocache++; + nfsd_stats_rc_nocache_inc(); goto out; } @@ -429,12 +446,12 @@ int nfsd_cache_lookup(struct svc_rqst *rqstp) goto found_entry; } - nfsdstats.rcmisses++; + nfsd_stats_rc_misses_inc(); rqstp->rq_cacherep = rp; rp->c_state = RC_INPROG; atomic_inc(&nn->num_drc_entries); - nn->drc_mem_usage += sizeof(*rp); + nfsd_stats_drc_mem_usage_add(nn, sizeof(*rp)); /* go ahead and prune the cache */ prune_bucket(b, nn); @@ -446,7 +463,7 @@ out: found_entry: /* We found a matching entry which is either in progress or done. */ - nfsdstats.rchits++; + nfsd_stats_rc_hits_inc(); rtn = RC_DROPIT; /* Request being processed */ @@ -548,7 +565,7 @@ void nfsd_cache_update(struct svc_rqst *rqstp, int cachetype, __be32 *statp) return; } spin_lock(&b->cache_lock); - nn->drc_mem_usage += bufsize; + nfsd_stats_drc_mem_usage_add(nn, bufsize); lru_put_end(b, rp); rp->c_secure = test_bit(RQ_SECURE, &rqstp->rq_flags); rp->c_type = cachetype; @@ -588,13 +605,18 @@ static int nfsd_reply_cache_stats_show(struct seq_file *m, void *v) seq_printf(m, "max entries: %u\n", nn->max_drc_entries); seq_printf(m, "num entries: %u\n", - atomic_read(&nn->num_drc_entries)); + atomic_read(&nn->num_drc_entries)); seq_printf(m, "hash buckets: %u\n", 1 << nn->maskbits); - seq_printf(m, "mem usage: %u\n", nn->drc_mem_usage); - seq_printf(m, "cache hits: %u\n", nfsdstats.rchits); - seq_printf(m, "cache misses: %u\n", nfsdstats.rcmisses); - seq_printf(m, "not cached: %u\n", nfsdstats.rcnocache); - seq_printf(m, "payload misses: %u\n", nn->payload_misses); + seq_printf(m, "mem usage: %lld\n", + percpu_counter_sum_positive(&nn->counter[NFSD_NET_DRC_MEM_USAGE])); + seq_printf(m, "cache hits: %lld\n", + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_RC_HITS])); + seq_printf(m, "cache misses: %lld\n", + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_RC_MISSES])); + seq_printf(m, "not cached: %lld\n", + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_RC_NOCACHE])); + seq_printf(m, "payload misses: %lld\n", + percpu_counter_sum_positive(&nn->counter[NFSD_NET_PAYLOAD_MISSES])); seq_printf(m, "longest chain len: %u\n", nn->longest_chain); seq_printf(m, "cachesize at longest: %u\n", nn->longest_chain_cachesize); return 0; diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index f6d5d783f4a4..258605ee49b8 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -1534,7 +1534,9 @@ static int __init init_nfsd(void) retval = nfsd4_init_pnfs(); if (retval) goto out_free_slabs; - nfsd_stat_init(); /* Statistics */ + retval = nfsd_stat_init(); /* Statistics */ + if (retval) + goto out_free_pnfs; retval = nfsd_drc_slab_create(); if (retval) goto out_free_stat; @@ -1554,6 +1556,7 @@ out_free_lockd: nfsd_drc_slab_free(); out_free_stat: nfsd_stat_shutdown(); +out_free_pnfs: nfsd4_exit_pnfs(); out_free_slabs: nfsd4_free_slabs(); diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 66f2ef67792a..9e31b2b5c6d2 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -422,7 +422,7 @@ skip_pseudoflavor_check: } out: if (error == nfserr_stale) - nfsdstats.fh_stale++; + nfsd_stats_fh_stale_inc(); return error; } diff --git a/fs/nfsd/stats.c b/fs/nfsd/stats.c index e928e224205a..1d3b881e7382 100644 --- a/fs/nfsd/stats.c +++ b/fs/nfsd/stats.c @@ -36,13 +36,13 @@ static int nfsd_proc_show(struct seq_file *seq, void *v) { int i; - seq_printf(seq, "rc %u %u %u\nfh %u 0 0 0 0\nio %u %u\n", - nfsdstats.rchits, - nfsdstats.rcmisses, - nfsdstats.rcnocache, - nfsdstats.fh_stale, - nfsdstats.io_read, - nfsdstats.io_write); + seq_printf(seq, "rc %lld %lld %lld\nfh %lld 0 0 0 0\nio %lld %lld\n", + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_RC_HITS]), + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_RC_MISSES]), + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_RC_NOCACHE]), + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_FH_STALE]), + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_IO_READ]), + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_IO_WRITE])); /* thread usage: */ seq_printf(seq, "th %u 0", nfsdstats.th_cnt); @@ -61,8 +61,10 @@ static int nfsd_proc_show(struct seq_file *seq, void *v) /* Show count for individual nfsv4 operations */ /* Writing operation numbers 0 1 2 also for maintaining uniformity */ seq_printf(seq,"proc4ops %u", LAST_NFS4_OP + 1); - for (i = 0; i <= LAST_NFS4_OP; i++) - seq_printf(seq, " %u", nfsdstats.nfs4_opcount[i]); + for (i = 0; i <= LAST_NFS4_OP; i++) { + seq_printf(seq, " %lld", + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_NFS4_OP(i)])); + } seq_putc(seq, '\n'); #endif @@ -82,14 +84,63 @@ static const struct proc_ops nfsd_proc_ops = { .proc_release = single_release, }; -void -nfsd_stat_init(void) +int nfsd_percpu_counters_init(struct percpu_counter counters[], int num) { + int i, err = 0; + + for (i = 0; !err && i < num; i++) + err = percpu_counter_init(&counters[i], 0, GFP_KERNEL); + + if (!err) + return 0; + + for (; i > 0; i--) + percpu_counter_destroy(&counters[i-1]); + + return err; +} + +void nfsd_percpu_counters_reset(struct percpu_counter counters[], int num) +{ + int i; + + for (i = 0; i < num; i++) + percpu_counter_set(&counters[i], 0); +} + +void nfsd_percpu_counters_destroy(struct percpu_counter counters[], int num) +{ + int i; + + for (i = 0; i < num; i++) + percpu_counter_destroy(&counters[i]); +} + +static int nfsd_stat_counters_init(void) +{ + return nfsd_percpu_counters_init(nfsdstats.counter, NFSD_STATS_COUNTERS_NUM); +} + +static void nfsd_stat_counters_destroy(void) +{ + nfsd_percpu_counters_destroy(nfsdstats.counter, NFSD_STATS_COUNTERS_NUM); +} + +int nfsd_stat_init(void) +{ + int err; + + err = nfsd_stat_counters_init(); + if (err) + return err; + svc_proc_register(&init_net, &nfsd_svcstats, &nfsd_proc_ops); + + return 0; } -void -nfsd_stat_shutdown(void) +void nfsd_stat_shutdown(void) { + nfsd_stat_counters_destroy(); svc_proc_unregister(&init_net, "nfsd"); } diff --git a/fs/nfsd/stats.h b/fs/nfsd/stats.h index 5e3cdf21556a..87c3150c200f 100644 --- a/fs/nfsd/stats.h +++ b/fs/nfsd/stats.h @@ -8,27 +8,85 @@ #define _NFSD_STATS_H #include +#include -struct nfsd_stats { - unsigned int rchits; /* repcache hits */ - unsigned int rcmisses; /* repcache hits */ - unsigned int rcnocache; /* uncached reqs */ - unsigned int fh_stale; /* FH stale error */ - unsigned int io_read; /* bytes returned to read requests */ - unsigned int io_write; /* bytes passed in write requests */ - unsigned int th_cnt; /* number of available threads */ +enum { + NFSD_STATS_RC_HITS, /* repcache hits */ + NFSD_STATS_RC_MISSES, /* repcache misses */ + NFSD_STATS_RC_NOCACHE, /* uncached reqs */ + NFSD_STATS_FH_STALE, /* FH stale error */ + NFSD_STATS_IO_READ, /* bytes returned to read requests */ + NFSD_STATS_IO_WRITE, /* bytes passed in write requests */ #ifdef CONFIG_NFSD_V4 - unsigned int nfs4_opcount[LAST_NFS4_OP + 1]; /* count of individual nfsv4 operations */ + NFSD_STATS_FIRST_NFS4_OP, /* count of individual nfsv4 operations */ + NFSD_STATS_LAST_NFS4_OP = NFSD_STATS_FIRST_NFS4_OP + LAST_NFS4_OP, +#define NFSD_STATS_NFS4_OP(op) (NFSD_STATS_FIRST_NFS4_OP + (op)) #endif + NFSD_STATS_COUNTERS_NUM +}; + +struct nfsd_stats { + struct percpu_counter counter[NFSD_STATS_COUNTERS_NUM]; + /* Protected by nfsd_mutex */ + unsigned int th_cnt; /* number of available threads */ }; extern struct nfsd_stats nfsdstats; + extern struct svc_stat nfsd_svcstats; -void nfsd_stat_init(void); -void nfsd_stat_shutdown(void); +int nfsd_percpu_counters_init(struct percpu_counter counters[], int num); +void nfsd_percpu_counters_reset(struct percpu_counter counters[], int num); +void nfsd_percpu_counters_destroy(struct percpu_counter counters[], int num); +int nfsd_stat_init(void); +void nfsd_stat_shutdown(void); + +static inline void nfsd_stats_rc_hits_inc(void) +{ + percpu_counter_inc(&nfsdstats.counter[NFSD_STATS_RC_HITS]); +} + +static inline void nfsd_stats_rc_misses_inc(void) +{ + percpu_counter_inc(&nfsdstats.counter[NFSD_STATS_RC_MISSES]); +} + +static inline void nfsd_stats_rc_nocache_inc(void) +{ + percpu_counter_inc(&nfsdstats.counter[NFSD_STATS_RC_NOCACHE]); +} + +static inline void nfsd_stats_fh_stale_inc(void) +{ + percpu_counter_inc(&nfsdstats.counter[NFSD_STATS_FH_STALE]); +} + +static inline void nfsd_stats_io_read_add(s64 amount) +{ + percpu_counter_add(&nfsdstats.counter[NFSD_STATS_IO_READ], amount); +} + +static inline void nfsd_stats_io_write_add(s64 amount) +{ + percpu_counter_add(&nfsdstats.counter[NFSD_STATS_IO_WRITE], amount); +} + +static inline void nfsd_stats_payload_misses_inc(struct nfsd_net *nn) +{ + percpu_counter_inc(&nn->counter[NFSD_NET_PAYLOAD_MISSES]); +} + +static inline void nfsd_stats_drc_mem_usage_add(struct nfsd_net *nn, s64 amount) +{ + percpu_counter_add(&nn->counter[NFSD_NET_DRC_MEM_USAGE], amount); +} + +static inline void nfsd_stats_drc_mem_usage_sub(struct nfsd_net *nn, s64 amount) +{ + percpu_counter_sub(&nn->counter[NFSD_NET_DRC_MEM_USAGE], amount); +} #endif /* _NFSD_STATS_H */ diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 04937e51de56..d560c1bb2ec2 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -889,7 +889,7 @@ static __be32 nfsd_finish_read(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned long *count, u32 *eof, ssize_t host_err) { if (host_err >= 0) { - nfsdstats.io_read += host_err; + nfsd_stats_io_read_add(host_err); *eof = nfsd_eof_on_read(file, offset, host_err, *count); *count = host_err; fsnotify_access(file); @@ -1040,7 +1040,7 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file *nf, goto out_nfserr; } *cnt = host_err; - nfsdstats.io_write += *cnt; + nfsd_stats_io_write_add(*cnt); fsnotify_modify(file); if (stable && use_wgather) { -- cgit v1.2.3 From 20ad856e47323e208ae8d6a9ecfe5bf0be6f505e Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Wed, 6 Jan 2021 09:52:36 +0200 Subject: nfsd: report per-export stats Collect some nfsd stats per export in addition to the global stats. A new nfsdfs export_stats file is created. It uses the same ops as the exports file to iterate the export entries and we use the file's name to determine the reported info per export. For example: $ cat /proc/fs/nfsd/export_stats # Version 1.1 # Path Client Start-time # Stats /test localhost 92 fh_stale: 0 io_read: 9 io_write: 1 Every export entry reports the start time when stats collection started, so stats collecting scripts can know if stats where reset between samples. Signed-off-by: Amir Goldstein Signed-off-by: Chuck Lever --- fs/nfsd/export.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++++------- fs/nfsd/export.h | 15 +++++++++++++ fs/nfsd/nfsctl.c | 3 +++ fs/nfsd/nfsd.h | 2 +- fs/nfsd/nfsfh.c | 4 ++-- fs/nfsd/stats.h | 12 +++++++--- fs/nfsd/vfs.c | 4 ++-- 7 files changed, 92 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 81e7bb12aca6..7c863f2c21e0 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -331,12 +331,29 @@ static void nfsd4_fslocs_free(struct nfsd4_fs_locations *fsloc) fsloc->locations = NULL; } +static int export_stats_init(struct export_stats *stats) +{ + stats->start_time = ktime_get_seconds(); + return nfsd_percpu_counters_init(stats->counter, EXP_STATS_COUNTERS_NUM); +} + +static void export_stats_reset(struct export_stats *stats) +{ + nfsd_percpu_counters_reset(stats->counter, EXP_STATS_COUNTERS_NUM); +} + +static void export_stats_destroy(struct export_stats *stats) +{ + nfsd_percpu_counters_destroy(stats->counter, EXP_STATS_COUNTERS_NUM); +} + static void svc_export_put(struct kref *ref) { struct svc_export *exp = container_of(ref, struct svc_export, h.ref); path_put(&exp->ex_path); auth_domain_put(exp->ex_client); nfsd4_fslocs_free(&exp->ex_fslocs); + export_stats_destroy(&exp->ex_stats); kfree(exp->ex_uuid); kfree_rcu(exp, ex_rcu); } @@ -692,22 +709,47 @@ static void exp_flags(struct seq_file *m, int flag, int fsid, kuid_t anonu, kgid_t anong, struct nfsd4_fs_locations *fslocs); static void show_secinfo(struct seq_file *m, struct svc_export *exp); +static int is_export_stats_file(struct seq_file *m) +{ + /* + * The export_stats file uses the same ops as the exports file. + * We use the file's name to determine the reported info per export. + * There is no rename in nsfdfs, so d_name.name is stable. + */ + return !strcmp(m->file->f_path.dentry->d_name.name, "export_stats"); +} + static int svc_export_show(struct seq_file *m, struct cache_detail *cd, struct cache_head *h) { - struct svc_export *exp ; + struct svc_export *exp; + bool export_stats = is_export_stats_file(m); - if (h ==NULL) { - seq_puts(m, "#path domain(flags)\n"); + if (h == NULL) { + if (export_stats) + seq_puts(m, "#path domain start-time\n#\tstats\n"); + else + seq_puts(m, "#path domain(flags)\n"); return 0; } exp = container_of(h, struct svc_export, h); seq_path(m, &exp->ex_path, " \t\n\\"); seq_putc(m, '\t'); seq_escape(m, exp->ex_client->name, " \t\n\\"); + if (export_stats) { + seq_printf(m, "\t%lld\n", exp->ex_stats.start_time); + seq_printf(m, "\tfh_stale: %lld\n", + percpu_counter_sum_positive(&exp->ex_stats.counter[EXP_STATS_FH_STALE])); + seq_printf(m, "\tio_read: %lld\n", + percpu_counter_sum_positive(&exp->ex_stats.counter[EXP_STATS_IO_READ])); + seq_printf(m, "\tio_write: %lld\n", + percpu_counter_sum_positive(&exp->ex_stats.counter[EXP_STATS_IO_WRITE])); + seq_putc(m, '\n'); + return 0; + } seq_putc(m, '('); - if (test_bit(CACHE_VALID, &h->flags) && + if (test_bit(CACHE_VALID, &h->flags) && !test_bit(CACHE_NEGATIVE, &h->flags)) { exp_flags(m, exp->ex_flags, exp->ex_fsid, exp->ex_anon_uid, exp->ex_anon_gid, &exp->ex_fslocs); @@ -748,6 +790,7 @@ static void svc_export_init(struct cache_head *cnew, struct cache_head *citem) new->ex_layout_types = 0; new->ex_uuid = NULL; new->cd = item->cd; + export_stats_reset(&new->ex_stats); } static void export_update(struct cache_head *cnew, struct cache_head *citem) @@ -780,10 +823,15 @@ static void export_update(struct cache_head *cnew, struct cache_head *citem) static struct cache_head *svc_export_alloc(void) { struct svc_export *i = kmalloc(sizeof(*i), GFP_KERNEL); - if (i) - return &i->h; - else + if (!i) + return NULL; + + if (export_stats_init(&i->ex_stats)) { + kfree(i); return NULL; + } + + return &i->h; } static const struct cache_detail svc_export_cache_template = { @@ -1245,10 +1293,14 @@ static int e_show(struct seq_file *m, void *p) struct cache_head *cp = p; struct svc_export *exp = container_of(cp, struct svc_export, h); struct cache_detail *cd = m->private; + bool export_stats = is_export_stats_file(m); if (p == SEQ_START_TOKEN) { seq_puts(m, "# Version 1.1\n"); - seq_puts(m, "# Path Client(Flags) # IPs\n"); + if (export_stats) + seq_puts(m, "# Path Client Start-time\n#\tStats\n"); + else + seq_puts(m, "# Path Client(Flags) # IPs\n"); return 0; } diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h index e7daa1f246f0..ee0e3aba4a6e 100644 --- a/fs/nfsd/export.h +++ b/fs/nfsd/export.h @@ -6,6 +6,7 @@ #define NFSD_EXPORT_H #include +#include #include #include @@ -46,6 +47,19 @@ struct exp_flavor_info { u32 flags; }; +/* Per-export stats */ +enum { + EXP_STATS_FH_STALE, + EXP_STATS_IO_READ, + EXP_STATS_IO_WRITE, + EXP_STATS_COUNTERS_NUM +}; + +struct export_stats { + time64_t start_time; + struct percpu_counter counter[EXP_STATS_COUNTERS_NUM]; +}; + struct svc_export { struct cache_head h; struct auth_domain * ex_client; @@ -62,6 +76,7 @@ struct svc_export { struct nfsd4_deviceid_map *ex_devid_map; struct cache_detail *cd; struct rcu_head ex_rcu; + struct export_stats ex_stats; }; /* an "export key" (expkey) maps a filehandlefragement to an diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 258605ee49b8..4f6e514192bd 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -32,6 +32,7 @@ enum { NFSD_Root = 1, NFSD_List, + NFSD_Export_Stats, NFSD_Export_features, NFSD_Fh, NFSD_FO_UnlockIP, @@ -1348,6 +1349,8 @@ static int nfsd_fill_super(struct super_block *sb, struct fs_context *fc) static const struct tree_descr nfsd_files[] = { [NFSD_List] = {"exports", &exports_nfsd_operations, S_IRUGO}, + /* Per-export io stats use same ops as exports file */ + [NFSD_Export_Stats] = {"export_stats", &exports_nfsd_operations, S_IRUGO}, [NFSD_Export_features] = {"export_features", &export_features_operations, S_IRUGO}, [NFSD_FO_UnlockIP] = {"unlock_ip", diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index d63cf8196fed..8bdc37aa2c2e 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -24,8 +24,8 @@ #include #include "netns.h" -#include "stats.h" #include "export.h" +#include "stats.h" #undef ifdebug #ifdef CONFIG_SUNRPC_DEBUG diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 9e31b2b5c6d2..4744a276058d 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -349,7 +349,7 @@ out: __be32 fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) { - struct svc_export *exp; + struct svc_export *exp = NULL; struct dentry *dentry; __be32 error; @@ -422,7 +422,7 @@ skip_pseudoflavor_check: } out: if (error == nfserr_stale) - nfsd_stats_fh_stale_inc(); + nfsd_stats_fh_stale_inc(exp); return error; } diff --git a/fs/nfsd/stats.h b/fs/nfsd/stats.h index 87c3150c200f..51ecda852e23 100644 --- a/fs/nfsd/stats.h +++ b/fs/nfsd/stats.h @@ -59,19 +59,25 @@ static inline void nfsd_stats_rc_nocache_inc(void) percpu_counter_inc(&nfsdstats.counter[NFSD_STATS_RC_NOCACHE]); } -static inline void nfsd_stats_fh_stale_inc(void) +static inline void nfsd_stats_fh_stale_inc(struct svc_export *exp) { percpu_counter_inc(&nfsdstats.counter[NFSD_STATS_FH_STALE]); + if (exp) + percpu_counter_inc(&exp->ex_stats.counter[EXP_STATS_FH_STALE]); } -static inline void nfsd_stats_io_read_add(s64 amount) +static inline void nfsd_stats_io_read_add(struct svc_export *exp, s64 amount) { percpu_counter_add(&nfsdstats.counter[NFSD_STATS_IO_READ], amount); + if (exp) + percpu_counter_add(&exp->ex_stats.counter[EXP_STATS_IO_READ], amount); } -static inline void nfsd_stats_io_write_add(s64 amount) +static inline void nfsd_stats_io_write_add(struct svc_export *exp, s64 amount) { percpu_counter_add(&nfsdstats.counter[NFSD_STATS_IO_WRITE], amount); + if (exp) + percpu_counter_add(&exp->ex_stats.counter[EXP_STATS_IO_WRITE], amount); } static inline void nfsd_stats_payload_misses_inc(struct nfsd_net *nn) diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index d560c1bb2ec2..d316e11923c5 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -889,7 +889,7 @@ static __be32 nfsd_finish_read(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned long *count, u32 *eof, ssize_t host_err) { if (host_err >= 0) { - nfsd_stats_io_read_add(host_err); + nfsd_stats_io_read_add(fhp->fh_export, host_err); *eof = nfsd_eof_on_read(file, offset, host_err, *count); *count = host_err; fsnotify_access(file); @@ -1040,7 +1040,7 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file *nf, goto out_nfserr; } *cnt = host_err; - nfsd_stats_io_write_add(*cnt); + nfsd_stats_io_write_add(exp, *cnt); fsnotify_modify(file); if (stable && use_wgather) { -- cgit v1.2.3 From 33311873adb0d55c287b164117b5b4bb7b1bdc40 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 21 Jan 2021 17:57:37 -0500 Subject: nfsd4: simplify process_lookup1 This STALE_CLIENTID check is redundant with the one in lookup_clientid(). There's a difference in behavior is in case of memory allocation failure, which I think isn't a big deal. Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 1d2cd6a88f61..f9f89229dba6 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -4680,8 +4680,6 @@ nfsd4_process_open1(struct nfsd4_compound_state *cstate, struct nfs4_openowner *oo = NULL; __be32 status; - if (STALE_CLIENTID(&open->op_clientid, nn)) - return nfserr_stale_clientid; /* * In case we need it later, after we've already created the * file and don't want to risk a further failure: -- cgit v1.2.3 From a9d53a75cf574d6aa41f3cb4968fffe4f64e0fad Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 21 Jan 2021 17:57:38 -0500 Subject: nfsd: simplify process_lock Similarly, this STALE_CLIENTID check is already handled by: nfs4_preprocess_confirmed_seqid_op()-> nfs4_preprocess_seqid_op()-> nfsd4_lookup_stateid()-> set_client()-> STALE_CLIENTID() (This may cause it to return a different error in some cases where there are multiple things wrong; pynfs test SEQ10 regressed on this commit because of that, but I think that's the test's fault, and I've fixed it separately.) Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index f9f89229dba6..7ea63d7cec4d 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -6697,10 +6697,6 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, &cstate->session->se_client->cl_clientid, sizeof(clientid_t)); - status = nfserr_stale_clientid; - if (STALE_CLIENTID(&lock->lk_new_clientid, nn)) - goto out; - /* validate and update open stateid and open seqid */ status = nfs4_preprocess_confirmed_seqid_op(cstate, lock->lk_new_open_seqid, -- cgit v1.2.3 From b4587eb2cf4b6271f67fb93b75f7de2a2026e853 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 21 Jan 2021 17:57:39 -0500 Subject: nfsd: simplify nfsd_renew You can take the single-exit thing too far, I think. Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 7ea63d7cec4d..ba955bbf21df 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -5300,15 +5300,12 @@ nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, trace_nfsd_clid_renew(clid); status = lookup_clientid(clid, cstate, nn, false); if (status) - goto out; + return status; clp = cstate->clp; - status = nfserr_cb_path_down; if (!list_empty(&clp->cl_delegations) && clp->cl_cb_state != NFSD4_CB_UP) - goto out; - status = nfs_ok; -out: - return status; + return nfserr_cb_path_down; + return nfs_ok; } void -- cgit v1.2.3 From 460d27091ae2c23e7ac959a61cd481c58832db58 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 21 Jan 2021 17:57:40 -0500 Subject: nfsd: rename lookup_clientid->set_client I think this is a better name, and I'm going to reuse elsewhere the code that does the lookup itself. Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index ba955bbf21df..4bdd90074e24 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -4633,7 +4633,7 @@ static __be32 nfsd4_check_seqid(struct nfsd4_compound_state *cstate, struct nfs4 return nfserr_bad_seqid; } -static __be32 lookup_clientid(clientid_t *clid, +static __be32 set_client(clientid_t *clid, struct nfsd4_compound_state *cstate, struct nfsd_net *nn, bool sessions) @@ -4688,7 +4688,7 @@ nfsd4_process_open1(struct nfsd4_compound_state *cstate, if (open->op_file == NULL) return nfserr_jukebox; - status = lookup_clientid(clientid, cstate, nn, false); + status = set_client(clientid, cstate, nn, false); if (status) return status; clp = cstate->clp; @@ -5298,7 +5298,7 @@ nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); trace_nfsd_clid_renew(clid); - status = lookup_clientid(clid, cstate, nn, false); + status = set_client(clid, cstate, nn, false); if (status) return status; clp = cstate->clp; @@ -5681,8 +5681,7 @@ nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate, if (ZERO_STATEID(stateid) || ONE_STATEID(stateid) || CLOSE_STATEID(stateid)) return nfserr_bad_stateid; - status = lookup_clientid(&stateid->si_opaque.so_clid, cstate, nn, - false); + status = set_client(&stateid->si_opaque.so_clid, cstate, nn, false); if (status == nfserr_stale_clientid) { if (cstate->session) return nfserr_bad_stateid; @@ -5821,7 +5820,7 @@ static __be32 find_cpntf_state(struct nfsd_net *nn, stateid_t *st, cps->cpntf_time = ktime_get_boottime_seconds(); memset(&cstate, 0, sizeof(cstate)); - status = lookup_clientid(&cps->cp_p_clid, &cstate, nn, true); + status = set_client(&cps->cp_p_clid, &cstate, nn, true); if (status) goto out; status = nfsd4_lookup_stateid(&cstate, &cps->cp_p_stateid, @@ -6900,8 +6899,7 @@ nfsd4_lockt(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, return nfserr_inval; if (!nfsd4_has_session(cstate)) { - status = lookup_clientid(&lockt->lt_clientid, cstate, nn, - false); + status = set_client(&lockt->lt_clientid, cstate, nn, false); if (status) goto out; } @@ -7085,7 +7083,7 @@ nfsd4_release_lockowner(struct svc_rqst *rqstp, dprintk("nfsd4_release_lockowner clientid: (%08x/%08x):\n", clid->cl_boot, clid->cl_id); - status = lookup_clientid(clid, cstate, nn, false); + status = set_client(clid, cstate, nn, false); if (status) return status; @@ -7232,7 +7230,7 @@ nfs4_check_open_reclaim(clientid_t *clid, __be32 status; /* find clientid in conf_id_hashtbl */ - status = lookup_clientid(clid, cstate, nn, false); + status = set_client(clid, cstate, nn, false); if (status) return nfserr_reclaim_bad; -- cgit v1.2.3 From 7950b5316e40d99dcb85ab81a2d1dbb913d7c1c8 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 21 Jan 2021 17:57:41 -0500 Subject: nfsd: refactor set_client This'll be useful elsewhere. Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 4bdd90074e24..c74bf3b5b0de 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -4633,40 +4633,40 @@ static __be32 nfsd4_check_seqid(struct nfsd4_compound_state *cstate, struct nfs4 return nfserr_bad_seqid; } +static struct nfs4_client *lookup_clientid(clientid_t *clid, bool sessions, + struct nfsd_net *nn) +{ + struct nfs4_client *found; + + spin_lock(&nn->client_lock); + found = find_confirmed_client(clid, sessions, nn); + if (found) + atomic_inc(&found->cl_rpc_users); + spin_unlock(&nn->client_lock); + return found; +} + static __be32 set_client(clientid_t *clid, struct nfsd4_compound_state *cstate, struct nfsd_net *nn, bool sessions) { - struct nfs4_client *found; - if (cstate->clp) { - found = cstate->clp; - if (!same_clid(&found->cl_clientid, clid)) + if (!same_clid(&cstate->clp->cl_clientid, clid)) return nfserr_stale_clientid; return nfs_ok; } - if (STALE_CLIENTID(clid, nn)) return nfserr_stale_clientid; - /* * For v4.1+ we get the client in the SEQUENCE op. If we don't have one * cached already then we know this is for is for v4.0 and "sessions" * will be false. */ WARN_ON_ONCE(cstate->session); - spin_lock(&nn->client_lock); - found = find_confirmed_client(clid, sessions, nn); - if (!found) { - spin_unlock(&nn->client_lock); + cstate->clp = lookup_clientid(clid, sessions, nn); + if (!cstate->clp) return nfserr_expired; - } - atomic_inc(&found->cl_rpc_users); - spin_unlock(&nn->client_lock); - - /* Cache the nfs4_client in cstate! */ - cstate->clp = found; return nfs_ok; } -- cgit v1.2.3 From 47fdb22dacae78f37701d82a94c16a014186d34e Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 21 Jan 2021 17:57:42 -0500 Subject: nfsd: find_cpntf_state cleanup I think this unusual use of struct compound_state could cause confusion. It's not that much more complicated just to open-code this stateid lookup. The only change in behavior should be a different error return in the case the copy is using a source stateid that is a revoked delegation, but I doubt that matters. Signed-off-by: J. Bruce Fields [ cel: squashed in fix reported by Coverity ] Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index c74bf3b5b0de..b7a5ac5a81ac 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -5812,21 +5812,27 @@ static __be32 find_cpntf_state(struct nfsd_net *nn, stateid_t *st, { __be32 status; struct nfs4_cpntf_state *cps = NULL; - struct nfsd4_compound_state cstate; + struct nfs4_client *found; status = manage_cpntf_state(nn, st, NULL, &cps); if (status) return status; cps->cpntf_time = ktime_get_boottime_seconds(); - memset(&cstate, 0, sizeof(cstate)); - status = set_client(&cps->cp_p_clid, &cstate, nn, true); - if (status) + + status = nfserr_expired; + found = lookup_clientid(&cps->cp_p_clid, true, nn); + if (!found) goto out; - status = nfsd4_lookup_stateid(&cstate, &cps->cp_p_stateid, - NFS4_DELEG_STID|NFS4_OPEN_STID|NFS4_LOCK_STID, - stid, nn); - put_client_renew(cstate.clp); + + *stid = find_stateid_by_type(found, &cps->cp_p_stateid, + NFS4_DELEG_STID|NFS4_OPEN_STID|NFS4_LOCK_STID); + if (*stid) + status = nfs_ok; + else + status = nfserr_bad_stateid; + + put_client_renew(found); out: nfs4_put_cpntf_state(nn, cps); return status; -- cgit v1.2.3 From 0f1d344feb534555a0dcd0beafb7211a37c5355e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 9 Jan 2021 16:02:57 +0000 Subject: splice: don't generate zero-len segement bvecs iter_file_splice_write() may spawn bvec segments with zero-length. In preparation for prohibiting them, filter out by hand at splice level. Reviewed-by: Christoph Hellwig Signed-off-by: Pavel Begunkov Reviewed-by: Ming Lei Signed-off-by: Jens Axboe --- fs/splice.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/splice.c b/fs/splice.c index 866d5c2367b2..474fb8b5562a 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -662,12 +662,14 @@ iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, /* build the vector */ left = sd.total_len; - for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++, n++) { + for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) { struct pipe_buffer *buf = &pipe->bufs[tail & mask]; size_t this_len = buf->len; - if (this_len > left) - this_len = left; + /* zero-length bvecs are not supported, skip them */ + if (!this_len) + continue; + this_len = min(this_len, left); ret = pipe_buf_confirm(pipe, buf); if (unlikely(ret)) { @@ -680,6 +682,7 @@ iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, array[n].bv_len = this_len; array[n].bv_offset = buf->offset; left -= this_len; + n++; } iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left); -- cgit v1.2.3 From 0cf41e5e9bafc185490624c3e321c915885a91f3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 9 Jan 2021 16:02:59 +0000 Subject: block/psi: remove PSI annotations from direct IO Direct IO does not operate on the current working set of pages managed by the kernel, so it should not be accounted as memory stall to PSI infrastructure. The block layer and iomap direct IO use bio_iov_iter_get_pages() to build bios, and they are the only users of it, so to avoid PSI tracking for them clear out BIO_WORKINGSET flag. Do same for dio_bio_submit() because fs/direct_io constructs bios by hand directly calling bio_add_page(). Reported-by: Christoph Hellwig Suggested-by: Christoph Hellwig Suggested-by: Johannes Weiner Reviewed-by: Christoph Hellwig Signed-off-by: Pavel Begunkov Reviewed-by: Ming Lei Signed-off-by: Jens Axboe --- block/bio.c | 6 ++++++ fs/direct-io.c | 2 ++ 2 files changed, 8 insertions(+) (limited to 'fs') diff --git a/block/bio.c b/block/bio.c index 56a06f94fb63..1cd8a2e79048 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1081,6 +1081,9 @@ static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter) * fit into the bio, or are requested in @iter, whatever is smaller. If * MM encounters an error pinning the requested pages, it stops. Error * is returned only if 0 pages could be pinned. + * + * It's intended for direct IO, so doesn't do PSI tracking, the caller is + * responsible for setting BIO_WORKINGSET if necessary. */ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) { @@ -1105,6 +1108,9 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) if (is_bvec) bio_set_flag(bio, BIO_NO_PAGE_REF); + + /* don't account direct I/O as memory stall */ + bio_clear_flag(bio, BIO_WORKINGSET); return bio->bi_vcnt ? 0 : ret; } EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages); diff --git a/fs/direct-io.c b/fs/direct-io.c index 2660e744da2d..aa1083ecd623 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -426,6 +426,8 @@ static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) unsigned long flags; bio->bi_private = dio; + /* don't account direct I/O as memory stall */ + bio_clear_flag(bio, BIO_WORKINGSET); spin_lock_irqsave(&dio->bio_lock, flags); dio->refcount++; -- cgit v1.2.3 From 3e1a88ec96259282b9a8b45c3f1fda7a3ff4f6ea Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 9 Jan 2021 16:03:02 +0000 Subject: bio: add a helper calculating nr segments to alloc Add a helper function calculating the number of bvec segments we need to allocate to construct a bio. It doesn't change anything functionally, but will be used to not duplicate special cases in the future. Reviewed-by: Christoph Hellwig Signed-off-by: Pavel Begunkov Reviewed-by: Ming Lei Signed-off-by: Jens Axboe --- fs/block_dev.c | 7 ++++--- fs/iomap/direct-io.c | 9 ++++----- include/linux/bio.h | 10 ++++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/block_dev.c b/fs/block_dev.c index 3b8963e228a1..6f5bd9950baf 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -416,7 +416,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) dio->size += bio->bi_iter.bi_size; pos += bio->bi_iter.bi_size; - nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES); + nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_PAGES); if (!nr_pages) { bool polled = false; @@ -481,9 +481,10 @@ blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) { int nr_pages; - nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1); - if (!nr_pages) + if (!iov_iter_count(iter)) return 0; + + nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_PAGES + 1); if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES) return __blkdev_direct_IO_simple(iocb, iter, nr_pages); diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 933f234d5bec..ea1e8f696076 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -250,11 +250,8 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, orig_count = iov_iter_count(dio->submit.iter); iov_iter_truncate(dio->submit.iter, length); - nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES); - if (nr_pages <= 0) { - ret = nr_pages; + if (!iov_iter_count(dio->submit.iter)) goto out; - } if (need_zeroout) { /* zero out from the start of the block to the write offset */ @@ -263,6 +260,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, iomap_dio_zero(dio, iomap, pos - pad, pad); } + nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_PAGES); do { size_t n; if (dio->error) { @@ -308,7 +306,8 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, dio->size += n; copied += n; - nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES); + nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, + BIO_MAX_PAGES); iomap_dio_submit_bio(dio, iomap, bio, pos); pos += n; } while (nr_pages); diff --git a/include/linux/bio.h b/include/linux/bio.h index e135b500df5d..9ddb19801a03 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -10,6 +10,7 @@ #include /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */ #include +#include #define BIO_DEBUG @@ -441,6 +442,15 @@ static inline void bio_wouldblock_error(struct bio *bio) bio_endio(bio); } +/* + * Calculate number of bvec segments that should be allocated to fit data + * pointed by @iter. + */ +static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) +{ + return iov_iter_npages(iter, max_segs); +} + struct request_queue; extern int submit_bio_wait(struct bio *bio); -- cgit v1.2.3 From 0a6dc67a6aa45f19bd4ff89b4f468fc50c4b8daa Mon Sep 17 00:00:00 2001 From: Pan Bian Date: Mon, 18 Jan 2021 04:04:55 -0800 Subject: isofs: release buffer head before return Release the buffer_head before returning error code in do_isofs_readdir() and isofs_find_entry(). Fixes: 2deb1acc653c ("isofs: fix access to unallocated memory when reading corrupted filesystem") Link: https://lore.kernel.org/r/20210118120455.118955-1-bianpan2016@163.com Signed-off-by: Pan Bian Signed-off-by: Jan Kara --- fs/isofs/dir.c | 1 + fs/isofs/namei.c | 1 + 2 files changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/isofs/dir.c b/fs/isofs/dir.c index f0fe641893a5..b9e6a7ec78be 100644 --- a/fs/isofs/dir.c +++ b/fs/isofs/dir.c @@ -152,6 +152,7 @@ static int do_isofs_readdir(struct inode *inode, struct file *file, printk(KERN_NOTICE "iso9660: Corrupted directory entry" " in block %lu of inode %lu\n", block, inode->i_ino); + brelse(bh); return -EIO; } diff --git a/fs/isofs/namei.c b/fs/isofs/namei.c index 402769881c32..58f80e1b3ac0 100644 --- a/fs/isofs/namei.c +++ b/fs/isofs/namei.c @@ -102,6 +102,7 @@ isofs_find_entry(struct inode *dir, struct dentry *dentry, printk(KERN_NOTICE "iso9660: Corrupted directory entry" " in block %lu of inode %lu\n", block, dir->i_ino); + brelse(bh); return 0; } -- cgit v1.2.3 From 63c9e47a1642fc817654a1bc18a6ec4bbcc0f056 Mon Sep 17 00:00:00 2001 From: "Steven J. Magnani" Date: Thu, 7 Jan 2021 17:41:16 -0600 Subject: udf: fix silent AED tagLocation corruption When extending a file, udf_do_extend_file() may enter following empty indirect extent. At the end of udf_do_extend_file() we revert prev_epos to point to the last written extent. However if we end up not adding any further extent in udf_do_extend_file(), the reverting points prev_epos into the header area of the AED and following updates of the extents (in udf_update_extents()) will corrupt the header. Make sure that we do not follow indirect extent if we are not going to add any more extents so that returning back to the last written extent works correctly. Link: https://lore.kernel.org/r/20210107234116.6190-2-magnani@ieee.org Signed-off-by: Steven J. Magnani Signed-off-by: Jan Kara --- fs/udf/inode.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/udf/inode.c b/fs/udf/inode.c index bb89c3e43212..0dd2f93ac048 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -544,11 +544,14 @@ static int udf_do_extend_file(struct inode *inode, udf_write_aext(inode, last_pos, &last_ext->extLocation, last_ext->extLength, 1); + /* - * We've rewritten the last extent but there may be empty - * indirect extent after it - enter it. + * We've rewritten the last extent. If we are going to add + * more extents, we may need to enter possible following + * empty indirect extent. */ - udf_next_aext(inode, last_pos, &tmploc, &tmplen, 0); + if (new_block_bytes || prealloc_len) + udf_next_aext(inode, last_pos, &tmploc, &tmplen, 0); } /* Managed to do everything necessary? */ -- cgit v1.2.3 From 2f96e40212d435b328459ba6b3956395eed8fa9f Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 15 Jan 2021 16:26:17 -0500 Subject: btrfs: fix possible free space tree corruption with online conversion While running btrfs/011 in a loop I would often ASSERT() while trying to add a new free space entry that already existed, or get an EEXIST while adding a new block to the extent tree, which is another indication of double allocation. This occurs because when we do the free space tree population, we create the new root and then populate the tree and commit the transaction. The problem is when you create a new root, the root node and commit root node are the same. During this initial transaction commit we will run all of the delayed refs that were paused during the free space tree generation, and thus begin to cache block groups. While caching block groups the caching thread will be reading from the main root for the free space tree, so as we make allocations we'll be changing the free space tree, which can cause us to add the same range twice which results in either the ASSERT(ret != -EEXIST); in __btrfs_add_free_space, or in a variety of different errors when running delayed refs because of a double allocation. Fix this by marking the fs_info as unsafe to load the free space tree, and fall back on the old slow method. We could be smarter than this, for example caching the block group while we're populating the free space tree, but since this is a serious problem I've opted for the simplest solution. CC: stable@vger.kernel.org # 4.9+ Fixes: a5ed91828518 ("Btrfs: implement the free space B-tree") Reviewed-by: Filipe Manana Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 10 +++++++++- fs/btrfs/ctree.h | 3 +++ fs/btrfs/free-space-tree.c | 10 +++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 0886e81e5540..48ebc106a606 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -673,7 +673,15 @@ static noinline void caching_thread(struct btrfs_work *work) wake_up(&caching_ctl->wait); } - if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) + /* + * If we are in the transaction that populated the free space tree we + * can't actually cache from the free space tree as our commit root and + * real root are the same, so we could change the contents of the blocks + * while caching. Instead do the slow caching in this case, and after + * the transaction has committed we will be safe. + */ + if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) && + !(test_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags))) ret = load_free_space_tree(caching_ctl); else ret = load_extent_tree_free(caching_ctl); diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 0225c5208f44..47ca8edafb5e 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -564,6 +564,9 @@ enum { /* Indicate that we need to cleanup space cache v1 */ BTRFS_FS_CLEANUP_SPACE_CACHE_V1, + + /* Indicate that we can't trust the free space tree for caching yet */ + BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, }; /* diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c index e33a65bd9a0c..a33bca94d133 100644 --- a/fs/btrfs/free-space-tree.c +++ b/fs/btrfs/free-space-tree.c @@ -1150,6 +1150,7 @@ int btrfs_create_free_space_tree(struct btrfs_fs_info *fs_info) return PTR_ERR(trans); set_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags); + set_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags); free_space_root = btrfs_create_tree(trans, BTRFS_FREE_SPACE_TREE_OBJECTID); if (IS_ERR(free_space_root)) { @@ -1171,11 +1172,18 @@ int btrfs_create_free_space_tree(struct btrfs_fs_info *fs_info) btrfs_set_fs_compat_ro(fs_info, FREE_SPACE_TREE); btrfs_set_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID); clear_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags); + ret = btrfs_commit_transaction(trans); - return btrfs_commit_transaction(trans); + /* + * Now that we've committed the transaction any reading of our commit + * root will be safe, so we can cache from the free space tree now. + */ + clear_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags); + return ret; abort: clear_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags); + clear_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags); btrfs_abort_transaction(trans, ret); btrfs_end_transaction(trans); return ret; -- cgit v1.2.3 From c41ec4529d3448df8998950d7bada757a1b321cf Mon Sep 17 00:00:00 2001 From: Su Yue Date: Thu, 21 Jan 2021 19:39:10 +0800 Subject: btrfs: fix lockdep warning due to seqcount_mutex on 32bit arch This effectively reverts commit d5c8238849e7 ("btrfs: convert data_seqcount to seqcount_mutex_t"). While running fstests on 32 bits test box, many tests failed because of warnings in dmesg. One of those warnings (btrfs/003): [66.441317] WARNING: CPU: 6 PID: 9251 at include/linux/seqlock.h:279 btrfs_remove_chunk+0x58b/0x7b0 [btrfs] [66.441446] CPU: 6 PID: 9251 Comm: btrfs Tainted: G O 5.11.0-rc4-custom+ #5 [66.441449] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ArchLinux 1.14.0-1 04/01/2014 [66.441451] EIP: btrfs_remove_chunk+0x58b/0x7b0 [btrfs] [66.441472] EAX: 00000000 EBX: 00000001 ECX: c576070c EDX: c6b15803 [66.441475] ESI: 10000000 EDI: 00000000 EBP: c56fbcfc ESP: c56fbc70 [66.441477] DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068 EFLAGS: 00010246 [66.441481] CR0: 80050033 CR2: 05c8da20 CR3: 04b20000 CR4: 00350ed0 [66.441485] Call Trace: [66.441510] btrfs_relocate_chunk+0xb1/0x100 [btrfs] [66.441529] ? btrfs_lookup_block_group+0x17/0x20 [btrfs] [66.441562] btrfs_balance+0x8ed/0x13b0 [btrfs] [66.441586] ? btrfs_ioctl_balance+0x333/0x3c0 [btrfs] [66.441619] ? __this_cpu_preempt_check+0xf/0x11 [66.441643] btrfs_ioctl_balance+0x333/0x3c0 [btrfs] [66.441664] ? btrfs_ioctl_get_supported_features+0x30/0x30 [btrfs] [66.441683] btrfs_ioctl+0x414/0x2ae0 [btrfs] [66.441700] ? __lock_acquire+0x35f/0x2650 [66.441717] ? lockdep_hardirqs_on+0x87/0x120 [66.441720] ? lockdep_hardirqs_on_prepare+0xd0/0x1e0 [66.441724] ? call_rcu+0x2d3/0x530 [66.441731] ? __might_fault+0x41/0x90 [66.441736] ? kvm_sched_clock_read+0x15/0x50 [66.441740] ? sched_clock+0x8/0x10 [66.441745] ? sched_clock_cpu+0x13/0x180 [66.441750] ? btrfs_ioctl_get_supported_features+0x30/0x30 [btrfs] [66.441750] ? btrfs_ioctl_get_supported_features+0x30/0x30 [btrfs] [66.441768] __ia32_sys_ioctl+0x165/0x8a0 [66.441773] ? __this_cpu_preempt_check+0xf/0x11 [66.441785] ? __might_fault+0x89/0x90 [66.441791] __do_fast_syscall_32+0x54/0x80 [66.441796] do_fast_syscall_32+0x32/0x70 [66.441801] do_SYSENTER_32+0x15/0x20 [66.441805] entry_SYSENTER_32+0x9f/0xf2 [66.441808] EIP: 0xab7b5549 [66.441814] EAX: ffffffda EBX: 00000003 ECX: c4009420 EDX: bfa91f5c [66.441816] ESI: 00000003 EDI: 00000001 EBP: 00000000 ESP: bfa91e98 [66.441818] DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 007b EFLAGS: 00000292 [66.441833] irq event stamp: 42579 [66.441835] hardirqs last enabled at (42585): [] console_unlock+0x495/0x590 [66.441838] hardirqs last disabled at (42590): [] console_unlock+0x405/0x590 [66.441840] softirqs last enabled at (41698): [] call_on_stack+0x1c/0x60 [66.441843] softirqs last disabled at (41681): [] call_on_stack+0x1c/0x60 ======================================================================== btrfs_remove_chunk+0x58b/0x7b0: __seqprop_mutex_assert at linux/./include/linux/seqlock.h:279 (inlined by) btrfs_device_set_bytes_used at linux/fs/btrfs/volumes.h:212 (inlined by) btrfs_remove_chunk at linux/fs/btrfs/volumes.c:2994 ======================================================================== The warning is produced by lockdep_assert_held() in __seqprop_mutex_assert() if CONFIG_LOCKDEP is enabled. And "olumes.c:2994 is btrfs_device_set_bytes_used() with mutex lock fs_info->chunk_mutex held already. After adding some debug prints, the cause was found that many __alloc_device() are called with NULL @fs_info (during scanning ioctl). Inside the function, btrfs_device_data_ordered_init() is expanded to seqcount_mutex_init(). In this scenario, its second parameter info->chunk_mutex is &NULL->chunk_mutex which equals to offsetof(struct btrfs_fs_info, chunk_mutex) unexpectedly. Thus, seqcount_mutex_init() is called in wrong way. And later btrfs_device_get/set helpers trigger lockdep warnings. The device and filesystem object lifetimes are different and we'd have to synchronize initialization of the btrfs_device::data_seqcount with the fs_info, possibly using some additional synchronization. It would still not prevent concurrent access to the seqcount lock when it's used for read and initialization. Commit d5c8238849e7 ("btrfs: convert data_seqcount to seqcount_mutex_t") does not mention a particular problem being fixed so revert should not cause any harm and we'll get the lockdep warning fixed. Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=210139 Reported-by: Erhard F Fixes: d5c8238849e7 ("btrfs: convert data_seqcount to seqcount_mutex_t") CC: stable@vger.kernel.org # 5.10 CC: Davidlohr Bueso Signed-off-by: Su Yue Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 2 +- fs/btrfs/volumes.h | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 0c7f4f6237e8..b900cc7849b6 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -433,7 +433,7 @@ static struct btrfs_device *__alloc_device(struct btrfs_fs_info *fs_info) atomic_set(&dev->reada_in_flight, 0); atomic_set(&dev->dev_stats_ccnt, 0); - btrfs_device_data_ordered_init(dev, fs_info); + btrfs_device_data_ordered_init(dev); INIT_RADIX_TREE(&dev->reada_zones, GFP_NOFS & ~__GFP_DIRECT_RECLAIM); INIT_RADIX_TREE(&dev->reada_extents, GFP_NOFS & ~__GFP_DIRECT_RECLAIM); extent_io_tree_init(fs_info, &dev->alloc_state, diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 1997a4649a66..c43663d9c22e 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -39,10 +39,10 @@ struct btrfs_io_geometry { #if BITS_PER_LONG==32 && defined(CONFIG_SMP) #include #define __BTRFS_NEED_DEVICE_DATA_ORDERED -#define btrfs_device_data_ordered_init(device, info) \ - seqcount_mutex_init(&device->data_seqcount, &info->chunk_mutex) +#define btrfs_device_data_ordered_init(device) \ + seqcount_init(&device->data_seqcount) #else -#define btrfs_device_data_ordered_init(device, info) do { } while (0) +#define btrfs_device_data_ordered_init(device) do { } while (0) #endif #define BTRFS_DEV_STATE_WRITEABLE (0) @@ -76,8 +76,7 @@ struct btrfs_device { blk_status_t last_flush_error; #ifdef __BTRFS_NEED_DEVICE_DATA_ORDERED - /* A seqcount_t with associated chunk_mutex (for lockdep) */ - seqcount_mutex_t data_seqcount; + seqcount_t data_seqcount; #endif /* the internal btrfs device id */ @@ -168,9 +167,11 @@ btrfs_device_get_##name(const struct btrfs_device *dev) \ static inline void \ btrfs_device_set_##name(struct btrfs_device *dev, u64 size) \ { \ + preempt_disable(); \ write_seqcount_begin(&dev->data_seqcount); \ dev->name = size; \ write_seqcount_end(&dev->data_seqcount); \ + preempt_enable(); \ } #elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) #define BTRFS_DEVICE_GETSET_FUNCS(name) \ -- cgit v1.2.3 From 9ad6d91f056b99dbe59a262810cb342519ea8d39 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 22 Jan 2021 19:07:45 +0000 Subject: btrfs: fix log replay failure due to race with space cache rebuild After a sudden power failure we may end up with a space cache on disk that is not valid and needs to be rebuilt from scratch. If that happens, during log replay when we attempt to pin an extent buffer from a log tree, at btrfs_pin_extent_for_log_replay(), we do not wait for the space cache to be rebuilt through the call to: btrfs_cache_block_group(cache, 1); That is because that only waits for the task (work queue job) that loads the space cache to change the cache state from BTRFS_CACHE_FAST to any other value. That is ok when the space cache on disk exists and is valid, but when the cache is not valid and needs to be rebuilt, it ends up returning as soon as the cache state changes to BTRFS_CACHE_STARTED (done at caching_thread()). So this means that we can end up trying to unpin a range which is not yet marked as free in the block group. This results in the call to btrfs_remove_free_space() to return -EINVAL to btrfs_pin_extent_for_log_replay(), which in turn makes the log replay fail as well as mounting the filesystem. More specifically the -EINVAL comes from free_space_cache.c:remove_from_bitmap(), because the requested range is not marked as free space (ones in the bitmap), we have the following condition triggered: static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl, (...) if (ret < 0 || search_start != *offset) return -EINVAL; (...) It's the "search_start != *offset" that results in the condition being evaluated to true. When this happens we got the following in dmesg/syslog: [72383.415114] BTRFS: device fsid 32b95b69-0ea9-496a-9f02-3f5a56dc9322 devid 1 transid 1432 /dev/sdb scanned by mount (3816007) [72383.417837] BTRFS info (device sdb): disk space caching is enabled [72383.418536] BTRFS info (device sdb): has skinny extents [72383.423846] BTRFS info (device sdb): start tree-log replay [72383.426416] BTRFS warning (device sdb): block group 30408704 has wrong amount of free space [72383.427686] BTRFS warning (device sdb): failed to load free space cache for block group 30408704, rebuilding it now [72383.454291] BTRFS: error (device sdb) in btrfs_recover_log_trees:6203: errno=-22 unknown (Failed to pin buffers while recovering log root tree.) [72383.456725] BTRFS: error (device sdb) in btrfs_replay_log:2253: errno=-22 unknown (Failed to recover log tree) [72383.460241] BTRFS error (device sdb): open_ctree failed We also mark the range for the extent buffer in the excluded extents io tree. That is fine when the space cache is valid on disk and we can load it, in which case it causes no problems. However, for the case where we need to rebuild the space cache, because it is either invalid or it is missing, having the extent buffer range marked in the excluded extents io tree leads to a -EINVAL failure from the call to btrfs_remove_free_space(), resulting in the log replay and mount to fail. This is because by having the range marked in the excluded extents io tree, the caching thread ends up never adding the range of the extent buffer as free space in the block group since the calls to add_new_free_space(), called from load_extent_tree_free(), filter out any ranges that are marked as excluded extents. So fix this by making sure that during log replay we wait for the caching task to finish completely when we need to rebuild a space cache, and also drop the need to mark the extent buffer range in the excluded extents io tree, as well as clearing ranges from that tree at btrfs_finish_extent_commit(). This started to happen with some frequency on large filesystems having block groups with a lot of fragmentation since the recent commit e747853cae3ae3 ("btrfs: load free space cache asynchronously"), but in fact the issue has been there for years, it was just much less likely to happen. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 61 +++++++++++++++----------------------------------- 1 file changed, 18 insertions(+), 43 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 30b1a630dc2f..0c335dae5af7 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -2602,8 +2602,6 @@ int btrfs_pin_extent_for_log_replay(struct btrfs_trans_handle *trans, struct btrfs_block_group *cache; int ret; - btrfs_add_excluded_extent(trans->fs_info, bytenr, num_bytes); - cache = btrfs_lookup_block_group(trans->fs_info, bytenr); if (!cache) return -EINVAL; @@ -2615,11 +2613,19 @@ int btrfs_pin_extent_for_log_replay(struct btrfs_trans_handle *trans, * the pinned extents. */ btrfs_cache_block_group(cache, 1); + /* + * Make sure we wait until the cache is completely built in case it is + * missing or is invalid and therefore needs to be rebuilt. + */ + ret = btrfs_wait_block_group_cache_done(cache); + if (ret) + goto out; pin_down_extent(trans, cache, bytenr, num_bytes, 0); /* remove us from the free space cache (if we're there at all) */ ret = btrfs_remove_free_space(cache, bytenr, num_bytes); +out: btrfs_put_block_group(cache); return ret; } @@ -2629,50 +2635,22 @@ static int __exclude_logged_extent(struct btrfs_fs_info *fs_info, { int ret; struct btrfs_block_group *block_group; - struct btrfs_caching_control *caching_ctl; block_group = btrfs_lookup_block_group(fs_info, start); if (!block_group) return -EINVAL; - btrfs_cache_block_group(block_group, 0); - caching_ctl = btrfs_get_caching_control(block_group); - - if (!caching_ctl) { - /* Logic error */ - BUG_ON(!btrfs_block_group_done(block_group)); - ret = btrfs_remove_free_space(block_group, start, num_bytes); - } else { - /* - * We must wait for v1 caching to finish, otherwise we may not - * remove our space. - */ - btrfs_wait_space_cache_v1_finished(block_group, caching_ctl); - mutex_lock(&caching_ctl->mutex); - - if (start >= caching_ctl->progress) { - ret = btrfs_add_excluded_extent(fs_info, start, - num_bytes); - } else if (start + num_bytes <= caching_ctl->progress) { - ret = btrfs_remove_free_space(block_group, - start, num_bytes); - } else { - num_bytes = caching_ctl->progress - start; - ret = btrfs_remove_free_space(block_group, - start, num_bytes); - if (ret) - goto out_lock; + btrfs_cache_block_group(block_group, 1); + /* + * Make sure we wait until the cache is completely built in case it is + * missing or is invalid and therefore needs to be rebuilt. + */ + ret = btrfs_wait_block_group_cache_done(block_group); + if (ret) + goto out; - num_bytes = (start + num_bytes) - - caching_ctl->progress; - start = caching_ctl->progress; - ret = btrfs_add_excluded_extent(fs_info, start, - num_bytes); - } -out_lock: - mutex_unlock(&caching_ctl->mutex); - btrfs_put_caching_control(caching_ctl); - } + ret = btrfs_remove_free_space(block_group, start, num_bytes); +out: btrfs_put_block_group(block_group); return ret; } @@ -2863,9 +2841,6 @@ int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans) mutex_unlock(&fs_info->unused_bg_unpin_mutex); break; } - if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) - clear_extent_bits(&fs_info->excluded_extents, start, - end, EXTENT_UPTODATE); if (btrfs_test_opt(fs_info, DISCARD_SYNC)) ret = btrfs_discard_extent(fs_info, start, -- cgit v1.2.3 From 313d64a35d36b4bb00edde418179ff1a5f342070 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 24 Jan 2021 23:49:04 -0500 Subject: do_splice_to(): move the logics for limiting the read length in Both callers have the identical logics limiting the amount of data we try to read into pipe - no more than would fit into that pipe. Move that into do_splice_to() itself. Signed-off-by: Al Viro --- fs/splice.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/splice.c b/fs/splice.c index 866d5c2367b2..c1ca2cc63b43 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -771,11 +771,16 @@ static long do_splice_to(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags) { + unsigned int p_space; int ret; if (unlikely(!(in->f_mode & FMODE_READ))) return -EBADF; + /* Don't try to read more the pipe has space for. */ + p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail); + len = min_t(size_t, len, p_space << PAGE_SHIFT); + ret = rw_verify_area(READ, in, ppos, len); if (unlikely(ret < 0)) return ret; @@ -856,15 +861,10 @@ ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail)); while (len) { - unsigned int p_space; size_t read_len; loff_t pos = sd->pos, prev_pos = pos; - /* Don't try to read more the pipe has space for. */ - p_space = pipe->max_usage - - pipe_occupancy(pipe->head, pipe->tail); - read_len = min_t(size_t, len, p_space << PAGE_SHIFT); - ret = do_splice_to(in, &pos, pipe, read_len, flags); + ret = do_splice_to(in, &pos, pipe, len, flags); if (unlikely(ret <= 0)) goto out_release; @@ -1083,15 +1083,8 @@ long do_splice(struct file *in, loff_t *off_in, struct file *out, pipe_lock(opipe); ret = wait_for_space(opipe, flags); - if (!ret) { - unsigned int p_space; - - /* Don't try to read more the pipe has space for. */ - p_space = opipe->max_usage - pipe_occupancy(opipe->head, opipe->tail); - len = min_t(size_t, len, p_space << PAGE_SHIFT); - + if (!ret) ret = do_splice_to(in, &offset, opipe, len, flags); - } pipe_unlock(opipe); if (ret > 0) wakeup_pipe_readers(opipe); -- cgit v1.2.3 From faa97c48c33454ac0107db930a491b692dd1dff1 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 25 Jan 2021 22:23:03 -0500 Subject: take the guts of file-to-pipe splice into a helper function Signed-off-by: Al Viro --- fs/splice.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/splice.c b/fs/splice.c index c1ca2cc63b43..74f968c65a93 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -1002,6 +1002,23 @@ static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, struct pipe_inode_info *opipe, size_t len, unsigned int flags); +static long splice_file_to_pipe(struct file *in, + struct pipe_inode_info *opipe, + loff_t *offset, + size_t len, unsigned int flags) +{ + long ret; + + pipe_lock(opipe); + ret = wait_for_space(opipe, flags); + if (!ret) + ret = do_splice_to(in, offset, opipe, len, flags); + pipe_unlock(opipe); + if (ret > 0) + wakeup_pipe_readers(opipe); + return ret; +} + /* * Determine where to splice to/from. */ @@ -1081,13 +1098,7 @@ long do_splice(struct file *in, loff_t *off_in, struct file *out, if (out->f_flags & O_NONBLOCK) flags |= SPLICE_F_NONBLOCK; - pipe_lock(opipe); - ret = wait_for_space(opipe, flags); - if (!ret) - ret = do_splice_to(in, &offset, opipe, len, flags); - pipe_unlock(opipe); - if (ret > 0) - wakeup_pipe_readers(opipe); + ret = splice_file_to_pipe(in, opipe, &offset, len, flags); if (!off_in) in->f_pos = offset; else -- cgit v1.2.3 From b964bf53e540262f2d12672b3cca10842c0172e7 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 25 Jan 2021 22:24:28 -0500 Subject: teach sendfile(2) to handle send-to-pipe directly no point going through the intermediate pipe Signed-off-by: Al Viro --- fs/internal.h | 9 +++++++++ fs/read_write.c | 19 +++++++++++++------ fs/splice.c | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/internal.h b/fs/internal.h index 77c50befbfbe..cff1f30cfefb 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -15,6 +15,7 @@ struct mount; struct shrink_control; struct fs_context; struct user_namespace; +struct pipe_inode_info; /* * block_dev.c @@ -193,3 +194,11 @@ int sb_init_dio_done_wq(struct super_block *sb); */ int do_statx(int dfd, const char __user *filename, unsigned flags, unsigned int mask, struct statx __user *buffer); + +/* + * fs/splice.c: + */ +long splice_file_to_pipe(struct file *in, + struct pipe_inode_info *opipe, + loff_t *offset, + size_t len, unsigned int flags); diff --git a/fs/read_write.c b/fs/read_write.c index 75f764b43418..9db7adf160d2 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -1188,6 +1188,7 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos, { struct fd in, out; struct inode *in_inode, *out_inode; + struct pipe_inode_info *opipe; loff_t pos; loff_t out_pos; ssize_t retval; @@ -1228,9 +1229,6 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos, in_inode = file_inode(in.file); out_inode = file_inode(out.file); out_pos = out.file->f_pos; - retval = rw_verify_area(WRITE, out.file, &out_pos, count); - if (retval < 0) - goto fput_out; if (!max) max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes); @@ -1253,9 +1251,18 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos, if (in.file->f_flags & O_NONBLOCK) fl = SPLICE_F_NONBLOCK; #endif - file_start_write(out.file); - retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl); - file_end_write(out.file); + opipe = get_pipe_info(out.file, true); + if (!opipe) { + retval = rw_verify_area(WRITE, out.file, &out_pos, count); + if (retval < 0) + goto fput_out; + file_start_write(out.file); + retval = do_splice_direct(in.file, &pos, out.file, &out_pos, + count, fl); + file_end_write(out.file); + } else { + retval = splice_file_to_pipe(in.file, opipe, &pos, count, fl); + } if (retval > 0) { add_rchar(current, retval); diff --git a/fs/splice.c b/fs/splice.c index 74f968c65a93..b06846f1e6ee 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -1002,7 +1002,7 @@ static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, struct pipe_inode_info *opipe, size_t len, unsigned int flags); -static long splice_file_to_pipe(struct file *in, +long splice_file_to_pipe(struct file *in, struct pipe_inode_info *opipe, loff_t *offset, size_t len, unsigned int flags) -- cgit v1.2.3 From a1bb3cd58913338e1b627ea6b8c03c2ae82d293f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 26 Jan 2021 15:28:26 +0000 Subject: io_uring: fix __io_uring_files_cancel() with TASK_UNINTERRUPTIBLE If the tctx inflight number haven't changed because of cancellation, __io_uring_task_cancel() will continue leaving the task in TASK_UNINTERRUPTIBLE state, that's not expected by __io_uring_files_cancel(). Ensure we always call finish_wait() before retrying. Cc: stable@vger.kernel.org # 5.9+ Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 2166c469789d..09aada153a71 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9124,16 +9124,15 @@ void __io_uring_task_cancel(void) prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); /* - * If we've seen completions, retry. This avoids a race where - * a completion comes in before we did prepare_to_wait(). + * If we've seen completions, retry without waiting. This + * avoids a race where a completion comes in before we did + * prepare_to_wait(). */ - if (inflight != tctx_inflight(tctx)) - continue; - schedule(); + if (inflight == tctx_inflight(tctx)) + schedule(); finish_wait(&tctx->wait, &wait); } while (1); - finish_wait(&tctx->wait, &wait); atomic_dec(&tctx->in_idle); io_uring_remove_task_files(tctx); -- cgit v1.2.3 From ca70f00bed6cb255b7a9b91aa18a2717c9217f70 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 26 Jan 2021 15:28:27 +0000 Subject: io_uring: fix cancellation taking mutex while TASK_UNINTERRUPTIBLE do not call blocking ops when !TASK_RUNNING; state=2 set at [<00000000ced9dbfc>] prepare_to_wait+0x1f4/0x3b0 kernel/sched/wait.c:262 WARNING: CPU: 1 PID: 19888 at kernel/sched/core.c:7853 __might_sleep+0xed/0x100 kernel/sched/core.c:7848 RIP: 0010:__might_sleep+0xed/0x100 kernel/sched/core.c:7848 Call Trace: __mutex_lock_common+0xc4/0x2ef0 kernel/locking/mutex.c:935 __mutex_lock kernel/locking/mutex.c:1103 [inline] mutex_lock_nested+0x1a/0x20 kernel/locking/mutex.c:1118 io_wq_submit_work+0x39a/0x720 fs/io_uring.c:6411 io_run_cancel fs/io-wq.c:856 [inline] io_wqe_cancel_pending_work fs/io-wq.c:990 [inline] io_wq_cancel_cb+0x614/0xcb0 fs/io-wq.c:1027 io_uring_cancel_files fs/io_uring.c:8874 [inline] io_uring_cancel_task_requests fs/io_uring.c:8952 [inline] __io_uring_files_cancel+0x115d/0x19e0 fs/io_uring.c:9038 io_uring_files_cancel include/linux/io_uring.h:51 [inline] do_exit+0x2e6/0x2490 kernel/exit.c:780 do_group_exit+0x168/0x2d0 kernel/exit.c:922 get_signal+0x16b5/0x2030 kernel/signal.c:2770 arch_do_signal_or_restart+0x8e/0x6a0 arch/x86/kernel/signal.c:811 handle_signal_work kernel/entry/common.c:147 [inline] exit_to_user_mode_loop kernel/entry/common.c:171 [inline] exit_to_user_mode_prepare+0xac/0x1e0 kernel/entry/common.c:201 __syscall_exit_to_user_mode_work kernel/entry/common.c:291 [inline] syscall_exit_to_user_mode+0x48/0x190 kernel/entry/common.c:302 entry_SYSCALL_64_after_hwframe+0x44/0xa9 Rewrite io_uring_cancel_files() to mimic __io_uring_task_cancel()'s counting scheme, so it does all the heavy work before setting TASK_UNINTERRUPTIBLE. Cc: stable@vger.kernel.org # 5.9+ Reported-by: syzbot+f655445043a26a7cfab8@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov [axboe: fix inverted task check] Signed-off-by: Jens Axboe --- fs/io_uring.c | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 09aada153a71..bb0270eeb8cb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8873,30 +8873,31 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx, } } +static int io_uring_count_inflight(struct io_ring_ctx *ctx, + struct task_struct *task, + struct files_struct *files) +{ + struct io_kiocb *req; + int cnt = 0; + + spin_lock_irq(&ctx->inflight_lock); + list_for_each_entry(req, &ctx->inflight_list, inflight_entry) + cnt += io_match_task(req, task, files); + spin_unlock_irq(&ctx->inflight_lock); + return cnt; +} + static void io_uring_cancel_files(struct io_ring_ctx *ctx, struct task_struct *task, struct files_struct *files) { while (!list_empty_careful(&ctx->inflight_list)) { struct io_task_cancel cancel = { .task = task, .files = files }; - struct io_kiocb *req; DEFINE_WAIT(wait); - bool found = false; - - spin_lock_irq(&ctx->inflight_lock); - list_for_each_entry(req, &ctx->inflight_list, inflight_entry) { - if (!io_match_task(req, task, files)) - continue; - found = true; - break; - } - if (found) - prepare_to_wait(&task->io_uring->wait, &wait, - TASK_UNINTERRUPTIBLE); - spin_unlock_irq(&ctx->inflight_lock); + int inflight; - /* We need to keep going until we don't find a matching req */ - if (!found) + inflight = io_uring_count_inflight(ctx, task, files); + if (!inflight) break; io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true); @@ -8905,7 +8906,11 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, io_cqring_overflow_flush(ctx, true, task, files); /* cancellations _may_ trigger task work */ io_run_task_work(); - schedule(); + + prepare_to_wait(&task->io_uring->wait, &wait, + TASK_UNINTERRUPTIBLE); + if (inflight == io_uring_count_inflight(ctx, task, files)) + schedule(); finish_wait(&task->io_uring->wait, &wait); } } -- cgit v1.2.3 From 767630c63bb23acf022adb265574996ca39a4645 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 7 Jan 2021 16:40:34 +0100 Subject: bdev: Do not return EBUSY if bdev discard races with write blkdev_fallocate() tries to detect whether a discard raced with an overlapping write by calling invalidate_inode_pages2_range(). However this check can give both false negatives (when writing using direct IO or when writeback already writes out the written pagecache range) and false positives (when write is not actually overlapping but ends in the same page when blocksize < pagesize). This actually causes issues for qemu which is getting confused by EBUSY errors. Fix the problem by removing this conflicting write detection since it is inherently racy and thus of little use anyway. Reported-by: Maxim Levitsky CC: "Darrick J. Wong" Link: https://lore.kernel.org/qemu-devel/20201111153913.41840-1-mlevitsk@redhat.com Signed-off-by: Jan Kara Reviewed-by: Maxim Levitsky Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- fs/block_dev.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/block_dev.c b/fs/block_dev.c index 6f5bd9950baf..289c3dd923a4 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -1801,13 +1801,11 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start, return error; /* - * Invalidate again; if someone wandered in and dirtied a page, - * the caller will be given -EBUSY. The third argument is - * inclusive, so the rounding here is safe. + * Invalidate the page cache again; if someone wandered in and dirtied + * a page, we just discard it - userspace has no way of knowing whether + * the write happened before or after discard completing... */ - return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping, - start >> PAGE_SHIFT, - end >> PAGE_SHIFT); + return truncate_bdev_range(bdev, file->f_mode, start, end); } const struct file_operations def_blk_fops = { -- cgit v1.2.3 From 2c2b9fd6b496b3616e9b9537ea0258b3040914f3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 9 Jan 2021 12:13:32 +0100 Subject: block: unexport truncate_bdev_range truncate_bdev_range is only used in always built-in block layer code, so remove the export and the !CONFIG_BLOCK stub. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- fs/block_dev.c | 1 - include/linux/blkdev.h | 9 ++------- 2 files changed, 2 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/block_dev.c b/fs/block_dev.c index 289c3dd923a4..c1fe29dac485 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -126,7 +126,6 @@ int truncate_bdev_range(struct block_device *bdev, fmode_t mode, bd_abort_claiming(bdev, truncate_bdev_range); return 0; } -EXPORT_SYMBOL(truncate_bdev_range); static void set_init_blocksize(struct block_device *bdev) { diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 20f3706b6b2e..2491e17b61c4 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1999,21 +1999,16 @@ void bdev_add(struct block_device *bdev, dev_t dev); struct block_device *I_BDEV(struct inode *inode); struct block_device *bdgrab(struct block_device *bdev); void bdput(struct block_device *); +int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart, + loff_t lend); #ifdef CONFIG_BLOCK void invalidate_bdev(struct block_device *bdev); -int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart, - loff_t lend); int sync_blockdev(struct block_device *bdev); #else static inline void invalidate_bdev(struct block_device *bdev) { } -static inline int truncate_bdev_range(struct block_device *bdev, fmode_t mode, - loff_t lstart, loff_t lend) -{ - return 0; -} static inline int sync_blockdev(struct block_device *bdev) { return 0; -- cgit v1.2.3 From 907d1df30a51cc1a1d25414a00cde0494b83df7b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 26 Jan 2021 23:35:10 +0000 Subject: io_uring: fix wqe->lock/completion_lock deadlock Joseph reports following deadlock: CPU0: ... io_kill_linked_timeout // &ctx->completion_lock io_commit_cqring __io_queue_deferred __io_queue_async_work io_wq_enqueue io_wqe_enqueue // &wqe->lock CPU1: ... __io_uring_files_cancel io_wq_cancel_cb io_wqe_cancel_pending_work // &wqe->lock io_cancel_task_cb // &ctx->completion_lock Only __io_queue_deferred() calls queue_async_work() while holding ctx->completion_lock, enqueue drained requests via io_req_task_queue() instead. Cc: stable@vger.kernel.org # 5.9+ Reported-by: Joseph Qi Tested-by: Joseph Qi Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index bb0270eeb8cb..c218deaf73a9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1026,6 +1026,7 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, const struct iovec *fast_iov, struct iov_iter *iter, bool force); static void io_req_drop_files(struct io_kiocb *req); +static void io_req_task_queue(struct io_kiocb *req); static struct kmem_cache *req_cachep; @@ -1634,18 +1635,11 @@ static void __io_queue_deferred(struct io_ring_ctx *ctx) do { struct io_defer_entry *de = list_first_entry(&ctx->defer_list, struct io_defer_entry, list); - struct io_kiocb *link; if (req_need_defer(de->req, de->seq)) break; list_del_init(&de->list); - /* punt-init is done before queueing for defer */ - link = __io_queue_async_work(de->req); - if (link) { - __io_queue_linked_timeout(link); - /* drop submission reference */ - io_put_req_deferred(link, 1); - } + io_req_task_queue(de->req); kfree(de); } while (!list_empty(&ctx->defer_list)); } -- cgit v1.2.3 From 8dc932d3e8afb65e12eba7495f046c83884c49bf Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Tue, 26 Jan 2021 21:59:07 +0200 Subject: Revert "block: simplify set_init_blocksize" to regain lost performance The cited commit introduced a serious regression with SATA write speed, as found by bisecting. This patch reverts this commit, which restores write speed back to the values observed before this commit. The performance tests were done on a Helios4 NAS (2nd batch) with 4 HDDs (WD8003FFBX) using dd (bs=1M count=2000). "Direct" is a test with a single HDD, the rest are different RAID levels built over the first partitions of 4 HDDs. Test results are in MB/s, R is read, W is write. | Direct | RAID0 | RAID10 f2 | RAID10 n2 | RAID6 ----------------+--------+-------+-----------+-----------+-------- 9011495c9466 | R:256 | R:313 | R:276 | R:313 | R:323 (before faulty) | W:254 | W:253 | W:195 | W:204 | W:117 ----------------+--------+-------+-----------+-----------+-------- 5ff9f19231a0 | R:257 | R:398 | R:312 | R:344 | R:391 (faulty commit) | W:154 | W:122 | W:67.7 | W:66.6 | W:67.2 ----------------+--------+-------+-----------+-----------+-------- 5.10.10 | R:256 | R:401 | R:312 | R:356 | R:375 unpatched | W:149 | W:123 | W:64 | W:64.1 | W:61.5 ----------------+--------+-------+-----------+-----------+-------- 5.10.10 | R:255 | R:396 | R:312 | R:340 | R:393 patched | W:247 | W:274 | W:220 | W:225 | W:121 Applying this patch doesn't hurt read performance, while improves the write speed by 1.5x - 3.5x (more impact on RAID tests). The write speed is restored back to the state before the faulty commit, and even a bit higher in RAID tests (which aren't HDD-bound on this device) - that is likely related to other optimizations done between the faulty commit and 5.10.10 which also improved the read speed. Signed-off-by: Maxim Mikityanskiy Fixes: 5ff9f19231a0 ("block: simplify set_init_blocksize") Cc: Christoph Hellwig Cc: Jens Axboe Acked-by: Christoph Hellwig Signed-off-by: Jens Axboe --- fs/block_dev.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/block_dev.c b/fs/block_dev.c index 3b8963e228a1..235b5042672e 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -130,7 +130,15 @@ EXPORT_SYMBOL(truncate_bdev_range); static void set_init_blocksize(struct block_device *bdev) { - bdev->bd_inode->i_blkbits = blksize_bits(bdev_logical_block_size(bdev)); + unsigned int bsize = bdev_logical_block_size(bdev); + loff_t size = i_size_read(bdev->bd_inode); + + while (bsize < PAGE_SIZE) { + if (size & bsize) + break; + bsize <<= 1; + } + bdev->bd_inode->i_blkbits = blksize_bits(bsize); } int set_blocksize(struct block_device *bdev, int size) -- cgit v1.2.3 From 6195ba09822c87cad09189bbf550d0fbe714687a Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Wed, 27 Jan 2021 15:14:09 +0800 Subject: io_uring: fix flush cqring overflow list while TASK_INTERRUPTIBLE Abaci reported the follow warning: [ 27.073425] do not call blocking ops when !TASK_RUNNING; state=1 set at [] prepare_to_wait_exclusive+0x3a/0xc0 [ 27.075805] WARNING: CPU: 0 PID: 951 at kernel/sched/core.c:7853 __might_sleep+0x80/0xa0 [ 27.077604] Modules linked in: [ 27.078379] CPU: 0 PID: 951 Comm: a.out Not tainted 5.11.0-rc3+ #1 [ 27.079637] Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011 [ 27.080852] RIP: 0010:__might_sleep+0x80/0xa0 [ 27.081835] Code: 65 48 8b 04 25 80 71 01 00 48 8b 90 c0 15 00 00 48 8b 70 18 48 c7 c7 08 39 95 82 c6 05 f9 5f de 08 01 48 89 d1 e8 00 c6 fa ff 0b eb bf 41 0f b6 f5 48 c7 c7 40 23 c9 82 e8 f3 48 ec 00 eb a7 [ 27.084521] RSP: 0018:ffffc90000fe3ce8 EFLAGS: 00010286 [ 27.085350] RAX: 0000000000000000 RBX: ffffffff82956083 RCX: 0000000000000000 [ 27.086348] RDX: ffff8881057a0000 RSI: ffffffff8118cc9e RDI: ffff88813bc28570 [ 27.087598] RBP: 00000000000003a7 R08: 0000000000000001 R09: 0000000000000001 [ 27.088819] R10: ffffc90000fe3e00 R11: 00000000fffef9f0 R12: 0000000000000000 [ 27.089819] R13: 0000000000000000 R14: ffff88810576eb80 R15: ffff88810576e800 [ 27.091058] FS: 00007f7b144cf740(0000) GS:ffff88813bc00000(0000) knlGS:0000000000000000 [ 27.092775] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 27.093796] CR2: 00000000022da7b8 CR3: 000000010b928002 CR4: 00000000003706f0 [ 27.094778] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 27.095780] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 27.097011] Call Trace: [ 27.097685] __mutex_lock+0x5d/0xa30 [ 27.098565] ? prepare_to_wait_exclusive+0x71/0xc0 [ 27.099412] ? io_cqring_overflow_flush.part.101+0x6d/0x70 [ 27.100441] ? lockdep_hardirqs_on_prepare+0xe9/0x1c0 [ 27.101537] ? _raw_spin_unlock_irqrestore+0x2d/0x40 [ 27.102656] ? trace_hardirqs_on+0x46/0x110 [ 27.103459] ? io_cqring_overflow_flush.part.101+0x6d/0x70 [ 27.104317] io_cqring_overflow_flush.part.101+0x6d/0x70 [ 27.105113] io_cqring_wait+0x36e/0x4d0 [ 27.105770] ? find_held_lock+0x28/0xb0 [ 27.106370] ? io_uring_remove_task_files+0xa0/0xa0 [ 27.107076] __x64_sys_io_uring_enter+0x4fb/0x640 [ 27.107801] ? rcu_read_lock_sched_held+0x59/0xa0 [ 27.108562] ? lockdep_hardirqs_on_prepare+0xe9/0x1c0 [ 27.109684] ? syscall_enter_from_user_mode+0x26/0x70 [ 27.110731] do_syscall_64+0x2d/0x40 [ 27.111296] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 27.112056] RIP: 0033:0x7f7b13dc8239 [ 27.112663] Code: 01 00 48 81 c4 80 00 00 00 e9 f1 fe ff ff 0f 1f 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 3d 01 f0 ff ff 73 01 c3 48 8b 0d 27 ec 2c 00 f7 d8 64 89 01 48 [ 27.115113] RSP: 002b:00007ffd6d7f5c88 EFLAGS: 00000286 ORIG_RAX: 00000000000001aa [ 27.116562] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f7b13dc8239 [ 27.117961] RDX: 000000000000478e RSI: 0000000000000000 RDI: 0000000000000003 [ 27.118925] RBP: 00007ffd6d7f5cb0 R08: 0000000020000040 R09: 0000000000000008 [ 27.119773] R10: 0000000000000001 R11: 0000000000000286 R12: 0000000000400480 [ 27.120614] R13: 00007ffd6d7f5d90 R14: 0000000000000000 R15: 0000000000000000 [ 27.121490] irq event stamp: 5635 [ 27.121946] hardirqs last enabled at (5643): [] console_unlock+0x5c4/0x740 [ 27.123476] hardirqs last disabled at (5652): [] console_unlock+0x4e7/0x740 [ 27.125192] softirqs last enabled at (5272): [] __do_softirq+0x3c5/0x5aa [ 27.126430] softirqs last disabled at (5267): [] asm_call_irq_on_stack+0xf/0x20 [ 27.127634] ---[ end trace 289d7e28fa60f928 ]--- This is caused by calling io_cqring_overflow_flush() which may sleep after calling prepare_to_wait_exclusive() which set task state to TASK_INTERRUPTIBLE Reported-by: Abaci Fixes: 6c503150ae33 ("io_uring: patch up IOPOLL overflow_flush sync") Reviewed-by: Pavel Begunkov Signed-off-by: Hao Xu Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index c218deaf73a9..ae388cc52843 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7268,14 +7268,18 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, TASK_INTERRUPTIBLE); /* make sure we run task_work before checking for signals */ ret = io_run_task_work_sig(); - if (ret > 0) + if (ret > 0) { + finish_wait(&ctx->wait, &iowq.wq); continue; + } else if (ret < 0) break; if (io_should_wake(&iowq)) break; - if (test_bit(0, &ctx->cq_check_overflow)) + if (test_bit(0, &ctx->cq_check_overflow)) { + finish_wait(&ctx->wait, &iowq.wq); continue; + } if (uts) { timeout = schedule_timeout(timeout); if (timeout == 0) { -- cgit v1.2.3 From f91ca2a370bec58eb3d54315b5cfa3a2a9288acc Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 26 Jan 2021 15:52:31 +0100 Subject: zonefs: use bio_alloc in zonefs_file_dio_append Use bio_alloc instead of open coding it. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Reviewed-by: Chaitanya Kulkarni Acked-by: Damien Le Moal Signed-off-by: Jens Axboe --- fs/zonefs/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index bec47f2d074b..faea2ed34b4a 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -678,7 +678,7 @@ static ssize_t zonefs_file_dio_append(struct kiocb *iocb, struct iov_iter *from) if (!nr_pages) return 0; - bio = bio_alloc_bioset(GFP_NOFS, nr_pages, &fs_bio_set); + bio = bio_alloc(GFP_NOFS, nr_pages); if (!bio) return -ENOMEM; -- cgit v1.2.3 From 616c6a6884e273349cda19483dfd7f5b7fd3da52 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 26 Jan 2021 15:52:32 +0100 Subject: btrfs: use bio_kmalloc in __alloc_device Use bio_kmalloc instead of open coding it. Signed-off-by: Christoph Hellwig Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Reviewed-by: Chaitanya Kulkarni Acked-by: Damien Le Moal Signed-off-by: Jens Axboe --- fs/btrfs/volumes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 0a6de859eb22..584ba093cf49 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -421,7 +421,7 @@ static struct btrfs_device *__alloc_device(struct btrfs_fs_info *fs_info) * Preallocate a bio that's always going to be used for flushing device * barriers and matches the device lifespan */ - dev->flush_bio = bio_alloc_bioset(GFP_KERNEL, 0, NULL); + dev->flush_bio = bio_kmalloc(GFP_KERNEL, 0); if (!dev->flush_bio) { kfree(dev); return ERR_PTR(-ENOMEM); -- cgit v1.2.3 From c6bf3f0e25f4c0f0ecce6cf8d1c589bd9d74d3cf Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 26 Jan 2021 15:52:35 +0100 Subject: block: use an on-stack bio in blkdev_issue_flush There is no point in allocating memory for a synchronous flush. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Reviewed-by: Chaitanya Kulkarni Acked-by: Damien Le Moal Signed-off-by: Jens Axboe --- block/blk-flush.c | 17 ++++++----------- drivers/md/dm-zoned-metadata.c | 6 +++--- drivers/md/raid5-ppl.c | 2 +- drivers/nvme/target/io-cmd-bdev.c | 2 +- fs/block_dev.c | 2 +- fs/exfat/file.c | 2 +- fs/ext4/fast_commit.c | 4 ++-- fs/ext4/fsync.c | 2 +- fs/ext4/ialloc.c | 2 +- fs/ext4/super.c | 2 +- fs/fat/file.c | 2 +- fs/hfsplus/inode.c | 2 +- fs/hfsplus/super.c | 2 +- fs/jbd2/checkpoint.c | 2 +- fs/jbd2/commit.c | 4 ++-- fs/jbd2/recovery.c | 2 +- fs/libfs.c | 2 +- fs/nilfs2/the_nilfs.h | 2 +- fs/ocfs2/file.c | 2 +- fs/reiserfs/file.c | 2 +- fs/xfs/xfs_super.c | 2 +- fs/zonefs/super.c | 2 +- include/linux/blkdev.h | 4 ++-- 23 files changed, 33 insertions(+), 38 deletions(-) (limited to 'fs') diff --git a/block/blk-flush.c b/block/blk-flush.c index 76c1624cb06c..7942ca6ed321 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -432,23 +432,18 @@ void blk_insert_flush(struct request *rq) /** * blkdev_issue_flush - queue a flush * @bdev: blockdev to issue flush for - * @gfp_mask: memory allocation flags (for bio_alloc) * * Description: * Issue a flush for the block device in question. */ -int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask) +int blkdev_issue_flush(struct block_device *bdev) { - struct bio *bio; - int ret = 0; + struct bio bio; - bio = bio_alloc(gfp_mask, 0); - bio_set_dev(bio, bdev); - bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; - - ret = submit_bio_wait(bio); - bio_put(bio); - return ret; + bio_init(&bio, NULL, 0); + bio_set_dev(&bio, bdev); + bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; + return submit_bio_wait(&bio); } EXPORT_SYMBOL(blkdev_issue_flush); diff --git a/drivers/md/dm-zoned-metadata.c b/drivers/md/dm-zoned-metadata.c index b298fefb022e..039d17b28938 100644 --- a/drivers/md/dm-zoned-metadata.c +++ b/drivers/md/dm-zoned-metadata.c @@ -819,7 +819,7 @@ static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set) ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block, mblk->page); if (ret == 0) - ret = blkdev_issue_flush(dev->bdev, GFP_NOIO); + ret = blkdev_issue_flush(dev->bdev); return ret; } @@ -862,7 +862,7 @@ static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd, /* Flush drive cache (this will also sync data) */ if (ret == 0) - ret = blkdev_issue_flush(dev->bdev, GFP_NOIO); + ret = blkdev_issue_flush(dev->bdev); return ret; } @@ -933,7 +933,7 @@ int dmz_flush_metadata(struct dmz_metadata *zmd) /* If there are no dirty metadata blocks, just flush the device cache */ if (list_empty(&write_list)) { - ret = blkdev_issue_flush(dev->bdev, GFP_NOIO); + ret = blkdev_issue_flush(dev->bdev); goto err; } diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index d0f540296fe9..e8c118e05dfd 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -1037,7 +1037,7 @@ static int ppl_recover(struct ppl_log *log, struct ppl_header *pplhdr, } /* flush the disk cache after recovery if necessary */ - ret = blkdev_issue_flush(rdev->bdev, GFP_KERNEL); + ret = blkdev_issue_flush(rdev->bdev); out: __free_page(page); return ret; diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 125dde3f410e..bf6e0ac9ad28 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -333,7 +333,7 @@ static void nvmet_bdev_execute_flush(struct nvmet_req *req) u16 nvmet_bdev_flush(struct nvmet_req *req) { - if (blkdev_issue_flush(req->ns->bdev, GFP_KERNEL)) + if (blkdev_issue_flush(req->ns->bdev)) return NVME_SC_INTERNAL | NVME_SC_DNR; return 0; } diff --git a/fs/block_dev.c b/fs/block_dev.c index c1fe29dac485..9d4b1a884d76 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -680,7 +680,7 @@ int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync) * i_mutex and doing so causes performance issues with concurrent * O_SYNC writers to a block device. */ - error = blkdev_issue_flush(bdev, GFP_KERNEL); + error = blkdev_issue_flush(bdev); if (error == -EOPNOTSUPP) error = 0; diff --git a/fs/exfat/file.c b/fs/exfat/file.c index a92478eabfa4..183ffdf4d43c 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -361,7 +361,7 @@ int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) if (err) return err; - return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + return blkdev_issue_flush(inode->i_sb->s_bdev); } const struct file_operations exfat_file_operations = { diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 0a14a7c87bf8..6e8208acfc62 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -1076,7 +1076,7 @@ static int ext4_fc_perform_commit(journal_t *journal) * flush before we start writing fast commit blocks. */ if (journal->j_fs_dev != journal->j_dev) - blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS); + blkdev_issue_flush(journal->j_fs_dev); blk_start_plug(&plug); if (sbi->s_fc_bytes == 0) { @@ -1535,7 +1535,7 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl) out: iput(inode); if (!ret) - blkdev_issue_flush(sb->s_bdev, GFP_KERNEL); + blkdev_issue_flush(sb->s_bdev); return 0; } diff --git a/fs/ext4/fsync.c b/fs/ext4/fsync.c index 113bfb023a4a..027a7d7037a0 100644 --- a/fs/ext4/fsync.c +++ b/fs/ext4/fsync.c @@ -174,7 +174,7 @@ int ext4_sync_file(struct file *file, loff_t start, loff_t end, int datasync) ret = ext4_fsync_journal(inode, datasync, &needs_barrier); if (needs_barrier) { - err = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + err = blkdev_issue_flush(inode->i_sb->s_bdev); if (!ret) ret = err; } diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index b215c564bc31..20f2fcb799f5 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -1583,7 +1583,7 @@ int ext4_init_inode_table(struct super_block *sb, ext4_group_t group, if (ret < 0) goto err_out; if (barrier) - blkdev_issue_flush(sb->s_bdev, GFP_NOFS); + blkdev_issue_flush(sb->s_bdev); skip_zeroout: ext4_lock_group(sb, group); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 9a6f9875aa34..fb5985102c1d 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -5709,7 +5709,7 @@ static int ext4_sync_fs(struct super_block *sb, int wait) needs_barrier = true; if (needs_barrier) { int err; - err = blkdev_issue_flush(sb->s_bdev, GFP_KERNEL); + err = blkdev_issue_flush(sb->s_bdev); if (!ret) ret = err; } diff --git a/fs/fat/file.c b/fs/fat/file.c index f9ee27cf4d7c..5fee74f1ad61 100644 --- a/fs/fat/file.c +++ b/fs/fat/file.c @@ -195,7 +195,7 @@ int fat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) if (err) return err; - return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + return blkdev_issue_flush(inode->i_sb->s_bdev); } diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index e3da9e96b835..ca464328b79c 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -340,7 +340,7 @@ int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, } if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags)) - blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + blkdev_issue_flush(inode->i_sb->s_bdev); inode_unlock(inode); diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c index 807119ae5adf..b9e3db3f855f 100644 --- a/fs/hfsplus/super.c +++ b/fs/hfsplus/super.c @@ -239,7 +239,7 @@ out: mutex_unlock(&sbi->vh_mutex); if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags)) - blkdev_issue_flush(sb->s_bdev, GFP_KERNEL); + blkdev_issue_flush(sb->s_bdev); return error; } diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 472932b9e6bc..63b526d44886 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -416,7 +416,7 @@ int jbd2_cleanup_journal_tail(journal_t *journal) * jbd2_cleanup_journal_tail() doesn't get called all that often. */ if (journal->j_flags & JBD2_BARRIER) - blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS); + blkdev_issue_flush(journal->j_fs_dev); return __jbd2_update_log_tail(journal, first_tid, blocknr); } diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c index b121d7d434c6..3cc4ab2ba7f4 100644 --- a/fs/jbd2/commit.c +++ b/fs/jbd2/commit.c @@ -825,7 +825,7 @@ start_journal_io: if (commit_transaction->t_need_data_flush && (journal->j_fs_dev != journal->j_dev) && (journal->j_flags & JBD2_BARRIER)) - blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS); + blkdev_issue_flush(journal->j_fs_dev); /* Done it all: now write the commit record asynchronously. */ if (jbd2_has_feature_async_commit(journal)) { @@ -932,7 +932,7 @@ start_journal_io: stats.run.rs_blocks_logged++; if (jbd2_has_feature_async_commit(journal) && journal->j_flags & JBD2_BARRIER) { - blkdev_issue_flush(journal->j_dev, GFP_NOFS); + blkdev_issue_flush(journal->j_dev); } if (err) diff --git a/fs/jbd2/recovery.c b/fs/jbd2/recovery.c index dc0694fcfcd1..69f18fe20923 100644 --- a/fs/jbd2/recovery.c +++ b/fs/jbd2/recovery.c @@ -326,7 +326,7 @@ int jbd2_journal_recover(journal_t *journal) err = err2; /* Make sure all replayed data is on permanent storage */ if (journal->j_flags & JBD2_BARRIER) { - err2 = blkdev_issue_flush(journal->j_fs_dev, GFP_KERNEL); + err2 = blkdev_issue_flush(journal->j_fs_dev); if (!err) err = err2; } diff --git a/fs/libfs.c b/fs/libfs.c index d1c3bade9f30..8398a0efb401 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -1117,7 +1117,7 @@ int generic_file_fsync(struct file *file, loff_t start, loff_t end, err = __generic_file_fsync(file, start, end, datasync); if (err) return err; - return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + return blkdev_issue_flush(inode->i_sb->s_bdev); } EXPORT_SYMBOL(generic_file_fsync); diff --git a/fs/nilfs2/the_nilfs.h b/fs/nilfs2/the_nilfs.h index b55cdeb4d169..987c8ab02aee 100644 --- a/fs/nilfs2/the_nilfs.h +++ b/fs/nilfs2/the_nilfs.h @@ -375,7 +375,7 @@ static inline int nilfs_flush_device(struct the_nilfs *nilfs) */ smp_wmb(); - err = blkdev_issue_flush(nilfs->ns_bdev, GFP_KERNEL); + err = blkdev_issue_flush(nilfs->ns_bdev); if (err != -EIO) err = 0; return err; diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 85979e2214b3..df6d709d2ae3 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -194,7 +194,7 @@ static int ocfs2_sync_file(struct file *file, loff_t start, loff_t end, needs_barrier = true; err = jbd2_complete_transaction(journal, commit_tid); if (needs_barrier) { - ret = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + ret = blkdev_issue_flush(inode->i_sb->s_bdev); if (!err) err = ret; } diff --git a/fs/reiserfs/file.c b/fs/reiserfs/file.c index 0b641ae694f1..1db0254bc38b 100644 --- a/fs/reiserfs/file.c +++ b/fs/reiserfs/file.c @@ -159,7 +159,7 @@ static int reiserfs_sync_file(struct file *filp, loff_t start, loff_t end, barrier_done = reiserfs_commit_for_inode(inode); reiserfs_write_unlock(inode->i_sb); if (barrier_done != 1 && reiserfs_barrier_flush(inode->i_sb)) - blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + blkdev_issue_flush(inode->i_sb->s_bdev); inode_unlock(inode); if (barrier_done < 0) return barrier_done; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 813be879a5e5..c3e32789829f 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -342,7 +342,7 @@ void xfs_blkdev_issue_flush( xfs_buftarg_t *buftarg) { - blkdev_issue_flush(buftarg->bt_bdev, GFP_NOFS); + blkdev_issue_flush(buftarg->bt_bdev); } STATIC void diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index faea2ed34b4a..ab68e27bb322 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -541,7 +541,7 @@ static int zonefs_file_fsync(struct file *file, loff_t start, loff_t end, if (ZONEFS_I(inode)->i_ztype == ZONEFS_ZTYPE_CNV) ret = file_write_and_wait_range(file, start, end); if (!ret) - ret = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + ret = blkdev_issue_flush(inode->i_sb->s_bdev); if (ret) zonefs_io_error(inode, true); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 2491e17b61c4..0dea268bd61b 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1288,7 +1288,7 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk) !list_empty(&plug->cb_list)); } -int blkdev_issue_flush(struct block_device *, gfp_t); +int blkdev_issue_flush(struct block_device *bdev); long nr_blockdev_pages(void); #else /* CONFIG_BLOCK */ struct blk_plug { @@ -1316,7 +1316,7 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk) return false; } -static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask) +static inline int blkdev_issue_flush(struct block_device *bdev) { return 0; } -- cgit v1.2.3 From 25ac84262cb5d5031f2769988ae5977a633b3f45 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 26 Jan 2021 15:52:37 +0100 Subject: f2fs: use blkdev_issue_flush in __submit_flush_wait Use the blkdev_issue_flush helper instead of duplicating it. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Reviewed-by: Chaitanya Kulkarni Acked-by: Damien Le Moal Signed-off-by: Jens Axboe --- fs/f2fs/data.c | 3 ++- fs/f2fs/f2fs.h | 1 - fs/f2fs/segment.c | 12 +----------- 3 files changed, 3 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 8cbf03159752..0cf0c6059924 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -56,7 +56,8 @@ static inline struct bio *__f2fs_bio_alloc(gfp_t gfp_mask, return bio_alloc_bioset(gfp_mask, nr_iovecs, &f2fs_bioset); } -struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio) +static struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, + bool noio) { if (noio) { /* No failure on bio allocation */ diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index bb11759191dc..902bd3267c03 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -3424,7 +3424,6 @@ void f2fs_destroy_checkpoint_caches(void); */ int __init f2fs_init_bioset(void); void f2fs_destroy_bioset(void); -struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio); int f2fs_init_bio_entry_cache(void); void f2fs_destroy_bio_entry_cache(void); void f2fs_submit_bio(struct f2fs_sb_info *sbi, diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index deca74cb17df..c495f170ee40 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -566,17 +566,7 @@ do_sync: static int __submit_flush_wait(struct f2fs_sb_info *sbi, struct block_device *bdev) { - struct bio *bio; - int ret; - - bio = f2fs_bio_alloc(sbi, 0, false); - if (!bio) - return -ENOMEM; - - bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH; - bio_set_dev(bio, bdev); - ret = submit_bio_wait(bio); - bio_put(bio); + int ret = blkdev_issue_flush(bdev); trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), test_opt(sbi, FLUSH_MERGE), ret); -- cgit v1.2.3 From 67883ade7a98a7589ca50e97b1c7b7893886d30e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 26 Jan 2021 15:52:38 +0100 Subject: f2fs: remove FAULT_ALLOC_BIO Sleeping bio allocations do not fail, which means that injecting an error into sleeping bio allocations is a little silly. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Reviewed-by: Chaitanya Kulkarni Acked-by: Damien Le Moal Signed-off-by: Jens Axboe --- Documentation/filesystems/f2fs.rst | 1 - fs/f2fs/data.c | 29 ++++------------------------- fs/f2fs/f2fs.h | 1 - fs/f2fs/super.c | 1 - 4 files changed, 4 insertions(+), 28 deletions(-) (limited to 'fs') diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index dae15c96e659..624f5f3ed93e 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -179,7 +179,6 @@ fault_type=%d Support configuring fault injection type, should be FAULT_KVMALLOC 0x000000002 FAULT_PAGE_ALLOC 0x000000004 FAULT_PAGE_GET 0x000000008 - FAULT_ALLOC_BIO 0x000000010 FAULT_ALLOC_NID 0x000000020 FAULT_ORPHAN 0x000000040 FAULT_BLOCK 0x000000080 diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 0cf0c6059924..9fb6be65592b 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -50,28 +50,6 @@ void f2fs_destroy_bioset(void) bioset_exit(&f2fs_bioset); } -static inline struct bio *__f2fs_bio_alloc(gfp_t gfp_mask, - unsigned int nr_iovecs) -{ - return bio_alloc_bioset(gfp_mask, nr_iovecs, &f2fs_bioset); -} - -static struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, - bool noio) -{ - if (noio) { - /* No failure on bio allocation */ - return __f2fs_bio_alloc(GFP_NOIO, npages); - } - - if (time_to_inject(sbi, FAULT_ALLOC_BIO)) { - f2fs_show_injection_info(sbi, FAULT_ALLOC_BIO); - return NULL; - } - - return __f2fs_bio_alloc(GFP_KERNEL, npages); -} - static bool __is_cp_guaranteed(struct page *page) { struct address_space *mapping = page->mapping; @@ -433,7 +411,7 @@ static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) struct f2fs_sb_info *sbi = fio->sbi; struct bio *bio; - bio = f2fs_bio_alloc(sbi, npages, true); + bio = bio_alloc_bioset(GFP_NOIO, npages, &f2fs_bioset); f2fs_target_device(sbi, fio->new_blkaddr, bio); if (is_read_io(fio->op)) { @@ -1029,8 +1007,9 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, struct bio_post_read_ctx *ctx; unsigned int post_read_steps = 0; - bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), - for_write); + bio = bio_alloc_bioset(for_write ? GFP_NOIO : GFP_KERNEL, + min_t(int, nr_pages, BIO_MAX_PAGES), + &f2fs_bioset); if (!bio) return ERR_PTR(-ENOMEM); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 902bd3267c03..6c78365d80ce 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -43,7 +43,6 @@ enum { FAULT_KVMALLOC, FAULT_PAGE_ALLOC, FAULT_PAGE_GET, - FAULT_ALLOC_BIO, FAULT_ALLOC_NID, FAULT_ORPHAN, FAULT_BLOCK, diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index b4a07fe62d1a..3a312642907e 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -45,7 +45,6 @@ const char *f2fs_fault_name[FAULT_MAX] = { [FAULT_KVMALLOC] = "kvmalloc", [FAULT_PAGE_ALLOC] = "page alloc", [FAULT_PAGE_GET] = "page get", - [FAULT_ALLOC_BIO] = "alloc bio", [FAULT_ALLOC_NID] = "alloc nid", [FAULT_ORPHAN] = "orphan", [FAULT_BLOCK] = "no more block", -- cgit v1.2.3 From 6808f7af964be4ed6b04d2aa4ba884a2e47c6214 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 26 Jan 2021 15:52:45 +0100 Subject: nfs/blocklayout: remove cruft in bl_alloc_init_bio bio_alloc never returns NULL when it can sleep. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Reviewed-by: Chaitanya Kulkarni Acked-by: Damien Le Moal Signed-off-by: Jens Axboe --- fs/nfs/blocklayout/blocklayout.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'fs') diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c index 3be6836074ae..1a96ce28efb0 100644 --- a/fs/nfs/blocklayout/blocklayout.c +++ b/fs/nfs/blocklayout/blocklayout.c @@ -123,11 +123,6 @@ bl_alloc_init_bio(int npg, struct block_device *bdev, sector_t disk_sector, npg = min(npg, BIO_MAX_PAGES); bio = bio_alloc(GFP_NOIO, npg); - if (!bio && (current->flags & PF_MEMALLOC)) { - while (!bio && (npg /= 2)) - bio = bio_alloc(GFP_NOIO, npg); - } - if (bio) { bio->bi_iter.bi_sector = disk_sector; bio_set_dev(bio, bdev); -- cgit v1.2.3 From 64820ac6c6962f76d164fa690deaa688d59278e2 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 26 Jan 2021 15:52:46 +0100 Subject: nilfs2: remove cruft in nilfs_alloc_seg_bio bio_alloc never returns NULL when it can sleep. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Reviewed-by: Chaitanya Kulkarni Acked-by: Damien Le Moal Signed-off-by: Jens Axboe --- fs/nilfs2/segbuf.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'fs') diff --git a/fs/nilfs2/segbuf.c b/fs/nilfs2/segbuf.c index 1a8729eded8b..1e75417bfe6e 100644 --- a/fs/nilfs2/segbuf.c +++ b/fs/nilfs2/segbuf.c @@ -386,10 +386,6 @@ static struct bio *nilfs_alloc_seg_bio(struct the_nilfs *nilfs, sector_t start, struct bio *bio; bio = bio_alloc(GFP_NOIO, nr_vecs); - if (bio == NULL) { - while (!bio && (nr_vecs >>= 1)) - bio = bio_alloc(GFP_NOIO, nr_vecs); - } if (likely(bio)) { bio_set_dev(bio, nilfs->ns_bdev); bio->bi_iter.bi_sector = -- cgit v1.2.3 From 632faca72938f9f63049e48a8c438913828ac7a9 Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Wed, 23 Dec 2020 11:44:25 -0800 Subject: f2fs: handle unallocated section and zone on pinned/atgc If we have large section/zone, unallocated segment makes them corrupted. E.g., - Pinned file: -1 119304647 119304647 - ATGC data: -1 119304647 119304647 Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/segment.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index e81eb0748e2a..229814b4f4a6 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -101,11 +101,11 @@ static inline void sanity_check_seg_type(struct f2fs_sb_info *sbi, #define BLKS_PER_SEC(sbi) \ ((sbi)->segs_per_sec * (sbi)->blocks_per_seg) #define GET_SEC_FROM_SEG(sbi, segno) \ - ((segno) / (sbi)->segs_per_sec) + (((segno) == -1) ? -1: (segno) / (sbi)->segs_per_sec) #define GET_SEG_FROM_SEC(sbi, secno) \ ((secno) * (sbi)->segs_per_sec) #define GET_ZONE_FROM_SEC(sbi, secno) \ - ((secno) / (sbi)->secs_per_zone) + (((secno) == -1) ? -1: (secno) / (sbi)->secs_per_zone) #define GET_ZONE_FROM_SEG(sbi, segno) \ GET_ZONE_FROM_SEC(sbi, GET_SEC_FROM_SEG(sbi, segno)) -- cgit v1.2.3 From 36218b81f094648d929994399eb6eb5c97b991e5 Mon Sep 17 00:00:00 2001 From: Zheng Yongjun Date: Tue, 22 Dec 2020 21:34:15 +0800 Subject: f2fs: Replace expression with offsetof() Use the existing offsetof() macro instead of duplicating code. Signed-off-by: Zheng Yongjun Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/node.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 3a24423ac65f..5e3fabacefb5 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -2696,7 +2696,7 @@ retry: src = F2FS_INODE(page); dst = F2FS_INODE(ipage); - memcpy(dst, src, (unsigned long)&src->i_ext - (unsigned long)src); + memcpy(dst, src, offsetof(struct f2fs_inode, i_ext)); dst->i_size = 0; dst->i_blocks = cpu_to_le64(1); dst->i_links = cpu_to_le32(1); -- cgit v1.2.3 From a28d9aa1a2c7c774c38f2da1a662434bc29cb98e Mon Sep 17 00:00:00 2001 From: Weichao Guo Date: Mon, 14 Dec 2020 11:54:53 +0800 Subject: f2fs: fix to set inode->i_mode correctly for posix_acl_update_mode We should update the ~S_IRWXUGO part of inode->i_mode in __setattr_copy, because posix_acl_update_mode updates mode based on inode->i_mode, which finally overwrites the ~S_IRWXUGO part of i_acl_mode with old i_mode. Testcase to reproduce this bug: 0. adduser abc 1. mkfs.f2fs /dev/sdd 2. mount -t f2fs /dev/sdd /mnt/f2fs 3. mkdir /mnt/f2fs/test 4. setfacl -m u:abc:r /mnt/f2fs/test 5. chmod +s /mnt/f2fs/test Signed-off-by: Weichao Guo Signed-off-by: Bin Shu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index f585545277d7..eced14882fc1 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -850,6 +850,7 @@ static void __setattr_copy(struct inode *inode, const struct iattr *attr) if (!in_group_p(inode->i_gid) && !capable(CAP_FSETID)) mode &= ~S_ISGID; + inode->i_mode = (inode->i_mode & S_IRWXUGO) | (mode & ~S_IRWXUGO); set_acl_inode(inode, mode); } } -- cgit v1.2.3 From 17232e830afb800acdcc22ae8980bf9d330393ef Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 25 Dec 2020 16:52:27 +0800 Subject: f2fs: enhance to update i_mode and acl atomically in f2fs_setattr() Previously, in f2fs_setattr(), we don't update S_ISUID|S_ISGID|S_ISVTX bits with S_IRWXUGO bits and acl entries atomically, so in error path, chmod() may partially success, this patch enhances to make chmod() flow being atomical. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/acl.c | 23 ++++++++++++++++++++++- fs/f2fs/file.c | 7 ++++--- fs/f2fs/xattr.c | 15 +++++++++------ 3 files changed, 35 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c index 1e5e9b1136ee..732ec10e7890 100644 --- a/fs/f2fs/acl.c +++ b/fs/f2fs/acl.c @@ -200,6 +200,27 @@ struct posix_acl *f2fs_get_acl(struct inode *inode, int type) return __f2fs_get_acl(inode, type, NULL); } +static int f2fs_acl_update_mode(struct inode *inode, umode_t *mode_p, + struct posix_acl **acl) +{ + umode_t mode = inode->i_mode; + int error; + + if (is_inode_flag_set(inode, FI_ACL_MODE)) + mode = F2FS_I(inode)->i_acl_mode; + + error = posix_acl_equiv_mode(*acl, &mode); + if (error < 0) + return error; + if (error == 0) + *acl = NULL; + if (!in_group_p(inode->i_gid) && + !capable_wrt_inode_uidgid(inode, CAP_FSETID)) + mode &= ~S_ISGID; + *mode_p = mode; + return 0; +} + static int __f2fs_set_acl(struct inode *inode, int type, struct posix_acl *acl, struct page *ipage) { @@ -213,7 +234,7 @@ static int __f2fs_set_acl(struct inode *inode, int type, case ACL_TYPE_ACCESS: name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS; if (acl && !ipage) { - error = posix_acl_update_mode(inode, &mode, &acl); + error = f2fs_acl_update_mode(inode, &mode, &acl); if (error) return error; set_acl_inode(inode, mode); diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index eced14882fc1..2ddc4baaf173 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -850,7 +850,6 @@ static void __setattr_copy(struct inode *inode, const struct iattr *attr) if (!in_group_p(inode->i_gid) && !capable(CAP_FSETID)) mode &= ~S_ISGID; - inode->i_mode = (inode->i_mode & S_IRWXUGO) | (mode & ~S_IRWXUGO); set_acl_inode(inode, mode); } } @@ -950,8 +949,10 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) if (attr->ia_valid & ATTR_MODE) { err = posix_acl_chmod(inode, f2fs_get_inode_mode(inode)); - if (err || is_inode_flag_set(inode, FI_ACL_MODE)) { - inode->i_mode = F2FS_I(inode)->i_acl_mode; + + if (is_inode_flag_set(inode, FI_ACL_MODE)) { + if (!err) + inode->i_mode = F2FS_I(inode)->i_acl_mode; clear_inode_flag(inode, FI_ACL_MODE); } } diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index 65afcc3cc68a..2086bef6c154 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -673,7 +673,7 @@ static int __f2fs_setxattr(struct inode *inode, int index, } if (value && f2fs_xattr_value_same(here, value, size)) - goto exit; + goto same; } else if ((flags & XATTR_REPLACE)) { error = -ENODATA; goto exit; @@ -738,17 +738,20 @@ static int __f2fs_setxattr(struct inode *inode, int index, if (error) goto exit; - if (is_inode_flag_set(inode, FI_ACL_MODE)) { - inode->i_mode = F2FS_I(inode)->i_acl_mode; - inode->i_ctime = current_time(inode); - clear_inode_flag(inode, FI_ACL_MODE); - } if (index == F2FS_XATTR_INDEX_ENCRYPTION && !strcmp(name, F2FS_XATTR_NAME_ENCRYPTION_CONTEXT)) f2fs_set_encrypted_inode(inode); f2fs_mark_inode_dirty_sync(inode, true); if (!error && S_ISDIR(inode->i_mode)) set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_CP); + +same: + if (is_inode_flag_set(inode, FI_ACL_MODE)) { + inode->i_mode = F2FS_I(inode)->i_acl_mode; + inode->i_ctime = current_time(inode); + clear_inode_flag(inode, FI_ACL_MODE); + } + exit: kfree(base_addr); return error; -- cgit v1.2.3 From e0fcd01510ad025c9bbce704c5c2579294056141 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Sat, 26 Dec 2020 18:07:01 +0800 Subject: f2fs: enforce the immutable flag on open files This patch ports commit 02b016ca7f99 ("ext4: enforce the immutable flag on open files") to f2fs. According to the chattr man page, "a file with the 'i' attribute cannot be modified..." Historically, this was only enforced when the file was opened, per the rest of the description, "... and the file can not be opened in write mode". There is general agreement that we should standardize all file systems to prevent modifications even for files that were opened at the time the immutable flag is set. Eventually, a change to enforce this at the VFS layer should be landing in mainline. Cc: stable@kernel.org Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'fs') diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 2ddc4baaf173..d57b54643918 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -60,6 +60,9 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) bool need_alloc = true; int err = 0; + if (unlikely(IS_IMMUTABLE(inode))) + return VM_FAULT_SIGBUS; + if (unlikely(f2fs_cp_error(sbi))) { err = -EIO; goto err; @@ -865,6 +868,14 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr) if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) return -EIO; + if (unlikely(IS_IMMUTABLE(inode))) + return -EPERM; + + if (unlikely(IS_APPEND(inode) && + (attr->ia_valid & (ATTR_MODE | ATTR_UID | + ATTR_GID | ATTR_TIMES_SET)))) + return -EPERM; + if ((attr->ia_valid & ATTR_SIZE) && !f2fs_is_compress_backend_ready(inode)) return -EOPNOTSUPP; @@ -4351,6 +4362,11 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) inode_lock(inode); } + if (unlikely(IS_IMMUTABLE(inode))) { + ret = -EPERM; + goto unlock; + } + ret = generic_write_checks(iocb, from); if (ret > 0) { bool preallocated = false; @@ -4415,6 +4431,7 @@ write: if (ret > 0) f2fs_update_iostat(F2FS_I_SB(inode), APP_WRITE_IO, ret); } +unlock: inode_unlock(inode); out: trace_f2fs_file_write_iter(inode, iocb->ki_pos, -- cgit v1.2.3 From 0b979f1bded3e6808184842133e6afeba312a4ff Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Sat, 26 Dec 2020 18:07:41 +0800 Subject: f2fs: relocate f2fs_precache_extents() Relocate f2fs_precache_extents() in prior to check_swap_activate(), then extent cache can be enabled before its use. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index aa34d620bec9..57b9aab2b142 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -4108,12 +4108,13 @@ static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, if (!f2fs_disable_compressed_file(inode)) return -EINVAL; + f2fs_precache_extents(inode); + ret = check_swap_activate(sis, file, span); if (ret < 0) return ret; set_inode_flag(inode, FI_PIN_FILE); - f2fs_precache_extents(inode); f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); return ret; } -- cgit v1.2.3 From 32be0e97c71366a19d11d1965e3f0957ea0be609 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 22 Jan 2021 17:40:13 +0800 Subject: f2fs: compress: deny setting unsupported compress algorithm If kernel doesn't support certain kinds of compress algorithm, deny to set them as compress algorithm of f2fs via 'compress_algorithm=%s' mount option. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/super.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'fs') diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index b4a07fe62d1a..a275bd312ae5 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -882,17 +882,33 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) if (!name) return -ENOMEM; if (!strcmp(name, "lzo")) { +#ifdef CONFIG_F2FS_FS_LZO F2FS_OPTION(sbi).compress_algorithm = COMPRESS_LZO; +#else + f2fs_info(sbi, "kernel doesn't support lzo compression"); +#endif } else if (!strcmp(name, "lz4")) { +#ifdef CONFIG_F2FS_FS_LZ4 F2FS_OPTION(sbi).compress_algorithm = COMPRESS_LZ4; +#else + f2fs_info(sbi, "kernel doesn't support lz4 compression"); +#endif } else if (!strcmp(name, "zstd")) { +#ifdef CONFIG_F2FS_FS_ZSTD F2FS_OPTION(sbi).compress_algorithm = COMPRESS_ZSTD; +#else + f2fs_info(sbi, "kernel doesn't support zstd compression"); +#endif } else if (!strcmp(name, "lzo-rle")) { +#ifdef CONFIG_F2FS_FS_LZORLE F2FS_OPTION(sbi).compress_algorithm = COMPRESS_LZORLE; +#else + f2fs_info(sbi, "kernel doesn't support lzorle compression"); +#endif } else { kfree(name); return -EINVAL; -- cgit v1.2.3 From 3fde13f817e23f05ce407d136325df4cbc913e67 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 22 Jan 2021 17:46:43 +0800 Subject: f2fs: compress: support compress level Expand 'compress_algorithm' mount option to accept parameter as format of :, by this way, it gives a way to allow user to do more specified config on lz4 and zstd compression level, then f2fs compression can provide higher compress ratio. In order to set compress level for lz4 algorithm, it needs to set CONFIG_LZ4HC_COMPRESS and CONFIG_F2FS_FS_LZ4HC config to enable lz4hc compress algorithm. CR and performance number on lz4/lz4hc algorithm: dd if=enwik9 of=compressed_file conv=fsync Original blocks: 244382 lz4 lz4hc-9 compressed blocks 170647 163270 compress ratio 69.8% 66.8% speed 16.4207 s, 60.9 MB/s 26.7299 s, 37.4 MB/s compress ratio = after / before Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- Documentation/filesystems/f2fs.rst | 5 +++ fs/f2fs/Kconfig | 10 +++++ fs/f2fs/compress.c | 41 ++++++++++++++++-- fs/f2fs/f2fs.h | 9 ++++ fs/f2fs/super.c | 89 +++++++++++++++++++++++++++++++++++++- include/linux/f2fs_fs.h | 3 ++ 6 files changed, 152 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index dae15c96e659..5eff4009e77e 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -249,6 +249,11 @@ checkpoint=%s[:%u[%]] Set to "disable" to turn off checkpointing. Set to "enabl This space is reclaimed once checkpoint=enable. compress_algorithm=%s Control compress algorithm, currently f2fs supports "lzo", "lz4", "zstd" and "lzo-rle" algorithm. +compress_algorithm=%s:%d Control compress algorithm and its compress level, now, only + "lz4" and "zstd" support compress level config. + algorithm level range + lz4 3 - 16 + zstd 1 - 22 compress_log_size=%u Support configuring compress cluster size, the size will be 4KB * (1 << %u), 16KB is minimum size, also it's default size. diff --git a/fs/f2fs/Kconfig b/fs/f2fs/Kconfig index d13c5c6a9787..63c1fc1a0e3b 100644 --- a/fs/f2fs/Kconfig +++ b/fs/f2fs/Kconfig @@ -119,6 +119,16 @@ config F2FS_FS_LZ4 help Support LZ4 compress algorithm, if unsure, say Y. +config F2FS_FS_LZ4HC + bool "LZ4HC compression support" + depends on F2FS_FS_COMPRESSION + depends on F2FS_FS_LZ4 + select LZ4HC_COMPRESS + default y + help + Support LZ4HC compress algorithm, LZ4HC has compatible on-disk + layout with LZ4, if unsure, say Y. + config F2FS_FS_ZSTD bool "ZSTD compression support" depends on F2FS_FS_COMPRESSION diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 4bcbacfe3325..a345a41e2119 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -252,8 +252,14 @@ static const struct f2fs_compress_ops f2fs_lzo_ops = { #ifdef CONFIG_F2FS_FS_LZ4 static int lz4_init_compress_ctx(struct compress_ctx *cc) { - cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), - LZ4_MEM_COMPRESS, GFP_NOFS); + unsigned int size = LZ4_MEM_COMPRESS; + +#ifdef CONFIG_F2FS_FS_LZ4HC + if (F2FS_I(cc->inode)->i_compress_flag >> COMPRESS_LEVEL_OFFSET) + size = LZ4HC_MEM_COMPRESS; +#endif + + cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), size, GFP_NOFS); if (!cc->private) return -ENOMEM; @@ -272,10 +278,34 @@ static void lz4_destroy_compress_ctx(struct compress_ctx *cc) cc->private = NULL; } +#ifdef CONFIG_F2FS_FS_LZ4HC +static int lz4hc_compress_pages(struct compress_ctx *cc) +{ + unsigned char level = F2FS_I(cc->inode)->i_compress_flag >> + COMPRESS_LEVEL_OFFSET; + int len; + + if (level) + len = LZ4_compress_HC(cc->rbuf, cc->cbuf->cdata, cc->rlen, + cc->clen, level, cc->private); + else + len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, + cc->clen, cc->private); + if (!len) + return -EAGAIN; + + cc->clen = len; + return 0; +} +#endif + static int lz4_compress_pages(struct compress_ctx *cc) { int len; +#ifdef CONFIG_F2FS_FS_LZ4HC + return lz4hc_compress_pages(cc); +#endif len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, cc->clen, cc->private); if (!len) @@ -325,8 +355,13 @@ static int zstd_init_compress_ctx(struct compress_ctx *cc) ZSTD_CStream *stream; void *workspace; unsigned int workspace_size; + unsigned char level = F2FS_I(cc->inode)->i_compress_flag >> + COMPRESS_LEVEL_OFFSET; + + if (!level) + level = F2FS_ZSTD_DEFAULT_CLEVEL; - params = ZSTD_getParams(F2FS_ZSTD_DEFAULT_CLEVEL, cc->rlen, 0); + params = ZSTD_getParams(level, cc->rlen, 0); workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams); workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode), diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index bb11759191dc..36012181c17f 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -146,6 +146,7 @@ struct f2fs_mount_info { /* For compression */ unsigned char compress_algorithm; /* algorithm type */ unsigned char compress_log_size; /* cluster log size */ + unsigned char compress_level; /* compress level */ bool compress_chksum; /* compressed data chksum */ unsigned char compress_ext_cnt; /* extension count */ int compress_mode; /* compression mode */ @@ -735,6 +736,7 @@ struct f2fs_inode_info { atomic_t i_compr_blocks; /* # of compressed blocks */ unsigned char i_compress_algorithm; /* algorithm type */ unsigned char i_log_cluster_size; /* log of cluster size */ + unsigned char i_compress_level; /* compress level (lz4hc,zstd) */ unsigned short i_compress_flag; /* compress flag */ unsigned int i_cluster_size; /* cluster size */ }; @@ -1310,6 +1312,8 @@ struct compress_data { #define F2FS_COMPRESSED_PAGE_MAGIC 0xF5F2C000 +#define COMPRESS_LEVEL_OFFSET 8 + /* compress context */ struct compress_ctx { struct inode *inode; /* inode the context belong to */ @@ -3934,6 +3938,11 @@ static inline void set_compress_context(struct inode *inode) 1 << COMPRESS_CHKSUM : 0; F2FS_I(inode)->i_cluster_size = 1 << F2FS_I(inode)->i_log_cluster_size; + if (F2FS_I(inode)->i_compress_algorithm == COMPRESS_LZ4 && + F2FS_OPTION(sbi).compress_level) + F2FS_I(inode)->i_compress_flag |= + F2FS_OPTION(sbi).compress_level << + COMPRESS_LEVEL_OFFSET; F2FS_I(inode)->i_flags |= F2FS_COMPR_FL; set_inode_flag(inode, FI_COMPRESSED_FILE); stat_inc_compr_inode(inode); diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index a275bd312ae5..c8be27a9eed6 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include "f2fs.h" #include "node.h" @@ -464,6 +466,74 @@ static int f2fs_set_test_dummy_encryption(struct super_block *sb, return 0; } +#ifdef CONFIG_F2FS_FS_COMPRESSION +#ifdef CONFIG_F2FS_FS_LZ4 +static int f2fs_set_lz4hc_level(struct f2fs_sb_info *sbi, const char *str) +{ +#ifdef CONFIG_F2FS_FS_LZ4HC + unsigned int level; +#endif + + if (strlen(str) == 3) { + F2FS_OPTION(sbi).compress_level = 0; + return 0; + } + +#ifdef CONFIG_F2FS_FS_LZ4HC + str += 3; + + if (str[0] != ':') { + f2fs_info(sbi, "wrong format, e.g. :"); + return -EINVAL; + } + if (kstrtouint(str + 1, 10, &level)) + return -EINVAL; + + if (level < LZ4HC_MIN_CLEVEL || level > LZ4HC_MAX_CLEVEL) { + f2fs_info(sbi, "invalid lz4hc compress level: %d", level); + return -EINVAL; + } + + F2FS_OPTION(sbi).compress_level = level; + return 0; +#else + f2fs_info(sbi, "kernel doesn't support lz4hc compression"); + return -EINVAL; +#endif +} +#endif + +#ifdef CONFIG_F2FS_FS_ZSTD +static int f2fs_set_zstd_level(struct f2fs_sb_info *sbi, const char *str) +{ + unsigned int level; + int len = 4; + + if (strlen(str) == len) { + F2FS_OPTION(sbi).compress_level = 0; + return 0; + } + + str += len; + + if (str[0] != ':') { + f2fs_info(sbi, "wrong format, e.g. :"); + return -EINVAL; + } + if (kstrtouint(str + 1, 10, &level)) + return -EINVAL; + + if (!level || level > ZSTD_maxCLevel()) { + f2fs_info(sbi, "invalid zstd compress level: %d", level); + return -EINVAL; + } + + F2FS_OPTION(sbi).compress_level = level; + return 0; +} +#endif +#endif + static int parse_options(struct super_block *sb, char *options, bool is_remount) { struct f2fs_sb_info *sbi = F2FS_SB(sb); @@ -883,20 +953,31 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) return -ENOMEM; if (!strcmp(name, "lzo")) { #ifdef CONFIG_F2FS_FS_LZO + F2FS_OPTION(sbi).compress_level = 0; F2FS_OPTION(sbi).compress_algorithm = COMPRESS_LZO; #else f2fs_info(sbi, "kernel doesn't support lzo compression"); #endif - } else if (!strcmp(name, "lz4")) { + } else if (!strncmp(name, "lz4", 3)) { #ifdef CONFIG_F2FS_FS_LZ4 + ret = f2fs_set_lz4hc_level(sbi, name); + if (ret) { + kfree(name); + return -EINVAL; + } F2FS_OPTION(sbi).compress_algorithm = COMPRESS_LZ4; #else f2fs_info(sbi, "kernel doesn't support lz4 compression"); #endif - } else if (!strcmp(name, "zstd")) { + } else if (!strncmp(name, "zstd", 4)) { #ifdef CONFIG_F2FS_FS_ZSTD + ret = f2fs_set_zstd_level(sbi, name); + if (ret) { + kfree(name); + return -EINVAL; + } F2FS_OPTION(sbi).compress_algorithm = COMPRESS_ZSTD; #else @@ -904,6 +985,7 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) #endif } else if (!strcmp(name, "lzo-rle")) { #ifdef CONFIG_F2FS_FS_LZORLE + F2FS_OPTION(sbi).compress_level = 0; F2FS_OPTION(sbi).compress_algorithm = COMPRESS_LZORLE; #else @@ -1555,6 +1637,9 @@ static inline void f2fs_show_compress_options(struct seq_file *seq, } seq_printf(seq, ",compress_algorithm=%s", algtype); + if (F2FS_OPTION(sbi).compress_level) + seq_printf(seq, ":%d", F2FS_OPTION(sbi).compress_level); + seq_printf(seq, ",compress_log_size=%u", F2FS_OPTION(sbi).compress_log_size); diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index 7dc2a06cf19a..c6cc0a566ef5 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -274,6 +274,9 @@ struct f2fs_inode { __u8 i_compress_algorithm; /* compress algorithm */ __u8 i_log_cluster_size; /* log of cluster size */ __le16 i_compress_flag; /* compress flag */ + /* 0 bit: chksum flag + * [10,15] bits: compress level + */ __le32 i_extra_end[0]; /* for attribute size calculation */ } __packed; __le32 i_addr[DEF_ADDRS_PER_INODE]; /* Pointers to data blocks */ -- cgit v1.2.3 From 5d4daa579e56adc97fb77c7dfda6c1f747c9ef25 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Wed, 9 Dec 2020 16:43:27 +0800 Subject: f2fs: introduce a new per-sb directory in sysfs Add a new directory 'stat' in path of /sys/fs/f2fs//, later we can add new readonly stat sysfs file into this directory, it will make directory less mess. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/f2fs.h | 5 ++++- fs/f2fs/sysfs.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 36012181c17f..ccbbf86d14e5 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1545,9 +1545,12 @@ struct f2fs_sb_info { unsigned int node_io_flag; /* For sysfs suppport */ - struct kobject s_kobj; + struct kobject s_kobj; /* /sys/fs/f2fs/ */ struct completion s_kobj_unregister; + struct kobject s_stat_kobj; /* /sys/fs/f2fs//stat */ + struct completion s_stat_kobj_unregister; + /* For shrinker support */ struct list_head s_list; int s_ndevs; /* number of devices */ diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 30bae57428d1..bd1174ed2e6f 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -702,6 +702,11 @@ static struct attribute *f2fs_feat_attrs[] = { }; ATTRIBUTE_GROUPS(f2fs_feat); +static struct attribute *f2fs_stat_attrs[] = { + NULL, +}; +ATTRIBUTE_GROUPS(f2fs_stat); + static const struct sysfs_ops f2fs_attr_ops = { .show = f2fs_attr_show, .store = f2fs_attr_store, @@ -730,6 +735,44 @@ static struct kobject f2fs_feat = { .kset = &f2fs_kset, }; +static ssize_t f2fs_stat_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_stat_kobj); + struct f2fs_attr *a = container_of(attr, struct f2fs_attr, attr); + + return a->show ? a->show(a, sbi, buf) : 0; +} + +static ssize_t f2fs_stat_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t len) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_stat_kobj); + struct f2fs_attr *a = container_of(attr, struct f2fs_attr, attr); + + return a->store ? a->store(a, sbi, buf, len) : 0; +} + +static void f2fs_stat_kobj_release(struct kobject *kobj) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_stat_kobj); + complete(&sbi->s_stat_kobj_unregister); +} + +static const struct sysfs_ops f2fs_stat_attr_ops = { + .show = f2fs_stat_attr_show, + .store = f2fs_stat_attr_store, +}; + +static struct kobj_type f2fs_stat_ktype = { + .default_groups = f2fs_stat_groups, + .sysfs_ops = &f2fs_stat_attr_ops, + .release = f2fs_stat_kobj_release, +}; + static int __maybe_unused segment_info_seq_show(struct seq_file *seq, void *offset) { @@ -936,11 +979,15 @@ int f2fs_register_sysfs(struct f2fs_sb_info *sbi) init_completion(&sbi->s_kobj_unregister); err = kobject_init_and_add(&sbi->s_kobj, &f2fs_sb_ktype, NULL, "%s", sb->s_id); - if (err) { - kobject_put(&sbi->s_kobj); - wait_for_completion(&sbi->s_kobj_unregister); - return err; - } + if (err) + goto put_sb_kobj; + + sbi->s_stat_kobj.kset = &f2fs_kset; + init_completion(&sbi->s_stat_kobj_unregister); + err = kobject_init_and_add(&sbi->s_stat_kobj, &f2fs_stat_ktype, + &sbi->s_kobj, "stat"); + if (err) + goto put_stat_kobj; if (f2fs_proc_root) sbi->s_proc = proc_mkdir(sb->s_id, f2fs_proc_root); @@ -956,6 +1003,13 @@ int f2fs_register_sysfs(struct f2fs_sb_info *sbi) victim_bits_seq_show, sb); } return 0; +put_stat_kobj: + kobject_put(&sbi->s_stat_kobj); + wait_for_completion(&sbi->s_stat_kobj_unregister); +put_sb_kobj: + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); + return err; } void f2fs_unregister_sysfs(struct f2fs_sb_info *sbi) @@ -967,6 +1021,11 @@ void f2fs_unregister_sysfs(struct f2fs_sb_info *sbi) remove_proc_entry("victim_bits", sbi->s_proc); remove_proc_entry(sbi->sb->s_id, f2fs_proc_root); } + + kobject_del(&sbi->s_stat_kobj); + kobject_put(&sbi->s_stat_kobj); + wait_for_completion(&sbi->s_stat_kobj_unregister); + kobject_del(&sbi->s_kobj); kobject_put(&sbi->s_kobj); wait_for_completion(&sbi->s_kobj_unregister); -- cgit v1.2.3 From 0953fe864c4d05f5a5cde626a630a76918cf4f9c Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 14 Dec 2020 17:20:57 +0800 Subject: f2fs: fix to tag FIEMAP_EXTENT_MERGED in f2fs_fiemap() f2fs does not natively support extents in metadata, 'extent' in f2fs is used as a virtual concept, so in f2fs_fiemap() interface, it needs to tag FIEMAP_EXTENT_MERGED flag to indicated the extent status is a result of merging. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 57b9aab2b142..547c9d4b430b 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1964,6 +1964,7 @@ next: } if (size) { + flags |= FIEMAP_EXTENT_MERGED; if (IS_ENCRYPTED(inode)) flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; -- cgit v1.2.3 From 2562515f0ad7342bde6456602c491b64c63fe950 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Wed, 16 Dec 2020 17:15:23 +0800 Subject: f2fs: fix out-of-repair __setattr_copy() __setattr_copy() was copied from setattr_copy() in fs/attr.c, there is two missing patches doesn't cover this inner function, fix it. Commit 7fa294c8991c ("userns: Allow chown and setgid preservation") Commit 23adbe12ef7d ("fs,userns: Change inode_capable to capable_wrt_inode_uidgid") Fixes: fbfa2cc58d53 ("f2fs: add file operations") Cc: stable@vger.kernel.org Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index d57b54643918..4e6d4b9120a8 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -851,7 +851,8 @@ static void __setattr_copy(struct inode *inode, const struct iattr *attr) if (ia_valid & ATTR_MODE) { umode_t mode = attr->ia_mode; - if (!in_group_p(inode->i_gid) && !capable(CAP_FSETID)) + if (!in_group_p(inode->i_gid) && + !capable_wrt_inode_uidgid(inode, CAP_FSETID)) mode &= ~S_ISGID; set_acl_inode(inode, mode); } -- cgit v1.2.3 From cf7404036019fada99d99ea01f49cb5c3142099d Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Wed, 30 Dec 2020 16:38:35 +0800 Subject: f2fs: trival cleanup in move_data_block() Trival cleanups: - relocate set_summary() before its use - relocate "allocate block address" to correct place - remove unneeded f2fs_wait_on_page_writeback() Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/gc.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 3ef84e6ded41..39330ad3c44e 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1169,8 +1169,6 @@ static int move_data_block(struct inode *inode, block_t bidx, if (err) goto put_out; - set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version); - /* read page */ fio.page = page; fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr; @@ -1207,6 +1205,9 @@ static int move_data_block(struct inode *inode, block_t bidx, } } + set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version); + + /* allocate block address */ f2fs_allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr, &sum, type, NULL); @@ -1233,9 +1234,6 @@ static int move_data_block(struct inode *inode, block_t bidx, set_page_writeback(fio.encrypted_page); ClearPageError(page); - /* allocate block address */ - f2fs_wait_on_page_writeback(dn.node_page, NODE, true, true); - fio.op = REQ_OP_WRITE; fio.op_flags = REQ_SYNC; fio.new_blkaddr = newaddr; -- cgit v1.2.3 From 7f59b277f79e8aacaa2ec7e549be6c27985c27f2 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 4 Jan 2021 22:33:02 -0800 Subject: f2fs: clean up post-read processing Rework the post-read processing logic to be much easier to understand. At least one bug is fixed by this: if an I/O error occurred when reading from disk, decryption and verity would be performed on the uninitialized data, causing misleading messages in the kernel log. Signed-off-by: Eric Biggers Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/compress.c | 149 ++++++++++++++++------ fs/f2fs/data.c | 357 +++++++++++++++++++++-------------------------------- fs/f2fs/f2fs.h | 55 +++++++-- 3 files changed, 297 insertions(+), 264 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index a345a41e2119..1696f9183ff5 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -756,38 +756,27 @@ out: return ret; } -void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity) +static void f2fs_decompress_cluster(struct decompress_io_ctx *dic) { - struct decompress_io_ctx *dic = - (struct decompress_io_ctx *)page_private(page); struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode); - struct f2fs_inode_info *fi= F2FS_I(dic->inode); + struct f2fs_inode_info *fi = F2FS_I(dic->inode); const struct f2fs_compress_ops *cops = f2fs_cops[fi->i_compress_algorithm]; int ret; int i; - dec_page_count(sbi, F2FS_RD_DATA); - - if (bio->bi_status || PageError(page)) - dic->failed = true; - - if (atomic_dec_return(&dic->pending_pages)) - return; - trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx, dic->cluster_size, fi->i_compress_algorithm); - /* submit partial compressed pages */ if (dic->failed) { ret = -EIO; - goto out_free_dic; + goto out_end_io; } dic->tpages = page_array_alloc(dic->inode, dic->cluster_size); if (!dic->tpages) { ret = -ENOMEM; - goto out_free_dic; + goto out_end_io; } for (i = 0; i < dic->cluster_size; i++) { @@ -799,20 +788,20 @@ void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity) dic->tpages[i] = f2fs_compress_alloc_page(); if (!dic->tpages[i]) { ret = -ENOMEM; - goto out_free_dic; + goto out_end_io; } } if (cops->init_decompress_ctx) { ret = cops->init_decompress_ctx(dic); if (ret) - goto out_free_dic; + goto out_end_io; } dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size); if (!dic->rbuf) { ret = -ENOMEM; - goto destroy_decompress_ctx; + goto out_destroy_decompress_ctx; } dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages); @@ -851,18 +840,34 @@ out_vunmap_cbuf: vm_unmap_ram(dic->cbuf, dic->nr_cpages); out_vunmap_rbuf: vm_unmap_ram(dic->rbuf, dic->cluster_size); -destroy_decompress_ctx: +out_destroy_decompress_ctx: if (cops->destroy_decompress_ctx) cops->destroy_decompress_ctx(dic); -out_free_dic: - if (!verity) - f2fs_decompress_end_io(dic->rpages, dic->cluster_size, - ret, false); - +out_end_io: trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx, dic->clen, ret); - if (!verity) - f2fs_free_dic(dic); + f2fs_decompress_end_io(dic, ret); +} + +/* + * This is called when a page of a compressed cluster has been read from disk + * (or failed to be read from disk). It checks whether this page was the last + * page being waited on in the cluster, and if so, it decompresses the cluster + * (or in the case of a failure, cleans up without actually decompressing). + */ +void f2fs_end_read_compressed_page(struct page *page, bool failed) +{ + struct decompress_io_ctx *dic = + (struct decompress_io_ctx *)page_private(page); + struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode); + + dec_page_count(sbi, F2FS_RD_DATA); + + if (failed) + WRITE_ONCE(dic->failed, true); + + if (atomic_dec_and_test(&dic->remaining_pages)) + f2fs_decompress_cluster(dic); } static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index) @@ -1529,6 +1534,8 @@ destroy_out: return err; } +static void f2fs_free_dic(struct decompress_io_ctx *dic); + struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) { struct decompress_io_ctx *dic; @@ -1547,12 +1554,14 @@ struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) dic->magic = F2FS_COMPRESSED_PAGE_MAGIC; dic->inode = cc->inode; - atomic_set(&dic->pending_pages, cc->nr_cpages); + atomic_set(&dic->remaining_pages, cc->nr_cpages); dic->cluster_idx = cc->cluster_idx; dic->cluster_size = cc->cluster_size; dic->log_cluster_size = cc->log_cluster_size; dic->nr_cpages = cc->nr_cpages; + refcount_set(&dic->refcnt, 1); dic->failed = false; + dic->need_verity = f2fs_need_verity(cc->inode, start_idx); for (i = 0; i < dic->cluster_size; i++) dic->rpages[i] = cc->rpages[i]; @@ -1581,7 +1590,7 @@ out_free: return ERR_PTR(-ENOMEM); } -void f2fs_free_dic(struct decompress_io_ctx *dic) +static void f2fs_free_dic(struct decompress_io_ctx *dic) { int i; @@ -1609,30 +1618,88 @@ void f2fs_free_dic(struct decompress_io_ctx *dic) kmem_cache_free(dic_entry_slab, dic); } -void f2fs_decompress_end_io(struct page **rpages, - unsigned int cluster_size, bool err, bool verity) +static void f2fs_put_dic(struct decompress_io_ctx *dic) +{ + if (refcount_dec_and_test(&dic->refcnt)) + f2fs_free_dic(dic); +} + +/* + * Update and unlock the cluster's pagecache pages, and release the reference to + * the decompress_io_ctx that was being held for I/O completion. + */ +static void __f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed) { int i; - for (i = 0; i < cluster_size; i++) { - struct page *rpage = rpages[i]; + for (i = 0; i < dic->cluster_size; i++) { + struct page *rpage = dic->rpages[i]; if (!rpage) continue; - if (err || PageError(rpage)) - goto clear_uptodate; - - if (!verity || fsverity_verify_page(rpage)) { + /* PG_error was set if verity failed. */ + if (failed || PageError(rpage)) { + ClearPageUptodate(rpage); + /* will re-read again later */ + ClearPageError(rpage); + } else { SetPageUptodate(rpage); - goto unlock; } -clear_uptodate: - ClearPageUptodate(rpage); - ClearPageError(rpage); -unlock: unlock_page(rpage); } + + f2fs_put_dic(dic); +} + +static void f2fs_verify_cluster(struct work_struct *work) +{ + struct decompress_io_ctx *dic = + container_of(work, struct decompress_io_ctx, verity_work); + int i; + + /* Verify the cluster's decompressed pages with fs-verity. */ + for (i = 0; i < dic->cluster_size; i++) { + struct page *rpage = dic->rpages[i]; + + if (rpage && !fsverity_verify_page(rpage)) + SetPageError(rpage); + } + + __f2fs_decompress_end_io(dic, false); +} + +/* + * This is called when a compressed cluster has been decompressed + * (or failed to be read and/or decompressed). + */ +void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed) +{ + if (!failed && dic->need_verity) { + /* + * Note that to avoid deadlocks, the verity work can't be done + * on the decompression workqueue. This is because verifying + * the data pages can involve reading metadata pages from the + * file, and these metadata pages may be compressed. + */ + INIT_WORK(&dic->verity_work, f2fs_verify_cluster); + fsverity_enqueue_verify_work(&dic->verity_work); + } else { + __f2fs_decompress_end_io(dic, failed); + } +} + +/* + * Put a reference to a compressed page's decompress_io_ctx. + * + * This is called when the page is no longer needed and can be freed. + */ +void f2fs_put_page_dic(struct page *page) +{ + struct decompress_io_ctx *dic = + (struct decompress_io_ctx *)page_private(page); + + f2fs_put_dic(dic); } int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 547c9d4b430b..4d80f00e5e40 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -115,10 +115,21 @@ static enum count_type __read_io_type(struct page *page) /* postprocessing steps for read bios */ enum bio_post_read_step { - STEP_DECRYPT, - STEP_DECOMPRESS_NOWQ, /* handle normal cluster data inplace */ - STEP_DECOMPRESS, /* handle compressed cluster data in workqueue */ - STEP_VERITY, +#ifdef CONFIG_FS_ENCRYPTION + STEP_DECRYPT = 1 << 0, +#else + STEP_DECRYPT = 0, /* compile out the decryption-related code */ +#endif +#ifdef CONFIG_F2FS_FS_COMPRESSION + STEP_DECOMPRESS = 1 << 1, +#else + STEP_DECOMPRESS = 0, /* compile out the decompression-related code */ +#endif +#ifdef CONFIG_FS_VERITY + STEP_VERITY = 1 << 2, +#else + STEP_VERITY = 0, /* compile out the verity-related code */ +#endif }; struct bio_post_read_ctx { @@ -128,25 +139,26 @@ struct bio_post_read_ctx { unsigned int enabled_steps; }; -static void __read_end_io(struct bio *bio, bool compr, bool verity) +static void f2fs_finish_read_bio(struct bio *bio) { - struct page *page; struct bio_vec *bv; struct bvec_iter_all iter_all; + /* + * Update and unlock the bio's pagecache pages, and put the + * decompression context for any compressed pages. + */ bio_for_each_segment_all(bv, bio, iter_all) { - page = bv->bv_page; + struct page *page = bv->bv_page; -#ifdef CONFIG_F2FS_FS_COMPRESSION - if (compr && f2fs_is_compressed_page(page)) { - f2fs_decompress_pages(bio, page, verity); + if (f2fs_is_compressed_page(page)) { + if (bio->bi_status) + f2fs_end_read_compressed_page(page, true); + f2fs_put_page_dic(page); continue; } - if (verity) - continue; -#endif - /* PG_error was set if any post_read step failed */ + /* PG_error was set if decryption or verity failed. */ if (bio->bi_status || PageError(page)) { ClearPageUptodate(page); /* will re-read again later */ @@ -157,181 +169,141 @@ static void __read_end_io(struct bio *bio, bool compr, bool verity) dec_page_count(F2FS_P_SB(page), __read_io_type(page)); unlock_page(page); } -} - -static void f2fs_release_read_bio(struct bio *bio); -static void __f2fs_read_end_io(struct bio *bio, bool compr, bool verity) -{ - if (!compr) - __read_end_io(bio, false, verity); - f2fs_release_read_bio(bio); -} - -static void f2fs_decompress_bio(struct bio *bio, bool verity) -{ - __read_end_io(bio, true, verity); -} - -static void bio_post_read_processing(struct bio_post_read_ctx *ctx); - -static void f2fs_decrypt_work(struct bio_post_read_ctx *ctx) -{ - fscrypt_decrypt_bio(ctx->bio); -} - -static void f2fs_decompress_work(struct bio_post_read_ctx *ctx) -{ - f2fs_decompress_bio(ctx->bio, ctx->enabled_steps & (1 << STEP_VERITY)); -} - -#ifdef CONFIG_F2FS_FS_COMPRESSION -static void f2fs_verify_pages(struct page **rpages, unsigned int cluster_size) -{ - f2fs_decompress_end_io(rpages, cluster_size, false, true); -} - -static void f2fs_verify_bio(struct bio *bio) -{ - struct bio_vec *bv; - struct bvec_iter_all iter_all; - - bio_for_each_segment_all(bv, bio, iter_all) { - struct page *page = bv->bv_page; - struct decompress_io_ctx *dic; - - dic = (struct decompress_io_ctx *)page_private(page); - - if (dic) { - if (atomic_dec_return(&dic->verity_pages)) - continue; - f2fs_verify_pages(dic->rpages, - dic->cluster_size); - f2fs_free_dic(dic); - continue; - } - - if (bio->bi_status || PageError(page)) - goto clear_uptodate; - if (fsverity_verify_page(page)) { - SetPageUptodate(page); - goto unlock; - } -clear_uptodate: - ClearPageUptodate(page); - ClearPageError(page); -unlock: - dec_page_count(F2FS_P_SB(page), __read_io_type(page)); - unlock_page(page); - } + if (bio->bi_private) + mempool_free(bio->bi_private, bio_post_read_ctx_pool); + bio_put(bio); } -#endif -static void f2fs_verity_work(struct work_struct *work) +static void f2fs_verify_bio(struct work_struct *work) { struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); struct bio *bio = ctx->bio; -#ifdef CONFIG_F2FS_FS_COMPRESSION - unsigned int enabled_steps = ctx->enabled_steps; -#endif + bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); /* * fsverity_verify_bio() may call readpages() again, and while verity - * will be disabled for this, decryption may still be needed, resulting - * in another bio_post_read_ctx being allocated. So to prevent - * deadlocks we need to release the current ctx to the mempool first. - * This assumes that verity is the last post-read step. + * will be disabled for this, decryption and/or decompression may still + * be needed, resulting in another bio_post_read_ctx being allocated. + * So to prevent deadlocks we need to release the current ctx to the + * mempool first. This assumes that verity is the last post-read step. */ mempool_free(ctx, bio_post_read_ctx_pool); bio->bi_private = NULL; -#ifdef CONFIG_F2FS_FS_COMPRESSION - /* previous step is decompression */ - if (enabled_steps & (1 << STEP_DECOMPRESS)) { - f2fs_verify_bio(bio); - f2fs_release_read_bio(bio); - return; + /* + * Verify the bio's pages with fs-verity. Exclude compressed pages, + * as those were handled separately by f2fs_end_read_compressed_page(). + */ + if (may_have_compressed_pages) { + struct bio_vec *bv; + struct bvec_iter_all iter_all; + + bio_for_each_segment_all(bv, bio, iter_all) { + struct page *page = bv->bv_page; + + if (!f2fs_is_compressed_page(page) && + !PageError(page) && !fsverity_verify_page(page)) + SetPageError(page); + } + } else { + fsverity_verify_bio(bio); } -#endif - fsverity_verify_bio(bio); - __f2fs_read_end_io(bio, false, false); + f2fs_finish_read_bio(bio); } -static void f2fs_post_read_work(struct work_struct *work) +/* + * If the bio's data needs to be verified with fs-verity, then enqueue the + * verity work for the bio. Otherwise finish the bio now. + * + * Note that to avoid deadlocks, the verity work can't be done on the + * decryption/decompression workqueue. This is because verifying the data pages + * can involve reading verity metadata pages from the file, and these verity + * metadata pages may be encrypted and/or compressed. + */ +static void f2fs_verify_and_finish_bio(struct bio *bio) { - struct bio_post_read_ctx *ctx = - container_of(work, struct bio_post_read_ctx, work); - - if (ctx->enabled_steps & (1 << STEP_DECRYPT)) - f2fs_decrypt_work(ctx); + struct bio_post_read_ctx *ctx = bio->bi_private; - if (ctx->enabled_steps & (1 << STEP_DECOMPRESS)) - f2fs_decompress_work(ctx); - - if (ctx->enabled_steps & (1 << STEP_VERITY)) { - INIT_WORK(&ctx->work, f2fs_verity_work); + if (ctx && (ctx->enabled_steps & STEP_VERITY)) { + INIT_WORK(&ctx->work, f2fs_verify_bio); fsverity_enqueue_verify_work(&ctx->work); - return; + } else { + f2fs_finish_read_bio(bio); } - - __f2fs_read_end_io(ctx->bio, - ctx->enabled_steps & (1 << STEP_DECOMPRESS), false); } -static void f2fs_enqueue_post_read_work(struct f2fs_sb_info *sbi, - struct work_struct *work) -{ - queue_work(sbi->post_read_wq, work); -} - -static void bio_post_read_processing(struct bio_post_read_ctx *ctx) +/* + * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last + * remaining page was read by @ctx->bio. + * + * Note that a bio may span clusters (even a mix of compressed and uncompressed + * clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates + * that the bio includes at least one compressed page. The actual decompression + * is done on a per-cluster basis, not a per-bio basis. + */ +static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx) { - /* - * We use different work queues for decryption and for verity because - * verity may require reading metadata pages that need decryption, and - * we shouldn't recurse to the same workqueue. - */ + struct bio_vec *bv; + struct bvec_iter_all iter_all; + bool all_compressed = true; - if (ctx->enabled_steps & (1 << STEP_DECRYPT) || - ctx->enabled_steps & (1 << STEP_DECOMPRESS)) { - INIT_WORK(&ctx->work, f2fs_post_read_work); - f2fs_enqueue_post_read_work(ctx->sbi, &ctx->work); - return; - } + bio_for_each_segment_all(bv, ctx->bio, iter_all) { + struct page *page = bv->bv_page; - if (ctx->enabled_steps & (1 << STEP_VERITY)) { - INIT_WORK(&ctx->work, f2fs_verity_work); - fsverity_enqueue_verify_work(&ctx->work); - return; + /* PG_error was set if decryption failed. */ + if (f2fs_is_compressed_page(page)) + f2fs_end_read_compressed_page(page, PageError(page)); + else + all_compressed = false; } - __f2fs_read_end_io(ctx->bio, false, false); + /* + * Optimization: if all the bio's pages are compressed, then scheduling + * the per-bio verity work is unnecessary, as verity will be fully + * handled at the compression cluster level. + */ + if (all_compressed) + ctx->enabled_steps &= ~STEP_VERITY; } -static bool f2fs_bio_post_read_required(struct bio *bio) +static void f2fs_post_read_work(struct work_struct *work) { - return bio->bi_private; + struct bio_post_read_ctx *ctx = + container_of(work, struct bio_post_read_ctx, work); + + if (ctx->enabled_steps & STEP_DECRYPT) + fscrypt_decrypt_bio(ctx->bio); + + if (ctx->enabled_steps & STEP_DECOMPRESS) + f2fs_handle_step_decompress(ctx); + + f2fs_verify_and_finish_bio(ctx->bio); } static void f2fs_read_end_io(struct bio *bio) { struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio)); + struct bio_post_read_ctx *ctx = bio->bi_private; if (time_to_inject(sbi, FAULT_READ_IO)) { f2fs_show_injection_info(sbi, FAULT_READ_IO); bio->bi_status = BLK_STS_IOERR; } - if (f2fs_bio_post_read_required(bio)) { - struct bio_post_read_ctx *ctx = bio->bi_private; - - bio_post_read_processing(ctx); + if (bio->bi_status) { + f2fs_finish_read_bio(bio); return; } - __f2fs_read_end_io(bio, false, false); + if (ctx && (ctx->enabled_steps & (STEP_DECRYPT | STEP_DECOMPRESS))) { + INIT_WORK(&ctx->work, f2fs_post_read_work); + queue_work(ctx->sbi->post_read_wq, &ctx->work); + } else { + f2fs_verify_and_finish_bio(bio); + } } static void f2fs_write_end_io(struct bio *bio) @@ -1022,16 +994,9 @@ out: up_write(&io->io_rwsem); } -static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx) -{ - return fsverity_active(inode) && - idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE); -} - static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, unsigned nr_pages, unsigned op_flag, - pgoff_t first_idx, bool for_write, - bool for_verity) + pgoff_t first_idx, bool for_write) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct bio *bio; @@ -1050,13 +1015,19 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, bio_set_op_attrs(bio, REQ_OP_READ, op_flag); if (fscrypt_inode_uses_fs_layer_crypto(inode)) - post_read_steps |= 1 << STEP_DECRYPT; - if (f2fs_compressed_file(inode)) - post_read_steps |= 1 << STEP_DECOMPRESS_NOWQ; - if (for_verity && f2fs_need_verity(inode, first_idx)) - post_read_steps |= 1 << STEP_VERITY; + post_read_steps |= STEP_DECRYPT; + + if (f2fs_need_verity(inode, first_idx)) + post_read_steps |= STEP_VERITY; + + /* + * STEP_DECOMPRESS is handled specially, since a compressed file might + * contain both compressed and uncompressed clusters. We'll allocate a + * bio_post_read_ctx if the file is compressed, but the caller is + * responsible for enabling STEP_DECOMPRESS if it's actually needed. + */ - if (post_read_steps) { + if (post_read_steps || f2fs_compressed_file(inode)) { /* Due to the mempool, this never fails. */ ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS); ctx->bio = bio; @@ -1068,13 +1039,6 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, return bio; } -static void f2fs_release_read_bio(struct bio *bio) -{ - if (bio->bi_private) - mempool_free(bio->bi_private, bio_post_read_ctx_pool); - bio_put(bio); -} - /* This can handle encryption stuffs */ static int f2fs_submit_page_read(struct inode *inode, struct page *page, block_t blkaddr, int op_flags, bool for_write) @@ -1083,7 +1047,7 @@ static int f2fs_submit_page_read(struct inode *inode, struct page *page, struct bio *bio; bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags, - page->index, for_write, true); + page->index, for_write); if (IS_ERR(bio)) return PTR_ERR(bio); @@ -2122,7 +2086,7 @@ submit_and_realloc: if (bio == NULL) { bio = f2fs_grab_read_bio(inode, block_nr, nr_pages, is_readahead ? REQ_RAHEAD : 0, page->index, - false, true); + false); if (IS_ERR(bio)) { ret = PTR_ERR(bio); bio = NULL; @@ -2168,8 +2132,6 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, sector_t last_block_in_file; const unsigned blocksize = blks_to_bytes(inode, 1); struct decompress_io_ctx *dic = NULL; - struct bio_post_read_ctx *ctx; - bool for_verity = false; int i; int ret = 0; @@ -2235,29 +2197,10 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, goto out_put_dnode; } - /* - * It's possible to enable fsverity on the fly when handling a cluster, - * which requires complicated error handling. Instead of adding more - * complexity, let's give a rule where end_io post-processes fsverity - * per cluster. In order to do that, we need to submit bio, if previous - * bio sets a different post-process policy. - */ - if (fsverity_active(cc->inode)) { - atomic_set(&dic->verity_pages, cc->nr_cpages); - for_verity = true; - - if (bio) { - ctx = bio->bi_private; - if (!(ctx->enabled_steps & (1 << STEP_VERITY))) { - __submit_bio(sbi, bio, DATA); - bio = NULL; - } - } - } - for (i = 0; i < dic->nr_cpages; i++) { struct page *page = dic->cpages[i]; block_t blkaddr; + struct bio_post_read_ctx *ctx; blkaddr = data_blkaddr(dn.inode, dn.node_page, dn.ofs_in_node + i + 1); @@ -2273,31 +2216,10 @@ submit_and_realloc: if (!bio) { bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages, is_readahead ? REQ_RAHEAD : 0, - page->index, for_write, for_verity); + page->index, for_write); if (IS_ERR(bio)) { - unsigned int remained = dic->nr_cpages - i; - bool release = false; - ret = PTR_ERR(bio); - dic->failed = true; - - if (for_verity) { - if (!atomic_sub_return(remained, - &dic->verity_pages)) - release = true; - } else { - if (!atomic_sub_return(remained, - &dic->pending_pages)) - release = true; - } - - if (release) { - f2fs_decompress_end_io(dic->rpages, - cc->cluster_size, true, - false); - f2fs_free_dic(dic); - } - + f2fs_decompress_end_io(dic, ret); f2fs_put_dnode(&dn); *bio_ret = NULL; return ret; @@ -2309,10 +2231,9 @@ submit_and_realloc: if (bio_add_page(bio, page, blocksize, 0) < blocksize) goto submit_and_realloc; - /* tag STEP_DECOMPRESS to handle IO in wq */ ctx = bio->bi_private; - if (!(ctx->enabled_steps & (1 << STEP_DECOMPRESS))) - ctx->enabled_steps |= 1 << STEP_DECOMPRESS; + ctx->enabled_steps |= STEP_DECOMPRESS; + refcount_inc(&dic->refcnt); inc_page_count(sbi, F2FS_RD_DATA); f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE); @@ -2329,7 +2250,13 @@ submit_and_realloc: out_put_dnode: f2fs_put_dnode(&dn); out: - f2fs_decompress_end_io(cc->rpages, cc->cluster_size, true, false); + for (i = 0; i < cc->cluster_size; i++) { + if (cc->rpages[i]) { + ClearPageUptodate(cc->rpages[i]); + ClearPageError(cc->rpages[i]); + unlock_page(cc->rpages[i]); + } + } *bio_ret = bio; return ret; } diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index ccbbf86d14e5..980e061f7968 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1341,7 +1341,7 @@ struct compress_io_ctx { atomic_t pending_pages; /* in-flight compressed page count */ }; -/* decompress io context for read IO path */ +/* Context for decompressing one cluster on the read IO path */ struct decompress_io_ctx { u32 magic; /* magic number to indicate page is compressed */ struct inode *inode; /* inode the context belong to */ @@ -1357,11 +1357,37 @@ struct decompress_io_ctx { struct compress_data *cbuf; /* virtual mapped address on cpages */ size_t rlen; /* valid data length in rbuf */ size_t clen; /* valid data length in cbuf */ - atomic_t pending_pages; /* in-flight compressed page count */ - atomic_t verity_pages; /* in-flight page count for verity */ - bool failed; /* indicate IO error during decompression */ + + /* + * The number of compressed pages remaining to be read in this cluster. + * This is initially nr_cpages. It is decremented by 1 each time a page + * has been read (or failed to be read). When it reaches 0, the cluster + * is decompressed (or an error is reported). + * + * If an error occurs before all the pages have been submitted for I/O, + * then this will never reach 0. In this case the I/O submitter is + * responsible for calling f2fs_decompress_end_io() instead. + */ + atomic_t remaining_pages; + + /* + * Number of references to this decompress_io_ctx. + * + * One reference is held for I/O completion. This reference is dropped + * after the pagecache pages are updated and unlocked -- either after + * decompression (and verity if enabled), or after an error. + * + * In addition, each compressed page holds a reference while it is in a + * bio. These references are necessary prevent compressed pages from + * being freed while they are still in a bio. + */ + refcount_t refcnt; + + bool failed; /* IO error occurred before decompression? */ + bool need_verity; /* need fs-verity verification after decompression? */ void *private; /* payload buffer for specified decompression algorithm */ void *private2; /* extra payload buffer */ + struct work_struct verity_work; /* work to verify the decompressed pages */ }; #define NULL_CLUSTER ((unsigned int)(~0)) @@ -3883,7 +3909,7 @@ void f2fs_compress_write_end_io(struct bio *bio, struct page *page); bool f2fs_is_compress_backend_ready(struct inode *inode); int f2fs_init_compress_mempool(void); void f2fs_destroy_compress_mempool(void); -void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity); +void f2fs_end_read_compressed_page(struct page *page, bool failed); bool f2fs_cluster_is_empty(struct compress_ctx *cc); bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index); void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page); @@ -3896,9 +3922,8 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, unsigned nr_pages, sector_t *last_block_in_bio, bool is_readahead, bool for_write); struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc); -void f2fs_free_dic(struct decompress_io_ctx *dic); -void f2fs_decompress_end_io(struct page **rpages, - unsigned int cluster_size, bool err, bool verity); +void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed); +void f2fs_put_page_dic(struct page *page); int f2fs_init_compress_ctx(struct compress_ctx *cc); void f2fs_destroy_compress_ctx(struct compress_ctx *cc); void f2fs_init_compress_info(struct f2fs_sb_info *sbi); @@ -3922,6 +3947,14 @@ static inline struct page *f2fs_compress_control_page(struct page *page) } static inline int f2fs_init_compress_mempool(void) { return 0; } static inline void f2fs_destroy_compress_mempool(void) { } +static inline void f2fs_end_read_compressed_page(struct page *page, bool failed) +{ + WARN_ON_ONCE(1); +} +static inline void f2fs_put_page_dic(struct page *page) +{ + WARN_ON_ONCE(1); +} static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; } static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { } static inline int __init f2fs_init_compress_cache(void) { return 0; } @@ -4126,6 +4159,12 @@ static inline bool f2fs_force_buffered_io(struct inode *inode, return false; } +static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx) +{ + return fsverity_active(inode) && + idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE); +} + #ifdef CONFIG_F2FS_FAULT_INJECTION extern void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate, unsigned int type); -- cgit v1.2.3 From df0736d70c4fa6ed711ba103b61880fe72bb4777 Mon Sep 17 00:00:00 2001 From: Daeho Jeong Date: Wed, 6 Jan 2021 08:49:28 +0900 Subject: f2fs: fix null page reference in redirty_blocks By Colin's static analysis, we found out there is a null page reference under low memory situation in redirty_blocks. I've made the page finding loop stop immediately and return an error not to cause further memory pressure when we run into a failure to find a page under low memory condition. Signed-off-by: Daeho Jeong Reported-by: Colin Ian King Fixes: 5fdb322ff2c2 ("f2fs: add F2FS_IOC_DECOMPRESS_FILE and F2FS_IOC_COMPRESS_FILE") Reviewed-by: Colin Ian King Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 4e6d4b9120a8..e3a5b620b50a 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -4057,8 +4057,10 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len) for (i = 0; i < page_len; i++, redirty_idx++) { page = find_lock_page(mapping, redirty_idx); - if (!page) - ret = -ENOENT; + if (!page) { + ret = -ENOMEM; + break; + } set_page_dirty(page); f2fs_put_page(page, 1); f2fs_put_page(page, 0); -- cgit v1.2.3 From 46085f37fc9e12d5c3539fb768b5ad7951e72acf Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 12 Jan 2021 09:55:09 +0800 Subject: f2fs: fix to set/clear I_LINKABLE under i_lock fsstress + fault injection test case reports a warning message as below: WARNING: CPU: 13 PID: 6226 at fs/inode.c:361 inc_nlink+0x32/0x40 Call Trace: f2fs_init_inode_metadata+0x25c/0x4a0 [f2fs] f2fs_add_inline_entry+0x153/0x3b0 [f2fs] f2fs_add_dentry+0x75/0x80 [f2fs] f2fs_do_add_link+0x108/0x160 [f2fs] f2fs_rename2+0x6ab/0x14f0 [f2fs] vfs_rename+0x70c/0x940 do_renameat2+0x4d8/0x4f0 __x64_sys_renameat2+0x4b/0x60 do_syscall_64+0x33/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xa9 Following race case can cause this: Thread A Kworker - f2fs_rename - f2fs_create_whiteout - __f2fs_tmpfile - f2fs_i_links_write - f2fs_mark_inode_dirty_sync - mark_inode_dirty_sync - writeback_single_inode - __writeback_single_inode - spin_lock(&inode->i_lock) - inode->i_state |= I_LINKABLE - inode->i_state &= ~dirty - spin_unlock(&inode->i_lock) - f2fs_add_link - f2fs_do_add_link - f2fs_add_dentry - f2fs_add_inline_entry - f2fs_init_inode_metadata - f2fs_i_links_write - inc_nlink - WARN_ON(!(inode->i_state & I_LINKABLE)) Fix to add i_lock to avoid i_state update race condition. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/namei.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'fs') diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index 6edb1ab579a1..887804968576 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -855,7 +855,11 @@ static int __f2fs_tmpfile(struct inode *dir, struct dentry *dentry, if (whiteout) { f2fs_i_links_write(inode, false); + + spin_lock(&inode->i_lock); inode->i_state |= I_LINKABLE; + spin_unlock(&inode->i_lock); + *whiteout = inode; } else { d_tmpfile(dentry, inode); @@ -1041,7 +1045,11 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry, err = f2fs_add_link(old_dentry, whiteout); if (err) goto put_out_dir; + + spin_lock(&whiteout->i_lock); whiteout->i_state &= ~I_LINKABLE; + spin_unlock(&whiteout->i_lock); + iput(whiteout); } -- cgit v1.2.3 From 794c43f716845e2d48ce195ed5c4179a4e05ce5f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Dec 2020 15:25:29 -0800 Subject: libfs: unexport generic_ci_d_compare() and generic_ci_d_hash() Now that generic_set_encrypted_ci_d_ops() has been added and ext4 and f2fs are using it, it's no longer necessary to export generic_ci_d_compare() and generic_ci_d_hash() to filesystems. Reviewed-by: Gabriel Krisman Bertazi Signed-off-by: Eric Biggers Signed-off-by: Jaegeuk Kim --- fs/libfs.c | 8 +++----- include/linux/fs.h | 5 ----- 2 files changed, 3 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/libfs.c b/fs/libfs.c index d1c3bade9f30..79721571e014 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -1388,8 +1388,8 @@ static bool needs_casefold(const struct inode *dir) * * Return: 0 if names match, 1 if mismatch, or -ERRNO */ -int generic_ci_d_compare(const struct dentry *dentry, unsigned int len, - const char *str, const struct qstr *name) +static int generic_ci_d_compare(const struct dentry *dentry, unsigned int len, + const char *str, const struct qstr *name) { const struct dentry *parent = READ_ONCE(dentry->d_parent); const struct inode *dir = READ_ONCE(parent->d_inode); @@ -1426,7 +1426,6 @@ fallback: return 1; return !!memcmp(str, name->name, len); } -EXPORT_SYMBOL(generic_ci_d_compare); /** * generic_ci_d_hash - generic d_hash implementation for casefolding filesystems @@ -1435,7 +1434,7 @@ EXPORT_SYMBOL(generic_ci_d_compare); * * Return: 0 if hash was successful or unchanged, and -EINVAL on error */ -int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str) +static int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str) { const struct inode *dir = READ_ONCE(dentry->d_inode); struct super_block *sb = dentry->d_sb; @@ -1450,7 +1449,6 @@ int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str) return -EINVAL; return 0; } -EXPORT_SYMBOL(generic_ci_d_hash); static const struct dentry_operations generic_ci_dentry_ops = { .d_hash = generic_ci_d_hash, diff --git a/include/linux/fs.h b/include/linux/fs.h index fd47deea7c17..6d8b1e7337e4 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3192,11 +3192,6 @@ extern int generic_file_fsync(struct file *, loff_t, loff_t, int); extern int generic_check_addressable(unsigned, u64); -#ifdef CONFIG_UNICODE -extern int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str); -extern int generic_ci_d_compare(const struct dentry *dentry, unsigned int len, - const char *str, const struct qstr *name); -#endif extern void generic_set_encrypted_ci_d_ops(struct dentry *dentry); #ifdef CONFIG_MIGRATION -- cgit v1.2.3 From 3afae09ffea5e08f523823be99a784675995d6bb Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 11 Jan 2021 17:42:53 +0800 Subject: f2fs: compress: fix potential deadlock generic/269 reports a hangtask issue, the root cause is ABBA deadlock described as below: Thread A Thread B - down_write(&sbi->gc_lock) -- A - f2fs_write_data_pages - lock all pages in cluster -- B - f2fs_write_multi_pages - f2fs_write_raw_pages - f2fs_write_single_data_page - f2fs_balance_fs - down_write(&sbi->gc_lock) -- A - f2fs_gc - do_garbage_collect - ra_data_block - pagecache_get_page -- B To fix this, it needs to avoid calling f2fs_balance_fs() if there is still cluster pages been locked in context of cluster writeback, so instead, let's call f2fs_balance_fs() in the end of f2fs_write_raw_pages() when all cluster pages were unlocked. Fixes: 4c8ff7095bef ("f2fs: support data compression") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/compress.c | 5 ++++- fs/f2fs/data.c | 10 ++++++---- fs/f2fs/f2fs.h | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 1696f9183ff5..77fa342de38f 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -1455,7 +1455,7 @@ retry_write: ret = f2fs_write_single_data_page(cc->rpages[i], &_submitted, NULL, NULL, wbc, io_type, - compr_blocks); + compr_blocks, false); if (ret) { if (ret == AOP_WRITEPAGE_ACTIVATE) { unlock_page(cc->rpages[i]); @@ -1490,6 +1490,9 @@ retry_write: *submitted += _submitted; } + + f2fs_balance_fs(F2FS_M_SB(mapping), true); + return 0; out_err: for (++i; i < cc->cluster_size; i++) { diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 4d80f00e5e40..c7bb07dd9a20 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2671,7 +2671,8 @@ int f2fs_write_single_data_page(struct page *page, int *submitted, sector_t *last_block, struct writeback_control *wbc, enum iostat_type io_type, - int compr_blocks) + int compr_blocks, + bool allow_balance) { struct inode *inode = page->mapping->host; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); @@ -2809,7 +2810,7 @@ out: } unlock_page(page); if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) && - !F2FS_I(inode)->cp_task) + !F2FS_I(inode)->cp_task && allow_balance) f2fs_balance_fs(sbi, need_balance_fs); if (unlikely(f2fs_cp_error(sbi))) { @@ -2856,7 +2857,7 @@ out: #endif return f2fs_write_single_data_page(page, NULL, NULL, NULL, - wbc, FS_DATA_IO, 0); + wbc, FS_DATA_IO, 0, true); } /* @@ -3024,7 +3025,8 @@ continue_unlock: } #endif ret = f2fs_write_single_data_page(page, &submitted, - &bio, &last_block, wbc, io_type, 0); + &bio, &last_block, wbc, io_type, + 0, true); if (ret == AOP_WRITEPAGE_ACTIVATE) unlock_page(page); #ifdef CONFIG_F2FS_FS_COMPRESSION diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 980e061f7968..63852404151e 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -3502,7 +3502,7 @@ int f2fs_write_single_data_page(struct page *page, int *submitted, struct bio **bio, sector_t *last_block, struct writeback_control *wbc, enum iostat_type io_type, - int compr_blocks); + int compr_blocks, bool allow_balance); void f2fs_invalidate_page(struct page *page, unsigned int offset, unsigned int length); int f2fs_release_page(struct page *page, gfp_t wait); -- cgit v1.2.3 From 6d1451bf7f84ea45035553ae566b3c91661d902b Mon Sep 17 00:00:00 2001 From: Chengguang Xu Date: Wed, 13 Jan 2021 13:21:54 +0800 Subject: f2fs: fix to use per-inode maxbytes F2FS inode may have different max size, e.g. compressed file have less blkaddr entries in all its direct-node blocks, result in being with less max filesize. So change to use per-inode maxbytes. Suggested-by: Chao Yu Signed-off-by: Chengguang Xu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 2 +- fs/f2fs/f2fs.h | 2 +- fs/f2fs/file.c | 9 ++++++--- fs/f2fs/super.c | 12 ++++++++---- 4 files changed, 16 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index c7bb07dd9a20..9aa458c01101 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -3761,7 +3761,7 @@ static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) filemap_write_and_wait(mapping); /* Block number less than F2FS MAX BLOCKS */ - if (unlikely(block >= F2FS_I_SB(inode)->max_file_blocks)) + if (unlikely(block >= max_file_blocks(inode))) goto out; if (f2fs_compressed_file(inode)) { diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 63852404151e..ca5f1ff14dab 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1474,7 +1474,6 @@ struct f2fs_sb_info { unsigned int total_sections; /* total section count */ unsigned int total_node_count; /* total node block count */ unsigned int total_valid_node_count; /* valid node block count */ - loff_t max_file_blocks; /* max block index of file */ int dir_level; /* directory level */ int readdir_ra; /* readahead inode in readdir */ u64 max_io_bytes; /* max io bytes to merge IOs */ @@ -3265,6 +3264,7 @@ int f2fs_inode_dirtied(struct inode *inode, bool sync); void f2fs_inode_synced(struct inode *inode); int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly); int f2fs_quota_sync(struct super_block *sb, int type); +loff_t max_file_blocks(struct inode *inode); void f2fs_quota_off_umount(struct super_block *sb); int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover); int f2fs_sync_fs(struct super_block *sb, int sync); diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index e3a5b620b50a..e768371c6575 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -486,6 +486,9 @@ static loff_t f2fs_llseek(struct file *file, loff_t offset, int whence) struct inode *inode = file->f_mapping->host; loff_t maxbytes = inode->i_sb->s_maxbytes; + if (f2fs_compressed_file(inode)) + maxbytes = max_file_blocks(inode) << F2FS_BLKSIZE_BITS; + switch (whence) { case SEEK_SET: case SEEK_CUR: @@ -670,7 +673,7 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) free_from = (pgoff_t)F2FS_BLK_ALIGN(from); - if (free_from >= sbi->max_file_blocks) + if (free_from >= max_file_blocks(inode)) goto free_partial; if (lock) @@ -2744,7 +2747,7 @@ static int f2fs_ioc_defragment(struct file *filp, unsigned long arg) return -EINVAL; if (unlikely((range.start + range.len) >> PAGE_SHIFT > - sbi->max_file_blocks)) + max_file_blocks(inode))) return -EINVAL; err = mnt_want_write_file(filp); @@ -3307,7 +3310,7 @@ int f2fs_precache_extents(struct inode *inode) map.m_next_extent = &m_next_extent; map.m_seg_type = NO_CHECK_TYPE; map.m_may_create = false; - end = F2FS_I_SB(inode)->max_file_blocks; + end = max_file_blocks(inode); while (map.m_lblk < end) { map.m_len = end - map.m_lblk; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index c8be27a9eed6..9749c9ad374f 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2739,10 +2739,10 @@ static const struct export_operations f2fs_export_ops = { .get_parent = f2fs_get_parent, }; -static loff_t max_file_blocks(void) +loff_t max_file_blocks(struct inode *inode) { loff_t result = 0; - loff_t leaf_count = DEF_ADDRS_PER_BLOCK; + loff_t leaf_count; /* * note: previously, result is equal to (DEF_ADDRS_PER_INODE - @@ -2751,6 +2751,11 @@ static loff_t max_file_blocks(void) * result as zero. */ + if (inode && f2fs_compressed_file(inode)) + leaf_count = ADDRS_PER_BLOCK(inode); + else + leaf_count = DEF_ADDRS_PER_BLOCK; + /* two direct node blocks */ result += (leaf_count * 2); @@ -3634,8 +3639,7 @@ try_onemore: if (err) goto free_options; - sbi->max_file_blocks = max_file_blocks(); - sb->s_maxbytes = sbi->max_file_blocks << + sb->s_maxbytes = max_file_blocks(NULL) << le32_to_cpu(raw_super->log_blocksize); sb->s_max_links = F2FS_LINK_MAX; -- cgit v1.2.3 From 0bfe9f790448012ef38abf4e78feb2e691e2d366 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Thu, 14 Jan 2021 09:41:27 +0800 Subject: f2fs: introduce sb_status sysfs node Introduce /sys/fs/f2fs//stat/sb_status to show superblock status in real time as a hexadecimal value. value sb status macro description 0x1 SBI_IS_DIRTY, /* dirty flag for checkpoint */ 0x2 SBI_IS_CLOSE, /* specify unmounting */ 0x4 SBI_NEED_FSCK, /* need fsck.f2fs to fix */ 0x8 SBI_POR_DOING, /* recovery is doing or not */ 0x10 SBI_NEED_SB_WRITE, /* need to recover superblock */ 0x20 SBI_NEED_CP, /* need to checkpoint */ 0x40 SBI_IS_SHUTDOWN, /* shutdown by ioctl */ 0x80 SBI_IS_RECOVERED, /* recovered orphan/data */ 0x100 SBI_CP_DISABLED, /* CP was disabled last mount */ 0x200 SBI_CP_DISABLED_QUICK, /* CP was disabled quickly */ 0x400 SBI_QUOTA_NEED_FLUSH, /* need to flush quota info in CP */ 0x800 SBI_QUOTA_SKIP_FLUSH, /* skip flushing quota in current CP */ 0x1000 SBI_QUOTA_NEED_REPAIR, /* quota file may be corrupted */ 0x2000 SBI_IS_RESIZEFS, /* resizefs is in process */ Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- Documentation/ABI/testing/sysfs-fs-f2fs | 23 +++++++++++++++++++++++ fs/f2fs/sysfs.c | 8 ++++++++ 2 files changed, 31 insertions(+) (limited to 'fs') diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index 3dfee94e0618..362803901614 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -377,3 +377,26 @@ Description: This gives a control to limit the bio size in f2fs. Default is zero, which will follow underlying block layer limit, whereas, if it has a certain bytes value, f2fs won't submit a bio larger than that size. + +What: /sys/fs/f2fs//stat/sb_status +Date: December 2020 +Contact: "Chao Yu" +Description: Show status of f2fs superblock in real time. + + ====== ===================== ================================= + value sb status macro description + 0x1 SBI_IS_DIRTY dirty flag for checkpoint + 0x2 SBI_IS_CLOSE specify unmounting + 0x4 SBI_NEED_FSCK need fsck.f2fs to fix + 0x8 SBI_POR_DOING recovery is doing or not + 0x10 SBI_NEED_SB_WRITE need to recover superblock + 0x20 SBI_NEED_CP need to checkpoint + 0x40 SBI_IS_SHUTDOWN shutdown by ioctl + 0x80 SBI_IS_RECOVERED recovered orphan/data + 0x100 SBI_CP_DISABLED CP was disabled last mount + 0x200 SBI_CP_DISABLED_QUICK CP was disabled quickly + 0x400 SBI_QUOTA_NEED_FLUSH need to flush quota info in CP + 0x800 SBI_QUOTA_SKIP_FLUSH skip flushing quota in current CP + 0x1000 SBI_QUOTA_NEED_REPAIR quota file may be corrupted + 0x2000 SBI_IS_RESIZEFS resizefs is in process + ====== ===================== ================================= diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index bd1174ed2e6f..f39874d512ea 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -96,6 +96,12 @@ static ssize_t lifetime_write_kbytes_show(struct f2fs_attr *a, sbi->sectors_written_start) >> 1))); } +static ssize_t sb_status_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sprintf(buf, "%lx\n", sbi->s_flag); +} + static ssize_t features_show(struct f2fs_attr *a, struct f2fs_sb_info *sbi, char *buf) { @@ -702,7 +708,9 @@ static struct attribute *f2fs_feat_attrs[] = { }; ATTRIBUTE_GROUPS(f2fs_feat); +F2FS_GENERAL_RO_ATTR(sb_status); static struct attribute *f2fs_stat_attrs[] = { + ATTR_LIST(sb_status), NULL, }; ATTRIBUTE_GROUPS(f2fs_stat); -- cgit v1.2.3 From deaa965fb01173478a1234f4305c71fffa4b5dc4 Mon Sep 17 00:00:00 2001 From: Jack Qiu Date: Wed, 13 Jan 2021 17:58:53 +0800 Subject: f2fs: remove unused stat_{inc, dec}_atomic_write Just clean code, no logical change. Signed-off-by: Jack Qiu Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/f2fs.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index ca5f1ff14dab..d37883ce9002 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -3748,8 +3748,6 @@ void f2fs_update_sit_info(struct f2fs_sb_info *sbi); #define stat_dec_compr_inode(inode) do { } while (0) #define stat_add_compr_blocks(inode, blocks) do { } while (0) #define stat_sub_compr_blocks(inode, blocks) do { } while (0) -#define stat_inc_atomic_write(inode) do { } while (0) -#define stat_dec_atomic_write(inode) do { } while (0) #define stat_update_max_atomic_write(inode) do { } while (0) #define stat_inc_volatile_write(inode) do { } while (0) #define stat_dec_volatile_write(inode) do { } while (0) -- cgit v1.2.3 From 12699fb781574d50871ec6a4d96ac5e0f0ede03e Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 14 Jan 2021 19:00:51 +0000 Subject: f2fs: Remove readahead collision detection With the new ->readahead operation, locked pages are added to the page cache, preventing two threads from racing with each other to read the same chunk of file, so this is dead code. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 25 ------------------------- fs/f2fs/f2fs.h | 1 - fs/f2fs/super.c | 2 -- 3 files changed, 28 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 9aa458c01101..d9a063d8a63d 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2265,11 +2265,6 @@ out: /* * This function was originally taken from fs/mpage.c, and customized for f2fs. * Major change was from block_size == page_size in f2fs by default. - * - * Note that the aops->readpages() function is ONLY used for read-ahead. If - * this function ever deviates from doing just read-ahead, it should either - * use ->readpage() or do the necessary surgery to decouple ->readpages() - * from read-ahead. */ static int f2fs_mpage_readpages(struct inode *inode, struct readahead_control *rac, struct page *page) @@ -2292,7 +2287,6 @@ static int f2fs_mpage_readpages(struct inode *inode, unsigned nr_pages = rac ? readahead_count(rac) : 1; unsigned max_nr_pages = nr_pages; int ret = 0; - bool drop_ra = false; map.m_pblk = 0; map.m_lblk = 0; @@ -2303,26 +2297,10 @@ static int f2fs_mpage_readpages(struct inode *inode, map.m_seg_type = NO_CHECK_TYPE; map.m_may_create = false; - /* - * Two readahead threads for same address range can cause race condition - * which fragments sequential read IOs. So let's avoid each other. - */ - if (rac && readahead_count(rac)) { - if (READ_ONCE(F2FS_I(inode)->ra_offset) == readahead_index(rac)) - drop_ra = true; - else - WRITE_ONCE(F2FS_I(inode)->ra_offset, - readahead_index(rac)); - } - for (; nr_pages; nr_pages--) { if (rac) { page = readahead_page(rac); prefetchw(&page->flags); - if (drop_ra) { - f2fs_put_page(page, 1); - continue; - } } #ifdef CONFIG_F2FS_FS_COMPRESSION @@ -2385,9 +2363,6 @@ next_page: } if (bio) __submit_bio(F2FS_I_SB(inode), bio, DATA); - - if (rac && readahead_count(rac) && !drop_ra) - WRITE_ONCE(F2FS_I(inode)->ra_offset, -1); return ret; } diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index d37883ce9002..a2e520a13630 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -718,7 +718,6 @@ struct f2fs_inode_info { struct list_head inmem_pages; /* inmemory pages managed by f2fs */ struct task_struct *inmem_task; /* store inmemory task */ struct mutex inmem_lock; /* lock for inmemory pages */ - pgoff_t ra_offset; /* ongoing readahead offset */ struct extent_tree *extent_tree; /* cached extent_tree entry */ /* avoid racing between foreground op and gc */ diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 9749c9ad374f..6a30876ff374 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -1174,8 +1174,6 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb) /* Will be used by directory only */ fi->i_dir_level = F2FS_SB(sb)->dir_level; - fi->ra_offset = -1; - return &fi->vfs_inode; } -- cgit v1.2.3 From d5f7bc0064e0541164bd3deeafad16bbb5992433 Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Thu, 14 Jan 2021 13:59:09 -0800 Subject: f2fs: deprecate f2fs_trace_io This patch deprecates f2fs_trace_io, since f2fs uses page->private more broadly, resulting in more buggy cases. Acked-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/Kconfig | 10 ---- fs/f2fs/Makefile | 1 - fs/f2fs/checkpoint.c | 3 - fs/f2fs/data.c | 4 -- fs/f2fs/file.c | 2 - fs/f2fs/node.c | 2 - fs/f2fs/segment.c | 3 - fs/f2fs/super.c | 6 -- fs/f2fs/trace.c | 165 --------------------------------------------------- fs/f2fs/trace.h | 43 -------------- 10 files changed, 239 deletions(-) delete mode 100644 fs/f2fs/trace.c delete mode 100644 fs/f2fs/trace.h (limited to 'fs') diff --git a/fs/f2fs/Kconfig b/fs/f2fs/Kconfig index 63c1fc1a0e3b..62e638a49bbf 100644 --- a/fs/f2fs/Kconfig +++ b/fs/f2fs/Kconfig @@ -76,16 +76,6 @@ config F2FS_CHECK_FS If you want to improve the performance, say N. -config F2FS_IO_TRACE - bool "F2FS IO tracer" - depends on F2FS_FS - depends on FUNCTION_TRACER - help - F2FS IO trace is based on a function trace, which gathers process - information and block IO patterns in the filesystem level. - - If unsure, say N. - config F2FS_FAULT_INJECTION bool "F2FS fault injection facility" depends on F2FS_FS diff --git a/fs/f2fs/Makefile b/fs/f2fs/Makefile index ee7316b42f69..e5295746208b 100644 --- a/fs/f2fs/Makefile +++ b/fs/f2fs/Makefile @@ -7,6 +7,5 @@ f2fs-y += shrinker.o extent_cache.o sysfs.o f2fs-$(CONFIG_F2FS_STAT_FS) += debug.o f2fs-$(CONFIG_F2FS_FS_XATTR) += xattr.o f2fs-$(CONFIG_F2FS_FS_POSIX_ACL) += acl.o -f2fs-$(CONFIG_F2FS_IO_TRACE) += trace.o f2fs-$(CONFIG_FS_VERITY) += verity.o f2fs-$(CONFIG_F2FS_FS_COMPRESSION) += compress.o diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 897edb7c951a..8c79ba0566b1 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -17,7 +17,6 @@ #include "f2fs.h" #include "node.h" #include "segment.h" -#include "trace.h" #include static struct kmem_cache *ino_entry_slab; @@ -443,7 +442,6 @@ static int f2fs_set_meta_page_dirty(struct page *page) __set_page_dirty_nobuffers(page); inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_META); f2fs_set_page_private(page, 0); - f2fs_trace_pid(page); return 1; } return 0; @@ -1017,7 +1015,6 @@ void f2fs_update_dirty_page(struct inode *inode, struct page *page) spin_unlock(&sbi->inode_lock[type]); f2fs_set_page_private(page, 0); - f2fs_trace_pid(page); } void f2fs_remove_dirty_inode(struct inode *inode) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index d9a063d8a63d..38476d0d3916 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -25,7 +25,6 @@ #include "f2fs.h" #include "node.h" #include "segment.h" -#include "trace.h" #include #define NUM_PREALLOC_POST_READ_CTXS 128 @@ -679,7 +678,6 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio) return -EFSCORRUPTED; trace_f2fs_submit_page_bio(page, fio); - f2fs_trace_ios(fio, 0); /* Allocate a new bio */ bio = __bio_alloc(fio, 1); @@ -884,7 +882,6 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio) return -EFSCORRUPTED; trace_f2fs_submit_page_bio(page, fio); - f2fs_trace_ios(fio, 0); if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block, fio->new_blkaddr)) @@ -981,7 +978,6 @@ alloc_new: wbc_account_cgroup_owner(fio->io_wbc, bio_page, PAGE_SIZE); io->last_block_in_bio = fio->new_blkaddr; - f2fs_trace_ios(fio, 0); trace_f2fs_submit_page_write(fio->page, fio); skip: diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index e768371c6575..7db27c81d034 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -29,7 +29,6 @@ #include "xattr.h" #include "acl.h" #include "gc.h" -#include "trace.h" #include #include @@ -369,7 +368,6 @@ flush_out: f2fs_update_time(sbi, REQ_TIME); out: trace_f2fs_sync_file_exit(inode, cp_reason, datasync, ret); - f2fs_trace_ios(NULL, 1); return ret; } diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 5e3fabacefb5..a8a0fb890e8d 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -17,7 +17,6 @@ #include "node.h" #include "segment.h" #include "xattr.h" -#include "trace.h" #include #define on_f2fs_build_free_nids(nmi) mutex_is_locked(&(nm_i)->build_lock) @@ -2089,7 +2088,6 @@ static int f2fs_set_node_page_dirty(struct page *page) __set_page_dirty_nobuffers(page); inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_NODES); f2fs_set_page_private(page, 0); - f2fs_trace_pid(page); return 1; } return 0; diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index deca74cb17df..7d34f1cacdee 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -20,7 +20,6 @@ #include "segment.h" #include "node.h" #include "gc.h" -#include "trace.h" #include #define __reverse_ffz(x) __reverse_ffs(~(x)) @@ -187,8 +186,6 @@ void f2fs_register_inmem_page(struct inode *inode, struct page *page) { struct inmem_pages *new; - f2fs_trace_pid(page); - f2fs_set_page_private(page, ATOMIC_WRITTEN_PAGE); new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS); diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 6a30876ff374..fc343243799c 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -33,7 +33,6 @@ #include "segment.h" #include "xattr.h" #include "gc.h" -#include "trace.h" #define CREATE_TRACE_POINTS #include @@ -1448,8 +1447,6 @@ int f2fs_sync_fs(struct super_block *sb, int sync) err = f2fs_write_checkpoint(sbi, &cpc); up_write(&sbi->gc_lock); } - f2fs_trace_ios(NULL, 1); - return err; } @@ -4127,8 +4124,6 @@ static int __init init_f2fs_fs(void) return -EINVAL; } - f2fs_build_trace_ios(); - err = init_inodecache(); if (err) goto fail; @@ -4221,7 +4216,6 @@ static void __exit exit_f2fs_fs(void) f2fs_destroy_segment_manager_caches(); f2fs_destroy_node_manager_caches(); destroy_inodecache(); - f2fs_destroy_trace_ios(); } module_init(init_f2fs_fs) diff --git a/fs/f2fs/trace.c b/fs/f2fs/trace.c deleted file mode 100644 index d0ab533a9ce8..000000000000 --- a/fs/f2fs/trace.c +++ /dev/null @@ -1,165 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * f2fs IO tracer - * - * Copyright (c) 2014 Motorola Mobility - * Copyright (c) 2014 Jaegeuk Kim - */ -#include -#include -#include -#include - -#include "f2fs.h" -#include "trace.h" - -static RADIX_TREE(pids, GFP_ATOMIC); -static spinlock_t pids_lock; -static struct last_io_info last_io; - -static inline void __print_last_io(void) -{ - if (!last_io.len) - return; - - trace_printk("%3x:%3x %4x %-16s %2x %5x %5x %12x %4x\n", - last_io.major, last_io.minor, - last_io.pid, "----------------", - last_io.type, - last_io.fio.op, last_io.fio.op_flags, - last_io.fio.new_blkaddr, - last_io.len); - memset(&last_io, 0, sizeof(last_io)); -} - -static int __file_type(struct inode *inode, pid_t pid) -{ - if (f2fs_is_atomic_file(inode)) - return __ATOMIC_FILE; - else if (f2fs_is_volatile_file(inode)) - return __VOLATILE_FILE; - else if (S_ISDIR(inode->i_mode)) - return __DIR_FILE; - else if (inode->i_ino == F2FS_NODE_INO(F2FS_I_SB(inode))) - return __NODE_FILE; - else if (inode->i_ino == F2FS_META_INO(F2FS_I_SB(inode))) - return __META_FILE; - else if (pid) - return __NORMAL_FILE; - else - return __MISC_FILE; -} - -void f2fs_trace_pid(struct page *page) -{ - struct inode *inode = page->mapping->host; - pid_t pid = task_pid_nr(current); - void *p; - - set_page_private(page, (unsigned long)pid); - -retry: - if (radix_tree_preload(GFP_NOFS)) - return; - - spin_lock(&pids_lock); - p = radix_tree_lookup(&pids, pid); - if (p == current) - goto out; - if (p) - radix_tree_delete(&pids, pid); - - if (radix_tree_insert(&pids, pid, current)) { - spin_unlock(&pids_lock); - radix_tree_preload_end(); - cond_resched(); - goto retry; - } - - trace_printk("%3x:%3x %4x %-16s\n", - MAJOR(inode->i_sb->s_dev), MINOR(inode->i_sb->s_dev), - pid, current->comm); -out: - spin_unlock(&pids_lock); - radix_tree_preload_end(); -} - -void f2fs_trace_ios(struct f2fs_io_info *fio, int flush) -{ - struct inode *inode; - pid_t pid; - int major, minor; - - if (flush) { - __print_last_io(); - return; - } - - inode = fio->page->mapping->host; - pid = page_private(fio->page); - - major = MAJOR(inode->i_sb->s_dev); - minor = MINOR(inode->i_sb->s_dev); - - if (last_io.major == major && last_io.minor == minor && - last_io.pid == pid && - last_io.type == __file_type(inode, pid) && - last_io.fio.op == fio->op && - last_io.fio.op_flags == fio->op_flags && - last_io.fio.new_blkaddr + last_io.len == - fio->new_blkaddr) { - last_io.len++; - return; - } - - __print_last_io(); - - last_io.major = major; - last_io.minor = minor; - last_io.pid = pid; - last_io.type = __file_type(inode, pid); - last_io.fio = *fio; - last_io.len = 1; - return; -} - -void f2fs_build_trace_ios(void) -{ - spin_lock_init(&pids_lock); -} - -#define PIDVEC_SIZE 128 -static unsigned int gang_lookup_pids(pid_t *results, unsigned long first_index, - unsigned int max_items) -{ - struct radix_tree_iter iter; - void **slot; - unsigned int ret = 0; - - if (unlikely(!max_items)) - return 0; - - radix_tree_for_each_slot(slot, &pids, &iter, first_index) { - results[ret] = iter.index; - if (++ret == max_items) - break; - } - return ret; -} - -void f2fs_destroy_trace_ios(void) -{ - pid_t pid[PIDVEC_SIZE]; - pid_t next_pid = 0; - unsigned int found; - - spin_lock(&pids_lock); - while ((found = gang_lookup_pids(pid, next_pid, PIDVEC_SIZE))) { - unsigned idx; - - next_pid = pid[found - 1] + 1; - for (idx = 0; idx < found; idx++) - radix_tree_delete(&pids, pid[idx]); - } - spin_unlock(&pids_lock); -} diff --git a/fs/f2fs/trace.h b/fs/f2fs/trace.h deleted file mode 100644 index 789f6aa727fc..000000000000 --- a/fs/f2fs/trace.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * f2fs IO tracer - * - * Copyright (c) 2014 Motorola Mobility - * Copyright (c) 2014 Jaegeuk Kim - */ -#ifndef __F2FS_TRACE_H__ -#define __F2FS_TRACE_H__ - -#ifdef CONFIG_F2FS_IO_TRACE -#include - -enum file_type { - __NORMAL_FILE, - __DIR_FILE, - __NODE_FILE, - __META_FILE, - __ATOMIC_FILE, - __VOLATILE_FILE, - __MISC_FILE, -}; - -struct last_io_info { - int major, minor; - pid_t pid; - enum file_type type; - struct f2fs_io_info fio; - block_t len; -}; - -extern void f2fs_trace_pid(struct page *); -extern void f2fs_trace_ios(struct f2fs_io_info *, int); -extern void f2fs_build_trace_ios(void); -extern void f2fs_destroy_trace_ios(void); -#else -#define f2fs_trace_pid(p) -#define f2fs_trace_ios(i, n) -#define f2fs_build_trace_ios() -#define f2fs_destroy_trace_ios() - -#endif -#endif /* __F2FS_TRACE_H__ */ -- cgit v1.2.3 From 9efb069de4ba748d284f6129e71de239f801053a Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Thu, 28 Jan 2021 10:22:48 +0100 Subject: ovl: add warning on user_ns mismatch Currently there's no way to create an overlay filesystem outside of the current user namespace. Make sure that if this assumption changes it doesn't go unnoticed. Reported-by: "Eric W. Biederman" Signed-off-by: Miklos Szeredi --- fs/overlayfs/super.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'fs') diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 2bd570cbe8a4..82cd6d55a5a1 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1923,6 +1923,10 @@ static int ovl_fill_super(struct super_block *sb, void *data, int silent) unsigned int numlower; int err; + err = -EIO; + if (WARN_ON(sb->s_user_ns != current_user_ns())) + goto out; + sb->s_d_op = &ovl_dentry_operations; err = -ENOMEM; -- cgit v1.2.3 From 554677b97257b0b69378bd74e521edb7e94769ff Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Thu, 28 Jan 2021 10:22:48 +0100 Subject: ovl: perform vfs_getxattr() with mounter creds The vfs_getxattr() in ovl_xattr_set() is used to check whether an xattr exist on a lower layer file that is to be removed. If the xattr does not exist, then no need to copy up the file. This call of vfs_getxattr() wasn't wrapped in credential override, and this is probably okay. But for consitency wrap this instance as well. Reported-by: "Eric W. Biederman" Signed-off-by: Miklos Szeredi --- fs/overlayfs/inode.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index d739e14c6814..cf41bcb664bc 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -352,7 +352,9 @@ int ovl_xattr_set(struct dentry *dentry, struct inode *inode, const char *name, goto out; if (!value && !upperdentry) { + old_cred = ovl_override_creds(dentry->d_sb); err = vfs_getxattr(realdentry, name, NULL, 0); + revert_creds(old_cred); if (err < 0) goto out_drop_write; } -- cgit v1.2.3 From b854cc659dcb80f172cb35dbedc15d39d49c383f Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Tue, 5 Jan 2021 08:36:11 +0800 Subject: ovl: avoid deadlock on directory ioctl The function ovl_dir_real_file() currently uses the inode lock to serialize writes to the od->upperfile field. However, this function will get called by ovl_ioctl_set_flags(), which utilizes the inode lock too. In this case ovl_dir_real_file() will try to claim a lock that is owned by a function in its call stack, which won't get released before ovl_dir_real_file() returns. Fix by replacing the open coded compare and exchange by an explicit atomic op. Fixes: 61536bed2149 ("ovl: support [S|G]ETFLAGS and FS[S|G]ETXATTR ioctls for directories") Cc: stable@vger.kernel.org # v5.10 Reported-by: Icenowy Zheng Tested-by: Icenowy Zheng Signed-off-by: Miklos Szeredi --- fs/overlayfs/readdir.c | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 01620ebae1bd..60d751f28fea 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -865,7 +865,7 @@ struct file *ovl_dir_real_file(const struct file *file, bool want_upper) struct ovl_dir_file *od = file->private_data; struct dentry *dentry = file->f_path.dentry; - struct file *realfile = od->realfile; + struct file *old, *realfile = od->realfile; if (!OVL_TYPE_UPPER(ovl_path_type(dentry))) return want_upper ? NULL : realfile; @@ -874,29 +874,20 @@ struct file *ovl_dir_real_file(const struct file *file, bool want_upper) * Need to check if we started out being a lower dir, but got copied up */ if (!od->is_upper) { - struct inode *inode = file_inode(file); - realfile = READ_ONCE(od->upperfile); if (!realfile) { struct path upperpath; ovl_path_upper(dentry, &upperpath); realfile = ovl_dir_open_realfile(file, &upperpath); + if (IS_ERR(realfile)) + return realfile; - inode_lock(inode); - if (!od->upperfile) { - if (IS_ERR(realfile)) { - inode_unlock(inode); - return realfile; - } - smp_store_release(&od->upperfile, realfile); - } else { - /* somebody has beaten us to it */ - if (!IS_ERR(realfile)) - fput(realfile); - realfile = od->upperfile; + old = cmpxchg_release(&od->upperfile, NULL, realfile); + if (old) { + fput(realfile); + realfile = old; } - inode_unlock(inode); } } -- cgit v1.2.3 From e04527fefba6e4e66492f122cf8cc6314f3cf3bf Mon Sep 17 00:00:00 2001 From: Liangyan Date: Tue, 22 Dec 2020 11:06:26 +0800 Subject: ovl: fix dentry leak in ovl_get_redirect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need to lock d_parent->d_lock before dget_dlock, or this may have d_lockref updated parallelly like calltrace below which will cause dentry->d_lockref leak and risk a crash. CPU 0 CPU 1 ovl_set_redirect lookup_fast ovl_get_redirect __d_lookup dget_dlock //no lock protection here spin_lock(&dentry->d_lock) dentry->d_lockref.count++ dentry->d_lockref.count++ [   49.799059] PGD 800000061fed7067 P4D 800000061fed7067 PUD 61fec5067 PMD 0 [   49.799689] Oops: 0002 [#1] SMP PTI [   49.800019] CPU: 2 PID: 2332 Comm: node Not tainted 4.19.24-7.20.al7.x86_64 #1 [   49.800678] Hardware name: Alibaba Cloud Alibaba Cloud ECS, BIOS 8a46cfe 04/01/2014 [   49.801380] RIP: 0010:_raw_spin_lock+0xc/0x20 [   49.803470] RSP: 0018:ffffac6fc5417e98 EFLAGS: 00010246 [   49.803949] RAX: 0000000000000000 RBX: ffff93b8da3446c0 RCX: 0000000a00000000 [   49.804600] RDX: 0000000000000001 RSI: 000000000000000a RDI: 0000000000000088 [   49.805252] RBP: 0000000000000000 R08: 0000000000000000 R09: ffffffff993cf040 [   49.805898] R10: ffff93b92292e580 R11: ffffd27f188a4b80 R12: 0000000000000000 [   49.806548] R13: 00000000ffffff9c R14: 00000000fffffffe R15: ffff93b8da3446c0 [   49.807200] FS:  00007ffbedffb700(0000) GS:ffff93b927880000(0000) knlGS:0000000000000000 [   49.807935] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [   49.808461] CR2: 0000000000000088 CR3: 00000005e3f74006 CR4: 00000000003606a0 [   49.809113] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [   49.809758] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [   49.810410] Call Trace: [   49.810653]  d_delete+0x2c/0xb0 [   49.810951]  vfs_rmdir+0xfd/0x120 [   49.811264]  do_rmdir+0x14f/0x1a0 [   49.811573]  do_syscall_64+0x5b/0x190 [   49.811917]  entry_SYSCALL_64_after_hwframe+0x44/0xa9 [   49.812385] RIP: 0033:0x7ffbf505ffd7 [   49.814404] RSP: 002b:00007ffbedffada8 EFLAGS: 00000297 ORIG_RAX: 0000000000000054 [   49.815098] RAX: ffffffffffffffda RBX: 00007ffbedffb640 RCX: 00007ffbf505ffd7 [   49.815744] RDX: 0000000004449700 RSI: 0000000000000000 RDI: 0000000006c8cd50 [   49.816394] RBP: 00007ffbedffaea0 R08: 0000000000000000 R09: 0000000000017d0b [   49.817038] R10: 0000000000000000 R11: 0000000000000297 R12: 0000000000000012 [   49.817687] R13: 00000000072823d8 R14: 00007ffbedffb700 R15: 00000000072823d8 [   49.818338] Modules linked in: pvpanic cirrusfb button qemu_fw_cfg atkbd libps2 i8042 [   49.819052] CR2: 0000000000000088 [   49.819368] ---[ end trace 4e652b8aa299aa2d ]--- [   49.819796] RIP: 0010:_raw_spin_lock+0xc/0x20 [   49.821880] RSP: 0018:ffffac6fc5417e98 EFLAGS: 00010246 [   49.822363] RAX: 0000000000000000 RBX: ffff93b8da3446c0 RCX: 0000000a00000000 [   49.823008] RDX: 0000000000000001 RSI: 000000000000000a RDI: 0000000000000088 [   49.823658] RBP: 0000000000000000 R08: 0000000000000000 R09: ffffffff993cf040 [   49.825404] R10: ffff93b92292e580 R11: ffffd27f188a4b80 R12: 0000000000000000 [   49.827147] R13: 00000000ffffff9c R14: 00000000fffffffe R15: ffff93b8da3446c0 [   49.828890] FS:  00007ffbedffb700(0000) GS:ffff93b927880000(0000) knlGS:0000000000000000 [   49.830725] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [   49.832359] CR2: 0000000000000088 CR3: 00000005e3f74006 CR4: 00000000003606a0 [   49.834085] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [   49.835792] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Cc: Fixes: a6c606551141 ("ovl: redirect on rename-dir") Signed-off-by: Liangyan Reviewed-by: Joseph Qi Suggested-by: Al Viro Signed-off-by: Miklos Szeredi --- fs/overlayfs/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index 28a075b5f5b2..d1efa3a5a503 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -992,8 +992,8 @@ static char *ovl_get_redirect(struct dentry *dentry, bool abs_redirect) buflen -= thislen; memcpy(&buf[buflen], name, thislen); - tmp = dget_dlock(d->d_parent); spin_unlock(&d->d_lock); + tmp = dget_parent(d); dput(d); d = tmp; -- cgit v1.2.3 From 03fedf93593c82538b18476d8c4f0e8f8435ea70 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sat, 19 Dec 2020 12:16:08 +0200 Subject: ovl: skip getxattr of security labels When inode has no listxattr op of its own (e.g. squashfs) vfs_listxattr calls the LSM inode_listsecurity hooks to list the xattrs that LSMs will intercept in inode_getxattr hooks. When selinux LSM is installed but not initialized, it will list the security.selinux xattr in inode_listsecurity, but will not intercept it in inode_getxattr. This results in -ENODATA for a getxattr call for an xattr returned by listxattr. This situation was manifested as overlayfs failure to copy up lower files from squashfs when selinux is built-in but not initialized, because ovl_copy_xattr() iterates the lower inode xattrs by vfs_listxattr() and vfs_getxattr(). ovl_copy_xattr() skips copy up of security labels that are indentified by inode_copy_up_xattr LSM hooks, but it does that after vfs_getxattr(). Since we are not going to copy them, skip vfs_getxattr() of the security labels. Reported-by: Michael Labriola Tested-by: Michael Labriola Link: https://lore.kernel.org/linux-unionfs/2nv9d47zt7.fsf@aldarion.sourceruckus.org/ Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/overlayfs/copy_up.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index e5b616c93e11..0fed532efa68 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -84,6 +84,14 @@ int ovl_copy_xattr(struct super_block *sb, struct dentry *old, if (ovl_is_private_xattr(sb, name)) continue; + + error = security_inode_copy_up_xattr(name); + if (error < 0 && error != -EOPNOTSUPP) + break; + if (error == 1) { + error = 0; + continue; /* Discard */ + } retry: size = vfs_getxattr(old, name, value, value_size); if (size == -ERANGE) @@ -107,13 +115,6 @@ retry: goto retry; } - error = security_inode_copy_up_xattr(name); - if (error < 0 && error != -EOPNOTSUPP) - break; - if (error == 1) { - error = 0; - continue; /* Discard */ - } error = vfs_setxattr(new, name, value, size, 0); if (error) { if (error != -EOPNOTSUPP || ovl_must_copy_xattr(name)) -- cgit v1.2.3 From 335d3fc57941e5c6164c69d439aec1cb7a800876 Mon Sep 17 00:00:00 2001 From: Sargun Dhillon Date: Thu, 7 Jan 2021 16:10:43 -0800 Subject: ovl: implement volatile-specific fsync error behaviour Overlayfs's volatile option allows the user to bypass all forced sync calls to the upperdir filesystem. This comes at the cost of safety. We can never ensure that the user's data is intact, but we can make a best effort to expose whether or not the data is likely to be in a bad state. The best way to handle this in the time being is that if an overlayfs's upperdir experiences an error after a volatile mount occurs, that error will be returned on fsync, fdatasync, sync, and syncfs. This is contradictory to the traditional behaviour of VFS which fails the call once, and only raises an error if a subsequent fsync error has occurred, and been raised by the filesystem. One awkward aspect of the patch is that we have to manually set the superblock's errseq_t after the sync_fs callback as opposed to just returning an error from syncfs. This is because the call chain looks something like this: sys_syncfs -> sync_filesystem -> __sync_filesystem -> /* The return value is ignored here sb->s_op->sync_fs(sb) _sync_blockdev /* Where the VFS fetches the error to raise to userspace */ errseq_check_and_advance Because of this we call errseq_set every time the sync_fs callback occurs. Due to the nature of this seen / unseen dichotomy, if the upperdir is an inconsistent state at the initial mount time, overlayfs will refuse to mount, as overlayfs cannot get a snapshot of the upperdir's errseq that will increment on error until the user calls syncfs. Signed-off-by: Sargun Dhillon Suggested-by: Amir Goldstein Reviewed-by: Amir Goldstein Fixes: c86243b090bc ("ovl: provide a mount option "volatile"") Cc: stable@vger.kernel.org Reviewed-by: Vivek Goyal Reviewed-by: Jeff Layton Signed-off-by: Miklos Szeredi --- Documentation/filesystems/overlayfs.rst | 8 ++++++++ fs/overlayfs/file.c | 5 +++-- fs/overlayfs/overlayfs.h | 1 + fs/overlayfs/ovl_entry.h | 2 ++ fs/overlayfs/readdir.c | 5 +++-- fs/overlayfs/super.c | 34 ++++++++++++++++++++++++++------- fs/overlayfs/util.c | 27 ++++++++++++++++++++++++++ 7 files changed, 71 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/Documentation/filesystems/overlayfs.rst b/Documentation/filesystems/overlayfs.rst index 587a93973929..78240e29b0bb 100644 --- a/Documentation/filesystems/overlayfs.rst +++ b/Documentation/filesystems/overlayfs.rst @@ -586,6 +586,14 @@ without significant effort. The advantage of mounting with the "volatile" option is that all forms of sync calls to the upper filesystem are omitted. +In order to avoid a giving a false sense of safety, the syncfs (and fsync) +semantics of volatile mounts are slightly different than that of the rest of +VFS. If any writeback error occurs on the upperdir's filesystem after a +volatile mount takes place, all sync functions will return an error. Once this +condition is reached, the filesystem will not recover, and every subsequent sync +call will return an error, even if the upperdir has not experience a new error +since the last sync call. + When overlay is mounted with "volatile" option, the directory "$workdir/work/incompat/volatile" is created. During next mount, overlay checks for this directory and refuses to mount if present. This is a strong diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c index bd9dd38347ae..077d3ad343f6 100644 --- a/fs/overlayfs/file.c +++ b/fs/overlayfs/file.c @@ -398,8 +398,9 @@ static int ovl_fsync(struct file *file, loff_t start, loff_t end, int datasync) const struct cred *old_cred; int ret; - if (!ovl_should_sync(OVL_FS(file_inode(file)->i_sb))) - return 0; + ret = ovl_sync_status(OVL_FS(file_inode(file)->i_sb)); + if (ret <= 0) + return ret; ret = ovl_real_fdget_meta(file, &real, !datasync); if (ret) diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index b487e48c7fd4..cb4e2d60ecf9 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -324,6 +324,7 @@ int ovl_check_metacopy_xattr(struct ovl_fs *ofs, struct dentry *dentry); bool ovl_is_metacopy_dentry(struct dentry *dentry); char *ovl_get_redirect_xattr(struct ovl_fs *ofs, struct dentry *dentry, int padding); +int ovl_sync_status(struct ovl_fs *ofs); static inline bool ovl_is_impuredir(struct super_block *sb, struct dentry *dentry) diff --git a/fs/overlayfs/ovl_entry.h b/fs/overlayfs/ovl_entry.h index fbd5e27ce66b..63efee554f69 100644 --- a/fs/overlayfs/ovl_entry.h +++ b/fs/overlayfs/ovl_entry.h @@ -81,6 +81,8 @@ struct ovl_fs { atomic_long_t last_ino; /* Whiteout dentry cache */ struct dentry *whiteout; + /* r/o snapshot of upperdir sb's only taken on volatile mounts */ + errseq_t errseq; }; static inline struct vfsmount *ovl_upper_mnt(struct ovl_fs *ofs) diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 60d751f28fea..f404a78e6b60 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -900,8 +900,9 @@ static int ovl_dir_fsync(struct file *file, loff_t start, loff_t end, struct file *realfile; int err; - if (!ovl_should_sync(OVL_FS(file->f_path.dentry->d_sb))) - return 0; + err = ovl_sync_status(OVL_FS(file->f_path.dentry->d_sb)); + if (err <= 0) + return err; realfile = ovl_dir_real_file(file, true); err = PTR_ERR_OR_ZERO(realfile); diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 82cd6d55a5a1..d58b8f2bf9d0 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -264,11 +264,20 @@ static int ovl_sync_fs(struct super_block *sb, int wait) struct super_block *upper_sb; int ret; - if (!ovl_upper_mnt(ofs)) - return 0; + ret = ovl_sync_status(ofs); + /* + * We have to always set the err, because the return value isn't + * checked in syncfs, and instead indirectly return an error via + * the sb's writeback errseq, which VFS inspects after this call. + */ + if (ret < 0) { + errseq_set(&sb->s_wb_err, -EIO); + return -EIO; + } + + if (!ret) + return ret; - if (!ovl_should_sync(ofs)) - return 0; /* * Not called for sync(2) call or an emergency sync (SB_I_SKIP_SYNC). * All the super blocks will be iterated, including upper_sb. @@ -1993,6 +2002,8 @@ static int ovl_fill_super(struct super_block *sb, void *data, int silent) sb->s_op = &ovl_super_operations; if (ofs->config.upperdir) { + struct super_block *upper_sb; + if (!ofs->config.workdir) { pr_err("missing 'workdir'\n"); goto out_err; @@ -2002,6 +2013,16 @@ static int ovl_fill_super(struct super_block *sb, void *data, int silent) if (err) goto out_err; + upper_sb = ovl_upper_mnt(ofs)->mnt_sb; + if (!ovl_should_sync(ofs)) { + ofs->errseq = errseq_sample(&upper_sb->s_wb_err); + if (errseq_check(&upper_sb->s_wb_err, ofs->errseq)) { + err = -EIO; + pr_err("Cannot mount volatile when upperdir has an unseen error. Sync upperdir fs to clear state.\n"); + goto out_err; + } + } + err = ovl_get_workdir(sb, ofs, &upperpath); if (err) goto out_err; @@ -2009,9 +2030,8 @@ static int ovl_fill_super(struct super_block *sb, void *data, int silent) if (!ofs->workdir) sb->s_flags |= SB_RDONLY; - sb->s_stack_depth = ovl_upper_mnt(ofs)->mnt_sb->s_stack_depth; - sb->s_time_gran = ovl_upper_mnt(ofs)->mnt_sb->s_time_gran; - + sb->s_stack_depth = upper_sb->s_stack_depth; + sb->s_time_gran = upper_sb->s_time_gran; } oe = ovl_get_lowerstack(sb, splitlower, numlower, ofs, layers); err = PTR_ERR(oe); diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 6569031af3cd..9826b003f1d2 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -962,3 +962,30 @@ err_free: kfree(buf); return ERR_PTR(res); } + +/* + * ovl_sync_status() - Check fs sync status for volatile mounts + * + * Returns 1 if this is not a volatile mount and a real sync is required. + * + * Returns 0 if syncing can be skipped because mount is volatile, and no errors + * have occurred on the upperdir since the mount. + * + * Returns -errno if it is a volatile mount, and the error that occurred since + * the last mount. If the error code changes, it'll return the latest error + * code. + */ + +int ovl_sync_status(struct ovl_fs *ofs) +{ + struct vfsmount *mnt; + + if (ovl_should_sync(ofs)) + return 1; + + mnt = ovl_upper_mnt(ofs); + if (!mnt) + return 0; + + return errseq_check(&mnt->mnt_sb->s_wb_err, ofs->errseq); +} -- cgit v1.2.3 From f71475ba8c2a77fff8051903cf4b7d826c3d1693 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 21 Jan 2021 17:57:43 -0500 Subject: nfsd: remove unused set_client argument Every caller is setting this argument to false, so we don't need it. Also cut this comment a bit and remove an unnecessary warning. Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index b7a5ac5a81ac..677586d9169a 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -4648,8 +4648,7 @@ static struct nfs4_client *lookup_clientid(clientid_t *clid, bool sessions, static __be32 set_client(clientid_t *clid, struct nfsd4_compound_state *cstate, - struct nfsd_net *nn, - bool sessions) + struct nfsd_net *nn) { if (cstate->clp) { if (!same_clid(&cstate->clp->cl_clientid, clid)) @@ -4659,12 +4658,10 @@ static __be32 set_client(clientid_t *clid, if (STALE_CLIENTID(clid, nn)) return nfserr_stale_clientid; /* - * For v4.1+ we get the client in the SEQUENCE op. If we don't have one - * cached already then we know this is for is for v4.0 and "sessions" - * will be false. + * We're in the 4.0 case (otherwise the SEQUENCE op would have + * set cstate->clp), so session = false: */ - WARN_ON_ONCE(cstate->session); - cstate->clp = lookup_clientid(clid, sessions, nn); + cstate->clp = lookup_clientid(clid, false, nn); if (!cstate->clp) return nfserr_expired; return nfs_ok; @@ -4688,7 +4685,7 @@ nfsd4_process_open1(struct nfsd4_compound_state *cstate, if (open->op_file == NULL) return nfserr_jukebox; - status = set_client(clientid, cstate, nn, false); + status = set_client(clientid, cstate, nn); if (status) return status; clp = cstate->clp; @@ -5298,7 +5295,7 @@ nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); trace_nfsd_clid_renew(clid); - status = set_client(clid, cstate, nn, false); + status = set_client(clid, cstate, nn); if (status) return status; clp = cstate->clp; @@ -5681,7 +5678,7 @@ nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate, if (ZERO_STATEID(stateid) || ONE_STATEID(stateid) || CLOSE_STATEID(stateid)) return nfserr_bad_stateid; - status = set_client(&stateid->si_opaque.so_clid, cstate, nn, false); + status = set_client(&stateid->si_opaque.so_clid, cstate, nn); if (status == nfserr_stale_clientid) { if (cstate->session) return nfserr_bad_stateid; @@ -6905,7 +6902,7 @@ nfsd4_lockt(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, return nfserr_inval; if (!nfsd4_has_session(cstate)) { - status = set_client(&lockt->lt_clientid, cstate, nn, false); + status = set_client(&lockt->lt_clientid, cstate, nn); if (status) goto out; } @@ -7089,7 +7086,7 @@ nfsd4_release_lockowner(struct svc_rqst *rqstp, dprintk("nfsd4_release_lockowner clientid: (%08x/%08x):\n", clid->cl_boot, clid->cl_id); - status = set_client(clid, cstate, nn, false); + status = set_client(clid, cstate, nn); if (status) return status; @@ -7236,7 +7233,7 @@ nfs4_check_open_reclaim(clientid_t *clid, __be32 status; /* find clientid in conf_id_hashtbl */ - status = set_client(clid, cstate, nn, false); + status = set_client(clid, cstate, nn); if (status) return nfserr_reclaim_bad; -- cgit v1.2.3 From 1722b04624806ced51693f546edb83e8b2297a77 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 21 Jan 2021 17:57:44 -0500 Subject: nfsd: simplify nfsd4_check_open_reclaim The set_client() was already taken care of by process_open1(). The comments here are mostly redundant with the code. Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfs4proc.c | 3 +-- fs/nfsd/nfs4state.c | 18 +++--------------- fs/nfsd/state.h | 3 +-- 3 files changed, 5 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 8273a99359d1..21d35b0339e6 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -428,8 +428,7 @@ nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, goto out; break; case NFS4_OPEN_CLAIM_PREVIOUS: - status = nfs4_check_open_reclaim(&open->op_clientid, - cstate, nn); + status = nfs4_check_open_reclaim(cstate->clp); if (status) goto out; open->op_openowner->oo_flags |= NFS4_OO_CONFIRMED; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 677586d9169a..e55c779dee5f 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -7222,25 +7222,13 @@ nfsd4_find_reclaim_client(struct xdr_netobj name, struct nfsd_net *nn) return NULL; } -/* -* Called from OPEN. Look for clientid in reclaim list. -*/ __be32 -nfs4_check_open_reclaim(clientid_t *clid, - struct nfsd4_compound_state *cstate, - struct nfsd_net *nn) +nfs4_check_open_reclaim(struct nfs4_client *clp) { - __be32 status; - - /* find clientid in conf_id_hashtbl */ - status = set_client(clid, cstate, nn); - if (status) - return nfserr_reclaim_bad; - - if (test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &cstate->clp->cl_flags)) + if (test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &clp->cl_flags)) return nfserr_no_grace; - if (nfsd4_client_record_check(cstate->clp)) + if (nfsd4_client_record_check(clp)) return nfserr_reclaim_bad; return nfs_ok; diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index 9eae11a9d21c..73deea353169 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -649,8 +649,7 @@ void nfs4_remove_reclaim_record(struct nfs4_client_reclaim *, struct nfsd_net *) extern void nfs4_release_reclaim(struct nfsd_net *); extern struct nfs4_client_reclaim *nfsd4_find_reclaim_client(struct xdr_netobj name, struct nfsd_net *nn); -extern __be32 nfs4_check_open_reclaim(clientid_t *clid, - struct nfsd4_compound_state *cstate, struct nfsd_net *nn); +extern __be32 nfs4_check_open_reclaim(struct nfs4_client *); extern void nfsd4_probe_callback(struct nfs4_client *clp); extern void nfsd4_probe_callback_sync(struct nfs4_client *clp); extern void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *); -- cgit v1.2.3 From ec59659b4972ec25851aa03b4b5baba6764a62e4 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 21 Jan 2021 17:57:45 -0500 Subject: nfsd: cstate->session->se_client -> cstate->clp I'm not sure why we're writing this out the hard way in so many places. Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfs4proc.c | 5 ++--- fs/nfsd/nfs4state.c | 16 ++++++++-------- 2 files changed, 10 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 21d35b0339e6..acdb3cd806a1 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -378,8 +378,7 @@ nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, * Before RECLAIM_COMPLETE done, server should deny new lock */ if (nfsd4_has_session(cstate) && - !test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, - &cstate->session->se_client->cl_flags) && + !test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &cstate->clp->cl_flags) && open->op_claim_type != NFS4_OPEN_CLAIM_PREVIOUS) return nfserr_grace; @@ -1887,7 +1886,7 @@ nfsd4_getdeviceinfo(struct svc_rqst *rqstp, nfserr = nfs_ok; if (gdp->gd_maxcount != 0) { nfserr = ops->proc_getdeviceinfo(exp->ex_path.mnt->mnt_sb, - rqstp, cstate->session->se_client, gdp); + rqstp, cstate->clp, gdp); } gdp->gd_notify_types &= ops->notify_types; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index e55c779dee5f..423fd6683f3a 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -3891,6 +3891,7 @@ nfsd4_reclaim_complete(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) { struct nfsd4_reclaim_complete *rc = &u->reclaim_complete; + struct nfs4_client *clp = cstate->clp; __be32 status = 0; if (rc->rca_one_fs) { @@ -3904,12 +3905,11 @@ nfsd4_reclaim_complete(struct svc_rqst *rqstp, } status = nfserr_complete_already; - if (test_and_set_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, - &cstate->session->se_client->cl_flags)) + if (test_and_set_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &clp->cl_flags)) goto out; status = nfserr_stale_clientid; - if (is_client_expired(cstate->session->se_client)) + if (is_client_expired(clp)) /* * The following error isn't really legal. * But we only get here if the client just explicitly @@ -3920,8 +3920,8 @@ nfsd4_reclaim_complete(struct svc_rqst *rqstp, goto out; status = nfs_ok; - nfsd4_client_record_create(cstate->session->se_client); - inc_reclaim_complete(cstate->session->se_client); + nfsd4_client_record_create(clp); + inc_reclaim_complete(clp); out: return status; } @@ -5918,7 +5918,7 @@ nfsd4_test_stateid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, { struct nfsd4_test_stateid *test_stateid = &u->test_stateid; struct nfsd4_test_stateid_id *stateid; - struct nfs4_client *cl = cstate->session->se_client; + struct nfs4_client *cl = cstate->clp; list_for_each_entry(stateid, &test_stateid->ts_stateid_list, ts_id_list) stateid->ts_id_status = @@ -5964,7 +5964,7 @@ nfsd4_free_stateid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, stateid_t *stateid = &free_stateid->fr_stateid; struct nfs4_stid *s; struct nfs4_delegation *dp; - struct nfs4_client *cl = cstate->session->se_client; + struct nfs4_client *cl = cstate->clp; __be32 ret = nfserr_bad_stateid; spin_lock(&cl->cl_lock); @@ -6693,7 +6693,7 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (nfsd4_has_session(cstate)) /* See rfc 5661 18.10.3: given clientid is ignored: */ memcpy(&lock->lk_new_clientid, - &cstate->session->se_client->cl_clientid, + &cstate->clp->cl_clientid, sizeof(clientid_t)); /* validate and update open stateid and open seqid */ -- cgit v1.2.3 From 02591f9febd5f69bb4c266a4abf899c4cf21964f Mon Sep 17 00:00:00 2001 From: Dai Ngo Date: Thu, 28 Jan 2021 01:42:26 -0500 Subject: NFSv4_2: SSC helper should use its own config. Currently NFSv4_2 SSC helper, nfs_ssc, incorrectly uses GRACE_PERIOD as its config. Fix by adding new config NFS_V4_2_SSC_HELPER which depends on NFS_V4_2 and is automatically selected when NFSD_V4 is enabled. Also removed the file name from a comment in nfs_ssc.c. Signed-off-by: Dai Ngo Signed-off-by: Chuck Lever --- fs/Kconfig | 4 ++++ fs/nfs/nfs4file.c | 4 ++++ fs/nfs/super.c | 12 ++++++++++++ fs/nfs_common/Makefile | 2 +- fs/nfs_common/nfs_ssc.c | 2 -- fs/nfsd/Kconfig | 1 + 6 files changed, 22 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/Kconfig b/fs/Kconfig index aa4c12282301..a55bda4233bb 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -333,6 +333,10 @@ config NFS_COMMON depends on NFSD || NFS_FS || LOCKD default y +config NFS_V4_2_SSC_HELPER + tristate + default y if NFS_V4=y || NFS_FS=y + source "net/sunrpc/Kconfig" source "fs/ceph/Kconfig" source "fs/cifs/Kconfig" diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c index 57b3821d975a..441a2fa073c8 100644 --- a/fs/nfs/nfs4file.c +++ b/fs/nfs/nfs4file.c @@ -420,7 +420,9 @@ static const struct nfs4_ssc_client_ops nfs4_ssc_clnt_ops_tbl = { */ void nfs42_ssc_register_ops(void) { +#ifdef CONFIG_NFSD_V4 nfs42_ssc_register(&nfs4_ssc_clnt_ops_tbl); +#endif } /** @@ -431,7 +433,9 @@ void nfs42_ssc_register_ops(void) */ void nfs42_ssc_unregister_ops(void) { +#ifdef CONFIG_NFSD_V4 nfs42_ssc_unregister(&nfs4_ssc_clnt_ops_tbl); +#endif } #endif /* CONFIG_NFS_V4_2 */ diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 4034102010f0..c7a924580eec 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -86,9 +86,11 @@ const struct super_operations nfs_sops = { }; EXPORT_SYMBOL_GPL(nfs_sops); +#ifdef CONFIG_NFS_V4_2 static const struct nfs_ssc_client_ops nfs_ssc_clnt_ops_tbl = { .sco_sb_deactive = nfs_sb_deactive, }; +#endif #if IS_ENABLED(CONFIG_NFS_V4) static int __init register_nfs4_fs(void) @@ -111,15 +113,21 @@ static void unregister_nfs4_fs(void) } #endif +#ifdef CONFIG_NFS_V4_2 static void nfs_ssc_register_ops(void) { +#ifdef CONFIG_NFSD_V4 nfs_ssc_register(&nfs_ssc_clnt_ops_tbl); +#endif } static void nfs_ssc_unregister_ops(void) { +#ifdef CONFIG_NFSD_V4 nfs_ssc_unregister(&nfs_ssc_clnt_ops_tbl); +#endif } +#endif /* CONFIG_NFS_V4_2 */ static struct shrinker acl_shrinker = { .count_objects = nfs_access_cache_count, @@ -148,7 +156,9 @@ int __init register_nfs_fs(void) ret = register_shrinker(&acl_shrinker); if (ret < 0) goto error_3; +#ifdef CONFIG_NFS_V4_2 nfs_ssc_register_ops(); +#endif return 0; error_3: nfs_unregister_sysctl(); @@ -168,7 +178,9 @@ void __exit unregister_nfs_fs(void) unregister_shrinker(&acl_shrinker); nfs_unregister_sysctl(); unregister_nfs4_fs(); +#ifdef CONFIG_NFS_V4_2 nfs_ssc_unregister_ops(); +#endif unregister_filesystem(&nfs_fs_type); } diff --git a/fs/nfs_common/Makefile b/fs/nfs_common/Makefile index fa82f5aaa6d9..119c75ab9fd0 100644 --- a/fs/nfs_common/Makefile +++ b/fs/nfs_common/Makefile @@ -7,4 +7,4 @@ obj-$(CONFIG_NFS_ACL_SUPPORT) += nfs_acl.o nfs_acl-objs := nfsacl.o obj-$(CONFIG_GRACE_PERIOD) += grace.o -obj-$(CONFIG_GRACE_PERIOD) += nfs_ssc.o +obj-$(CONFIG_NFS_V4_2_SSC_HELPER) += nfs_ssc.o diff --git a/fs/nfs_common/nfs_ssc.c b/fs/nfs_common/nfs_ssc.c index f43bbb373913..7c1509e968c8 100644 --- a/fs/nfs_common/nfs_ssc.c +++ b/fs/nfs_common/nfs_ssc.c @@ -1,7 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * fs/nfs_common/nfs_ssc_comm.c - * * Helper for knfsd's SSC to access ops in NFS client modules * * Author: Dai Ngo diff --git a/fs/nfsd/Kconfig b/fs/nfsd/Kconfig index dbbc583d6273..821e5913faee 100644 --- a/fs/nfsd/Kconfig +++ b/fs/nfsd/Kconfig @@ -76,6 +76,7 @@ config NFSD_V4 select CRYPTO_MD5 select CRYPTO_SHA256 select GRACE_PERIOD + select NFS_V4_2_SSC_HELPER if NFS_V4_2 help This option enables support in your system's NFS server for version 4 of the NFS protocol (RFC 3530). -- cgit v1.2.3 From 19d51588125fb7abe258e85b412710486a3d9219 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 27 Jan 2021 13:44:35 -0800 Subject: cifs: ignore auto and noauto options if given In 24e0a1eff9e2, the noauto and auto options were missed when migrating to the new mount API. As a result, users with noauto in their fstab mount options are now unable to mount cifs filesystems, as they'll receive an "Unknown parameter" error. This restores the old behaviour of ignoring noauto and auto if they're given. Fixes: 24e0a1eff9e2 ("cifs: switch to new mount api") Signed-off-by: Adam Harvey Reviewed-by: Ronnie Sahlberg Signed-off-by: Steve French --- fs/cifs/fs_context.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 076bcadc756a..62818b142e2e 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -175,6 +175,7 @@ const struct fs_parameter_spec smb3_fs_parameters[] = { fsparam_flag_no("exec", Opt_ignore), fsparam_flag_no("dev", Opt_ignore), fsparam_flag_no("mand", Opt_ignore), + fsparam_flag_no("auto", Opt_ignore), fsparam_string("cred", Opt_ignore), fsparam_string("credentials", Opt_ignore), {} -- cgit v1.2.3 From c9b8cd6a39c48b2827c0925b648b221b5f8ef25d Mon Sep 17 00:00:00 2001 From: Steve French Date: Thu, 28 Jan 2021 11:20:22 -0600 Subject: cifs: fix mounts to subdirectories of target The "prefixpath" mount option needs to be ignored which was missed in the recent conversion to the new mount API (prefixpath would be set by the mount helper if mounting a subdirectory of the root of a share e.g. //server/share/subdir) Fixes: 24e0a1eff9e2 ("cifs: switch to new mount api") Suggested-by: Ronnie Sahlberg Signed-off-by: Steve French Reviewed-by: Ronnie Sahlberg --- fs/cifs/fs_context.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 62818b142e2e..818c413db82d 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -178,6 +178,7 @@ const struct fs_parameter_spec smb3_fs_parameters[] = { fsparam_flag_no("auto", Opt_ignore), fsparam_string("cred", Opt_ignore), fsparam_string("credentials", Opt_ignore), + fsparam_string("prefixpath", Opt_ignore), {} }; -- cgit v1.2.3 From f609cbb8911e40e15f9055e8f945f926ac906924 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 28 Jan 2021 18:39:24 +0000 Subject: io_uring: fix list corruption for splice file_get kernel BUG at lib/list_debug.c:29! Call Trace: __list_add include/linux/list.h:67 [inline] list_add include/linux/list.h:86 [inline] io_file_get+0x8cc/0xdb0 fs/io_uring.c:6466 __io_splice_prep+0x1bc/0x530 fs/io_uring.c:3866 io_splice_prep fs/io_uring.c:3920 [inline] io_req_prep+0x3546/0x4e80 fs/io_uring.c:6081 io_queue_sqe+0x609/0x10d0 fs/io_uring.c:6628 io_submit_sqe fs/io_uring.c:6705 [inline] io_submit_sqes+0x1495/0x2720 fs/io_uring.c:6953 __do_sys_io_uring_enter+0x107d/0x1f30 fs/io_uring.c:9353 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9 io_file_get() may be called from splice, and so REQ_F_INFLIGHT may already be set. Fixes: 02a13674fa0e8 ("io_uring: account io_uring internal files as REQ_F_INFLIGHT") Cc: stable@vger.kernel.org # 5.9+ Reported-by: syzbot+6879187cf57845801267@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index ae388cc52843..39ae1f821cef 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6460,7 +6460,8 @@ static struct file *io_file_get(struct io_submit_state *state, file = __io_file_get(state, fd); } - if (file && file->f_op == &io_uring_fops) { + if (file && file->f_op == &io_uring_fops && + !(req->flags & REQ_F_INFLIGHT)) { io_req_init_async(req); req->flags |= REQ_F_INFLIGHT; -- cgit v1.2.3 From 70b2c60d3797bffe182dddb9bb55975b9be5889a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 28 Jan 2021 18:39:25 +0000 Subject: io_uring: fix sqo ownership false positive warning WARNING: CPU: 0 PID: 21359 at fs/io_uring.c:9042 io_uring_cancel_task_requests+0xe55/0x10c0 fs/io_uring.c:9042 Call Trace: io_uring_flush+0x47b/0x6e0 fs/io_uring.c:9227 filp_close+0xb4/0x170 fs/open.c:1295 close_files fs/file.c:403 [inline] put_files_struct fs/file.c:418 [inline] put_files_struct+0x1cc/0x350 fs/file.c:415 exit_files+0x7e/0xa0 fs/file.c:435 do_exit+0xc22/0x2ae0 kernel/exit.c:820 do_group_exit+0x125/0x310 kernel/exit.c:922 get_signal+0x427/0x20f0 kernel/signal.c:2773 arch_do_signal_or_restart+0x2a8/0x1eb0 arch/x86/kernel/signal.c:811 handle_signal_work kernel/entry/common.c:147 [inline] exit_to_user_mode_loop kernel/entry/common.c:171 [inline] exit_to_user_mode_prepare+0x148/0x250 kernel/entry/common.c:201 __syscall_exit_to_user_mode_work kernel/entry/common.c:291 [inline] syscall_exit_to_user_mode+0x19/0x50 kernel/entry/common.c:302 entry_SYSCALL_64_after_hwframe+0x44/0xa9 Now io_uring_cancel_task_requests() can be called not through file notes but directly, remove a WARN_ONCE() there that give us false positives. That check is not very important and we catch it in other places. Fixes: 84965ff8a84f0 ("io_uring: if we see flush on exit, cancel related tasks") Cc: stable@vger.kernel.org # 5.9+ Reported-by: syzbot+3e3d9bd0c6ce9efbc3ef@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 39ae1f821cef..12bf7180c0f1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8967,8 +8967,6 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, struct task_struct *task = current; if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { - /* for SQPOLL only sqo_task has task notes */ - WARN_ON_ONCE(ctx->sqo_task != current); io_disable_sqo_submit(ctx); task = ctx->sq_data->thread; atomic_inc(&task->io_uring->in_idle); -- cgit v1.2.3 From bd2f0b43c1c864fa653342c5c074bfcd29f10934 Mon Sep 17 00:00:00 2001 From: Steve French Date: Thu, 28 Jan 2021 16:43:03 -0600 Subject: cifs: returning mount parm processing errors correctly During additional testing of the updated cifs.ko with the new mount API support, we found a few additional cases where we were logging errors, but not returning them to the user. For example: a) invalid security mechanisms b) invalid cache options c) unsupported rdma d) invalid smb dialect requested Fixes: 24e0a1eff9e2 ("cifs: switch to new mount api") Acked-by: Ronnie Sahlberg Signed-off-by: Steve French --- fs/cifs/fs_context.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 818c413db82d..27354417e988 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -533,7 +533,7 @@ static int smb3_fs_context_validate(struct fs_context *fc) if (ctx->rdma && ctx->vals->protocol_id < SMB30_PROT_ID) { cifs_dbg(VFS, "SMB Direct requires Version >=3.0\n"); - return -1; + return -EOPNOTSUPP; } #ifndef CONFIG_KEYS @@ -556,7 +556,7 @@ static int smb3_fs_context_validate(struct fs_context *fc) /* make sure UNC has a share name */ if (strlen(ctx->UNC) < 3 || !strchr(ctx->UNC + 3, '\\')) { cifs_dbg(VFS, "Malformed UNC. Unable to find share name.\n"); - return -1; + return -ENOENT; } if (!ctx->got_ip) { @@ -570,7 +570,7 @@ static int smb3_fs_context_validate(struct fs_context *fc) if (!cifs_convert_address((struct sockaddr *)&ctx->dstaddr, &ctx->UNC[2], len)) { pr_err("Unable to determine destination address\n"); - return -1; + return -EHOSTUNREACH; } } @@ -1265,7 +1265,7 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, return 0; cifs_parse_mount_err: - return 1; + return -EINVAL; } int smb3_init_fs_context(struct fs_context *fc) -- cgit v1.2.3 From 3a7efd1ad269ccaf9c1423364d97c9661ba6dafa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 28 Jan 2021 23:23:42 +0000 Subject: io_uring: reinforce cancel on flush during exit What 84965ff8a84f0 ("io_uring: if we see flush on exit, cancel related tasks") really wants is to cancel all relevant REQ_F_INFLIGHT requests reliably. That can be achieved by io_uring_cancel_files(), but we'll miss it calling io_uring_cancel_task_requests(files=NULL) from io_uring_flush(), because it will go through __io_uring_cancel_task_requests(). Just always call io_uring_cancel_files() during cancel, it's good enough for now. Cc: stable@vger.kernel.org # 5.9+ Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 12bf7180c0f1..38c6cbe1ab38 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8976,10 +8976,9 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, io_cancel_defer_files(ctx, task, files); io_cqring_overflow_flush(ctx, true, task, files); + io_uring_cancel_files(ctx, task, files); if (!files) __io_uring_cancel_task_requests(ctx, task); - else - io_uring_cancel_files(ctx, task, files); if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { atomic_dec(&task->io_uring->in_idle); -- cgit v1.2.3 From 0d4873f9aa4ff8fc1d63a5755395b794d32ce046 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Thu, 28 Jan 2021 21:35:10 -0600 Subject: cifs: fix dfs domain referrals The new mount API requires additional changes to how DFS is handled. Additional testing of DFS uncovered problems with domain based DFS referrals (a follow on patch addresses DFS links) which this patch addresses. Signed-off-by: Ronnie Sahlberg Signed-off-by: Paulo Alcantara (SUSE) Signed-off-by: Steve French --- fs/cifs/cifs_dfs_ref.c | 12 ++++++++---- fs/cifs/cifsfs.c | 2 +- fs/cifs/cifsproto.h | 6 ++++-- fs/cifs/connect.c | 32 ++++++++++++++++++++++++++------ fs/cifs/dfs_cache.c | 8 +++++--- fs/cifs/fs_context.c | 31 +++++++++++++++++++++++++++++++ 6 files changed, 75 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifs_dfs_ref.c b/fs/cifs/cifs_dfs_ref.c index e4c6ae47a796..6b1ce4efb591 100644 --- a/fs/cifs/cifs_dfs_ref.c +++ b/fs/cifs/cifs_dfs_ref.c @@ -133,8 +133,9 @@ cifs_build_devname(char *nodename, const char *prepath) * Caller is responsible for freeing returned value if it is not error. */ char *cifs_compose_mount_options(const char *sb_mountdata, - const char *fullpath, - const struct dfs_info3_param *ref) + const char *fullpath, + const struct dfs_info3_param *ref, + char **devname) { int rc; char *name; @@ -231,7 +232,10 @@ char *cifs_compose_mount_options(const char *sb_mountdata, strcat(mountdata, "ip="); strcat(mountdata, srvIP); - kfree(name); + if (devname) + *devname = name; + else + kfree(name); /*cifs_dbg(FYI, "%s: parent mountdata: %s\n", __func__, sb_mountdata);*/ /*cifs_dbg(FYI, "%s: submount mountdata: %s\n", __func__, mountdata );*/ @@ -278,7 +282,7 @@ static struct vfsmount *cifs_dfs_do_mount(struct dentry *mntpt, /* strip first '\' from fullpath */ mountdata = cifs_compose_mount_options(cifs_sb->ctx->mount_options, - fullpath + 1, NULL); + fullpath + 1, NULL, NULL); if (IS_ERR(mountdata)) { kfree(devname); return (struct vfsmount *)mountdata; diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index ce0d0037fd0a..e46da536ed33 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -822,7 +822,7 @@ cifs_smb3_do_mount(struct file_system_type *fs_type, goto out; } - rc = cifs_setup_volume_info(cifs_sb->ctx); + rc = cifs_setup_volume_info(cifs_sb->ctx, NULL, old_ctx->UNC); if (rc) { root = ERR_PTR(rc); goto out; diff --git a/fs/cifs/cifsproto.h b/fs/cifs/cifsproto.h index 340ff81ee87b..32f7a013402e 100644 --- a/fs/cifs/cifsproto.h +++ b/fs/cifs/cifsproto.h @@ -78,7 +78,8 @@ extern char *cifs_build_path_to_root(struct smb3_fs_context *ctx, int add_treename); extern char *build_wildcard_path_from_dentry(struct dentry *direntry); extern char *cifs_compose_mount_options(const char *sb_mountdata, - const char *fullpath, const struct dfs_info3_param *ref); + const char *fullpath, const struct dfs_info3_param *ref, + char **devname); /* extern void renew_parental_timestamps(struct dentry *direntry);*/ extern struct mid_q_entry *AllocMidQEntry(const struct smb_hdr *smb_buffer, struct TCP_Server_Info *server); @@ -89,6 +90,7 @@ extern void cifs_wake_up_task(struct mid_q_entry *mid); extern int cifs_handle_standard(struct TCP_Server_Info *server, struct mid_q_entry *mid); extern int smb3_parse_devname(const char *devname, struct smb3_fs_context *ctx); +extern int smb3_parse_opt(const char *options, const char *key, char **val); extern bool cifs_match_ipaddr(struct sockaddr *srcaddr, struct sockaddr *rhs); extern int cifs_discard_remaining_data(struct TCP_Server_Info *server); extern int cifs_call_async(struct TCP_Server_Info *server, @@ -549,7 +551,7 @@ extern int SMBencrypt(unsigned char *passwd, const unsigned char *c8, unsigned char *p24); extern int -cifs_setup_volume_info(struct smb3_fs_context *ctx); +cifs_setup_volume_info(struct smb3_fs_context *ctx, const char *mntopts, const char *devname); extern struct TCP_Server_Info * cifs_find_tcp_session(struct smb3_fs_context *ctx); diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index c8ef24bac94f..10fe6d6d2dee 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -2972,17 +2972,20 @@ expand_dfs_referral(const unsigned int xid, struct cifs_ses *ses, rc = dfs_cache_find(xid, ses, cifs_sb->local_nls, cifs_remap(cifs_sb), ref_path, &referral, NULL); if (!rc) { + char *fake_devname = NULL; + mdata = cifs_compose_mount_options(cifs_sb->ctx->mount_options, - full_path + 1, &referral); + full_path + 1, &referral, + &fake_devname); free_dfs_info_param(&referral); if (IS_ERR(mdata)) { rc = PTR_ERR(mdata); mdata = NULL; } else { - smb3_cleanup_fs_context_contents(ctx); - rc = cifs_setup_volume_info(ctx); + rc = cifs_setup_volume_info(ctx, mdata, fake_devname); } + kfree(fake_devname); kfree(cifs_sb->ctx->mount_options); cifs_sb->ctx->mount_options = mdata; } @@ -3036,6 +3039,7 @@ static int setup_dfs_tgt_conn(const char *path, const char *full_path, struct dfs_info3_param ref = {0}; char *mdata = NULL; struct smb3_fs_context fake_ctx = {NULL}; + char *fake_devname = NULL; cifs_dbg(FYI, "%s: dfs path: %s\n", __func__, path); @@ -3044,16 +3048,18 @@ static int setup_dfs_tgt_conn(const char *path, const char *full_path, return rc; mdata = cifs_compose_mount_options(cifs_sb->ctx->mount_options, - full_path + 1, &ref); + full_path + 1, &ref, + &fake_devname); free_dfs_info_param(&ref); if (IS_ERR(mdata)) { rc = PTR_ERR(mdata); mdata = NULL; } else - rc = cifs_setup_volume_info(&fake_ctx); + rc = cifs_setup_volume_info(&fake_ctx, mdata, fake_devname); kfree(mdata); + kfree(fake_devname); if (!rc) { /* @@ -3122,10 +3128,24 @@ static int do_dfs_failover(const char *path, const char *full_path, struct cifs_ * we should pass a clone of the original context? */ int -cifs_setup_volume_info(struct smb3_fs_context *ctx) +cifs_setup_volume_info(struct smb3_fs_context *ctx, const char *mntopts, const char *devname) { int rc = 0; + smb3_parse_devname(devname, ctx); + + if (mntopts) { + char *ip; + + cifs_dbg(FYI, "%s: mntopts=%s\n", __func__, mntopts); + rc = smb3_parse_opt(mntopts, "ip", &ip); + if (!rc && !cifs_convert_address((struct sockaddr *)&ctx->dstaddr, ip, + strlen(ip))) { + cifs_dbg(VFS, "%s: failed to convert ip address\n", __func__); + return -EINVAL; + } + } + if (ctx->nullauth) { cifs_dbg(FYI, "Anonymous login\n"); kfree(ctx->username); diff --git a/fs/cifs/dfs_cache.c b/fs/cifs/dfs_cache.c index 0fdb0de7ff86..4950ab0486ae 100644 --- a/fs/cifs/dfs_cache.c +++ b/fs/cifs/dfs_cache.c @@ -1417,7 +1417,7 @@ static struct cifs_ses *find_root_ses(struct vol_info *vi, int rc; struct cache_entry *ce; struct dfs_info3_param ref = {0}; - char *mdata = NULL; + char *mdata = NULL, *devname = NULL; struct TCP_Server_Info *server; struct cifs_ses *ses; struct smb3_fs_context ctx = {NULL}; @@ -1444,7 +1444,8 @@ static struct cifs_ses *find_root_ses(struct vol_info *vi, up_read(&htable_rw_lock); - mdata = cifs_compose_mount_options(vi->mntdata, rpath, &ref); + mdata = cifs_compose_mount_options(vi->mntdata, rpath, &ref, + &devname); free_dfs_info_param(&ref); if (IS_ERR(mdata)) { @@ -1453,7 +1454,7 @@ static struct cifs_ses *find_root_ses(struct vol_info *vi, goto out; } - rc = cifs_setup_volume_info(&ctx); + rc = cifs_setup_volume_info(&ctx, NULL, devname); if (rc) { ses = ERR_PTR(rc); @@ -1472,6 +1473,7 @@ out: smb3_cleanup_fs_context_contents(&ctx); kfree(mdata); kfree(rpath); + kfree(devname); return ses; } diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 27354417e988..5111aadfdb6b 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -401,6 +401,37 @@ cifs_parse_smb_version(char *value, struct smb3_fs_context *ctx, bool is_smb3) return 0; } +int smb3_parse_opt(const char *options, const char *key, char **val) +{ + int rc = -ENOENT; + char *opts, *orig, *p; + + orig = opts = kstrdup(options, GFP_KERNEL); + if (!opts) + return -ENOMEM; + + while ((p = strsep(&opts, ","))) { + char *nval; + + if (!*p) + continue; + if (strncasecmp(p, key, strlen(key))) + continue; + nval = strchr(p, '='); + if (nval) { + if (nval == p) + continue; + *nval++ = 0; + *val = kstrndup(nval, strlen(nval), GFP_KERNEL); + rc = !*val ? -ENOMEM : 0; + goto out; + } + } +out: + kfree(orig); + return rc; +} + /* * Parse a devname into substrings and populate the ctx->UNC and ctx->prepath * fields with the result. Returns 0 on success and an error otherwise -- cgit v1.2.3 From be65de6b03aa638c46ea51e9d11a92e4914d8103 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 14 Jan 2021 17:05:31 +0530 Subject: fs: Remove dcookies support The dcookies stuff was only used by the kernel's old oprofile code. Now that oprofile's support is removed from the kernel, there is no need for dcookies as well. Remove it. Suggested-by: Christoph Hellwig Suggested-by: Linus Torvalds Signed-off-by: Viresh Kumar Acked-by: Robert Richter Acked-by: William Cohen Acked-by: Al Viro Acked-by: Thomas Gleixner --- fs/Makefile | 1 - fs/dcookies.c | 356 ----------------------------------------------- include/linux/dcookies.h | 69 --------- kernel/sys.c | 1 - 4 files changed, 427 deletions(-) delete mode 100644 fs/dcookies.c delete mode 100644 include/linux/dcookies.h (limited to 'fs') diff --git a/fs/Makefile b/fs/Makefile index 999d1a23f036..3215fe205256 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -64,7 +64,6 @@ obj-$(CONFIG_SYSFS) += sysfs/ obj-$(CONFIG_CONFIGFS_FS) += configfs/ obj-y += devpts/ -obj-$(CONFIG_PROFILING) += dcookies.o obj-$(CONFIG_DLM) += dlm/ # Do not add any filesystems before this line diff --git a/fs/dcookies.c b/fs/dcookies.c deleted file mode 100644 index 6eeb61100a09..000000000000 --- a/fs/dcookies.c +++ /dev/null @@ -1,356 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * dcookies.c - * - * Copyright 2002 John Levon - * - * Persistent cookie-path mappings. These are used by - * profilers to convert a per-task EIP value into something - * non-transitory that can be processed at a later date. - * This is done by locking the dentry/vfsmnt pair in the - * kernel until released by the tasks needing the persistent - * objects. The tag is simply an unsigned long that refers - * to the pair and can be looked up from userspace. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* The dcookies are allocated from a kmem_cache and - * hashed onto a small number of lists. None of the - * code here is particularly performance critical - */ -struct dcookie_struct { - struct path path; - struct list_head hash_list; -}; - -static LIST_HEAD(dcookie_users); -static DEFINE_MUTEX(dcookie_mutex); -static struct kmem_cache *dcookie_cache __read_mostly; -static struct list_head *dcookie_hashtable __read_mostly; -static size_t hash_size __read_mostly; - -static inline int is_live(void) -{ - return !(list_empty(&dcookie_users)); -} - - -/* The dentry is locked, its address will do for the cookie */ -static inline unsigned long dcookie_value(struct dcookie_struct * dcs) -{ - return (unsigned long)dcs->path.dentry; -} - - -static size_t dcookie_hash(unsigned long dcookie) -{ - return (dcookie >> L1_CACHE_SHIFT) & (hash_size - 1); -} - - -static struct dcookie_struct * find_dcookie(unsigned long dcookie) -{ - struct dcookie_struct *found = NULL; - struct dcookie_struct * dcs; - struct list_head * pos; - struct list_head * list; - - list = dcookie_hashtable + dcookie_hash(dcookie); - - list_for_each(pos, list) { - dcs = list_entry(pos, struct dcookie_struct, hash_list); - if (dcookie_value(dcs) == dcookie) { - found = dcs; - break; - } - } - - return found; -} - - -static void hash_dcookie(struct dcookie_struct * dcs) -{ - struct list_head * list = dcookie_hashtable + dcookie_hash(dcookie_value(dcs)); - list_add(&dcs->hash_list, list); -} - - -static struct dcookie_struct *alloc_dcookie(const struct path *path) -{ - struct dcookie_struct *dcs = kmem_cache_alloc(dcookie_cache, - GFP_KERNEL); - struct dentry *d; - if (!dcs) - return NULL; - - d = path->dentry; - spin_lock(&d->d_lock); - d->d_flags |= DCACHE_COOKIE; - spin_unlock(&d->d_lock); - - dcs->path = *path; - path_get(path); - hash_dcookie(dcs); - return dcs; -} - - -/* This is the main kernel-side routine that retrieves the cookie - * value for a dentry/vfsmnt pair. - */ -int get_dcookie(const struct path *path, unsigned long *cookie) -{ - int err = 0; - struct dcookie_struct * dcs; - - mutex_lock(&dcookie_mutex); - - if (!is_live()) { - err = -EINVAL; - goto out; - } - - if (path->dentry->d_flags & DCACHE_COOKIE) { - dcs = find_dcookie((unsigned long)path->dentry); - } else { - dcs = alloc_dcookie(path); - if (!dcs) { - err = -ENOMEM; - goto out; - } - } - - *cookie = dcookie_value(dcs); - -out: - mutex_unlock(&dcookie_mutex); - return err; -} - - -/* And here is where the userspace process can look up the cookie value - * to retrieve the path. - */ -static int do_lookup_dcookie(u64 cookie64, char __user *buf, size_t len) -{ - unsigned long cookie = (unsigned long)cookie64; - int err = -EINVAL; - char * kbuf; - char * path; - size_t pathlen; - struct dcookie_struct * dcs; - - /* we could leak path information to users - * without dir read permission without this - */ - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - mutex_lock(&dcookie_mutex); - - if (!is_live()) { - err = -EINVAL; - goto out; - } - - if (!(dcs = find_dcookie(cookie))) - goto out; - - err = -ENOMEM; - kbuf = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!kbuf) - goto out; - - /* FIXME: (deleted) ? */ - path = d_path(&dcs->path, kbuf, PAGE_SIZE); - - mutex_unlock(&dcookie_mutex); - - if (IS_ERR(path)) { - err = PTR_ERR(path); - goto out_free; - } - - err = -ERANGE; - - pathlen = kbuf + PAGE_SIZE - path; - if (pathlen <= len) { - err = pathlen; - if (copy_to_user(buf, path, pathlen)) - err = -EFAULT; - } - -out_free: - kfree(kbuf); - return err; -out: - mutex_unlock(&dcookie_mutex); - return err; -} - -SYSCALL_DEFINE3(lookup_dcookie, u64, cookie64, char __user *, buf, size_t, len) -{ - return do_lookup_dcookie(cookie64, buf, len); -} - -#ifdef CONFIG_COMPAT -COMPAT_SYSCALL_DEFINE4(lookup_dcookie, u32, w0, u32, w1, char __user *, buf, compat_size_t, len) -{ -#ifdef __BIG_ENDIAN - return do_lookup_dcookie(((u64)w0 << 32) | w1, buf, len); -#else - return do_lookup_dcookie(((u64)w1 << 32) | w0, buf, len); -#endif -} -#endif - -static int dcookie_init(void) -{ - struct list_head * d; - unsigned int i, hash_bits; - int err = -ENOMEM; - - dcookie_cache = kmem_cache_create("dcookie_cache", - sizeof(struct dcookie_struct), - 0, 0, NULL); - - if (!dcookie_cache) - goto out; - - dcookie_hashtable = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!dcookie_hashtable) - goto out_kmem; - - err = 0; - - /* - * Find the power-of-two list-heads that can fit into the allocation.. - * We don't guarantee that "sizeof(struct list_head)" is necessarily - * a power-of-two. - */ - hash_size = PAGE_SIZE / sizeof(struct list_head); - hash_bits = 0; - do { - hash_bits++; - } while ((hash_size >> hash_bits) != 0); - hash_bits--; - - /* - * Re-calculate the actual number of entries and the mask - * from the number of bits we can fit. - */ - hash_size = 1UL << hash_bits; - - /* And initialize the newly allocated array */ - d = dcookie_hashtable; - i = hash_size; - do { - INIT_LIST_HEAD(d); - d++; - i--; - } while (i); - -out: - return err; -out_kmem: - kmem_cache_destroy(dcookie_cache); - goto out; -} - - -static void free_dcookie(struct dcookie_struct * dcs) -{ - struct dentry *d = dcs->path.dentry; - - spin_lock(&d->d_lock); - d->d_flags &= ~DCACHE_COOKIE; - spin_unlock(&d->d_lock); - - path_put(&dcs->path); - kmem_cache_free(dcookie_cache, dcs); -} - - -static void dcookie_exit(void) -{ - struct list_head * list; - struct list_head * pos; - struct list_head * pos2; - struct dcookie_struct * dcs; - size_t i; - - for (i = 0; i < hash_size; ++i) { - list = dcookie_hashtable + i; - list_for_each_safe(pos, pos2, list) { - dcs = list_entry(pos, struct dcookie_struct, hash_list); - list_del(&dcs->hash_list); - free_dcookie(dcs); - } - } - - kfree(dcookie_hashtable); - kmem_cache_destroy(dcookie_cache); -} - - -struct dcookie_user { - struct list_head next; -}; - -struct dcookie_user * dcookie_register(void) -{ - struct dcookie_user * user; - - mutex_lock(&dcookie_mutex); - - user = kmalloc(sizeof(struct dcookie_user), GFP_KERNEL); - if (!user) - goto out; - - if (!is_live() && dcookie_init()) - goto out_free; - - list_add(&user->next, &dcookie_users); - -out: - mutex_unlock(&dcookie_mutex); - return user; -out_free: - kfree(user); - user = NULL; - goto out; -} - - -void dcookie_unregister(struct dcookie_user * user) -{ - mutex_lock(&dcookie_mutex); - - list_del(&user->next); - kfree(user); - - if (!is_live()) - dcookie_exit(); - - mutex_unlock(&dcookie_mutex); -} - -EXPORT_SYMBOL_GPL(dcookie_register); -EXPORT_SYMBOL_GPL(dcookie_unregister); -EXPORT_SYMBOL_GPL(get_dcookie); diff --git a/include/linux/dcookies.h b/include/linux/dcookies.h deleted file mode 100644 index ddfdac20cad0..000000000000 --- a/include/linux/dcookies.h +++ /dev/null @@ -1,69 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * dcookies.h - * - * Persistent cookie-path mappings - * - * Copyright 2002 John Levon - */ - -#ifndef DCOOKIES_H -#define DCOOKIES_H - - -#ifdef CONFIG_PROFILING - -#include -#include - -struct dcookie_user; -struct path; - -/** - * dcookie_register - register a user of dcookies - * - * Register as a dcookie user. Returns %NULL on failure. - */ -struct dcookie_user * dcookie_register(void); - -/** - * dcookie_unregister - unregister a user of dcookies - * - * Unregister as a dcookie user. This may invalidate - * any dcookie values returned from get_dcookie(). - */ -void dcookie_unregister(struct dcookie_user * user); - -/** - * get_dcookie - acquire a dcookie - * - * Convert the given dentry/vfsmount pair into - * a cookie value. - * - * Returns -EINVAL if no living task has registered as a - * dcookie user. - * - * Returns 0 on success, with *cookie filled in - */ -int get_dcookie(const struct path *path, unsigned long *cookie); - -#else - -static inline struct dcookie_user * dcookie_register(void) -{ - return NULL; -} - -static inline void dcookie_unregister(struct dcookie_user * user) -{ - return; -} - -static inline int get_dcookie(const struct path *path, unsigned long *cookie) -{ - return -ENOSYS; -} - -#endif /* CONFIG_PROFILING */ - -#endif /* DCOOKIES_H */ diff --git a/kernel/sys.c b/kernel/sys.c index 51f00fe20e4d..6928d23c46ea 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include -- cgit v1.2.3 From 912efa17e5121693dfbadae29768f4144a3f9e62 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 Jan 2021 23:53:42 +0000 Subject: mm: proc: Invalidate TLB after clearing soft-dirty page state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit 0758cd830494 ("asm-generic/tlb: avoid potential double flush"), TLB invalidation is elided in tlb_finish_mmu() if no entries were batched via the tlb_remove_*() functions. Consequently, the page-table modifications performed by clear_refs_write() in response to a write to /proc//clear_refs do not perform TLB invalidation. Although this is fine when simply aging the ptes, in the case of clearing the "soft-dirty" state we can end up with entries where pte_write() is false, yet a writable mapping remains in the TLB. Fix this by avoiding the mmu_gather API altogether: managing both the 'tlb_flush_pending' flag on the 'mm_struct' and explicit TLB invalidation for the sort-dirty path, much like mprotect() does already. Fixes: 0758cd830494 ("asm-generic/tlb: avoid potential double flush”) Signed-off-by: Will Deacon Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yu Zhao Acked-by: Peter Zijlstra (Intel) Acked-by: Linus Torvalds Link: https://lkml.kernel.org/r/20210127235347.1402-2-will@kernel.org --- fs/proc/task_mmu.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 602e3a52884d..3cec6fbef725 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1210,7 +1210,6 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, struct mm_struct *mm; struct vm_area_struct *vma; enum clear_refs_types type; - struct mmu_gather tlb; int itype; int rv; @@ -1249,7 +1248,6 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, goto out_unlock; } - tlb_gather_mmu(&tlb, mm, 0, -1); if (type == CLEAR_REFS_SOFT_DIRTY) { for (vma = mm->mmap; vma; vma = vma->vm_next) { if (!(vma->vm_flags & VM_SOFTDIRTY)) @@ -1258,15 +1256,18 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, vma_set_page_prot(vma); } + inc_tlb_flush_pending(mm); mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY, 0, NULL, mm, 0, -1UL); mmu_notifier_invalidate_range_start(&range); } walk_page_range(mm, 0, mm->highest_vm_end, &clear_refs_walk_ops, &cp); - if (type == CLEAR_REFS_SOFT_DIRTY) + if (type == CLEAR_REFS_SOFT_DIRTY) { mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb, 0, -1); + flush_tlb_mm(mm); + dec_tlb_flush_pending(mm); + } out_unlock: mmap_write_unlock(mm); out_mm: -- cgit v1.2.3 From ae8eba8b5d723a4ca543024b6e51f4d0f4fb6b6b Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 Jan 2021 23:53:43 +0000 Subject: tlb: mmu_gather: Remove unused start/end arguments from tlb_finish_mmu() Since commit 7a30df49f63a ("mm: mmu_gather: remove __tlb_reset_range() for force flush"), the 'start' and 'end' arguments to tlb_finish_mmu() are no longer used, since we flush the whole mm in case of a nested invalidation. Remove the unused arguments and update all callers. Signed-off-by: Will Deacon Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yu Zhao Acked-by: Peter Zijlstra (Intel) Acked-by: Linus Torvalds Link: https://lkml.kernel.org/r/20210127235347.1402-3-will@kernel.org --- arch/ia64/include/asm/tlb.h | 2 +- arch/x86/kernel/ldt.c | 2 +- fs/exec.c | 2 +- include/linux/mm_types.h | 3 +-- mm/hugetlb.c | 2 +- mm/madvise.c | 6 +++--- mm/memory.c | 4 ++-- mm/mmap.c | 4 ++-- mm/mmu_gather.c | 5 +---- mm/oom_kill.c | 4 ++-- 10 files changed, 15 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h index 8d9da6f08a62..7059eb2e867a 100644 --- a/arch/ia64/include/asm/tlb.h +++ b/arch/ia64/include/asm/tlb.h @@ -36,7 +36,7 @@ * tlb_end_vma(tlb, vma); * } * } - * tlb_finish_mmu(tlb, start, end); // finish unmap for address space MM + * tlb_finish_mmu(tlb); // finish unmap for address space MM */ #include #include diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c index b8aee71840ae..0d4e1253c9c9 100644 --- a/arch/x86/kernel/ldt.c +++ b/arch/x86/kernel/ldt.c @@ -400,7 +400,7 @@ static void free_ldt_pgtables(struct mm_struct *mm) tlb_gather_mmu(&tlb, mm, start, end); free_pgd_range(&tlb, start, end, start, end); - tlb_finish_mmu(&tlb, start, end); + tlb_finish_mmu(&tlb); #endif } diff --git a/fs/exec.c b/fs/exec.c index 5d4d52039105..69d89a0c35e9 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -725,7 +725,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) free_pgd_range(&tlb, old_start, old_end, new_end, vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); } - tlb_finish_mmu(&tlb, old_start, old_end); + tlb_finish_mmu(&tlb); /* * Shrink the vma to just the new range. Always succeeds. diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 07d9acb5b19c..1fe6a51298a6 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -590,8 +590,7 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) struct mmu_gather; extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end); -extern void tlb_finish_mmu(struct mmu_gather *tlb, - unsigned long start, unsigned long end); +extern void tlb_finish_mmu(struct mmu_gather *tlb); static inline void init_tlb_flush_pending(struct mm_struct *mm) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 18f6ee317900..33db4fa62c7b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3985,7 +3985,7 @@ void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, tlb_gather_mmu(&tlb, mm, tlb_start, tlb_end); __unmap_hugepage_range(&tlb, vma, start, end, ref_page); - tlb_finish_mmu(&tlb, tlb_start, tlb_end); + tlb_finish_mmu(&tlb); } /* diff --git a/mm/madvise.c b/mm/madvise.c index 6a660858784b..1b68520ea3f4 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -508,7 +508,7 @@ static long madvise_cold(struct vm_area_struct *vma, lru_add_drain(); tlb_gather_mmu(&tlb, mm, start_addr, end_addr); madvise_cold_page_range(&tlb, vma, start_addr, end_addr); - tlb_finish_mmu(&tlb, start_addr, end_addr); + tlb_finish_mmu(&tlb); return 0; } @@ -560,7 +560,7 @@ static long madvise_pageout(struct vm_area_struct *vma, lru_add_drain(); tlb_gather_mmu(&tlb, mm, start_addr, end_addr); madvise_pageout_page_range(&tlb, vma, start_addr, end_addr); - tlb_finish_mmu(&tlb, start_addr, end_addr); + tlb_finish_mmu(&tlb); return 0; } @@ -732,7 +732,7 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, &madvise_free_walk_ops, &tlb); tlb_end_vma(&tlb, vma); mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb, range.start, range.end); + tlb_finish_mmu(&tlb); return 0; } diff --git a/mm/memory.c b/mm/memory.c index feff48e1465a..7bd3f122bd10 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1540,7 +1540,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) unmap_single_vma(&tlb, vma, start, range.end, NULL); mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb, start, range.end); + tlb_finish_mmu(&tlb); } /** @@ -1566,7 +1566,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr mmu_notifier_invalidate_range_start(&range); unmap_single_vma(&tlb, vma, address, range.end, details); mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb, address, range.end); + tlb_finish_mmu(&tlb); } /** diff --git a/mm/mmap.c b/mm/mmap.c index dc7206032387..7a9f493a4b83 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2676,7 +2676,7 @@ static void unmap_region(struct mm_struct *mm, unmap_vmas(&tlb, vma, start, end); free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, next ? next->vm_start : USER_PGTABLES_CEILING); - tlb_finish_mmu(&tlb, start, end); + tlb_finish_mmu(&tlb); } /* @@ -3219,7 +3219,7 @@ void exit_mmap(struct mm_struct *mm) /* Use -1 here to ensure all VMAs in the mm are unmapped */ unmap_vmas(&tlb, vma, 0, -1); free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); - tlb_finish_mmu(&tlb, 0, -1); + tlb_finish_mmu(&tlb); /* * Walk the list again, actually closing and freeing it, diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 03c33c93a582..b0be5a7aa08f 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -290,14 +290,11 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, /** * tlb_finish_mmu - finish an mmu_gather structure * @tlb: the mmu_gather structure to finish - * @start: start of the region that will be removed from the page-table - * @end: end of the region that will be removed from the page-table * * Called at the end of the shootdown operation to free up any resources that * were required. */ -void tlb_finish_mmu(struct mmu_gather *tlb, - unsigned long start, unsigned long end) +void tlb_finish_mmu(struct mmu_gather *tlb) { /* * If there are parallel threads are doing PTE changes on same range diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 04b19b7b5435..757e557211fb 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -548,13 +548,13 @@ bool __oom_reap_task_mm(struct mm_struct *mm) vma->vm_end); tlb_gather_mmu(&tlb, mm, range.start, range.end); if (mmu_notifier_invalidate_range_start_nonblock(&range)) { - tlb_finish_mmu(&tlb, range.start, range.end); + tlb_finish_mmu(&tlb); ret = false; continue; } unmap_page_range(&tlb, vma, range.start, range.end, NULL); mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb, range.start, range.end); + tlb_finish_mmu(&tlb); } } -- cgit v1.2.3 From a72afd873089c697053e9daa85ff343b3140d2e7 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 Jan 2021 23:53:45 +0000 Subject: tlb: mmu_gather: Remove start/end arguments from tlb_gather_mmu() The 'start' and 'end' arguments to tlb_gather_mmu() are no longer needed now that there is a separate function for 'fullmm' flushing. Remove the unused arguments and update all callers. Suggested-by: Linus Torvalds Signed-off-by: Will Deacon Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yu Zhao Acked-by: Peter Zijlstra (Intel) Acked-by: Linus Torvalds Link: https://lore.kernel.org/r/CAHk-=wjQWa14_4UpfDf=fiineNP+RH74kZeDMo_f1D35xNzq9w@mail.gmail.com --- arch/ia64/include/asm/tlb.h | 2 +- arch/x86/kernel/ldt.c | 2 +- fs/exec.c | 2 +- include/linux/mm_types.h | 3 +-- mm/hugetlb.c | 16 +--------------- mm/madvise.c | 6 +++--- mm/memory.c | 4 ++-- mm/mmap.c | 2 +- mm/mmu_gather.c | 22 ++++++++-------------- mm/oom_kill.c | 2 +- 10 files changed, 20 insertions(+), 41 deletions(-) (limited to 'fs') diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h index 7059eb2e867a..a15fe0809aae 100644 --- a/arch/ia64/include/asm/tlb.h +++ b/arch/ia64/include/asm/tlb.h @@ -23,7 +23,7 @@ * unmapping a portion of the virtual address space, these hooks are called according to * the following template: * - * tlb <- tlb_gather_mmu(mm, start, end); // start unmap for address space MM + * tlb <- tlb_gather_mmu(mm); // start unmap for address space MM * { * for each vma that needs a shootdown do { * tlb_start_vma(tlb, vma); diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c index 0d4e1253c9c9..7ad9834e0d95 100644 --- a/arch/x86/kernel/ldt.c +++ b/arch/x86/kernel/ldt.c @@ -398,7 +398,7 @@ static void free_ldt_pgtables(struct mm_struct *mm) if (!boot_cpu_has(X86_FEATURE_PTI)) return; - tlb_gather_mmu(&tlb, mm, start, end); + tlb_gather_mmu(&tlb, mm); free_pgd_range(&tlb, start, end, start, end); tlb_finish_mmu(&tlb); #endif diff --git a/fs/exec.c b/fs/exec.c index 69d89a0c35e9..5a853f03c233 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -708,7 +708,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) return -ENOMEM; lru_add_drain(); - tlb_gather_mmu(&tlb, mm, old_start, old_end); + tlb_gather_mmu(&tlb, mm); if (new_end > old_start) { /* * when the old and new regions overlap clear from new_end. diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index e49868bc12a7..0974ad501a47 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -588,8 +588,7 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) } struct mmu_gather; -extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, - unsigned long start, unsigned long end); +extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_finish_mmu(struct mmu_gather *tlb); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 33db4fa62c7b..89635f407232 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3967,23 +3967,9 @@ void __unmap_hugepage_range_final(struct mmu_gather *tlb, void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct page *ref_page) { - struct mm_struct *mm; struct mmu_gather tlb; - unsigned long tlb_start = start; - unsigned long tlb_end = end; - /* - * If shared PMDs were possibly used within this vma range, adjust - * start/end for worst case tlb flushing. - * Note that we can not be sure if PMDs are shared until we try to - * unmap pages. However, we want to make sure TLB flushing covers - * the largest possible range. - */ - adjust_range_if_pmd_sharing_possible(vma, &tlb_start, &tlb_end); - - mm = vma->vm_mm; - - tlb_gather_mmu(&tlb, mm, tlb_start, tlb_end); + tlb_gather_mmu(&tlb, vma->vm_mm); __unmap_hugepage_range(&tlb, vma, start, end, ref_page); tlb_finish_mmu(&tlb); } diff --git a/mm/madvise.c b/mm/madvise.c index 1b68520ea3f4..0938fd3ad228 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -506,7 +506,7 @@ static long madvise_cold(struct vm_area_struct *vma, return -EINVAL; lru_add_drain(); - tlb_gather_mmu(&tlb, mm, start_addr, end_addr); + tlb_gather_mmu(&tlb, mm); madvise_cold_page_range(&tlb, vma, start_addr, end_addr); tlb_finish_mmu(&tlb); @@ -558,7 +558,7 @@ static long madvise_pageout(struct vm_area_struct *vma, return 0; lru_add_drain(); - tlb_gather_mmu(&tlb, mm, start_addr, end_addr); + tlb_gather_mmu(&tlb, mm); madvise_pageout_page_range(&tlb, vma, start_addr, end_addr); tlb_finish_mmu(&tlb); @@ -723,7 +723,7 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, range.start, range.end); lru_add_drain(); - tlb_gather_mmu(&tlb, mm, range.start, range.end); + tlb_gather_mmu(&tlb, mm); update_hiwater_rss(mm); mmu_notifier_invalidate_range_start(&range); diff --git a/mm/memory.c b/mm/memory.c index 7bd3f122bd10..9e8576a83147 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1534,7 +1534,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, lru_add_drain(); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, start, start + size); - tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end); + tlb_gather_mmu(&tlb, vma->vm_mm); update_hiwater_rss(vma->vm_mm); mmu_notifier_invalidate_range_start(&range); for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) @@ -1561,7 +1561,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr lru_add_drain(); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, address, address + size); - tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end); + tlb_gather_mmu(&tlb, vma->vm_mm); update_hiwater_rss(vma->vm_mm); mmu_notifier_invalidate_range_start(&range); unmap_single_vma(&tlb, vma, address, range.end, details); diff --git a/mm/mmap.c b/mm/mmap.c index 4eac7c63edbe..90673febce6a 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2671,7 +2671,7 @@ static void unmap_region(struct mm_struct *mm, struct mmu_gather tlb; lru_add_drain(); - tlb_gather_mmu(&tlb, mm, start, end); + tlb_gather_mmu(&tlb, mm); update_hiwater_rss(mm); unmap_vmas(&tlb, vma, start, end); free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 5f5e45d9eb50..0dc7149b0c61 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -253,21 +253,17 @@ void tlb_flush_mmu(struct mmu_gather *tlb) * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down * @tlb: the mmu_gather structure to initialize * @mm: the mm_struct of the target address space - * @start: start of the region that will be removed from the page-table - * @end: end of the region that will be removed from the page-table + * @fullmm: @mm is without users and we're going to destroy the full address + * space (exit/execve) * * Called to initialize an (on-stack) mmu_gather structure for page-table - * tear-down from @mm. The @start and @end are set to 0 and -1 - * respectively when @mm is without users and we're going to destroy - * the full address space (exit/execve). + * tear-down from @mm. */ static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, - unsigned long start, unsigned long end) + bool fullmm) { tlb->mm = mm; - - /* Is it from 0 to ~0? */ - tlb->fullmm = !(start | (end+1)); + tlb->fullmm = fullmm; #ifndef CONFIG_MMU_GATHER_NO_GATHER tlb->need_flush_all = 0; @@ -287,16 +283,14 @@ static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, inc_tlb_flush_pending(tlb->mm); } -void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, - unsigned long start, unsigned long end) +void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm) { - WARN_ON(!(start | (end + 1))); /* Use _fullmm() instead */ - __tlb_gather_mmu(tlb, mm, start, end); + __tlb_gather_mmu(tlb, mm, false); } void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm) { - __tlb_gather_mmu(tlb, mm, 0, -1); + __tlb_gather_mmu(tlb, mm, true); } /** diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 757e557211fb..c9a33ffe38b7 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -546,7 +546,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, vma->vm_start, vma->vm_end); - tlb_gather_mmu(&tlb, mm, range.start, range.end); + tlb_gather_mmu(&tlb, mm); if (mmu_notifier_invalidate_range_start_nonblock(&range)) { tlb_finish_mmu(&tlb); ret = false; -- cgit v1.2.3 From 5399d52233c47905bbf97dcbaa2d7a9cc31670ba Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 29 Jan 2021 23:53:50 +0000 Subject: rxrpc: Fix deadlock around release of dst cached on udp tunnel AF_RXRPC sockets use UDP ports in encap mode. This causes socket and dst from an incoming packet to get stolen and attached to the UDP socket from whence it is leaked when that socket is closed. When a network namespace is removed, the wait for dst records to be cleaned up happens before the cleanup of the rxrpc and UDP socket, meaning that the wait never finishes. Fix this by moving the rxrpc (and, by dependence, the afs) private per-network namespace registrations to the device group rather than subsys group. This allows cached rxrpc local endpoints to be cleared and their UDP sockets closed before we try waiting for the dst records. The symptom is that lines looking like the following: unregister_netdevice: waiting for lo to become free get emitted at regular intervals after running something like the referenced syzbot test. Thanks to Vadim for tracking this down and work out the fix. Reported-by: syzbot+df400f2f24a1677cd7e0@syzkaller.appspotmail.com Reported-by: Vadim Fedorenko Fixes: 5271953cad31 ("rxrpc: Use the UDP encap_rcv hook") Signed-off-by: David Howells Acked-by: Vadim Fedorenko Link: https://lore.kernel.org/r/161196443016.3868642.5577440140646403533.stgit@warthog.procyon.org.uk Signed-off-by: Jakub Kicinski --- fs/afs/main.c | 6 +++--- net/rxrpc/af_rxrpc.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/afs/main.c b/fs/afs/main.c index accdd8970e7c..b2975256dadb 100644 --- a/fs/afs/main.c +++ b/fs/afs/main.c @@ -193,7 +193,7 @@ static int __init afs_init(void) goto error_cache; #endif - ret = register_pernet_subsys(&afs_net_ops); + ret = register_pernet_device(&afs_net_ops); if (ret < 0) goto error_net; @@ -213,7 +213,7 @@ static int __init afs_init(void) error_proc: afs_fs_exit(); error_fs: - unregister_pernet_subsys(&afs_net_ops); + unregister_pernet_device(&afs_net_ops); error_net: #ifdef CONFIG_AFS_FSCACHE fscache_unregister_netfs(&afs_cache_netfs); @@ -244,7 +244,7 @@ static void __exit afs_exit(void) proc_remove(afs_proc_symlink); afs_fs_exit(); - unregister_pernet_subsys(&afs_net_ops); + unregister_pernet_device(&afs_net_ops); #ifdef CONFIG_AFS_FSCACHE fscache_unregister_netfs(&afs_cache_netfs); #endif diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c index 0a2f4817ec6c..41671af6b33f 100644 --- a/net/rxrpc/af_rxrpc.c +++ b/net/rxrpc/af_rxrpc.c @@ -990,7 +990,7 @@ static int __init af_rxrpc_init(void) goto error_security; } - ret = register_pernet_subsys(&rxrpc_net_ops); + ret = register_pernet_device(&rxrpc_net_ops); if (ret) goto error_pernet; @@ -1035,7 +1035,7 @@ error_key_type: error_sock: proto_unregister(&rxrpc_proto); error_proto: - unregister_pernet_subsys(&rxrpc_net_ops); + unregister_pernet_device(&rxrpc_net_ops); error_pernet: rxrpc_exit_security(); error_security: @@ -1057,7 +1057,7 @@ static void __exit af_rxrpc_exit(void) unregister_key_type(&key_type_rxrpc); sock_unregister(PF_RXRPC); proto_unregister(&rxrpc_proto); - unregister_pernet_subsys(&rxrpc_net_ops); + unregister_pernet_device(&rxrpc_net_ops); ASSERTCMP(atomic_read(&rxrpc_n_tx_skbs), ==, 0); ASSERTCMP(atomic_read(&rxrpc_n_rx_skbs), ==, 0); -- cgit v1.2.3 From 3cc55f4434b421d37300aa9a167ace7d60b45ccf Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Fri, 29 Jan 2021 14:26:29 -0500 Subject: nfs: use change attribute for NFS re-exports When exporting NFS, we may as well use the real change attribute returned by the original server instead of faking up a change attribute from the ctime. Note we can't do that by setting I_VERSION--that would also turn on the logic in iversion.h which treats the lower bit specially, and that doesn't make sense for NFS. So instead we define a new export operation for filesystems like NFS that want to manage the change attribute themselves. Signed-off-by: J. Bruce Fields Reviewed-by: Christoph Hellwig Signed-off-by: Chuck Lever --- fs/nfs/export.c | 18 ++++++++++++++++++ fs/nfsd/nfsfh.h | 5 ++++- include/linux/exportfs.h | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/nfs/export.c b/fs/nfs/export.c index 7412bb164fa7..f2b34cfe286c 100644 --- a/fs/nfs/export.c +++ b/fs/nfs/export.c @@ -167,10 +167,28 @@ out: return parent; } +static u64 nfs_fetch_iversion(struct inode *inode) +{ + struct nfs_server *server = NFS_SERVER(inode); + + /* Is this the right call?: */ + nfs_revalidate_inode(server, inode); + /* + * Also, note we're ignoring any returned error. That seems to be + * the practice for cache consistency information elsewhere in + * the server, but I'm not sure why. + */ + if (server->nfs_client->rpc_ops->version >= 4) + return inode_peek_iversion_raw(inode); + else + return time_to_chattr(&inode->i_ctime); +} + const struct export_operations nfs_export_ops = { .encode_fh = nfs_encode_fh, .fh_to_dentry = nfs_fh_to_dentry, .get_parent = nfs_get_parent, + .fetch_iversion = nfs_fetch_iversion, .flags = EXPORT_OP_NOWCC|EXPORT_OP_NOSUBTREECHK| EXPORT_OP_CLOSE_BEFORE_UNLINK|EXPORT_OP_REMOTE_FS| EXPORT_OP_NOATOMIC_ATTR, diff --git a/fs/nfsd/nfsfh.h b/fs/nfsd/nfsfh.h index cb20c2cd3469..f58933519f38 100644 --- a/fs/nfsd/nfsfh.h +++ b/fs/nfsd/nfsfh.h @@ -12,6 +12,7 @@ #include #include #include +#include static inline __u32 ino_t_to_u32(ino_t ino) { @@ -264,7 +265,9 @@ fh_clear_wcc(struct svc_fh *fhp) static inline u64 nfsd4_change_attribute(struct kstat *stat, struct inode *inode) { - if (IS_I_VERSION(inode)) { + if (inode->i_sb->s_export_op->fetch_iversion) + return inode->i_sb->s_export_op->fetch_iversion(inode); + else if (IS_I_VERSION(inode)) { u64 chattr; chattr = stat->ctime.tv_sec; diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h index 9f4d4bcbf251..fe848901fcc3 100644 --- a/include/linux/exportfs.h +++ b/include/linux/exportfs.h @@ -213,6 +213,7 @@ struct export_operations { bool write, u32 *device_generation); int (*commit_blocks)(struct inode *inode, struct iomap *iomaps, int nr_iomaps, struct iattr *iattr); + u64 (*fetch_iversion)(struct inode *); #define EXPORT_OP_NOWCC (0x1) /* don't collect v3 wcc data */ #define EXPORT_OP_NOSUBTREECHK (0x2) /* no subtree checking */ #define EXPORT_OP_CLOSE_BEFORE_UNLINK (0x4) /* close files before unlink */ -- cgit v1.2.3 From 428a23d2bf0ca8fd4d364a464c3e468f0e81671e Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Fri, 29 Jan 2021 14:27:01 -0500 Subject: nfsd: skip some unnecessary stats in the v4 case In the typical case of v4 and an i_version-supporting filesystem, we can skip a stat which is only required to fake up a change attribute from ctime. Signed-off-by: J. Bruce Fields Reviewed-by: Christoph Hellwig Signed-off-by: Chuck Lever --- fs/nfsd/nfs3xdr.c | 44 +++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 00a96054280a..9d9a01ce0b27 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -364,6 +364,11 @@ encode_wcc_data(struct svc_rqst *rqstp, __be32 *p, struct svc_fh *fhp) return encode_post_op_attr(rqstp, p, fhp); } +static bool fs_supports_change_attribute(struct super_block *sb) +{ + return sb->s_flags & SB_I_VERSION || sb->s_export_op->fetch_iversion; +} + /* * Fill in the pre_op attr for the wcc data */ @@ -372,24 +377,26 @@ void fill_pre_wcc(struct svc_fh *fhp) struct inode *inode; struct kstat stat; bool v4 = (fhp->fh_maxsize == NFS4_FHSIZE); - __be32 err; if (fhp->fh_no_wcc || fhp->fh_pre_saved) return; inode = d_inode(fhp->fh_dentry); - err = fh_getattr(fhp, &stat); - if (err) { - /* Grab the times from inode anyway */ - stat.mtime = inode->i_mtime; - stat.ctime = inode->i_ctime; - stat.size = inode->i_size; + if (fs_supports_change_attribute(inode->i_sb) || !v4) { + __be32 err = fh_getattr(fhp, &stat); + + if (err) { + /* Grab the times from inode anyway */ + stat.mtime = inode->i_mtime; + stat.ctime = inode->i_ctime; + stat.size = inode->i_size; + } + fhp->fh_pre_mtime = stat.mtime; + fhp->fh_pre_ctime = stat.ctime; + fhp->fh_pre_size = stat.size; } if (v4) fhp->fh_pre_change = nfsd4_change_attribute(&stat, inode); - fhp->fh_pre_mtime = stat.mtime; - fhp->fh_pre_ctime = stat.ctime; - fhp->fh_pre_size = stat.size; fhp->fh_pre_saved = true; } @@ -400,7 +407,6 @@ void fill_post_wcc(struct svc_fh *fhp) { bool v4 = (fhp->fh_maxsize == NFS4_FHSIZE); struct inode *inode = d_inode(fhp->fh_dentry); - __be32 err; if (fhp->fh_no_wcc) return; @@ -408,12 +414,16 @@ void fill_post_wcc(struct svc_fh *fhp) if (fhp->fh_post_saved) printk("nfsd: inode locked twice during operation.\n"); - err = fh_getattr(fhp, &fhp->fh_post_attr); - if (err) { - fhp->fh_post_saved = false; - fhp->fh_post_attr.ctime = inode->i_ctime; - } else - fhp->fh_post_saved = true; + fhp->fh_post_saved = true; + + if (fs_supports_change_attribute(inode->i_sb) || !v4) { + __be32 err = fh_getattr(fhp, &fhp->fh_post_attr); + + if (err) { + fhp->fh_post_saved = false; + fhp->fh_post_attr.ctime = inode->i_ctime; + } + } if (v4) fhp->fh_post_change = nfsd4_change_attribute(&fhp->fh_post_attr, inode); -- cgit v1.2.3 From c0da04ae079fb4979b921f8b46a6eeb15a4f803c Mon Sep 17 00:00:00 2001 From: Menglong Dong Date: Mon, 9 Nov 2020 03:12:03 -0500 Subject: fs/nfs: remove duplicate include 'nfs42.h' is already included above and can be removed here. Signed-off-by: Menglong Dong Signed-off-by: Anna Schumaker --- fs/nfs/nfs4proc.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'fs') diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 2f4679a62712..224773eb61b1 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -71,10 +71,6 @@ #include "nfs4trace.h" -#ifdef CONFIG_NFS_V4_2 -#include "nfs42.h" -#endif /* CONFIG_NFS_V4_2 */ - #define NFSDBG_FACILITY NFSDBG_PROC #define NFS4_BITMASK_SZ 3 -- cgit v1.2.3 From 3a81fd02045c329f25e5900fa61f613c9b317644 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 10 Dec 2020 12:25:36 -0700 Subject: io_uring: enable LOOKUP_CACHED path resolution for filename lookups Instead of being pessimistic and assume that path lookup will block, use LOOKUP_CACHED to attempt just a cached lookup. This ensures that the fast path is always done inline, and we only punt to async context if IO is needed to satisfy the lookup. For forced nonblock open attempts, mark the file O_NONBLOCK over the actual ->open() call as well. We can safely clear this again before doing fd_install(), so it'll never be user visible that we fiddled with it. This greatly improves the performance of file open where the dentry is already cached: ached 5.10-git 5.10-git+LOOKUP_CACHED Speedup --------------------------------------------------------------- 33% 1,014,975 900,474 1.1x 89% 545,466 292,937 1.9x 100% 435,636 151,475 2.9x The more cache hot we are, the faster the inline LOOKUP_CACHED optimization helps. This is unsurprising and expected, as a thread offload becomes a more dominant part of the total overhead. If we look at io_uring tracing, doing an IORING_OP_OPENAT on a file that isn't in the dentry cache will yield: 275.550481: io_uring_create: ring 00000000ddda6278, fd 3 sq size 8, cq size 16, flags 0 275.550491: io_uring_submit_sqe: ring 00000000ddda6278, op 18, data 0x0, non block 1, sq_thread 0 275.550498: io_uring_queue_async_work: ring 00000000ddda6278, request 00000000c0267d17, flags 69760, normal queue, work 000000003d683991 275.550502: io_uring_cqring_wait: ring 00000000ddda6278, min_events 1 275.550556: io_uring_complete: ring 00000000ddda6278, user_data 0x0, result 4 which shows a failed nonblock lookup, then punt to worker, and then we complete with fd == 4. This takes 65 usec in total. Re-running the same test case again: 281.253956: io_uring_create: ring 0000000008207252, fd 3 sq size 8, cq size 16, flags 0 281.253967: io_uring_submit_sqe: ring 0000000008207252, op 18, data 0x0, non block 1, sq_thread 0 281.253973: io_uring_complete: ring 0000000008207252, user_data 0x0, result 4 shows the same request completing inline, also returning fd == 4. This takes 6 usec. Signed-off-by: Jens Axboe --- fs/io_uring.c | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 38c6cbe1ab38..fbc6d2fb7c1d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -489,7 +489,6 @@ struct io_sr_msg { struct io_open { struct file *file; int dfd; - bool ignore_nonblock; struct filename *filename; struct open_how how; unsigned long nofile; @@ -4054,7 +4053,6 @@ static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe return ret; } req->open.nofile = rlimit(RLIMIT_NOFILE); - req->open.ignore_nonblock = false; req->flags |= REQ_F_NEED_CLEANUP; return 0; } @@ -4096,39 +4094,48 @@ static int io_openat2(struct io_kiocb *req, bool force_nonblock) { struct open_flags op; struct file *file; + bool nonblock_set; + bool resolve_nonblock; int ret; - if (force_nonblock && !req->open.ignore_nonblock) - return -EAGAIN; - ret = build_open_flags(&req->open.how, &op); if (ret) goto err; + nonblock_set = op.open_flag & O_NONBLOCK; + resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED; + if (force_nonblock) { + /* + * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, + * it'll always -EAGAIN + */ + if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE)) + return -EAGAIN; + op.lookup_flags |= LOOKUP_CACHED; + op.open_flag |= O_NONBLOCK; + } ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile); if (ret < 0) goto err; file = do_filp_open(req->open.dfd, req->open.filename, &op); + /* only retry if RESOLVE_CACHED wasn't already set by application */ + if ((!resolve_nonblock && force_nonblock) && file == ERR_PTR(-EAGAIN)) { + /* + * We could hang on to this 'fd', but seems like marginal + * gain for something that is now known to be a slower path. + * So just put it, and we'll get a new one when we retry. + */ + put_unused_fd(ret); + return -EAGAIN; + } + if (IS_ERR(file)) { put_unused_fd(ret); ret = PTR_ERR(file); - /* - * A work-around to ensure that /proc/self works that way - * that it should - if we get -EOPNOTSUPP back, then assume - * that proc_self_get_link() failed us because we're in async - * context. We should be safe to retry this from the task - * itself with force_nonblock == false set, as it should not - * block on lookup. Would be nice to know this upfront and - * avoid the async dance, but doesn't seem feasible. - */ - if (ret == -EOPNOTSUPP && io_wq_current_is_worker()) { - req->open.ignore_nonblock = true; - refcount_inc(&req->refs); - io_req_task_queue(req); - return 0; - } } else { + if (force_nonblock && !nonblock_set) + file->f_flags &= ~O_NONBLOCK; fsnotify_open(file); fd_install(ret, file); } -- cgit v1.2.3 From 0a96bbe49994a46c1fea34619a501ead46aa7584 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Wed, 6 Jan 2021 12:39:10 -0800 Subject: io_uring: modularize io_sqe_buffer_register Split io_sqe_buffer_register into two routines: - io_sqe_buffer_register() registers a single buffer - io_sqe_buffers_register iterates over all user specified buffers Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh Signed-off-by: Jens Axboe --- fs/io_uring.c | 210 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 107 insertions(+), 103 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index fbc6d2fb7c1d..ec70ba064774 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8370,7 +8370,7 @@ static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries) return pages; } -static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx) +static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) { int i, j; @@ -8488,14 +8488,103 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages, return ret; } -static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg, - unsigned nr_args) +static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, + struct io_mapped_ubuf *imu, + struct page **last_hpage) { struct vm_area_struct **vmas = NULL; struct page **pages = NULL; + unsigned long off, start, end, ubuf; + size_t size; + int ret, pret, nr_pages, i; + + ubuf = (unsigned long) iov->iov_base; + end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT; + start = ubuf >> PAGE_SHIFT; + nr_pages = end - start; + + ret = -ENOMEM; + + pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL); + if (!pages) + goto done; + + vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *), + GFP_KERNEL); + if (!vmas) + goto done; + + imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec), + GFP_KERNEL); + if (!imu->bvec) + goto done; + + ret = 0; + mmap_read_lock(current->mm); + pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM, + pages, vmas); + if (pret == nr_pages) { + /* don't support file backed memory */ + for (i = 0; i < nr_pages; i++) { + struct vm_area_struct *vma = vmas[i]; + + if (vma->vm_file && + !is_file_hugepages(vma->vm_file)) { + ret = -EOPNOTSUPP; + break; + } + } + } else { + ret = pret < 0 ? pret : -EFAULT; + } + mmap_read_unlock(current->mm); + if (ret) { + /* + * if we did partial map, or found file backed vmas, + * release any pages we did get + */ + if (pret > 0) + unpin_user_pages(pages, pret); + kvfree(imu->bvec); + goto done; + } + + ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage); + if (ret) { + unpin_user_pages(pages, pret); + kvfree(imu->bvec); + goto done; + } + + off = ubuf & ~PAGE_MASK; + size = iov->iov_len; + for (i = 0; i < nr_pages; i++) { + size_t vec_len; + + vec_len = min_t(size_t, size, PAGE_SIZE - off); + imu->bvec[i].bv_page = pages[i]; + imu->bvec[i].bv_len = vec_len; + imu->bvec[i].bv_offset = off; + off = 0; + size -= vec_len; + } + /* store original address for later verification */ + imu->ubuf = ubuf; + imu->len = iov->iov_len; + imu->nr_bvecs = nr_pages; + ret = 0; +done: + kvfree(pages); + kvfree(vmas); + return ret; +} + +static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, + unsigned int nr_args) +{ + int i, ret; + struct iovec iov; struct page *last_hpage = NULL; - int i, j, got_pages = 0; - int ret = -EINVAL; if (ctx->user_bufs) return -EBUSY; @@ -8509,14 +8598,10 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg, for (i = 0; i < nr_args; i++) { struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; - unsigned long off, start, end, ubuf; - int pret, nr_pages; - struct iovec iov; - size_t size; ret = io_copy_iov(ctx, &iov, arg, i); if (ret) - goto err; + break; /* * Don't impose further limits on the size and buffer @@ -8525,103 +8610,22 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg, */ ret = -EFAULT; if (!iov.iov_base || !iov.iov_len) - goto err; + break; /* arbitrary limit, but we need something */ if (iov.iov_len > SZ_1G) - goto err; - - ubuf = (unsigned long) iov.iov_base; - end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT; - start = ubuf >> PAGE_SHIFT; - nr_pages = end - start; - - ret = 0; - if (!pages || nr_pages > got_pages) { - kvfree(vmas); - kvfree(pages); - pages = kvmalloc_array(nr_pages, sizeof(struct page *), - GFP_KERNEL); - vmas = kvmalloc_array(nr_pages, - sizeof(struct vm_area_struct *), - GFP_KERNEL); - if (!pages || !vmas) { - ret = -ENOMEM; - goto err; - } - got_pages = nr_pages; - } - - imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec), - GFP_KERNEL); - ret = -ENOMEM; - if (!imu->bvec) - goto err; - - ret = 0; - mmap_read_lock(current->mm); - pret = pin_user_pages(ubuf, nr_pages, - FOLL_WRITE | FOLL_LONGTERM, - pages, vmas); - if (pret == nr_pages) { - /* don't support file backed memory */ - for (j = 0; j < nr_pages; j++) { - struct vm_area_struct *vma = vmas[j]; - - if (vma->vm_file && - !is_file_hugepages(vma->vm_file)) { - ret = -EOPNOTSUPP; - break; - } - } - } else { - ret = pret < 0 ? pret : -EFAULT; - } - mmap_read_unlock(current->mm); - if (ret) { - /* - * if we did partial map, or found file backed vmas, - * release any pages we did get - */ - if (pret > 0) - unpin_user_pages(pages, pret); - kvfree(imu->bvec); - goto err; - } - - ret = io_buffer_account_pin(ctx, pages, pret, imu, &last_hpage); - if (ret) { - unpin_user_pages(pages, pret); - kvfree(imu->bvec); - goto err; - } + break; - off = ubuf & ~PAGE_MASK; - size = iov.iov_len; - for (j = 0; j < nr_pages; j++) { - size_t vec_len; - - vec_len = min_t(size_t, size, PAGE_SIZE - off); - imu->bvec[j].bv_page = pages[j]; - imu->bvec[j].bv_len = vec_len; - imu->bvec[j].bv_offset = off; - off = 0; - size -= vec_len; - } - /* store original address for later verification */ - imu->ubuf = ubuf; - imu->len = iov.iov_len; - imu->nr_bvecs = nr_pages; + ret = io_sqe_buffer_register(ctx, &iov, imu, &last_hpage); + if (ret) + break; ctx->nr_user_bufs++; } - kvfree(pages); - kvfree(vmas); - return 0; -err: - kvfree(pages); - kvfree(vmas); - io_sqe_buffer_unregister(ctx); + + if (ret) + io_sqe_buffers_unregister(ctx); + return ret; } @@ -8675,7 +8679,7 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) static void io_ring_ctx_free(struct io_ring_ctx *ctx) { io_finish_async(ctx); - io_sqe_buffer_unregister(ctx); + io_sqe_buffers_unregister(ctx); if (ctx->sqo_task) { put_task_struct(ctx->sqo_task); @@ -10057,13 +10061,13 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, switch (opcode) { case IORING_REGISTER_BUFFERS: - ret = io_sqe_buffer_register(ctx, arg, nr_args); + ret = io_sqe_buffers_register(ctx, arg, nr_args); break; case IORING_UNREGISTER_BUFFERS: ret = -EINVAL; if (arg || nr_args) break; - ret = io_sqe_buffer_unregister(ctx); + ret = io_sqe_buffers_unregister(ctx); break; case IORING_REGISTER_FILES: ret = io_sqe_files_register(ctx, arg, nr_args); -- cgit v1.2.3 From 2b358604aa6e8c12d7efa14777fcc66c377682b0 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Wed, 6 Jan 2021 12:39:11 -0800 Subject: io_uring: modularize io_sqe_buffers_register Move allocation of buffer management structures, and validation of buffers into separate routines. Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh Signed-off-by: Jens Axboe --- fs/io_uring.c | 51 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index ec70ba064774..4cdf0f906f12 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8579,13 +8579,8 @@ done: return ret; } -static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, - unsigned int nr_args) +static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args) { - int i, ret; - struct iovec iov; - struct page *last_hpage = NULL; - if (ctx->user_bufs) return -EBUSY; if (!nr_args || nr_args > UIO_MAXIOV) @@ -8596,6 +8591,37 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, if (!ctx->user_bufs) return -ENOMEM; + return 0; +} + +static int io_buffer_validate(struct iovec *iov) +{ + /* + * Don't impose further limits on the size and buffer + * constraints here, we'll -EINVAL later when IO is + * submitted if they are wrong. + */ + if (!iov->iov_base || !iov->iov_len) + return -EFAULT; + + /* arbitrary limit, but we need something */ + if (iov->iov_len > SZ_1G) + return -EFAULT; + + return 0; +} + +static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, + unsigned int nr_args) +{ + int i, ret; + struct iovec iov; + struct page *last_hpage = NULL; + + ret = io_buffers_map_alloc(ctx, nr_args); + if (ret) + return ret; + for (i = 0; i < nr_args; i++) { struct io_mapped_ubuf *imu = &ctx->user_bufs[i]; @@ -8603,17 +8629,8 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, if (ret) break; - /* - * Don't impose further limits on the size and buffer - * constraints here, we'll -EINVAL later when IO is - * submitted if they are wrong. - */ - ret = -EFAULT; - if (!iov.iov_base || !iov.iov_len) - break; - - /* arbitrary limit, but we need something */ - if (iov.iov_len > SZ_1G) + ret = io_buffer_validate(&iov); + if (ret) break; ret = io_sqe_buffer_register(ctx, &iov, imu, &last_hpage); -- cgit v1.2.3 From 269bbe5fd4d2fdd3b0d3a82a3c3c1dd1209aa8b8 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:44 +0000 Subject: io_uring: rename file related variables to rsrc This is a prep rename patch for subsequent patches to generalize file registration. [io_uring_rsrc_update:: rename fds -> data] Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh [leave io_uring_files_update as struct] Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 228 ++++++++++++++++++++++-------------------- include/uapi/linux/io_uring.h | 7 ++ 2 files changed, 124 insertions(+), 111 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 4cdf0f906f12..95b3b8747a65 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -195,24 +195,29 @@ struct io_mapped_ubuf { unsigned long acct_pages; }; -struct fixed_file_table { +struct io_rsrc_put { + struct list_head list; + struct file *file; +}; + +struct fixed_rsrc_table { struct file **files; }; -struct fixed_file_ref_node { +struct fixed_rsrc_ref_node { struct percpu_ref refs; struct list_head node; - struct list_head file_list; - struct fixed_file_data *file_data; + struct list_head rsrc_list; + struct fixed_rsrc_data *rsrc_data; struct llist_node llist; bool done; }; -struct fixed_file_data { - struct fixed_file_table *table; +struct fixed_rsrc_data { + struct fixed_rsrc_table *table; struct io_ring_ctx *ctx; - struct fixed_file_ref_node *node; + struct fixed_rsrc_ref_node *node; struct percpu_ref refs; struct completion done; struct list_head ref_list; @@ -319,7 +324,7 @@ struct io_ring_ctx { * readers must ensure that ->refs is alive as long as the file* is * used. Only updated through io_uring_register(2). */ - struct fixed_file_data *file_data; + struct fixed_rsrc_data *file_data; unsigned nr_user_files; /* if used, fixed mapped user buffers */ @@ -384,8 +389,8 @@ struct io_ring_ctx { struct list_head inflight_list; } ____cacheline_aligned_in_smp; - struct delayed_work file_put_work; - struct llist_head file_put_llist; + struct delayed_work rsrc_put_work; + struct llist_head rsrc_put_llist; struct work_struct exit_work; struct io_restriction restrictions; @@ -494,7 +499,7 @@ struct io_open { unsigned long nofile; }; -struct io_files_update { +struct io_rsrc_update { struct file *file; u64 arg; u32 nr_args; @@ -688,7 +693,7 @@ struct io_kiocb { struct io_sr_msg sr_msg; struct io_open open; struct io_close close; - struct io_files_update files_update; + struct io_rsrc_update rsrc_update; struct io_fadvise fadvise; struct io_madvise madvise; struct io_epoll epoll; @@ -718,7 +723,7 @@ struct io_kiocb { u64 user_data; struct io_kiocb *link; - struct percpu_ref *fixed_file_refs; + struct percpu_ref *fixed_rsrc_refs; /* * 1. used with ctx->iopoll_list with reads/writes @@ -996,8 +1001,8 @@ enum io_mem_account { static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, struct task_struct *task); -static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node); -static struct fixed_file_ref_node *alloc_fixed_file_ref_node( +static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node); +static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( struct io_ring_ctx *ctx); static void __io_complete_rw(struct io_kiocb *req, long res, long res2, @@ -1010,13 +1015,13 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req); static void __io_queue_linked_timeout(struct io_kiocb *req); static void io_queue_linked_timeout(struct io_kiocb *req); static int __io_sqe_files_update(struct io_ring_ctx *ctx, - struct io_uring_files_update *ip, + struct io_uring_rsrc_update *ip, unsigned nr_args); static void __io_clean_op(struct io_kiocb *req); static struct file *io_file_get(struct io_submit_state *state, struct io_kiocb *req, int fd, bool fixed); static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs); -static void io_file_put_work(struct work_struct *work); +static void io_rsrc_put_work(struct work_struct *work); static ssize_t io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, struct iov_iter *iter, @@ -1057,9 +1062,9 @@ static inline void io_set_resource_node(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; - if (!req->fixed_file_refs) { - req->fixed_file_refs = &ctx->file_data->node->refs; - percpu_ref_get(req->fixed_file_refs); + if (!req->fixed_rsrc_refs) { + req->fixed_rsrc_refs = &ctx->file_data->node->refs; + percpu_ref_get(req->fixed_rsrc_refs); } } @@ -1330,8 +1335,8 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->timeout_list); spin_lock_init(&ctx->inflight_lock); INIT_LIST_HEAD(&ctx->inflight_list); - INIT_DELAYED_WORK(&ctx->file_put_work, io_file_put_work); - init_llist_head(&ctx->file_put_llist); + INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); + init_llist_head(&ctx->rsrc_put_llist); return ctx; err: if (ctx->fallback_req) @@ -2011,8 +2016,8 @@ static void io_dismantle_req(struct io_kiocb *req) kfree(req->async_data); if (req->file) io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE)); - if (req->fixed_file_refs) - percpu_ref_put(req->fixed_file_refs); + if (req->fixed_rsrc_refs) + percpu_ref_put(req->fixed_rsrc_refs); io_req_clean_work(req); } @@ -5988,7 +5993,7 @@ static int io_async_cancel(struct io_kiocb *req) return 0; } -static int io_files_update_prep(struct io_kiocb *req, +static int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { if (unlikely(req->ctx->flags & IORING_SETUP_SQPOLL)) @@ -5998,11 +6003,11 @@ static int io_files_update_prep(struct io_kiocb *req, if (sqe->ioprio || sqe->rw_flags) return -EINVAL; - req->files_update.offset = READ_ONCE(sqe->off); - req->files_update.nr_args = READ_ONCE(sqe->len); - if (!req->files_update.nr_args) + req->rsrc_update.offset = READ_ONCE(sqe->off); + req->rsrc_update.nr_args = READ_ONCE(sqe->len); + if (!req->rsrc_update.nr_args) return -EINVAL; - req->files_update.arg = READ_ONCE(sqe->addr); + req->rsrc_update.arg = READ_ONCE(sqe->addr); return 0; } @@ -6010,17 +6015,17 @@ static int io_files_update(struct io_kiocb *req, bool force_nonblock, struct io_comp_state *cs) { struct io_ring_ctx *ctx = req->ctx; - struct io_uring_files_update up; + struct io_uring_rsrc_update up; int ret; if (force_nonblock) return -EAGAIN; - up.offset = req->files_update.offset; - up.fds = req->files_update.arg; + up.offset = req->rsrc_update.offset; + up.data = req->rsrc_update.arg; mutex_lock(&ctx->uring_lock); - ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args); + ret = __io_sqe_files_update(ctx, &up, req->rsrc_update.nr_args); mutex_unlock(&ctx->uring_lock); if (ret < 0) @@ -6075,7 +6080,7 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) case IORING_OP_CLOSE: return io_close_prep(req, sqe); case IORING_OP_FILES_UPDATE: - return io_files_update_prep(req, sqe); + return io_rsrc_update_prep(req, sqe); case IORING_OP_STATX: return io_statx_prep(req, sqe); case IORING_OP_FADVISE: @@ -6444,7 +6449,7 @@ static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work) static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, int index) { - struct fixed_file_table *table; + struct fixed_rsrc_table *table; table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT]; return table->files[index & IORING_FILE_TABLE_MASK]; @@ -6840,7 +6845,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->ctx = ctx; req->flags = 0; req->link = NULL; - req->fixed_file_refs = NULL; + req->fixed_rsrc_refs = NULL; /* one is dropped after submission, the other at completion */ refcount_set(&req->refs, 2); req->task = current; @@ -7328,28 +7333,28 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) #endif } -static void io_file_ref_kill(struct percpu_ref *ref) +static void io_rsrc_ref_kill(struct percpu_ref *ref) { - struct fixed_file_data *data; + struct fixed_rsrc_data *data; - data = container_of(ref, struct fixed_file_data, refs); + data = container_of(ref, struct fixed_rsrc_data, refs); complete(&data->done); } -static void io_sqe_files_set_node(struct fixed_file_data *file_data, - struct fixed_file_ref_node *ref_node) +static void io_sqe_rsrc_set_node(struct fixed_rsrc_data *rsrc_data, + struct fixed_rsrc_ref_node *ref_node) { - spin_lock_bh(&file_data->lock); - file_data->node = ref_node; - list_add_tail(&ref_node->node, &file_data->ref_list); - spin_unlock_bh(&file_data->lock); - percpu_ref_get(&file_data->refs); + spin_lock_bh(&rsrc_data->lock); + rsrc_data->node = ref_node; + list_add_tail(&ref_node->node, &rsrc_data->ref_list); + spin_unlock_bh(&rsrc_data->lock); + percpu_ref_get(&rsrc_data->refs); } static int io_sqe_files_unregister(struct io_ring_ctx *ctx) { - struct fixed_file_data *data = ctx->file_data; - struct fixed_file_ref_node *backup_node, *ref_node = NULL; + struct fixed_rsrc_data *data = ctx->file_data; + struct fixed_rsrc_ref_node *backup_node, *ref_node = NULL; unsigned nr_tables, i; int ret; @@ -7368,7 +7373,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) percpu_ref_kill(&data->refs); /* wait for all refs nodes to complete */ - flush_delayed_work(&ctx->file_put_work); + flush_delayed_work(&ctx->rsrc_put_work); do { ret = wait_for_completion_interruptible(&data->done); if (!ret) @@ -7377,7 +7382,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (ret < 0) { percpu_ref_resurrect(&data->refs); reinit_completion(&data->done); - io_sqe_files_set_node(data, backup_node); + io_sqe_rsrc_set_node(data, backup_node); return ret; } } while (1); @@ -7391,7 +7396,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) kfree(data); ctx->file_data = NULL; ctx->nr_user_files = 0; - destroy_fixed_file_ref_node(backup_node); + destroy_fixed_rsrc_ref_node(backup_node); return 0; } @@ -7614,13 +7619,13 @@ static int io_sqe_files_scm(struct io_ring_ctx *ctx) } #endif -static int io_sqe_alloc_file_tables(struct fixed_file_data *file_data, +static int io_sqe_alloc_file_tables(struct fixed_rsrc_data *file_data, unsigned nr_tables, unsigned nr_files) { int i; for (i = 0; i < nr_tables; i++) { - struct fixed_file_table *table = &file_data->table[i]; + struct fixed_rsrc_table *table = &file_data->table[i]; unsigned this_files; this_files = min(nr_files, IORING_MAX_FILES_TABLE); @@ -7635,7 +7640,7 @@ static int io_sqe_alloc_file_tables(struct fixed_file_data *file_data, return 0; for (i = 0; i < nr_tables; i++) { - struct fixed_file_table *table = &file_data->table[i]; + struct fixed_rsrc_table *table = &file_data->table[i]; kfree(table->files); } return 1; @@ -7703,56 +7708,51 @@ static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file) #endif } -struct io_file_put { - struct list_head list; - struct file *file; -}; - -static void __io_file_put_work(struct fixed_file_ref_node *ref_node) +static void __io_rsrc_put_work(struct fixed_rsrc_ref_node *ref_node) { - struct fixed_file_data *file_data = ref_node->file_data; - struct io_ring_ctx *ctx = file_data->ctx; - struct io_file_put *pfile, *tmp; + struct fixed_rsrc_data *rsrc_data = ref_node->rsrc_data; + struct io_ring_ctx *ctx = rsrc_data->ctx; + struct io_rsrc_put *prsrc, *tmp; - list_for_each_entry_safe(pfile, tmp, &ref_node->file_list, list) { - list_del(&pfile->list); - io_ring_file_put(ctx, pfile->file); - kfree(pfile); + list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) { + list_del(&prsrc->list); + io_ring_file_put(ctx, prsrc->file); + kfree(prsrc); } percpu_ref_exit(&ref_node->refs); kfree(ref_node); - percpu_ref_put(&file_data->refs); + percpu_ref_put(&rsrc_data->refs); } -static void io_file_put_work(struct work_struct *work) +static void io_rsrc_put_work(struct work_struct *work) { struct io_ring_ctx *ctx; struct llist_node *node; - ctx = container_of(work, struct io_ring_ctx, file_put_work.work); - node = llist_del_all(&ctx->file_put_llist); + ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work); + node = llist_del_all(&ctx->rsrc_put_llist); while (node) { - struct fixed_file_ref_node *ref_node; + struct fixed_rsrc_ref_node *ref_node; struct llist_node *next = node->next; - ref_node = llist_entry(node, struct fixed_file_ref_node, llist); - __io_file_put_work(ref_node); + ref_node = llist_entry(node, struct fixed_rsrc_ref_node, llist); + __io_rsrc_put_work(ref_node); node = next; } } -static void io_file_data_ref_zero(struct percpu_ref *ref) +static void io_rsrc_data_ref_zero(struct percpu_ref *ref) { - struct fixed_file_ref_node *ref_node; - struct fixed_file_data *data; + struct fixed_rsrc_ref_node *ref_node; + struct fixed_rsrc_data *data; struct io_ring_ctx *ctx; bool first_add = false; int delay = HZ; - ref_node = container_of(ref, struct fixed_file_ref_node, refs); - data = ref_node->file_data; + ref_node = container_of(ref, struct fixed_rsrc_ref_node, refs); + data = ref_node->rsrc_data; ctx = data->ctx; spin_lock_bh(&data->lock); @@ -7760,12 +7760,12 @@ static void io_file_data_ref_zero(struct percpu_ref *ref) while (!list_empty(&data->ref_list)) { ref_node = list_first_entry(&data->ref_list, - struct fixed_file_ref_node, node); + struct fixed_rsrc_ref_node, node); /* recycle ref nodes in order */ if (!ref_node->done) break; list_del(&ref_node->node); - first_add |= llist_add(&ref_node->llist, &ctx->file_put_llist); + first_add |= llist_add(&ref_node->llist, &ctx->rsrc_put_llist); } spin_unlock_bh(&data->lock); @@ -7773,33 +7773,33 @@ static void io_file_data_ref_zero(struct percpu_ref *ref) delay = 0; if (!delay) - mod_delayed_work(system_wq, &ctx->file_put_work, 0); + mod_delayed_work(system_wq, &ctx->rsrc_put_work, 0); else if (first_add) - queue_delayed_work(system_wq, &ctx->file_put_work, delay); + queue_delayed_work(system_wq, &ctx->rsrc_put_work, delay); } -static struct fixed_file_ref_node *alloc_fixed_file_ref_node( +static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( struct io_ring_ctx *ctx) { - struct fixed_file_ref_node *ref_node; + struct fixed_rsrc_ref_node *ref_node; ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL); if (!ref_node) return NULL; - if (percpu_ref_init(&ref_node->refs, io_file_data_ref_zero, + if (percpu_ref_init(&ref_node->refs, io_rsrc_data_ref_zero, 0, GFP_KERNEL)) { kfree(ref_node); return NULL; } INIT_LIST_HEAD(&ref_node->node); - INIT_LIST_HEAD(&ref_node->file_list); - ref_node->file_data = ctx->file_data; + INIT_LIST_HEAD(&ref_node->rsrc_list); + ref_node->rsrc_data = ctx->file_data; ref_node->done = false; return ref_node; } -static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node) +static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node) { percpu_ref_exit(&ref_node->refs); kfree(ref_node); @@ -7812,8 +7812,8 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_tables, i; struct file *file; int fd, ret = -ENOMEM; - struct fixed_file_ref_node *ref_node; - struct fixed_file_data *file_data; + struct fixed_rsrc_ref_node *ref_node; + struct fixed_rsrc_data *file_data; if (ctx->file_data) return -EBUSY; @@ -7836,7 +7836,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, if (!file_data->table) goto out_free; - if (percpu_ref_init(&file_data->refs, io_file_ref_kill, + if (percpu_ref_init(&file_data->refs, io_rsrc_ref_kill, PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) goto out_free; @@ -7845,7 +7845,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, ctx->file_data = file_data; for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { - struct fixed_file_table *table; + struct fixed_rsrc_table *table; unsigned index; if (copy_from_user(&fd, &fds[i], sizeof(fd))) { @@ -7889,7 +7889,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return -ENOMEM; } - io_sqe_files_set_node(file_data, ref_node); + io_sqe_rsrc_set_node(file_data, ref_node); return ret; out_fput: for (i = 0; i < ctx->nr_user_files; i++) { @@ -7952,28 +7952,34 @@ static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file, #endif } -static int io_queue_file_removal(struct fixed_file_data *data, - struct file *file) +static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, + struct file *rsrc) { - struct io_file_put *pfile; - struct fixed_file_ref_node *ref_node = data->node; + struct io_rsrc_put *prsrc; + struct fixed_rsrc_ref_node *ref_node = data->node; - pfile = kzalloc(sizeof(*pfile), GFP_KERNEL); - if (!pfile) + prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL); + if (!prsrc) return -ENOMEM; - pfile->file = file; - list_add(&pfile->list, &ref_node->file_list); + prsrc->file = rsrc; + list_add(&prsrc->list, &ref_node->rsrc_list); return 0; } +static inline int io_queue_file_removal(struct fixed_rsrc_data *data, + struct file *file) +{ + return io_queue_rsrc_removal(data, file); +} + static int __io_sqe_files_update(struct io_ring_ctx *ctx, - struct io_uring_files_update *up, + struct io_uring_rsrc_update *up, unsigned nr_args) { - struct fixed_file_data *data = ctx->file_data; - struct fixed_file_ref_node *ref_node; + struct fixed_rsrc_data *data = ctx->file_data; + struct fixed_rsrc_ref_node *ref_node; struct file *file; __s32 __user *fds; int fd, i, err; @@ -7990,9 +7996,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, return -ENOMEM; done = 0; - fds = u64_to_user_ptr(up->fds); + fds = u64_to_user_ptr(up->data); while (nr_args) { - struct fixed_file_table *table; + struct fixed_rsrc_table *table; unsigned index; err = 0; @@ -8045,9 +8051,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, if (needs_switch) { percpu_ref_kill(&data->node->refs); - io_sqe_files_set_node(data, ref_node); + io_sqe_rsrc_set_node(data, ref_node); } else - destroy_fixed_file_ref_node(ref_node); + destroy_fixed_rsrc_ref_node(ref_node); return done ? done : err; } @@ -8055,7 +8061,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) { - struct io_uring_files_update up; + struct io_uring_rsrc_update up; if (!ctx->file_data) return -ENXIO; @@ -9482,7 +9488,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1); seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files); for (i = 0; has_lock && i < ctx->nr_user_files; i++) { - struct fixed_file_table *table; + struct fixed_rsrc_table *table; struct file *f; table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index d31a2a1e8ef9..f9f106c54d90 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -285,12 +285,19 @@ enum { IORING_REGISTER_LAST }; +/* deprecated, see struct io_uring_rsrc_update */ struct io_uring_files_update { __u32 offset; __u32 resv; __aligned_u64 /* __s32 * */ fds; }; +struct io_uring_rsrc_update { + __u32 offset; + __u32 resv; + __aligned_u64 data; +}; + #define IO_URING_OP_SUPPORTED (1U << 0) struct io_uring_probe_op { -- cgit v1.2.3 From 5023853183699dd1e3e47622c03d7ae11343837a Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:45 +0000 Subject: io_uring: generalize io_queue_rsrc_removal Generalize io_queue_rsrc_removal to handle both files and buffers. Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh [remove io_mapped_ubuf from rsrc tables/etc. for now] Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 95b3b8747a65..e52800e19c60 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -195,9 +195,14 @@ struct io_mapped_ubuf { unsigned long acct_pages; }; +struct io_ring_ctx; + struct io_rsrc_put { struct list_head list; - struct file *file; + union { + void *rsrc; + struct file *file; + }; }; struct fixed_rsrc_table { @@ -209,6 +214,8 @@ struct fixed_rsrc_ref_node { struct list_head node; struct list_head rsrc_list; struct fixed_rsrc_data *rsrc_data; + void (*rsrc_put)(struct io_ring_ctx *ctx, + struct io_rsrc_put *prsrc); struct llist_node llist; bool done; }; @@ -7646,8 +7653,9 @@ static int io_sqe_alloc_file_tables(struct fixed_rsrc_data *file_data, return 1; } -static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file) +static void io_ring_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) { + struct file *file = prsrc->file; #if defined(CONFIG_UNIX) struct sock *sock = ctx->ring_sock->sk; struct sk_buff_head list, *head = &sock->sk_receive_queue; @@ -7716,7 +7724,7 @@ static void __io_rsrc_put_work(struct fixed_rsrc_ref_node *ref_node) list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) { list_del(&prsrc->list); - io_ring_file_put(ctx, prsrc->file); + ref_node->rsrc_put(ctx, prsrc); kfree(prsrc); } @@ -7795,6 +7803,7 @@ static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( INIT_LIST_HEAD(&ref_node->node); INIT_LIST_HEAD(&ref_node->rsrc_list); ref_node->rsrc_data = ctx->file_data; + ref_node->rsrc_put = io_ring_file_put; ref_node->done = false; return ref_node; } @@ -7952,8 +7961,7 @@ static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file, #endif } -static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, - struct file *rsrc) +static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, void *rsrc) { struct io_rsrc_put *prsrc; struct fixed_rsrc_ref_node *ref_node = data->node; @@ -7962,7 +7970,7 @@ static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, if (!prsrc) return -ENOMEM; - prsrc->file = rsrc; + prsrc->rsrc = rsrc; list_add(&prsrc->list, &ref_node->rsrc_list); return 0; @@ -7971,7 +7979,7 @@ static int io_queue_rsrc_removal(struct fixed_rsrc_data *data, static inline int io_queue_file_removal(struct fixed_rsrc_data *data, struct file *file) { - return io_queue_rsrc_removal(data, file); + return io_queue_rsrc_removal(data, (void *)file); } static int __io_sqe_files_update(struct io_ring_ctx *ctx, -- cgit v1.2.3 From d67d2263fb2350a68074f2cb4dd78549aeebbfae Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:46 +0000 Subject: io_uring: separate ref_list from fixed_rsrc_data Uplevel ref_list and make it common to all resources. This is to allow one common ref_list to be used for both files, and buffers in upcoming patches. Signed-off-by: Bijan Mottahedeh Reviewed-by: Pavel Begunkov Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index e52800e19c60..fcc7a3ed800a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -227,8 +227,6 @@ struct fixed_rsrc_data { struct fixed_rsrc_ref_node *node; struct percpu_ref refs; struct completion done; - struct list_head ref_list; - spinlock_t lock; }; struct io_buffer { @@ -398,6 +396,8 @@ struct io_ring_ctx { struct delayed_work rsrc_put_work; struct llist_head rsrc_put_llist; + struct list_head rsrc_ref_list; + spinlock_t rsrc_ref_lock; struct work_struct exit_work; struct io_restriction restrictions; @@ -1342,6 +1342,8 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->timeout_list); spin_lock_init(&ctx->inflight_lock); INIT_LIST_HEAD(&ctx->inflight_list); + spin_lock_init(&ctx->rsrc_ref_lock); + INIT_LIST_HEAD(&ctx->rsrc_ref_list); INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); return ctx; @@ -7348,13 +7350,14 @@ static void io_rsrc_ref_kill(struct percpu_ref *ref) complete(&data->done); } -static void io_sqe_rsrc_set_node(struct fixed_rsrc_data *rsrc_data, +static void io_sqe_rsrc_set_node(struct io_ring_ctx *ctx, + struct fixed_rsrc_data *rsrc_data, struct fixed_rsrc_ref_node *ref_node) { - spin_lock_bh(&rsrc_data->lock); + spin_lock_bh(&ctx->rsrc_ref_lock); rsrc_data->node = ref_node; - list_add_tail(&ref_node->node, &rsrc_data->ref_list); - spin_unlock_bh(&rsrc_data->lock); + list_add_tail(&ref_node->node, &ctx->rsrc_ref_list); + spin_unlock_bh(&ctx->rsrc_ref_lock); percpu_ref_get(&rsrc_data->refs); } @@ -7371,9 +7374,9 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (!backup_node) return -ENOMEM; - spin_lock_bh(&data->lock); + spin_lock_bh(&ctx->rsrc_ref_lock); ref_node = data->node; - spin_unlock_bh(&data->lock); + spin_unlock_bh(&ctx->rsrc_ref_lock); if (ref_node) percpu_ref_kill(&ref_node->refs); @@ -7389,7 +7392,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (ret < 0) { percpu_ref_resurrect(&data->refs); reinit_completion(&data->done); - io_sqe_rsrc_set_node(data, backup_node); + io_sqe_rsrc_set_node(ctx, data, backup_node); return ret; } } while (1); @@ -7763,11 +7766,11 @@ static void io_rsrc_data_ref_zero(struct percpu_ref *ref) data = ref_node->rsrc_data; ctx = data->ctx; - spin_lock_bh(&data->lock); + spin_lock_bh(&ctx->rsrc_ref_lock); ref_node->done = true; - while (!list_empty(&data->ref_list)) { - ref_node = list_first_entry(&data->ref_list, + while (!list_empty(&ctx->rsrc_ref_list)) { + ref_node = list_first_entry(&ctx->rsrc_ref_list, struct fixed_rsrc_ref_node, node); /* recycle ref nodes in order */ if (!ref_node->done) @@ -7775,7 +7778,7 @@ static void io_rsrc_data_ref_zero(struct percpu_ref *ref) list_del(&ref_node->node); first_add |= llist_add(&ref_node->llist, &ctx->rsrc_put_llist); } - spin_unlock_bh(&data->lock); + spin_unlock_bh(&ctx->rsrc_ref_lock); if (percpu_ref_is_dying(&data->refs)) delay = 0; @@ -7836,8 +7839,6 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return -ENOMEM; file_data->ctx = ctx; init_completion(&file_data->done); - INIT_LIST_HEAD(&file_data->ref_list); - spin_lock_init(&file_data->lock); nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE); file_data->table = kcalloc(nr_tables, sizeof(*file_data->table), @@ -7898,7 +7899,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return -ENOMEM; } - io_sqe_rsrc_set_node(file_data, ref_node); + io_sqe_rsrc_set_node(ctx, file_data, ref_node); return ret; out_fput: for (i = 0; i < ctx->nr_user_files; i++) { @@ -8059,7 +8060,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, if (needs_switch) { percpu_ref_kill(&data->node->refs); - io_sqe_rsrc_set_node(data, ref_node); + io_sqe_rsrc_set_node(ctx, data, ref_node); } else destroy_fixed_rsrc_ref_node(ref_node); -- cgit v1.2.3 From 2a63b2d9c30b2029892c368d11ede1434de6c565 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:47 +0000 Subject: io_uring: add rsrc_ref locking routines Encapsulate resource reference locking into separate routines. Signed-off-by: Bijan Mottahedeh Signed-off-by: Pavel Begunkov Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index fcc7a3ed800a..a129192c20d3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7350,14 +7350,24 @@ static void io_rsrc_ref_kill(struct percpu_ref *ref) complete(&data->done); } +static inline void io_rsrc_ref_lock(struct io_ring_ctx *ctx) +{ + spin_lock_bh(&ctx->rsrc_ref_lock); +} + +static inline void io_rsrc_ref_unlock(struct io_ring_ctx *ctx) +{ + spin_unlock_bh(&ctx->rsrc_ref_lock); +} + static void io_sqe_rsrc_set_node(struct io_ring_ctx *ctx, struct fixed_rsrc_data *rsrc_data, struct fixed_rsrc_ref_node *ref_node) { - spin_lock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_lock(ctx); rsrc_data->node = ref_node; list_add_tail(&ref_node->node, &ctx->rsrc_ref_list); - spin_unlock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_unlock(ctx); percpu_ref_get(&rsrc_data->refs); } @@ -7374,9 +7384,9 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (!backup_node) return -ENOMEM; - spin_lock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_lock(ctx); ref_node = data->node; - spin_unlock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_unlock(ctx); if (ref_node) percpu_ref_kill(&ref_node->refs); @@ -7766,7 +7776,7 @@ static void io_rsrc_data_ref_zero(struct percpu_ref *ref) data = ref_node->rsrc_data; ctx = data->ctx; - spin_lock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_lock(ctx); ref_node->done = true; while (!list_empty(&ctx->rsrc_ref_list)) { @@ -7778,7 +7788,7 @@ static void io_rsrc_data_ref_zero(struct percpu_ref *ref) list_del(&ref_node->node); first_add |= llist_add(&ref_node->llist, &ctx->rsrc_put_llist); } - spin_unlock_bh(&ctx->rsrc_ref_lock); + io_rsrc_ref_unlock(ctx); if (percpu_ref_is_dying(&data->refs)) delay = 0; -- cgit v1.2.3 From 6802535df7bf807c94de32a9d0bf0401d3109671 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:48 +0000 Subject: io_uring: split alloc_fixed_file_ref_node Split alloc_fixed_file_ref_node into resource generic/specific parts, to be leveraged for fixed buffers. Signed-off-by: Bijan Mottahedeh Signed-off-by: Pavel Begunkov Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index a129192c20d3..ab5bf1bf0779 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7799,7 +7799,7 @@ static void io_rsrc_data_ref_zero(struct percpu_ref *ref) queue_delayed_work(system_wq, &ctx->rsrc_put_work, delay); } -static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( +static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( struct io_ring_ctx *ctx) { struct fixed_rsrc_ref_node *ref_node; @@ -7815,9 +7815,21 @@ static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( } INIT_LIST_HEAD(&ref_node->node); INIT_LIST_HEAD(&ref_node->rsrc_list); + ref_node->done = false; + return ref_node; +} + +static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( + struct io_ring_ctx *ctx) +{ + struct fixed_rsrc_ref_node *ref_node; + + ref_node = alloc_fixed_rsrc_ref_node(ctx); + if (!ref_node) + return NULL; + ref_node->rsrc_data = ctx->file_data; ref_node->rsrc_put = io_ring_file_put; - ref_node->done = false; return ref_node; } -- cgit v1.2.3 From bc9744cd162b2f6c38d75dc49c310677dc13afa8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Jan 2021 17:37:49 +0000 Subject: io_uring: split ref_node alloc and init A simple prep patch allowing to set refnode callbacks after it was allocated. This needed to 1) keep ourself off of hi-level functions where it's not pretty and they are not necessary 2) amortise ref_node allocation in the future, e.g. for updates. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index ab5bf1bf0779..bb51f2abd009 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1009,8 +1009,10 @@ static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, struct task_struct *task); static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node); -static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( +static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( struct io_ring_ctx *ctx); +static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, + struct fixed_rsrc_ref_node *ref_node); static void __io_complete_rw(struct io_kiocb *req, long res, long res2, struct io_comp_state *cs); @@ -7380,9 +7382,10 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) if (!data) return -ENXIO; - backup_node = alloc_fixed_file_ref_node(ctx); + backup_node = alloc_fixed_rsrc_ref_node(ctx); if (!backup_node) return -ENOMEM; + init_fixed_file_ref_node(ctx, backup_node); io_rsrc_ref_lock(ctx); ref_node = data->node; @@ -7819,18 +7822,11 @@ static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( return ref_node; } -static struct fixed_rsrc_ref_node *alloc_fixed_file_ref_node( - struct io_ring_ctx *ctx) +static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, + struct fixed_rsrc_ref_node *ref_node) { - struct fixed_rsrc_ref_node *ref_node; - - ref_node = alloc_fixed_rsrc_ref_node(ctx); - if (!ref_node) - return NULL; - ref_node->rsrc_data = ctx->file_data; ref_node->rsrc_put = io_ring_file_put; - return ref_node; } static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node) @@ -7915,11 +7911,12 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, return ret; } - ref_node = alloc_fixed_file_ref_node(ctx); + ref_node = alloc_fixed_rsrc_ref_node(ctx); if (!ref_node) { io_sqe_files_unregister(ctx); return -ENOMEM; } + init_fixed_file_ref_node(ctx, ref_node); io_sqe_rsrc_set_node(ctx, file_data, ref_node); return ret; @@ -8022,9 +8019,10 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, if (done > ctx->nr_user_files) return -EINVAL; - ref_node = alloc_fixed_file_ref_node(ctx); + ref_node = alloc_fixed_rsrc_ref_node(ctx); if (!ref_node) return -ENOMEM; + init_fixed_file_ref_node(ctx, ref_node); done = 0; fds = u64_to_user_ptr(up->data); -- cgit v1.2.3 From d7954b2ba94639b7f5b08760d36e54c28544730f Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:50 +0000 Subject: io_uring: create common fixed_rsrc_ref_node handling routines Create common routines to be used for both files/buffers registration. [remove io_sqe_rsrc_set_node substitution] Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh [merge, quiesce only for files] Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index bb51f2abd009..727d0d3cdbcc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7373,20 +7373,13 @@ static void io_sqe_rsrc_set_node(struct io_ring_ctx *ctx, percpu_ref_get(&rsrc_data->refs); } -static int io_sqe_files_unregister(struct io_ring_ctx *ctx) +static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, + struct io_ring_ctx *ctx, + struct fixed_rsrc_ref_node *backup_node) { - struct fixed_rsrc_data *data = ctx->file_data; - struct fixed_rsrc_ref_node *backup_node, *ref_node = NULL; - unsigned nr_tables, i; + struct fixed_rsrc_ref_node *ref_node; int ret; - if (!data) - return -ENXIO; - backup_node = alloc_fixed_rsrc_ref_node(ctx); - if (!backup_node) - return -ENOMEM; - init_fixed_file_ref_node(ctx, backup_node); - io_rsrc_ref_lock(ctx); ref_node = data->node; io_rsrc_ref_unlock(ctx); @@ -7410,6 +7403,28 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) } } while (1); + destroy_fixed_rsrc_ref_node(backup_node); + return 0; +} + +static int io_sqe_files_unregister(struct io_ring_ctx *ctx) +{ + struct fixed_rsrc_data *data = ctx->file_data; + struct fixed_rsrc_ref_node *backup_node; + unsigned nr_tables, i; + int ret; + + if (!data) + return -ENXIO; + backup_node = alloc_fixed_rsrc_ref_node(ctx); + if (!backup_node) + return -ENOMEM; + init_fixed_file_ref_node(ctx, backup_node); + + ret = io_rsrc_ref_quiesce(data, ctx, backup_node); + if (ret) + return ret; + __io_sqe_files_unregister(ctx); nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE); for (i = 0; i < nr_tables; i++) @@ -7419,7 +7434,6 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) kfree(data); ctx->file_data = NULL; ctx->nr_user_files = 0; - destroy_fixed_rsrc_ref_node(backup_node); return 0; } -- cgit v1.2.3 From 1ad555c6ae6e28ec7b1acaa2af72a9904e6ba96a Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:51 +0000 Subject: io_uring: create common fixed_rsrc_data allocation routines Create common alloc/free fixed_rsrc_data routines for both files and buffers. Reviewed-by: Pavel Begunkov Signed-off-by: Bijan Mottahedeh [remove buffer part] Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 727d0d3cdbcc..8f7d95e0d240 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7407,6 +7407,31 @@ static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, return 0; } +static struct fixed_rsrc_data *alloc_fixed_rsrc_data(struct io_ring_ctx *ctx) +{ + struct fixed_rsrc_data *data; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + + if (percpu_ref_init(&data->refs, io_rsrc_ref_kill, + PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) { + kfree(data); + return NULL; + } + data->ctx = ctx; + init_completion(&data->done); + return data; +} + +static void free_fixed_rsrc_data(struct fixed_rsrc_data *data) +{ + percpu_ref_exit(&data->refs); + kfree(data->table); + kfree(data); +} + static int io_sqe_files_unregister(struct io_ring_ctx *ctx) { struct fixed_rsrc_data *data = ctx->file_data; @@ -7429,9 +7454,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE); for (i = 0; i < nr_tables; i++) kfree(data->table[i].files); - kfree(data->table); - percpu_ref_exit(&data->refs); - kfree(data); + free_fixed_rsrc_data(data); ctx->file_data = NULL; ctx->nr_user_files = 0; return 0; @@ -7866,11 +7889,9 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, if (nr_args > IORING_MAX_FIXED_FILES) return -EMFILE; - file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL); + file_data = alloc_fixed_rsrc_data(ctx); if (!file_data) return -ENOMEM; - file_data->ctx = ctx; - init_completion(&file_data->done); nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE); file_data->table = kcalloc(nr_tables, sizeof(*file_data->table), @@ -7878,12 +7899,8 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, if (!file_data->table) goto out_free; - if (percpu_ref_init(&file_data->refs, io_rsrc_ref_kill, - PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) - goto out_free; - if (io_sqe_alloc_file_tables(file_data, nr_tables, nr_args)) - goto out_ref; + goto out_free; ctx->file_data = file_data; for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { @@ -7943,11 +7960,8 @@ out_fput: for (i = 0; i < nr_tables; i++) kfree(file_data->table[i].files); ctx->nr_user_files = 0; -out_ref: - percpu_ref_exit(&file_data->refs); out_free: - kfree(file_data->table); - kfree(file_data); + free_fixed_rsrc_data(ctx->file_data); ctx->file_data = NULL; return ret; } -- cgit v1.2.3 From 00835dce1406e746fe5ab8c522cceb9594c78acb Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 15 Jan 2021 17:37:52 +0000 Subject: io_uring: make percpu_ref_release names consistent Make the percpu ref release function names consistent between rsrc data and nodes. Signed-off-by: Bijan Mottahedeh Reviewed-by: Pavel Begunkov Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 8f7d95e0d240..98789fece715 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7344,7 +7344,7 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) #endif } -static void io_rsrc_ref_kill(struct percpu_ref *ref) +static void io_rsrc_data_ref_zero(struct percpu_ref *ref) { struct fixed_rsrc_data *data; @@ -7415,7 +7415,7 @@ static struct fixed_rsrc_data *alloc_fixed_rsrc_data(struct io_ring_ctx *ctx) if (!data) return NULL; - if (percpu_ref_init(&data->refs, io_rsrc_ref_kill, + if (percpu_ref_init(&data->refs, io_rsrc_data_ref_zero, PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) { kfree(data); return NULL; @@ -7804,7 +7804,7 @@ static void io_rsrc_put_work(struct work_struct *work) } } -static void io_rsrc_data_ref_zero(struct percpu_ref *ref) +static void io_rsrc_node_ref_zero(struct percpu_ref *ref) { struct fixed_rsrc_ref_node *ref_node; struct fixed_rsrc_data *data; @@ -7848,7 +7848,7 @@ static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( if (!ref_node) return NULL; - if (percpu_ref_init(&ref_node->refs, io_rsrc_data_ref_zero, + if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero, 0, GFP_KERNEL)) { kfree(ref_node); return NULL; -- cgit v1.2.3 From bf6182b6d46e28c3e59b9c0d6097b379cae56b94 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:34 +0000 Subject: io_uring: optimise io_rw_reissue() The hot path is IO completing on the first try. Reshuffle io_rw_reissue() so it's checked first. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 98789fece715..4a8900d480c5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2738,12 +2738,13 @@ end_req: static bool io_rw_reissue(struct io_kiocb *req, long res) { #ifdef CONFIG_BLOCK - umode_t mode = file_inode(req->file)->i_mode; + umode_t mode; int ret; - if (!S_ISBLK(mode) && !S_ISREG(mode)) + if (res != -EAGAIN && res != -EOPNOTSUPP) return false; - if ((res != -EAGAIN && res != -EOPNOTSUPP) || io_wq_current_is_worker()) + mode = file_inode(req->file)->i_mode; + if ((!S_ISBLK(mode) && !S_ISREG(mode)) || io_wq_current_is_worker()) return false; lockdep_assert_held(&req->ctx->uring_lock); -- cgit v1.2.3 From dc2a6e9aa9c349d76c318d22bbe26006fda1ce97 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:35 +0000 Subject: io_uring: refactor io_resubmit_prep() It's awkward to pass return a value into a function for it to return it back. Check it at the caller site and clean up io_resubmit_prep() a bit. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 4a8900d480c5..be2760ae6c23 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2689,17 +2689,16 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res, } #ifdef CONFIG_BLOCK -static bool io_resubmit_prep(struct io_kiocb *req, int error) +static bool io_resubmit_prep(struct io_kiocb *req) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; ssize_t ret = -ECANCELED; struct iov_iter iter; int rw; - if (error) { - ret = error; - goto end_req; - } + /* already prepared */ + if (req->async_data) + return true; switch (req->opcode) { case IORING_OP_READV: @@ -2715,22 +2714,16 @@ static bool io_resubmit_prep(struct io_kiocb *req, int error) default: printk_once(KERN_WARNING "io_uring: bad opcode in resubmit %d\n", req->opcode); - goto end_req; + return false; } - if (!req->async_data) { - ret = io_import_iovec(rw, req, &iovec, &iter, false); - if (ret < 0) - goto end_req; - ret = io_setup_async_rw(req, iovec, inline_vecs, &iter, false); - if (!ret) - return true; - kfree(iovec); - } else { + ret = io_import_iovec(rw, req, &iovec, &iter, false); + if (ret < 0) + return false; + ret = io_setup_async_rw(req, iovec, inline_vecs, &iter, false); + if (!ret) return true; - } -end_req: - req_set_fail_links(req); + kfree(iovec); return false; } #endif @@ -2751,12 +2744,12 @@ static bool io_rw_reissue(struct io_kiocb *req, long res) ret = io_sq_thread_acquire_mm_files(req->ctx, req); - if (io_resubmit_prep(req, ret)) { + if (!ret && io_resubmit_prep(req)) { refcount_inc(&req->refs); io_queue_async_work(req); return true; } - + req_set_fail_links(req); #endif return false; } -- cgit v1.2.3 From 5c766a908d06e96d30e0ec2511a24fa311553d2c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:36 +0000 Subject: io_uring: cleanup personalities under uring_lock personality_idr is usually synchronised by uring_lock, the exception would be removing personalities in io_ring_ctx_wait_and_kill(), which is legit as refs are killed by that point but still would be more resilient to do it under the lock. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index be2760ae6c23..5e576878efd9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8867,6 +8867,7 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) ctx->cq_overflow_flushed = 1; if (ctx->rings) __io_cqring_overflow_flush(ctx, true, NULL, NULL); + idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx); mutex_unlock(&ctx->uring_lock); io_kill_timeouts(ctx, NULL, NULL); @@ -8877,7 +8878,6 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) /* if we failed setting up the ctx, we might not have any rings */ io_iopoll_try_reap_events(ctx); - idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx); /* * Do this upfront, so we won't have a grace period where the ring -- cgit v1.2.3 From 2d7e935809b7f740442ce79fc6f53e94a1f0b874 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:37 +0000 Subject: io_uring: inline io_async_submit() The name is confusing and it's used only in one place. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5e576878efd9..6eb4c25fa18b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1278,11 +1278,6 @@ static inline void io_req_init_async(struct io_kiocb *req) refcount_inc(&req->work.identity->count); } -static inline bool io_async_submit(struct io_ring_ctx *ctx) -{ - return ctx->flags & IORING_SETUP_SQPOLL; -} - static void io_ring_ctx_ref_free(struct percpu_ref *ref) { struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); @@ -6969,7 +6964,7 @@ fail_req: } trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, - true, io_async_submit(ctx)); + true, ctx->flags & IORING_SETUP_SQPOLL); err = io_submit_sqe(req, sqe, &link, &state.comp); if (err) goto fail_req; -- cgit v1.2.3 From ec30e04ba4a5c265f52482092a5f5f5232947c48 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:38 +0000 Subject: io_uring: inline __io_commit_cqring() Inline it in its only user, that's cleaner Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6eb4c25fa18b..347bdcd2c0fe 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1364,14 +1364,6 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) return false; } -static void __io_commit_cqring(struct io_ring_ctx *ctx) -{ - struct io_rings *rings = ctx->rings; - - /* order cqe stores with ring update */ - smp_store_release(&rings->cq.tail, ctx->cached_cq_tail); -} - static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req) { if (req->work.identity == &tctx->__identity) @@ -1693,7 +1685,9 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) static void io_commit_cqring(struct io_ring_ctx *ctx) { io_flush_timeouts(ctx); - __io_commit_cqring(ctx); + + /* order cqe stores with ring update */ + smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); if (unlikely(!list_empty(&ctx->defer_list))) __io_queue_deferred(ctx); -- cgit v1.2.3 From 888aae2eeddfe1d6c9731cf4af1a1b2605af6470 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:39 +0000 Subject: io_uring: further deduplicate #CQ events calc Apparently, there is one more place hand coded calculation of number of CQ events in the ring. Use __io_cqring_events() helper in io_get_cqring() as well. Naturally, assembly stays identical. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 347bdcd2c0fe..0a578c40b854 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1700,21 +1700,25 @@ static inline bool io_sqring_full(struct io_ring_ctx *ctx) return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == r->sq_ring_entries; } +static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) +{ + return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); +} + static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) { struct io_rings *rings = ctx->rings; unsigned tail; - tail = ctx->cached_cq_tail; /* * writes to the cq entry need to come after reading head; the * control dependency is enough as we're using WRITE_ONCE to * fill the cq entry */ - if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries) + if (__io_cqring_events(ctx) == rings->cq_ring_entries) return NULL; - ctx->cached_cq_tail++; + tail = ctx->cached_cq_tail++; return &rings->cqes[tail & ctx->cq_mask]; } @@ -1729,11 +1733,6 @@ static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) return io_wq_current_is_worker(); } -static inline unsigned __io_cqring_events(struct io_ring_ctx *ctx) -{ - return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); -} - static void io_cqring_ev_posted(struct io_ring_ctx *ctx) { /* see waitqueue_active() comment */ -- cgit v1.2.3 From 85bcb6c67ea145b8032089db891218e3339cbdb8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:40 +0000 Subject: io_uring: simplify io_alloc_req() Get rid of a label in io_alloc_req(), it's cleaner to do return directly. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 0a578c40b854..9ff84ceff4f9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1988,7 +1988,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx, if (unlikely(ret <= 0)) { state->reqs[0] = kmem_cache_alloc(req_cachep, gfp); if (!state->reqs[0]) - goto fallback; + return io_get_fallback_req(ctx); ret = 1; } state->free_reqs = ret; @@ -1996,8 +1996,6 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx, state->free_reqs--; return state->reqs[state->free_reqs]; -fallback: - return io_get_fallback_req(ctx); } static inline void io_put_file(struct io_kiocb *req, struct file *file, -- cgit v1.2.3 From 02b23a9af5ba4db0a85ebb81c8b376b2fe860d0f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:41 +0000 Subject: io_uring: remove __io_state_file_put The check in io_state_file_put() is optimised pretty well when called from __io_file_get(). Don't pollute the code with all these variants. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 9ff84ceff4f9..c3e0d6246d71 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2815,16 +2815,12 @@ static void io_iopoll_req_issued(struct io_kiocb *req, bool in_async) wake_up(&ctx->sq_data->wait); } -static inline void __io_state_file_put(struct io_submit_state *state) -{ - fput_many(state->file, state->file_refs); - state->file_refs = 0; -} - static inline void io_state_file_put(struct io_submit_state *state) { - if (state->file_refs) - __io_state_file_put(state); + if (state->file_refs) { + fput_many(state->file, state->file_refs); + state->file_refs = 0; + } } /* @@ -2842,7 +2838,7 @@ static struct file *__io_file_get(struct io_submit_state *state, int fd) state->file_refs--; return state->file; } - __io_state_file_put(state); + io_state_file_put(state); } state->file = fget_many(fd, state->ios_left); if (unlikely(!state->file)) -- cgit v1.2.3 From eab30c4d20dc761d463445e5130421863ff81505 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:42 +0000 Subject: io_uring: deduplicate failing task_work_add When io_req_task_work_add() fails, the request will be cancelled by enqueueing via task_works of io-wq. Extract a function for that. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index c3e0d6246d71..90c3cad1723b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2171,6 +2171,16 @@ static int io_req_task_work_add(struct io_kiocb *req) return ret; } +static void io_req_task_work_add_fallback(struct io_kiocb *req, + void (*cb)(struct callback_head *)) +{ + struct task_struct *tsk = io_wq_get_task(req->ctx->io_wq); + + init_task_work(&req->task_work, cb); + task_work_add(tsk, &req->task_work, TWA_NONE); + wake_up_process(tsk); +} + static void __io_req_task_cancel(struct io_kiocb *req, int error) { struct io_ring_ctx *ctx = req->ctx; @@ -2225,14 +2235,8 @@ static void io_req_task_queue(struct io_kiocb *req) percpu_ref_get(&req->ctx->refs); ret = io_req_task_work_add(req); - if (unlikely(ret)) { - struct task_struct *tsk; - - init_task_work(&req->task_work, io_req_task_cancel); - tsk = io_wq_get_task(req->ctx->io_wq); - task_work_add(tsk, &req->task_work, TWA_NONE); - wake_up_process(tsk); - } + if (unlikely(ret)) + io_req_task_work_add_fallback(req, io_req_task_cancel); } static inline void io_queue_next(struct io_kiocb *req) @@ -2350,13 +2354,8 @@ static void io_free_req_deferred(struct io_kiocb *req) init_task_work(&req->task_work, io_put_req_deferred_cb); ret = io_req_task_work_add(req); - if (unlikely(ret)) { - struct task_struct *tsk; - - tsk = io_wq_get_task(req->ctx->io_wq); - task_work_add(tsk, &req->task_work, TWA_NONE); - wake_up_process(tsk); - } + if (unlikely(ret)) + io_req_task_work_add_fallback(req, io_put_req_deferred_cb); } static inline void io_put_req_deferred(struct io_kiocb *req, int refs) @@ -3425,15 +3424,8 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, /* submit ref gets dropped, acquire a new one */ refcount_inc(&req->refs); ret = io_req_task_work_add(req); - if (unlikely(ret)) { - struct task_struct *tsk; - - /* queue just for cancelation */ - init_task_work(&req->task_work, io_req_task_cancel); - tsk = io_wq_get_task(req->ctx->io_wq); - task_work_add(tsk, &req->task_work, TWA_NONE); - wake_up_process(tsk); - } + if (unlikely(ret)) + io_req_task_work_add_fallback(req, io_req_task_cancel); return 1; } @@ -5153,12 +5145,8 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, */ ret = io_req_task_work_add(req); if (unlikely(ret)) { - struct task_struct *tsk; - WRITE_ONCE(poll->canceled, true); - tsk = io_wq_get_task(req->ctx->io_wq); - task_work_add(tsk, &req->task_work, TWA_NONE); - wake_up_process(tsk); + io_req_task_work_add_fallback(req, func); } return 1; } -- cgit v1.2.3 From 8662daec09edcdba2659799040aee1ba575c4799 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:44 +0000 Subject: io_uring: add a helper timeout mode calculation Deduplicates translation of timeout flags into hrtimer_mode. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 90c3cad1723b..4e167217c898 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5771,6 +5771,12 @@ static int io_timeout_remove_prep(struct io_kiocb *req, return 0; } +static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) +{ + return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS + : HRTIMER_MODE_REL; +} + /* * Remove or update an existing timeout command */ @@ -5781,14 +5787,11 @@ static int io_timeout_remove(struct io_kiocb *req) int ret; spin_lock_irq(&ctx->completion_lock); - if (req->timeout_rem.flags & IORING_TIMEOUT_UPDATE) { - enum hrtimer_mode mode = (tr->flags & IORING_TIMEOUT_ABS) - ? HRTIMER_MODE_ABS : HRTIMER_MODE_REL; - - ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode); - } else { + if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) ret = io_timeout_cancel(ctx, tr->addr); - } + else + ret = io_timeout_update(ctx, tr->addr, &tr->ts, + io_translate_timeout_mode(tr->flags)); io_cqring_fill_event(req, ret); io_commit_cqring(ctx); @@ -5828,11 +5831,7 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) return -EFAULT; - if (flags & IORING_TIMEOUT_ABS) - data->mode = HRTIMER_MODE_ABS; - else - data->mode = HRTIMER_MODE_REL; - + data->mode = io_translate_timeout_mode(flags); hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode); return 0; } -- cgit v1.2.3 From a38d68db6742c19a74141c0f56785ef67f51c504 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:45 +0000 Subject: io_uring: help inlining of io_req_complete() __io_req_complete() inlining is a bit weird, some compilers don't optimise out the non-NULL branch of it even when called as io_req_complete(). Help it a bit by extracting state and stateless helpers out of __io_req_complete(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 4e167217c898..f676b198ee1b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1886,7 +1886,8 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) __io_cqring_fill_event(req, res, 0); } -static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags) +static void io_req_complete_nostate(struct io_kiocb *req, long res, + unsigned int cflags) { struct io_ring_ctx *ctx = req->ctx; unsigned long flags; @@ -1897,6 +1898,7 @@ static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags) spin_unlock_irqrestore(&ctx->completion_lock, flags); io_cqring_ev_posted(ctx); + io_put_req(req); } static void io_submit_flush_completions(struct io_comp_state *cs) @@ -1932,23 +1934,27 @@ static void io_submit_flush_completions(struct io_comp_state *cs) cs->nr = 0; } -static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags, - struct io_comp_state *cs) +static void io_req_complete_state(struct io_kiocb *req, long res, + unsigned int cflags, struct io_comp_state *cs) { - if (!cs) { - io_cqring_add_event(req, res, cflags); - io_put_req(req); - } else { - io_clean_op(req); - req->result = res; - req->compl.cflags = cflags; - list_add_tail(&req->compl.list, &cs->list); - if (++cs->nr >= 32) - io_submit_flush_completions(cs); - } + io_clean_op(req); + req->result = res; + req->compl.cflags = cflags; + list_add_tail(&req->compl.list, &cs->list); + if (++cs->nr >= 32) + io_submit_flush_completions(cs); +} + +static inline void __io_req_complete(struct io_kiocb *req, long res, + unsigned cflags, struct io_comp_state *cs) +{ + if (!cs) + io_req_complete_nostate(req, res, cflags); + else + io_req_complete_state(req, res, cflags, cs); } -static void io_req_complete(struct io_kiocb *req, long res) +static inline void io_req_complete(struct io_kiocb *req, long res) { __io_req_complete(req, res, 0, NULL); } -- cgit v1.2.3 From 9affd664f0e0512d8997dbdddb1448a4faf9bc82 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:46 +0000 Subject: io_uring: don't flush CQEs deep down the stack io_submit_flush_completions() is called down the stack in the _state version of io_req_complete(), that's ok because is only called by io_uring opcode handler functions directly. Move it up to __io_queue_sqe() as preparation. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index f676b198ee1b..935a16a682a2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1941,8 +1941,7 @@ static void io_req_complete_state(struct io_kiocb *req, long res, req->result = res; req->compl.cflags = cflags; list_add_tail(&req->compl.list, &cs->list); - if (++cs->nr >= 32) - io_submit_flush_completions(cs); + cs->nr++; } static inline void __io_req_complete(struct io_kiocb *req, long res, @@ -6577,7 +6576,15 @@ again: io_queue_linked_timeout(linked_timeout); } else if (likely(!ret)) { /* drop submission reference */ - req = io_put_req_find_next(req); + if (cs) { + io_put_req(req); + if (cs->nr >= 32) + io_submit_flush_completions(cs); + req = NULL; + } else { + req = io_put_req_find_next(req); + } + if (linked_timeout) io_queue_linked_timeout(linked_timeout); -- cgit v1.2.3 From e342c807f556dbcee1370ab78af1d8faf497d771 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 19 Jan 2021 13:32:47 +0000 Subject: io_uring: save atomic dec for inline executed reqs When a request is completed with comp_state, its completion reference put is deferred to io_submit_flush_completions(), but the submission is put not far from there, so do it together to save one atomic dec per request. That targets requests that complete inline, e.g. buffered rw, send/recv. Proper benchmarking haven't been conducted but for nops(batch=32) it was around 7901 vs 8117 KIOPS (~2.7%), or ~4% per perf profiling. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 935a16a682a2..3f6d055eb6d4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -629,6 +629,7 @@ enum { REQ_F_NO_FILE_TABLE_BIT, REQ_F_WORK_INITIALIZED_BIT, REQ_F_LTIMEOUT_ACTIVE_BIT, + REQ_F_COMPLETE_INLINE_BIT, /* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT, @@ -672,6 +673,8 @@ enum { REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT), /* linked timeout is active, i.e. prepared by link's head */ REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT), + /* completion is deferred through io_comp_state */ + REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT), }; struct async_poll { @@ -1917,14 +1920,15 @@ static void io_submit_flush_completions(struct io_comp_state *cs) * io_free_req() doesn't care about completion_lock unless one * of these flags is set. REQ_F_WORK_INITIALIZED is in the list * because of a potential deadlock with req->work.fs->lock + * We defer both, completion and submission refs. */ if (req->flags & (REQ_F_FAIL_LINK|REQ_F_LINK_TIMEOUT |REQ_F_WORK_INITIALIZED)) { spin_unlock_irq(&ctx->completion_lock); - io_put_req(req); + io_double_put_req(req); spin_lock_irq(&ctx->completion_lock); } else { - io_put_req(req); + io_double_put_req(req); } } io_commit_cqring(ctx); @@ -1940,8 +1944,7 @@ static void io_req_complete_state(struct io_kiocb *req, long res, io_clean_op(req); req->result = res; req->compl.cflags = cflags; - list_add_tail(&req->compl.list, &cs->list); - cs->nr++; + req->flags |= REQ_F_COMPLETE_INLINE; } static inline void __io_req_complete(struct io_kiocb *req, long res, @@ -6576,9 +6579,9 @@ again: io_queue_linked_timeout(linked_timeout); } else if (likely(!ret)) { /* drop submission reference */ - if (cs) { - io_put_req(req); - if (cs->nr >= 32) + if (req->flags & REQ_F_COMPLETE_INLINE) { + list_add_tail(&req->compl.list, &cs->list); + if (++cs->nr >= 32) io_submit_flush_completions(cs); req = NULL; } else { -- cgit v1.2.3 From 53dec2ea74f2ef360e8455439be96a780baa6097 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 19 Jan 2021 15:41:52 -0700 Subject: fs: provide locked helper variant of close_fd_get_file() Assumes current->files->file_lock is already held on invocation. Helps the caller check the file before removing the fd, if it needs to. Signed-off-by: Jens Axboe --- fs/file.c | 36 +++++++++++++++++++++++++----------- fs/internal.h | 1 + 2 files changed, 26 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/file.c b/fs/file.c index dab120b71e44..f3a4bac2cbe9 100644 --- a/fs/file.c +++ b/fs/file.c @@ -22,6 +22,8 @@ #include #include +#include "internal.h" + unsigned int sysctl_nr_open __read_mostly = 1024*1024; unsigned int sysctl_nr_open_min = BITS_PER_LONG; /* our min() is unusable in constant expressions ;-/ */ @@ -732,36 +734,48 @@ int __close_range(unsigned fd, unsigned max_fd, unsigned int flags) } /* - * variant of close_fd that gets a ref on the file for later fput. - * The caller must ensure that filp_close() called on the file, and then - * an fput(). + * See close_fd_get_file() below, this variant assumes current->files->file_lock + * is held. */ -int close_fd_get_file(unsigned int fd, struct file **res) +int __close_fd_get_file(unsigned int fd, struct file **res) { struct files_struct *files = current->files; struct file *file; struct fdtable *fdt; - spin_lock(&files->file_lock); fdt = files_fdtable(files); if (fd >= fdt->max_fds) - goto out_unlock; + goto out_err; file = fdt->fd[fd]; if (!file) - goto out_unlock; + goto out_err; rcu_assign_pointer(fdt->fd[fd], NULL); __put_unused_fd(files, fd); - spin_unlock(&files->file_lock); get_file(file); *res = file; return 0; - -out_unlock: - spin_unlock(&files->file_lock); +out_err: *res = NULL; return -ENOENT; } +/* + * variant of close_fd that gets a ref on the file for later fput. + * The caller must ensure that filp_close() called on the file, and then + * an fput(). + */ +int close_fd_get_file(unsigned int fd, struct file **res) +{ + struct files_struct *files = current->files; + int ret; + + spin_lock(&files->file_lock); + ret = __close_fd_get_file(fd, res); + spin_unlock(&files->file_lock); + + return ret; +} + void do_close_on_exec(struct files_struct *files) { unsigned i; diff --git a/fs/internal.h b/fs/internal.h index 77c50befbfbe..c6c85f6ad598 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -132,6 +132,7 @@ extern struct file *do_file_open_root(struct dentry *, struct vfsmount *, const char *, const struct open_flags *); extern struct open_how build_open_how(int flags, umode_t mode); extern int build_open_flags(const struct open_how *how, struct open_flags *op); +extern int __close_fd_get_file(unsigned int fd, struct file **res); long do_sys_ftruncate(unsigned int fd, loff_t length, int small); int chmod_common(const struct path *path, umode_t mode); -- cgit v1.2.3 From 9eac1904d3364254d622bf2c771c4f85cd435fc2 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 19 Jan 2021 15:50:37 -0700 Subject: io_uring: get rid of intermediate IORING_OP_CLOSE stage We currently split the close into two, in case we have a ->flush op that we can't safely handle from non-blocking context. This requires us to flag the op as uncancelable if we do need to punt it async, and that means special handling for just this op type. Use __close_fd_get_file() and grab the files lock so we can get the file and check if we need to go async in one atomic operation. That gets rid of the need for splitting this into two steps, and hence the need for IO_WQ_WORK_NO_CANCEL. Signed-off-by: Jens Axboe --- fs/io_uring.c | 64 ++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 29 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 3f6d055eb6d4..4dd18c81789c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -423,7 +423,6 @@ struct io_poll_remove { struct io_close { struct file *file; - struct file *put_file; int fd; }; @@ -920,8 +919,6 @@ static const struct io_op_def io_op_defs[] = { IO_WQ_WORK_FS | IO_WQ_WORK_MM, }, [IORING_OP_CLOSE] = { - .needs_file = 1, - .needs_file_no_error = 1, .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG, }, [IORING_OP_FILES_UPDATE] = { @@ -4475,13 +4472,6 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock) static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - /* - * If we queue this for async, it must not be cancellable. That would - * leave the 'file' in an undeterminate state, and here need to modify - * io_wq_work.flags, so initialize io_wq_work firstly. - */ - io_req_init_async(req); - if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; if (sqe->ioprio || sqe->off || sqe->addr || sqe->len || @@ -4491,43 +4481,59 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EBADF; req->close.fd = READ_ONCE(sqe->fd); - if ((req->file && req->file->f_op == &io_uring_fops)) - return -EBADF; - - req->close.put_file = NULL; return 0; } static int io_close(struct io_kiocb *req, bool force_nonblock, struct io_comp_state *cs) { + struct files_struct *files = current->files; struct io_close *close = &req->close; + struct fdtable *fdt; + struct file *file; int ret; - /* might be already done during nonblock submission */ - if (!close->put_file) { - ret = close_fd_get_file(close->fd, &close->put_file); - if (ret < 0) - return (ret == -ENOENT) ? -EBADF : ret; + file = NULL; + ret = -EBADF; + spin_lock(&files->file_lock); + fdt = files_fdtable(files); + if (close->fd >= fdt->max_fds) { + spin_unlock(&files->file_lock); + goto err; + } + file = fdt->fd[close->fd]; + if (!file) { + spin_unlock(&files->file_lock); + goto err; + } + + if (file->f_op == &io_uring_fops) { + spin_unlock(&files->file_lock); + file = NULL; + goto err; } /* if the file has a flush method, be safe and punt to async */ - if (close->put_file->f_op->flush && force_nonblock) { - /* not safe to cancel at this point */ - req->work.flags |= IO_WQ_WORK_NO_CANCEL; - /* was never set, but play safe */ - req->flags &= ~REQ_F_NOWAIT; - /* avoid grabbing files - we don't need the files */ - req->flags |= REQ_F_NO_FILE_TABLE; + if (file->f_op->flush && force_nonblock) { + spin_unlock(&files->file_lock); return -EAGAIN; } + ret = __close_fd_get_file(close->fd, &file); + spin_unlock(&files->file_lock); + if (ret < 0) { + if (ret == -ENOENT) + ret = -EBADF; + goto err; + } + /* No ->flush() or already async, safely close from here */ - ret = filp_close(close->put_file, req->work.identity->files); + ret = filp_close(file, current->files); +err: if (ret < 0) req_set_fail_links(req); - fput(close->put_file); - close->put_file = NULL; + if (file) + fput(file); __io_req_complete(req, ret, 0, cs); return 0; } -- cgit v1.2.3 From 4014d943cb62db892eb023d385a966a3fce5ee4c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 19 Jan 2021 15:53:54 -0700 Subject: io_uring/io-wq: kill off now unused IO_WQ_WORK_NO_CANCEL It's no longer used as IORING_OP_CLOSE got rid for the need of flagging it as uncancelable, kill it of. Signed-off-by: Jens Axboe --- fs/io-wq.c | 1 - fs/io-wq.h | 1 - fs/io_uring.c | 5 +---- 3 files changed, 1 insertion(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index a564f36e260c..2e2f14f42bf2 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -944,7 +944,6 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data) */ spin_lock_irqsave(&worker->lock, flags); if (worker->cur_work && - !(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL) && match->fn(worker->cur_work, match->data)) { send_sig(SIGINT, worker->task, 1); match->nr_running++; diff --git a/fs/io-wq.h b/fs/io-wq.h index b158f8addcf3..e1ffb80a4a1d 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -9,7 +9,6 @@ enum { IO_WQ_WORK_CANCEL = 1, IO_WQ_WORK_HASHED = 2, IO_WQ_WORK_UNBOUND = 4, - IO_WQ_WORK_NO_CANCEL = 8, IO_WQ_WORK_CONCURRENT = 16, IO_WQ_WORK_FILES = 32, diff --git a/fs/io_uring.c b/fs/io_uring.c index 4dd18c81789c..be73f6ddbd9e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6386,11 +6386,8 @@ static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work) if (timeout) io_queue_linked_timeout(timeout); - /* if NO_CANCEL is set, we must still run the work */ - if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) == - IO_WQ_WORK_CANCEL) { + if (work->flags & IO_WQ_WORK_CANCEL) ret = -ECANCELED; - } if (!ret) { do { -- cgit v1.2.3 From 0bead8cd39b9c9c7c4e902018ccf129107ac50ef Mon Sep 17 00:00:00 2001 From: Yejune Deng Date: Thu, 24 Dec 2020 11:02:20 +0800 Subject: io_uring: simplify io_remove_personalities() The function io_remove_personalities() is very similar to io_unregister_personality(),so implement io_remove_personalities() calling io_unregister_personality(). Signed-off-by: Yejune Deng Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index be73f6ddbd9e..b05d37431c12 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8805,9 +8805,8 @@ static int io_uring_fasync(int fd, struct file *file, int on) return fasync_helper(fd, file, on, &ctx->cq_fasync); } -static int io_remove_personalities(int id, void *p, void *data) +static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) { - struct io_ring_ctx *ctx = data; struct io_identity *iod; iod = idr_remove(&ctx->personality_idr, id); @@ -8815,7 +8814,17 @@ static int io_remove_personalities(int id, void *p, void *data) put_cred(iod->creds); if (refcount_dec_and_test(&iod->count)) kfree(iod); + return 0; } + + return -EINVAL; +} + +static int io_remove_personalities(int id, void *p, void *data) +{ + struct io_ring_ctx *ctx = data; + + io_unregister_personality(ctx, id); return 0; } @@ -9951,21 +9960,6 @@ static int io_register_personality(struct io_ring_ctx *ctx) return ret; } -static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) -{ - struct io_identity *iod; - - iod = idr_remove(&ctx->personality_idr, id); - if (iod) { - put_cred(iod->creds); - if (refcount_dec_and_test(&iod->count)) - kfree(iod); - return 0; - } - - return -EINVAL; -} - static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, unsigned int nr_args) { -- cgit v1.2.3 From ecfc8492820732be652146280912554ced62c32b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 25 Jan 2021 11:42:20 +0000 Subject: io_uring: ensure only sqo_task has file notes For SQPOLL io_uring we want to have only one file note held by sqo_task. Add a warning to make sure it holds. It's deep in io_uring_add_task_file() out of hot path, so shouldn't hurt. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index b05d37431c12..68bf2c8c23a9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9099,6 +9099,10 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) fput(file); return ret; } + + /* one and only SQPOLL file note, held by sqo_task */ + WARN_ON_ONCE((ctx->flags & IORING_SETUP_SQPOLL) && + current != ctx->sqo_task); } tctx->last = file; } -- cgit v1.2.3 From 7c6607313f032b73638a6f752cb4adf50ba947cf Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 25 Jan 2021 11:42:21 +0000 Subject: io_uring: consolidate putting reqs task We grab a task for each request and while putting it it also have to do extra work like inflight accounting and waking up that task. This sequence is duplicated several time, it's good time to add a helper. More to that, the helper generates better code due to better locality and so not failing alias analysis. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 68bf2c8c23a9..6d45a0975d9c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2023,17 +2023,22 @@ static void io_dismantle_req(struct io_kiocb *req) io_req_clean_work(req); } +static inline void io_put_task(struct task_struct *task, int nr) +{ + struct io_uring_task *tctx = task->io_uring; + + percpu_counter_sub(&tctx->inflight, nr); + if (unlikely(atomic_read(&tctx->in_idle))) + wake_up(&tctx->wait); + put_task_struct_many(task, nr); +} + static void __io_free_req(struct io_kiocb *req) { - struct io_uring_task *tctx = req->task->io_uring; struct io_ring_ctx *ctx = req->ctx; io_dismantle_req(req); - - percpu_counter_dec(&tctx->inflight); - if (atomic_read(&tctx->in_idle)) - wake_up(&tctx->wait); - put_task_struct(req->task); + io_put_task(req->task, 1); if (likely(!io_is_fallback_req(req))) kmem_cache_free(req_cachep, req); @@ -2287,12 +2292,7 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx, if (rb->to_free) __io_req_free_batch_flush(ctx, rb); if (rb->task) { - struct io_uring_task *tctx = rb->task->io_uring; - - percpu_counter_sub(&tctx->inflight, rb->task_refs); - if (atomic_read(&tctx->in_idle)) - wake_up(&tctx->wait); - put_task_struct_many(rb->task, rb->task_refs); + io_put_task(rb->task, rb->task_refs); rb->task = NULL; } } @@ -2306,14 +2306,8 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) io_queue_next(req); if (req->task != rb->task) { - if (rb->task) { - struct io_uring_task *tctx = rb->task->io_uring; - - percpu_counter_sub(&tctx->inflight, rb->task_refs); - if (atomic_read(&tctx->in_idle)) - wake_up(&tctx->wait); - put_task_struct_many(rb->task, rb->task_refs); - } + if (rb->task) + io_put_task(rb->task, rb->task_refs); rb->task = req->task; rb->task_refs = 0; } -- cgit v1.2.3 From 67973b933e347c38478b591d6c9dc076bea7c9dc Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 26 Jan 2021 13:51:09 +0000 Subject: io_uring: cleanup files_update looping Replace a while with a simple for loop, that looks way more natural, and enables us to use "continue" as indexes are no more updated by hand in the end of the loop. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6d45a0975d9c..0ca99bd5c316 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8028,9 +8028,8 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, return -ENOMEM; init_fixed_file_ref_node(ctx, ref_node); - done = 0; fds = u64_to_user_ptr(up->data); - while (nr_args) { + for (done = 0; done < nr_args; done++) { struct fixed_rsrc_table *table; unsigned index; @@ -8039,7 +8038,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EFAULT; break; } - i = array_index_nospec(up->offset, ctx->nr_user_files); + i = array_index_nospec(up->offset + done, ctx->nr_user_files); table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; index = i & IORING_FILE_TABLE_MASK; if (table->files[index]) { @@ -8077,9 +8076,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, break; } } - nr_args--; - done++; - up->offset++; } if (needs_switch) { -- cgit v1.2.3 From 4e0377a1c5c633852f443a562ec55f7dfea65350 Mon Sep 17 00:00:00 2001 From: noah Date: Tue, 26 Jan 2021 15:23:28 -0500 Subject: io_uring: Add skip option for __io_sqe_files_update This patch adds support for skipping a file descriptor when using IORING_REGISTER_FILES_UPDATE. __io_sqe_files_update will skip fds set to IORING_REGISTER_FILES_SKIP. IORING_REGISTER_FILES_SKIP is inturn added as a #define in io_uring.h Signed-off-by: noah Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +++ include/uapi/linux/io_uring.h | 3 +++ 2 files changed, 6 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 0ca99bd5c316..dd83a64ba709 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8038,6 +8038,9 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EFAULT; break; } + if (fd == IORING_REGISTER_FILES_SKIP) + continue; + i = array_index_nospec(up->offset + done, ctx->nr_user_files); table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; index = i & IORING_FILE_TABLE_MASK; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index f9f106c54d90..ac4e1738a9af 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -298,6 +298,9 @@ struct io_uring_rsrc_update { __aligned_u64 data; }; +/* Skip updating fd indexes set to this value in the fd table */ +#define IORING_REGISTER_FILES_SKIP (-2) + #define IO_URING_OP_SUPPORTED (1U << 0) struct io_uring_probe_op { -- cgit v1.2.3 From 8b28fdf21193d35d6ec5a8430f0241f5f977c6ac Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Sun, 31 Jan 2021 22:39:04 +0800 Subject: io_uring: check kthread parked flag before sqthread goes to sleep Abaci reported this issue: #[ 605.170872] INFO: task kworker/u4:1:53 blocked for more than 143 seconds. [ 605.172123] Not tainted 5.10.0+ #1 [ 605.172811] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 605.173915] task:kworker/u4:1 state:D stack: 0 pid: 53 ppid: 2 flags:0x00004000 [ 605.175130] Workqueue: events_unbound io_ring_exit_work [ 605.175931] Call Trace: [ 605.176334] __schedule+0xe0e/0x25a0 [ 605.176971] ? firmware_map_remove+0x1a1/0x1a1 [ 605.177631] ? write_comp_data+0x2a/0x80 [ 605.178272] schedule+0xd0/0x270 [ 605.178811] schedule_timeout+0x6b6/0x940 [ 605.179415] ? mark_lock.part.0+0xca/0x1420 [ 605.180062] ? usleep_range+0x170/0x170 [ 605.180684] ? wait_for_completion+0x16d/0x280 [ 605.181392] ? mark_held_locks+0x9e/0xe0 [ 605.182079] ? rwlock_bug.part.0+0x90/0x90 [ 605.182853] ? lockdep_hardirqs_on_prepare+0x286/0x400 [ 605.183817] wait_for_completion+0x175/0x280 [ 605.184713] ? wait_for_completion_interruptible+0x340/0x340 [ 605.185611] ? _raw_spin_unlock_irq+0x24/0x30 [ 605.186307] ? migrate_swap_stop+0x9c0/0x9c0 [ 605.187046] kthread_park+0x127/0x1c0 [ 605.187738] io_sq_thread_stop+0xd5/0x530 [ 605.188459] io_ring_exit_work+0xb1/0x970 [ 605.189207] process_one_work+0x92c/0x1510 [ 605.189947] ? pwq_dec_nr_in_flight+0x360/0x360 [ 605.190682] ? rwlock_bug.part.0+0x90/0x90 [ 605.191430] ? write_comp_data+0x2a/0x80 [ 605.192207] worker_thread+0x9b/0xe20 [ 605.192900] ? process_one_work+0x1510/0x1510 [ 605.193599] kthread+0x353/0x460 [ 605.194154] ? _raw_spin_unlock_irq+0x24/0x30 [ 605.194910] ? kthread_create_on_node+0x100/0x100 [ 605.195821] ret_from_fork+0x1f/0x30 [ 605.196605] [ 605.196605] Showing all locks held in the system: [ 605.197598] 1 lock held by khungtaskd/25: [ 605.198301] #0: ffffffff8b5f76a0 (rcu_read_lock){....}-{1:2}, at: rcu_lock_acquire.constprop.0+0x0/0x30 [ 605.199914] 3 locks held by kworker/u4:1/53: [ 605.200609] #0: ffff888100109938 ((wq_completion)events_unbound){+.+.}-{0:0}, at: process_one_work+0x82a/0x1510 [ 605.202108] #1: ffff888100e47dc0 ((work_completion)(&ctx->exit_work)){+.+.}-{0:0}, at: process_one_work+0x85e/0x1510 [ 605.203681] #2: ffff888116931870 (&sqd->lock){+.+.}-{3:3}, at: io_sq_thread_park.part.0+0x19/0x50 [ 605.205183] 3 locks held by systemd-journal/161: [ 605.206037] 1 lock held by syslog-ng/254: [ 605.206674] 2 locks held by agetty/311: [ 605.207292] #0: ffff888101097098 (&tty->ldisc_sem){++++}-{0:0}, at: tty_ldisc_ref_wait+0x27/0x80 [ 605.208715] #1: ffffc900000332e8 (&ldata->atomic_read_lock){+.+.}-{3:3}, at: n_tty_read+0x222/0x1bb0 [ 605.210131] 2 locks held by bash/677: [ 605.210723] #0: ffff88810419a098 (&tty->ldisc_sem){++++}-{0:0}, at: tty_ldisc_ref_wait+0x27/0x80 [ 605.212105] #1: ffffc900000512e8 (&ldata->atomic_read_lock){+.+.}-{3:3}, at: n_tty_read+0x222/0x1bb0 [ 605.213777] [ 605.214151] ============================================= I believe this is caused by the follow race: (ctx_list is empty now) => io_put_sq_data | ==> kthread_park(sqd->thread); | ====> set KTHREAD_SHOULD_PARK | ====> wake_up_process(k) | sq thread is running | | | needs_sched is true since no ctx, | so TASK_INTERRUPTIBLE set and schedule | out then never wake up again | ====> wait_for_completion | (stuck here) So check if sqthread gets park flag right before schedule(). since ctx_list is always empty when this problem happens, here I put kthread_should_park() before setting the wakeup flag(ctx_list is empty so this for loop is fast), where is close enough to schedule(). The problem doesn't show again in my repro testing after this fix. Reported-by: Abaci Signed-off-by: Hao Xu Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index dd83a64ba709..a8bf867b6cf2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7117,9 +7117,6 @@ static int io_sq_thread(void *data) continue; } - if (kthread_should_park()) - continue; - needs_sched = true; prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { @@ -7134,7 +7131,7 @@ static int io_sq_thread(void *data) } } - if (needs_sched) { + if (needs_sched && !kthread_should_park()) { list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); -- cgit v1.2.3 From 4533fc631547213bd03fbdf0a96dd8eb6807d3a7 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Tue, 26 Jan 2021 19:14:55 -0800 Subject: xfs: fix unused log variable in xfs_log_cover() The log variable is only used in kernels with asserts enabled. Remove it and open code the dereference to avoid unused variable warnings. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_log.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 58699881c100..d8b814227734 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -1108,12 +1108,11 @@ static int xfs_log_cover( struct xfs_mount *mp) { - struct xlog *log = mp->m_log; int error = 0; bool need_covered; - ASSERT((xlog_cil_empty(log) && xlog_iclogs_empty(log) && - !xfs_ail_min_lsn(log->l_ailp)) || + ASSERT((xlog_cil_empty(mp->m_log) && xlog_iclogs_empty(mp->m_log) && + !xfs_ail_min_lsn(mp->m_log->l_ailp)) || XFS_FORCED_SHUTDOWN(mp)); if (!xfs_log_writable(mp)) -- cgit v1.2.3 From 560ab6c0d12ebccabb83638abe23a7875b946f9a Mon Sep 17 00:00:00 2001 From: Chandan Babu R Date: Wed, 27 Jan 2021 08:35:14 -0800 Subject: xfs: Fix 'set but not used' warning in xfs_bmap_compute_alignments() With both CONFIG_XFS_DEBUG and CONFIG_XFS_WARN disabled, the only reference to local variable "error" in xfs_bmap_compute_alignments() gets eliminated during pre-processing stage of the compilation process. This causes the compiler to generate a "set but not used" warning. Reported-by: kernel test robot Signed-off-by: Chandan Babu R Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster --- fs/xfs/libxfs/xfs_bmap.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 2cd24bb06040..7ea1dbbe3d0b 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -3471,7 +3471,6 @@ xfs_bmap_compute_alignments( struct xfs_mount *mp = args->mp; xfs_extlen_t align = 0; /* minimum allocation alignment */ int stripe_align = 0; - int error; /* stripe alignment for allocation is determined by mount parameters */ if (mp->m_swidth && (mp->m_flags & XFS_MOUNT_SWALLOC)) @@ -3484,10 +3483,10 @@ xfs_bmap_compute_alignments( else if (ap->datatype & XFS_ALLOC_USERDATA) align = xfs_get_extsz_hint(ap->ip); if (align) { - error = xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, - align, 0, ap->eof, 0, ap->conv, - &ap->offset, &ap->length); - ASSERT(!error); + if (xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, align, 0, + ap->eof, 0, ap->conv, &ap->offset, + &ap->length)) + ASSERT(0); ASSERT(ap->length); } -- cgit v1.2.3 From f50b8f475a2c70ae8309c16b6d4ecb305a4aa9d6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:27 -0800 Subject: xfs: factor out a xfs_ilock_iocb helper Add a helper to factor out the nowait locking logical for the read/write helpers. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_file.c | 55 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 26 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 7f18dae84584..13ca049c44fd 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -197,6 +197,23 @@ xfs_file_fsync( return error; } +static int +xfs_ilock_iocb( + struct kiocb *iocb, + unsigned int lock_mode) +{ + struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!xfs_ilock_nowait(ip, lock_mode)) + return -EAGAIN; + } else { + xfs_ilock(ip, lock_mode); + } + + return 0; +} + STATIC ssize_t xfs_file_dio_aio_read( struct kiocb *iocb, @@ -213,12 +230,9 @@ xfs_file_dio_aio_read( file_accessed(iocb->ki_filp); - if (iocb->ki_flags & IOCB_NOWAIT) { - if (!xfs_ilock_nowait(ip, XFS_IOLOCK_SHARED)) - return -EAGAIN; - } else { - xfs_ilock(ip, XFS_IOLOCK_SHARED); - } + ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED); + if (ret) + return ret; ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, 0); xfs_iunlock(ip, XFS_IOLOCK_SHARED); @@ -239,13 +253,9 @@ xfs_file_dax_read( if (!count) return 0; /* skip atime */ - if (iocb->ki_flags & IOCB_NOWAIT) { - if (!xfs_ilock_nowait(ip, XFS_IOLOCK_SHARED)) - return -EAGAIN; - } else { - xfs_ilock(ip, XFS_IOLOCK_SHARED); - } - + ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED); + if (ret) + return ret; ret = dax_iomap_rw(iocb, to, &xfs_read_iomap_ops); xfs_iunlock(ip, XFS_IOLOCK_SHARED); @@ -263,12 +273,9 @@ xfs_file_buffered_aio_read( trace_xfs_file_buffered_read(ip, iov_iter_count(to), iocb->ki_pos); - if (iocb->ki_flags & IOCB_NOWAIT) { - if (!xfs_ilock_nowait(ip, XFS_IOLOCK_SHARED)) - return -EAGAIN; - } else { - xfs_ilock(ip, XFS_IOLOCK_SHARED); - } + ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED); + if (ret) + return ret; ret = generic_file_read_iter(iocb, to); xfs_iunlock(ip, XFS_IOLOCK_SHARED); @@ -607,13 +614,9 @@ xfs_file_dax_write( size_t count; loff_t pos; - if (iocb->ki_flags & IOCB_NOWAIT) { - if (!xfs_ilock_nowait(ip, iolock)) - return -EAGAIN; - } else { - xfs_ilock(ip, iolock); - } - + ret = xfs_ilock_iocb(iocb, iolock); + if (ret) + return ret; ret = xfs_file_aio_write_checks(iocb, from, &iolock); if (ret) goto out; -- cgit v1.2.3 From 354be7e3b2baf32e63c0599cc131d393591ba299 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:28 -0800 Subject: xfs: make xfs_file_aio_write_checks IOCB_NOWAIT-aware Ensure we don't block on the iolock, or waiting for I/O in xfs_file_aio_write_checks if the caller asked to avoid that. Fixes: 29a5d29ec181 ("xfs: nowait aio support") Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_file.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 13ca049c44fd..df6a9a51b2d8 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -334,7 +334,14 @@ restart: if (error <= 0) return error; - error = xfs_break_layouts(inode, iolock, BREAK_WRITE); + if (iocb->ki_flags & IOCB_NOWAIT) { + error = break_layout(inode, false); + if (error == -EWOULDBLOCK) + error = -EAGAIN; + } else { + error = xfs_break_layouts(inode, iolock, BREAK_WRITE); + } + if (error) return error; @@ -345,7 +352,11 @@ restart: if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) { xfs_iunlock(ip, *iolock); *iolock = XFS_IOLOCK_EXCL; - xfs_ilock(ip, *iolock); + error = xfs_ilock_iocb(iocb, *iolock); + if (error) { + *iolock = 0; + return error; + } goto restart; } /* @@ -367,6 +378,10 @@ restart: isize = i_size_read(inode); if (iocb->ki_pos > isize) { spin_unlock(&ip->i_flags_lock); + + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; + if (!drained_dio) { if (*iolock == XFS_IOLOCK_SHARED) { xfs_iunlock(ip, *iolock); @@ -592,7 +607,8 @@ xfs_file_dio_aio_write( &xfs_dio_write_ops, unaligned_io ? IOMAP_DIO_FORCE_WAIT : 0); out: - xfs_iunlock(ip, iolock); + if (iolock) + xfs_iunlock(ip, iolock); /* * No fallback to buffered IO after short writes for XFS, direct I/O @@ -631,7 +647,8 @@ xfs_file_dax_write( error = xfs_setfilesize(ip, pos, ret); } out: - xfs_iunlock(ip, iolock); + if (iolock) + xfs_iunlock(ip, iolock); if (error) return error; -- cgit v1.2.3 From ee1b218b09560982010e4dfffa8f9d4f05f62220 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:28 -0800 Subject: xfs: cleanup the read/write helper naming Drop a few pointless aio_ prefixes. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_file.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index df6a9a51b2d8..3dc5f8ca64ec 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -215,7 +215,7 @@ xfs_ilock_iocb( } STATIC ssize_t -xfs_file_dio_aio_read( +xfs_file_dio_read( struct kiocb *iocb, struct iov_iter *to) { @@ -264,7 +264,7 @@ xfs_file_dax_read( } STATIC ssize_t -xfs_file_buffered_aio_read( +xfs_file_buffered_read( struct kiocb *iocb, struct iov_iter *to) { @@ -299,9 +299,9 @@ xfs_file_read_iter( if (IS_DAX(inode)) ret = xfs_file_dax_read(iocb, to); else if (iocb->ki_flags & IOCB_DIRECT) - ret = xfs_file_dio_aio_read(iocb, to); + ret = xfs_file_dio_read(iocb, to); else - ret = xfs_file_buffered_aio_read(iocb, to); + ret = xfs_file_buffered_read(iocb, to); if (ret > 0) XFS_STATS_ADD(mp, xs_read_bytes, ret); @@ -316,7 +316,7 @@ xfs_file_read_iter( * if called for a direct write beyond i_size. */ STATIC ssize_t -xfs_file_aio_write_checks( +xfs_file_write_checks( struct kiocb *iocb, struct iov_iter *from, int *iolock) @@ -501,7 +501,7 @@ static const struct iomap_dio_ops xfs_dio_write_ops = { }; /* - * xfs_file_dio_aio_write - handle direct IO writes + * xfs_file_dio_write - handle direct IO writes * * Lock the inode appropriately to prepare for and issue a direct IO write. * By separating it from the buffered write path we remove all the tricky to @@ -526,7 +526,7 @@ static const struct iomap_dio_ops xfs_dio_write_ops = { * negative return values. */ STATIC ssize_t -xfs_file_dio_aio_write( +xfs_file_dio_write( struct kiocb *iocb, struct iov_iter *from) { @@ -548,7 +548,7 @@ xfs_file_dio_aio_write( /* * Don't take the exclusive iolock here unless the I/O is unaligned to * the file system block size. We don't need to consider the EOF - * extension case here because xfs_file_aio_write_checks() will relock + * extension case here because xfs_file_write_checks() will relock * the inode as necessary for EOF zeroing cases and fill out the new * inode size as appropriate. */ @@ -579,7 +579,7 @@ xfs_file_dio_aio_write( xfs_ilock(ip, iolock); } - ret = xfs_file_aio_write_checks(iocb, from, &iolock); + ret = xfs_file_write_checks(iocb, from, &iolock); if (ret) goto out; count = iov_iter_count(from); @@ -589,7 +589,7 @@ xfs_file_dio_aio_write( * in-flight at the same time or we risk data corruption. Wait for all * other IO to drain before we submit. If the IO is aligned, demote the * iolock if we had to take the exclusive lock in - * xfs_file_aio_write_checks() for other reasons. + * xfs_file_write_checks() for other reasons. */ if (unaligned_io) { inode_dio_wait(inode); @@ -633,7 +633,7 @@ xfs_file_dax_write( ret = xfs_ilock_iocb(iocb, iolock); if (ret) return ret; - ret = xfs_file_aio_write_checks(iocb, from, &iolock); + ret = xfs_file_write_checks(iocb, from, &iolock); if (ret) goto out; @@ -662,7 +662,7 @@ out: } STATIC ssize_t -xfs_file_buffered_aio_write( +xfs_file_buffered_write( struct kiocb *iocb, struct iov_iter *from) { @@ -681,7 +681,7 @@ write_retry: iolock = XFS_IOLOCK_EXCL; xfs_ilock(ip, iolock); - ret = xfs_file_aio_write_checks(iocb, from, &iolock); + ret = xfs_file_write_checks(iocb, from, &iolock); if (ret) goto out; @@ -768,12 +768,12 @@ xfs_file_write_iter( * CoW. In all other directio scenarios we do not * allow an operation to fall back to buffered mode. */ - ret = xfs_file_dio_aio_write(iocb, from); + ret = xfs_file_dio_write(iocb, from); if (ret != -ENOTBLK) return ret; } - return xfs_file_buffered_aio_write(iocb, from); + return xfs_file_buffered_write(iocb, from); } static void -- cgit v1.2.3 From 670654b004b0bf7a0bc749f4f555fdefd5c89dcb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:29 -0800 Subject: xfs: remove the buffered I/O fallback assert The iomap code has been designed from the start not to do magic fallback, so remove the assert in preparation for further code cleanups. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_file.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 3dc5f8ca64ec..b38002ddfa6c 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -609,12 +609,6 @@ xfs_file_dio_write( out: if (iolock) xfs_iunlock(ip, iolock); - - /* - * No fallback to buffered IO after short writes for XFS, direct I/O - * will either complete fully or return an error. - */ - ASSERT(ret < 0 || ret == count); return ret; } -- cgit v1.2.3 From 3e40b13c3b57108a118de639d4af04805ac6873f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:29 -0800 Subject: xfs: simplify the read/write tracepoints Pass the iocb and iov_iter to the tracepoints and leave decoding of actual arguments to the code only run when tracing is enabled. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_file.c | 20 ++++++++------------ fs/xfs/xfs_trace.h | 18 +++++++++--------- 2 files changed, 17 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index b38002ddfa6c..27b81925f92a 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -220,12 +220,11 @@ xfs_file_dio_read( struct iov_iter *to) { struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); - size_t count = iov_iter_count(to); ssize_t ret; - trace_xfs_file_direct_read(ip, count, iocb->ki_pos); + trace_xfs_file_direct_read(iocb, to); - if (!count) + if (!iov_iter_count(to)) return 0; /* skip atime */ file_accessed(iocb->ki_filp); @@ -245,12 +244,11 @@ xfs_file_dax_read( struct iov_iter *to) { struct xfs_inode *ip = XFS_I(iocb->ki_filp->f_mapping->host); - size_t count = iov_iter_count(to); ssize_t ret = 0; - trace_xfs_file_dax_read(ip, count, iocb->ki_pos); + trace_xfs_file_dax_read(iocb, to); - if (!count) + if (!iov_iter_count(to)) return 0; /* skip atime */ ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED); @@ -271,7 +269,7 @@ xfs_file_buffered_read( struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); ssize_t ret; - trace_xfs_file_buffered_read(ip, iov_iter_count(to), iocb->ki_pos); + trace_xfs_file_buffered_read(iocb, to); ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED); if (ret) @@ -598,7 +596,7 @@ xfs_file_dio_write( iolock = XFS_IOLOCK_SHARED; } - trace_xfs_file_direct_write(ip, count, iocb->ki_pos); + trace_xfs_file_direct_write(iocb, from); /* * If unaligned, this is the only IO in-flight. Wait on it before we * release the iolock to prevent subsequent overlapping IO. @@ -621,7 +619,6 @@ xfs_file_dax_write( struct xfs_inode *ip = XFS_I(inode); int iolock = XFS_IOLOCK_EXCL; ssize_t ret, error = 0; - size_t count; loff_t pos; ret = xfs_ilock_iocb(iocb, iolock); @@ -632,9 +629,8 @@ xfs_file_dax_write( goto out; pos = iocb->ki_pos; - count = iov_iter_count(from); - trace_xfs_file_dax_write(ip, count, pos); + trace_xfs_file_dax_write(iocb, from); ret = dax_iomap_rw(iocb, from, &xfs_direct_write_iomap_ops); if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { i_size_write(inode, iocb->ki_pos); @@ -682,7 +678,7 @@ write_retry: /* We can write back this queue in page reclaim */ current->backing_dev_info = inode_to_bdi(inode); - trace_xfs_file_buffered_write(ip, iov_iter_count(from), iocb->ki_pos); + trace_xfs_file_buffered_write(iocb, from); ret = iomap_file_buffered_write(iocb, from, &xfs_buffered_write_iomap_ops); if (likely(ret >= 0)) diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 5a263ae3d4f0..a6d04d860a56 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -1287,8 +1287,8 @@ TRACE_EVENT(xfs_log_assign_tail_lsn, ) DECLARE_EVENT_CLASS(xfs_file_class, - TP_PROTO(struct xfs_inode *ip, size_t count, loff_t offset), - TP_ARGS(ip, count, offset), + TP_PROTO(struct kiocb *iocb, struct iov_iter *iter), + TP_ARGS(iocb, iter), TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_ino_t, ino) @@ -1297,11 +1297,11 @@ DECLARE_EVENT_CLASS(xfs_file_class, __field(size_t, count) ), TP_fast_assign( - __entry->dev = VFS_I(ip)->i_sb->s_dev; - __entry->ino = ip->i_ino; - __entry->size = ip->i_d.di_size; - __entry->offset = offset; - __entry->count = count; + __entry->dev = file_inode(iocb->ki_filp)->i_sb->s_dev; + __entry->ino = XFS_I(file_inode(iocb->ki_filp))->i_ino; + __entry->size = XFS_I(file_inode(iocb->ki_filp))->i_d.di_size; + __entry->offset = iocb->ki_pos; + __entry->count = iov_iter_count(iter); ), TP_printk("dev %d:%d ino 0x%llx size 0x%llx offset 0x%llx count 0x%zx", MAJOR(__entry->dev), MINOR(__entry->dev), @@ -1313,8 +1313,8 @@ DECLARE_EVENT_CLASS(xfs_file_class, #define DEFINE_RW_EVENT(name) \ DEFINE_EVENT(xfs_file_class, name, \ - TP_PROTO(struct xfs_inode *ip, size_t count, loff_t offset), \ - TP_ARGS(ip, count, offset)) + TP_PROTO(struct kiocb *iocb, struct iov_iter *iter), \ + TP_ARGS(iocb, iter)) DEFINE_RW_EVENT(xfs_file_buffered_read); DEFINE_RW_EVENT(xfs_file_direct_read); DEFINE_RW_EVENT(xfs_file_dax_read); -- cgit v1.2.3 From 896f72d067a5e9a90d2178b311bb89a18c7b5cfb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:30 -0800 Subject: xfs: improve the reflink_bounce_dio_write tracepoint Use a more suitable event class. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_file.c | 2 +- fs/xfs/xfs_trace.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 27b81925f92a..94bc2ea83734 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -559,7 +559,7 @@ xfs_file_dio_write( * files yet, as we can't unshare a partial block. */ if (xfs_is_cow_inode(ip)) { - trace_xfs_reflink_bounce_dio_write(ip, iocb->ki_pos, count); + trace_xfs_reflink_bounce_dio_write(iocb, from); return -ENOTBLK; } iolock = XFS_IOLOCK_EXCL; diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index a6d04d860a56..0cfd65cd67c1 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -1321,6 +1321,8 @@ DEFINE_RW_EVENT(xfs_file_dax_read); DEFINE_RW_EVENT(xfs_file_buffered_write); DEFINE_RW_EVENT(xfs_file_direct_write); DEFINE_RW_EVENT(xfs_file_dax_write); +DEFINE_RW_EVENT(xfs_reflink_bounce_dio_write); + DECLARE_EVENT_CLASS(xfs_imap_class, TP_PROTO(struct xfs_inode *ip, xfs_off_t offset, ssize_t count, @@ -3294,8 +3296,6 @@ DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_found); DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_enospc); DEFINE_INODE_IREC_EVENT(xfs_reflink_convert_cow); -DEFINE_SIMPLE_IO_EVENT(xfs_reflink_bounce_dio_write); - DEFINE_SIMPLE_IO_EVENT(xfs_reflink_cancel_cow_range); DEFINE_SIMPLE_IO_EVENT(xfs_reflink_end_cow); DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_remap); -- cgit v1.2.3 From caa89dbc4303d9bc7caa6c1124a84aa3efa47e00 Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Sat, 23 Jan 2021 10:06:30 -0800 Subject: xfs: split the unaligned DIO write code out The unaligned DIO write path is more convolted than the normal path, and we are about to make it more complex. Keep the block aligned fast path dio write code trim and simple by splitting out the unaligned DIO code from it. Signed-off-by: Dave Chinner [hch: rebased, fixed a few minor nits] Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_file.c | 170 +++++++++++++++++++++++++++--------------------------- 1 file changed, 85 insertions(+), 85 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 94bc2ea83734..c60ff7b5dd82 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -499,117 +499,117 @@ static const struct iomap_dio_ops xfs_dio_write_ops = { }; /* - * xfs_file_dio_write - handle direct IO writes - * - * Lock the inode appropriately to prepare for and issue a direct IO write. - * By separating it from the buffered write path we remove all the tricky to - * follow locking changes and looping. - * - * If there are cached pages or we're extending the file, we need IOLOCK_EXCL - * until we're sure the bytes at the new EOF have been zeroed and/or the cached - * pages are flushed out. - * - * In most cases the direct IO writes will be done holding IOLOCK_SHARED - * allowing them to be done in parallel with reads and other direct IO writes. - * However, if the IO is not aligned to filesystem blocks, the direct IO layer - * needs to do sub-block zeroing and that requires serialisation against other - * direct IOs to the same block. In this case we need to serialise the - * submission of the unaligned IOs so that we don't get racing block zeroing in - * the dio layer. To avoid the problem with aio, we also need to wait for - * outstanding IOs to complete so that unwritten extent conversion is completed - * before we try to map the overlapping block. This is currently implemented by - * hitting it with a big hammer (i.e. inode_dio_wait()). - * - * Returns with locks held indicated by @iolock and errors indicated by - * negative return values. + * Handle block aligned direct I/O writes */ -STATIC ssize_t -xfs_file_dio_write( +static noinline ssize_t +xfs_file_dio_write_aligned( + struct xfs_inode *ip, struct kiocb *iocb, struct iov_iter *from) { - struct file *file = iocb->ki_filp; - struct address_space *mapping = file->f_mapping; - struct inode *inode = mapping->host; - struct xfs_inode *ip = XFS_I(inode); - struct xfs_mount *mp = ip->i_mount; - ssize_t ret = 0; - int unaligned_io = 0; - int iolock; - size_t count = iov_iter_count(from); - struct xfs_buftarg *target = xfs_inode_buftarg(ip); + int iolock = XFS_IOLOCK_SHARED; + ssize_t ret; - /* DIO must be aligned to device logical sector size */ - if ((iocb->ki_pos | count) & target->bt_logical_sectormask) - return -EINVAL; + ret = xfs_ilock_iocb(iocb, iolock); + if (ret) + return ret; + ret = xfs_file_write_checks(iocb, from, &iolock); + if (ret) + goto out_unlock; /* - * Don't take the exclusive iolock here unless the I/O is unaligned to - * the file system block size. We don't need to consider the EOF - * extension case here because xfs_file_write_checks() will relock - * the inode as necessary for EOF zeroing cases and fill out the new - * inode size as appropriate. + * We don't need to hold the IOLOCK exclusively across the IO, so demote + * the iolock back to shared if we had to take the exclusive lock in + * xfs_file_write_checks() for other reasons. */ - if ((iocb->ki_pos & mp->m_blockmask) || - ((iocb->ki_pos + count) & mp->m_blockmask)) { - unaligned_io = 1; - - /* - * We can't properly handle unaligned direct I/O to reflink - * files yet, as we can't unshare a partial block. - */ - if (xfs_is_cow_inode(ip)) { - trace_xfs_reflink_bounce_dio_write(iocb, from); - return -ENOTBLK; - } - iolock = XFS_IOLOCK_EXCL; - } else { + if (iolock == XFS_IOLOCK_EXCL) { + xfs_ilock_demote(ip, XFS_IOLOCK_EXCL); iolock = XFS_IOLOCK_SHARED; } + trace_xfs_file_direct_write(iocb, from); + ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops, + &xfs_dio_write_ops, 0); +out_unlock: + if (iolock) + xfs_iunlock(ip, iolock); + return ret; +} - if (iocb->ki_flags & IOCB_NOWAIT) { - /* unaligned dio always waits, bail */ - if (unaligned_io) - return -EAGAIN; - if (!xfs_ilock_nowait(ip, iolock)) - return -EAGAIN; - } else { - xfs_ilock(ip, iolock); +/* + * Handle block unaligned direct I/O writes + * + * In most cases direct I/O writes will be done holding IOLOCK_SHARED, allowing + * them to be done in parallel with reads and other direct I/O writes. However, + * if the I/O is not aligned to filesystem blocks, the direct I/O layer may need + * to do sub-block zeroing and that requires serialisation against other direct + * I/O to the same block. In this case we need to serialise the submission of + * the unaligned I/O so that we don't get racing block zeroing in the dio layer. + * + * This means that unaligned dio writes always block. There is no "nowait" fast + * path in this code - if IOCB_NOWAIT is set we simply return -EAGAIN up front + * and we don't have to worry about that anymore. + */ +static noinline ssize_t +xfs_file_dio_write_unaligned( + struct xfs_inode *ip, + struct kiocb *iocb, + struct iov_iter *from) +{ + int iolock = XFS_IOLOCK_EXCL; + ssize_t ret; + + /* unaligned dio always waits, bail */ + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; + xfs_ilock(ip, iolock); + + /* + * We can't properly handle unaligned direct I/O to reflink files yet, + * as we can't unshare a partial block. + */ + if (xfs_is_cow_inode(ip)) { + trace_xfs_reflink_bounce_dio_write(iocb, from); + ret = -ENOTBLK; + goto out_unlock; } ret = xfs_file_write_checks(iocb, from, &iolock); if (ret) - goto out; - count = iov_iter_count(from); + goto out_unlock; /* - * If we are doing unaligned IO, we can't allow any other overlapping IO - * in-flight at the same time or we risk data corruption. Wait for all - * other IO to drain before we submit. If the IO is aligned, demote the - * iolock if we had to take the exclusive lock in - * xfs_file_write_checks() for other reasons. + * If we are doing unaligned I/O, this must be the only I/O in-flight. + * Otherwise we risk data corruption due to unwritten extent conversions + * from the AIO end_io handler. Wait for all other I/O to drain first. */ - if (unaligned_io) { - inode_dio_wait(inode); - } else if (iolock == XFS_IOLOCK_EXCL) { - xfs_ilock_demote(ip, XFS_IOLOCK_EXCL); - iolock = XFS_IOLOCK_SHARED; - } + inode_dio_wait(VFS_I(ip)); trace_xfs_file_direct_write(iocb, from); - /* - * If unaligned, this is the only IO in-flight. Wait on it before we - * release the iolock to prevent subsequent overlapping IO. - */ ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops, - &xfs_dio_write_ops, - unaligned_io ? IOMAP_DIO_FORCE_WAIT : 0); -out: + &xfs_dio_write_ops, IOMAP_DIO_FORCE_WAIT); +out_unlock: if (iolock) xfs_iunlock(ip, iolock); return ret; } +static ssize_t +xfs_file_dio_write( + struct kiocb *iocb, + struct iov_iter *from) +{ + struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); + struct xfs_buftarg *target = xfs_inode_buftarg(ip); + size_t count = iov_iter_count(from); + + /* direct I/O must be aligned to device logical sector size */ + if ((iocb->ki_pos | count) & target->bt_logical_sectormask) + return -EINVAL; + if ((iocb->ki_pos | count) & ip->i_mount->m_blockmask) + return xfs_file_dio_write_unaligned(ip, iocb, from); + return xfs_file_dio_write_aligned(ip, iocb, from); +} + static noinline ssize_t xfs_file_dax_write( struct kiocb *iocb, -- cgit v1.2.3 From ed1128c2d0c87e5ff49c40f5529f06bc35f4251b Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Sat, 23 Jan 2021 10:06:31 -0800 Subject: xfs: reduce exclusive locking on unaligned dio Attempt shared locking for unaligned DIO, but only if the the underlying extent is already allocated and in written state. On failure, retry with the existing exclusive locking. Test case is fio randrw of 512 byte IOs using AIO and an iodepth of 32 IOs. Vanilla: READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec Patched: READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec That's an improvement from ~18k IOPS to a ~150k IOPS, which is about the IOPS limit of the VM block device setup I'm testing on. 4kB block IO comparison: READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec Which is ~150k IOPS, same as what the test gets for sub-block AIO+DIO writes with this patch. Signed-off-by: Dave Chinner [hch: rebased, split unaligned from nowait] Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_file.c | 58 ++++++++++++++++++++++++++++++++++++++++++------------ fs/xfs/xfs_iomap.c | 29 +++++++++++++++++++-------- 2 files changed, 66 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index c60ff7b5dd82..39695b59dfcc 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -544,10 +544,13 @@ out_unlock: * to do sub-block zeroing and that requires serialisation against other direct * I/O to the same block. In this case we need to serialise the submission of * the unaligned I/O so that we don't get racing block zeroing in the dio layer. + * In the case where sub-block zeroing is not required, we can do concurrent + * sub-block dios to the same block successfully. * - * This means that unaligned dio writes always block. There is no "nowait" fast - * path in this code - if IOCB_NOWAIT is set we simply return -EAGAIN up front - * and we don't have to worry about that anymore. + * Optimistically submit the I/O using the shared lock first, but use the + * IOMAP_DIO_OVERWRITE_ONLY flag to tell the lower layers to return -EAGAIN + * if block allocation or partial block zeroing would be required. In that case + * we try again with the exclusive lock. */ static noinline ssize_t xfs_file_dio_write_unaligned( @@ -555,13 +558,28 @@ xfs_file_dio_write_unaligned( struct kiocb *iocb, struct iov_iter *from) { - int iolock = XFS_IOLOCK_EXCL; + size_t isize = i_size_read(VFS_I(ip)); + size_t count = iov_iter_count(from); + int iolock = XFS_IOLOCK_SHARED; + unsigned int flags = IOMAP_DIO_OVERWRITE_ONLY; ssize_t ret; - /* unaligned dio always waits, bail */ - if (iocb->ki_flags & IOCB_NOWAIT) - return -EAGAIN; - xfs_ilock(ip, iolock); + /* + * Extending writes need exclusivity because of the sub-block zeroing + * that the DIO code always does for partial tail blocks beyond EOF, so + * don't even bother trying the fast path in this case. + */ + if (iocb->ki_pos > isize || iocb->ki_pos + count >= isize) { +retry_exclusive: + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; + iolock = XFS_IOLOCK_EXCL; + flags = IOMAP_DIO_FORCE_WAIT; + } + + ret = xfs_ilock_iocb(iocb, iolock); + if (ret) + return ret; /* * We can't properly handle unaligned direct I/O to reflink files yet, @@ -578,15 +596,29 @@ xfs_file_dio_write_unaligned( goto out_unlock; /* - * If we are doing unaligned I/O, this must be the only I/O in-flight. - * Otherwise we risk data corruption due to unwritten extent conversions - * from the AIO end_io handler. Wait for all other I/O to drain first. + * If we are doing exclusive unaligned I/O, this must be the only I/O + * in-flight. Otherwise we risk data corruption due to unwritten extent + * conversions from the AIO end_io handler. Wait for all other I/O to + * drain first. */ - inode_dio_wait(VFS_I(ip)); + if (flags & IOMAP_DIO_FORCE_WAIT) + inode_dio_wait(VFS_I(ip)); trace_xfs_file_direct_write(iocb, from); ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops, - &xfs_dio_write_ops, IOMAP_DIO_FORCE_WAIT); + &xfs_dio_write_ops, flags); + + /* + * Retry unaligned I/O with exclusive blocking semantics if the DIO + * layer rejected it for mapping or locking reasons. If we are doing + * nonblocking user I/O, propagate the error. + */ + if (ret == -EAGAIN && !(iocb->ki_flags & IOCB_NOWAIT)) { + ASSERT(flags & IOMAP_DIO_OVERWRITE_ONLY); + xfs_iunlock(ip, iolock); + goto retry_exclusive; + } + out_unlock: if (iolock) xfs_iunlock(ip, iolock); diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 7b9ff824e82d..ef76f775fabf 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -784,15 +784,28 @@ xfs_direct_write_iomap_begin( goto allocate_blocks; /* - * NOWAIT IO needs to span the entire requested IO with a single map so - * that we avoid partial IO failures due to the rest of the IO range not - * covered by this map triggering an EAGAIN condition when it is - * subsequently mapped and aborting the IO. + * NOWAIT and OVERWRITE I/O needs to span the entire requested I/O with + * a single map so that we avoid partial IO failures due to the rest of + * the I/O range not covered by this map triggering an EAGAIN condition + * when it is subsequently mapped and aborting the I/O. */ - if ((flags & IOMAP_NOWAIT) && - !imap_spans_range(&imap, offset_fsb, end_fsb)) { + if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY)) { error = -EAGAIN; - goto out_unlock; + if (!imap_spans_range(&imap, offset_fsb, end_fsb)) + goto out_unlock; + } + + /* + * For overwrite only I/O, we cannot convert unwritten extents without + * requiring sub-block zeroing. This can only be done under an + * exclusive IOLOCK, hence return -EAGAIN if this is not a written + * extent to tell the caller to try again. + */ + if (flags & IOMAP_OVERWRITE_ONLY) { + error = -EAGAIN; + if (imap.br_state != XFS_EXT_NORM && + ((offset | length) & mp->m_blockmask)) + goto out_unlock; } xfs_iunlock(ip, lockmode); @@ -801,7 +814,7 @@ xfs_direct_write_iomap_begin( allocate_blocks: error = -EAGAIN; - if (flags & IOMAP_NOWAIT) + if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY)) goto out_unlock; /* -- cgit v1.2.3 From ffb81717a166b3c4a676ada61283b3121448e503 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Fri, 20 Nov 2020 12:26:46 -0600 Subject: nfs: Fix fall-through warnings for Clang In preparation to enable -Wimplicit-fallthrough for Clang, fix multiple warnings by explicitly add multiple break/goto/return/fallthrough statements instead of just letting the code fall through to the next case. Link: https://github.com/KSPP/linux/issues/115 Signed-off-by: Gustavo A. R. Silva Signed-off-by: Anna Schumaker --- fs/nfs/nfs3acl.c | 1 + fs/nfs/nfs4client.c | 1 + fs/nfs/nfs4proc.c | 2 ++ fs/nfs/nfs4state.c | 1 + fs/nfs/pnfs.c | 2 ++ 5 files changed, 7 insertions(+) (limited to 'fs') diff --git a/fs/nfs/nfs3acl.c b/fs/nfs/nfs3acl.c index c6c863382f37..68e206cf4c4e 100644 --- a/fs/nfs/nfs3acl.c +++ b/fs/nfs/nfs3acl.c @@ -111,6 +111,7 @@ struct posix_acl *nfs3_get_acl(struct inode *inode, int type) fallthrough; case -ENOTSUPP: status = -EOPNOTSUPP; + goto getout; default: goto getout; } diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index 86acffe7335c..889a9f4c0310 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -609,6 +609,7 @@ found: * changed. Schedule recovery! */ nfs4_schedule_path_down_recovery(pos); + goto out; default: goto out; } diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 224773eb61b1..34570a7785fa 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -2227,6 +2227,7 @@ static int nfs4_handle_delegation_recall_error(struct nfs_server *server, struct default: printk(KERN_ERR "NFS: %s: unhandled error " "%d.\n", __func__, err); + fallthrough; case 0: case -ENOENT: case -EAGAIN: @@ -9701,6 +9702,7 @@ nfs4_layoutcommit_done(struct rpc_task *task, void *calldata) case -NFS4ERR_BADLAYOUT: /* no layout */ case -NFS4ERR_GRACE: /* loca_recalim always false */ task->tk_status = 0; + break; case 0: break; default: diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index 4bf10792cb5b..3a51351bdc6a 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -1125,6 +1125,7 @@ static void nfs_increment_seqid(int status, struct nfs_seqid *seqid) " sequence-id error on an" " unconfirmed sequence %p!\n", seqid->sequence); + return; case -NFS4ERR_STALE_CLIENTID: case -NFS4ERR_STALE_STATEID: case -NFS4ERR_BAD_STATEID: diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index af64b4e6fd1f..102b66e0bdef 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -2875,6 +2875,7 @@ pnfs_do_write(struct nfs_pageio_descriptor *desc, switch (trypnfs) { case PNFS_NOT_ATTEMPTED: pnfs_write_through_mds(desc, hdr); + break; case PNFS_ATTEMPTED: break; case PNFS_TRY_AGAIN: @@ -3019,6 +3020,7 @@ pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) switch (trypnfs) { case PNFS_NOT_ATTEMPTED: pnfs_read_through_mds(desc, hdr); + break; case PNFS_ATTEMPTED: break; case PNFS_TRY_AGAIN: -- cgit v1.2.3 From 49dee70052b89498cc0fc61e0e193cefeee40989 Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Thu, 28 Jan 2021 09:54:59 -0500 Subject: NFS: Clean up nfs_readpage() and nfs_readpages() In prep for the new fscache netfs API, refactor nfs_readpage() and nfs_readpages() for future patches. No functional change. Signed-off-by: Dave Wysochanski Signed-off-by: Anna Schumaker --- fs/nfs/read.c | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) (limited to 'fs') diff --git a/fs/nfs/read.c b/fs/nfs/read.c index eb854f1f86e2..dd92156e27c5 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -314,7 +314,7 @@ int nfs_readpage(struct file *file, struct page *page) { struct nfs_open_context *ctx; struct inode *inode = page_file_mapping(page)->host; - int error; + int ret; dprintk("NFS: nfs_readpage (%p %ld@%lu)\n", page, PAGE_SIZE, page_index(page)); @@ -328,18 +328,18 @@ int nfs_readpage(struct file *file, struct page *page) * be any new pending writes generated at this point * for this page (other pages can be written to). */ - error = nfs_wb_page(inode, page); - if (error) + ret = nfs_wb_page(inode, page); + if (ret) goto out_unlock; if (PageUptodate(page)) goto out_unlock; - error = -ESTALE; + ret = -ESTALE; if (NFS_STALE(inode)) goto out_unlock; if (file == NULL) { - error = -EBADF; + ret = -EBADF; ctx = nfs_find_open_context(inode, NULL, FMODE_READ); if (ctx == NULL) goto out_unlock; @@ -347,24 +347,24 @@ int nfs_readpage(struct file *file, struct page *page) ctx = get_nfs_open_context(nfs_file_open_context(file)); if (!IS_SYNC(inode)) { - error = nfs_readpage_from_fscache(ctx, inode, page); - if (error == 0) + ret = nfs_readpage_from_fscache(ctx, inode, page); + if (ret == 0) goto out; } xchg(&ctx->error, 0); - error = nfs_readpage_async(ctx, inode, page); - if (!error) { - error = wait_on_page_locked_killable(page); - if (!PageUptodate(page) && !error) - error = xchg(&ctx->error, 0); + ret = nfs_readpage_async(ctx, inode, page); + if (!ret) { + ret = wait_on_page_locked_killable(page); + if (!PageUptodate(page) && !ret) + ret = xchg(&ctx->error, 0); } out: put_nfs_open_context(ctx); - return error; + return ret; out_unlock: unlock_page(page); - return error; + return ret; } struct nfs_readdesc { @@ -404,17 +404,15 @@ out: return error; } -int nfs_readpages(struct file *filp, struct address_space *mapping, +int nfs_readpages(struct file *file, struct address_space *mapping, struct list_head *pages, unsigned nr_pages) { struct nfs_pageio_descriptor pgio; struct nfs_pgio_mirror *pgm; - struct nfs_readdesc desc = { - .pgio = &pgio, - }; + struct nfs_readdesc desc; struct inode *inode = mapping->host; unsigned long npages; - int ret = -ESTALE; + int ret; dprintk("NFS: nfs_readpages (%s/%Lu %d)\n", inode->i_sb->s_id, @@ -422,15 +420,17 @@ int nfs_readpages(struct file *filp, struct address_space *mapping, nr_pages); nfs_inc_stats(inode, NFSIOS_VFSREADPAGES); + ret = -ESTALE; if (NFS_STALE(inode)) goto out; - if (filp == NULL) { + if (file == NULL) { + ret = -EBADF; desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ); if (desc.ctx == NULL) - return -EBADF; + goto out; } else - desc.ctx = get_nfs_open_context(nfs_file_open_context(filp)); + desc.ctx = get_nfs_open_context(nfs_file_open_context(file)); /* attempt to read as many of the pages as possible from the cache * - this returns -ENOBUFS immediately if the cookie is negative @@ -440,6 +440,7 @@ int nfs_readpages(struct file *filp, struct address_space *mapping, if (ret == 0) goto read_complete; /* all pages were read */ + desc.pgio = &pgio; nfs_pageio_init_read(&pgio, inode, false, &nfs_async_read_completion_ops); -- cgit v1.2.3 From 6ddfd213f4ea22ac955bcd82100c57cd294494d2 Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Thu, 28 Jan 2021 09:55:00 -0500 Subject: NFS: In nfs_readpage() only increment NFSIOS_READPAGES when read succeeds There is a small inconsistency with nfs_readpage() vs nfs_readpages() with regards to NFSIOS_READPAGES. In readpage we unconditionally increment NFSIOS_READPAGES at the top, which means even if the read fails. In readpages, we increment NFSIOS_READPAGES at the bottom based on how many pages were successfully read. Change readpage to be consistent with readpages and so NFSIOS_READPAGES only reflects successful, non-fscache reads. Signed-off-by: Dave Wysochanski Signed-off-by: Anna Schumaker --- fs/nfs/read.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/nfs/read.c b/fs/nfs/read.c index dd92156e27c5..464077daf62f 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -319,7 +319,6 @@ int nfs_readpage(struct file *file, struct page *page) dprintk("NFS: nfs_readpage (%p %ld@%lu)\n", page, PAGE_SIZE, page_index(page)); nfs_inc_stats(inode, NFSIOS_VFSREADPAGE); - nfs_add_stats(inode, NFSIOS_READPAGES, 1); /* * Try to flush any pending writes to the file.. @@ -359,6 +358,7 @@ int nfs_readpage(struct file *file, struct page *page) if (!PageUptodate(page) && !ret) ret = xchg(&ctx->error, 0); } + nfs_add_stats(inode, NFSIOS_READPAGES, 1); out: put_nfs_open_context(ctx); return ret; -- cgit v1.2.3 From 1af7e7f8c12f521c111bd7cf0d138be7e15b51a5 Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Thu, 28 Jan 2021 09:55:01 -0500 Subject: NFS: Refactor nfs_readpage() and nfs_readpage_async() to use nfs_readdesc Both nfs_readpage() and nfs_readpages() use similar code. This patch should be no functional change, and refactors nfs_readpage_async() to use nfs_readdesc to enable future merging of nfs_readpage_async() and nfs_readpage_async_filler(). Signed-off-by: Dave Wysochanski Signed-off-by: Anna Schumaker --- fs/nfs/read.c | 62 ++++++++++++++++++++++++-------------------------- include/linux/nfs_fs.h | 3 +-- 2 files changed, 31 insertions(+), 34 deletions(-) (limited to 'fs') diff --git a/fs/nfs/read.c b/fs/nfs/read.c index 464077daf62f..8c05e56dab65 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -114,18 +114,23 @@ static void nfs_readpage_release(struct nfs_page *req, int error) nfs_release_request(req); } -int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode, +struct nfs_readdesc { + struct nfs_pageio_descriptor pgio; + struct nfs_open_context *ctx; +}; + +int nfs_readpage_async(void *data, struct inode *inode, struct page *page) { + struct nfs_readdesc *desc = data; struct nfs_page *new; unsigned int len; - struct nfs_pageio_descriptor pgio; struct nfs_pgio_mirror *pgm; len = nfs_page_length(page); if (len == 0) return nfs_return_empty_page(page); - new = nfs_create_request(ctx, page, 0, len); + new = nfs_create_request(desc->ctx, page, 0, len); if (IS_ERR(new)) { unlock_page(page); return PTR_ERR(new); @@ -133,21 +138,21 @@ int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode, if (len < PAGE_SIZE) zero_user_segment(page, len, PAGE_SIZE); - nfs_pageio_init_read(&pgio, inode, false, + nfs_pageio_init_read(&desc->pgio, inode, false, &nfs_async_read_completion_ops); - if (!nfs_pageio_add_request(&pgio, new)) { + if (!nfs_pageio_add_request(&desc->pgio, new)) { nfs_list_remove_request(new); - nfs_readpage_release(new, pgio.pg_error); + nfs_readpage_release(new, desc->pgio.pg_error); } - nfs_pageio_complete(&pgio); + nfs_pageio_complete(&desc->pgio); /* It doesn't make sense to do mirrored reads! */ - WARN_ON_ONCE(pgio.pg_mirror_count != 1); + WARN_ON_ONCE(desc->pgio.pg_mirror_count != 1); - pgm = &pgio.pg_mirrors[0]; + pgm = &desc->pgio.pg_mirrors[0]; NFS_I(inode)->read_io += pgm->pg_bytes_written; - return pgio.pg_error < 0 ? pgio.pg_error : 0; + return desc->pgio.pg_error < 0 ? desc->pgio.pg_error : 0; } static void nfs_page_group_set_uptodate(struct nfs_page *req) @@ -312,7 +317,7 @@ static void nfs_readpage_result(struct rpc_task *task, */ int nfs_readpage(struct file *file, struct page *page) { - struct nfs_open_context *ctx; + struct nfs_readdesc desc; struct inode *inode = page_file_mapping(page)->host; int ret; @@ -339,39 +344,34 @@ int nfs_readpage(struct file *file, struct page *page) if (file == NULL) { ret = -EBADF; - ctx = nfs_find_open_context(inode, NULL, FMODE_READ); - if (ctx == NULL) + desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ); + if (desc.ctx == NULL) goto out_unlock; } else - ctx = get_nfs_open_context(nfs_file_open_context(file)); + desc.ctx = get_nfs_open_context(nfs_file_open_context(file)); if (!IS_SYNC(inode)) { - ret = nfs_readpage_from_fscache(ctx, inode, page); + ret = nfs_readpage_from_fscache(desc.ctx, inode, page); if (ret == 0) goto out; } - xchg(&ctx->error, 0); - ret = nfs_readpage_async(ctx, inode, page); + xchg(&desc.ctx->error, 0); + ret = nfs_readpage_async(&desc, inode, page); if (!ret) { ret = wait_on_page_locked_killable(page); if (!PageUptodate(page) && !ret) - ret = xchg(&ctx->error, 0); + ret = xchg(&desc.ctx->error, 0); } nfs_add_stats(inode, NFSIOS_READPAGES, 1); out: - put_nfs_open_context(ctx); + put_nfs_open_context(desc.ctx); return ret; out_unlock: unlock_page(page); return ret; } -struct nfs_readdesc { - struct nfs_pageio_descriptor *pgio; - struct nfs_open_context *ctx; -}; - static int readpage_async_filler(void *data, struct page *page) { @@ -390,9 +390,9 @@ readpage_async_filler(void *data, struct page *page) if (len < PAGE_SIZE) zero_user_segment(page, len, PAGE_SIZE); - if (!nfs_pageio_add_request(desc->pgio, new)) { + if (!nfs_pageio_add_request(&desc->pgio, new)) { nfs_list_remove_request(new); - error = desc->pgio->pg_error; + error = desc->pgio.pg_error; nfs_readpage_release(new, error); goto out; } @@ -407,7 +407,6 @@ out: int nfs_readpages(struct file *file, struct address_space *mapping, struct list_head *pages, unsigned nr_pages) { - struct nfs_pageio_descriptor pgio; struct nfs_pgio_mirror *pgm; struct nfs_readdesc desc; struct inode *inode = mapping->host; @@ -440,17 +439,16 @@ int nfs_readpages(struct file *file, struct address_space *mapping, if (ret == 0) goto read_complete; /* all pages were read */ - desc.pgio = &pgio; - nfs_pageio_init_read(&pgio, inode, false, + nfs_pageio_init_read(&desc.pgio, inode, false, &nfs_async_read_completion_ops); ret = read_cache_pages(mapping, pages, readpage_async_filler, &desc); - nfs_pageio_complete(&pgio); + nfs_pageio_complete(&desc.pgio); /* It doesn't make sense to do mirrored reads! */ - WARN_ON_ONCE(pgio.pg_mirror_count != 1); + WARN_ON_ONCE(desc.pgio.pg_mirror_count != 1); - pgm = &pgio.pg_mirrors[0]; + pgm = &desc.pgio.pg_mirrors[0]; NFS_I(inode)->read_io += pgm->pg_bytes_written; npages = (pgm->pg_bytes_written + PAGE_SIZE - 1) >> PAGE_SHIFT; diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 681ed98e4ba8..cb0248a34518 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -570,8 +570,7 @@ nfs_have_writebacks(struct inode *inode) extern int nfs_readpage(struct file *, struct page *); extern int nfs_readpages(struct file *, struct address_space *, struct list_head *, unsigned); -extern int nfs_readpage_async(struct nfs_open_context *, struct inode *, - struct page *); +extern int nfs_readpage_async(void *, struct inode *, struct page *); /* * inline functions -- cgit v1.2.3 From 0c119e3a18f994251c74c751e1657e4ef8da0c00 Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Thu, 28 Jan 2021 09:55:02 -0500 Subject: NFS: Call readpage_async_filler() from nfs_readpage_async() Refactor slightly so nfs_readpage_async() calls into readpage_async_filler(). Signed-off-by: Dave Wysochanski Signed-off-by: Anna Schumaker --- fs/nfs/read.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/nfs/read.c b/fs/nfs/read.c index 8c05e56dab65..0ed79e6bc486 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -119,31 +119,22 @@ struct nfs_readdesc { struct nfs_open_context *ctx; }; +static int readpage_async_filler(void *data, struct page *page); + int nfs_readpage_async(void *data, struct inode *inode, struct page *page) { struct nfs_readdesc *desc = data; - struct nfs_page *new; - unsigned int len; struct nfs_pgio_mirror *pgm; - - len = nfs_page_length(page); - if (len == 0) - return nfs_return_empty_page(page); - new = nfs_create_request(desc->ctx, page, 0, len); - if (IS_ERR(new)) { - unlock_page(page); - return PTR_ERR(new); - } - if (len < PAGE_SIZE) - zero_user_segment(page, len, PAGE_SIZE); + int error; nfs_pageio_init_read(&desc->pgio, inode, false, &nfs_async_read_completion_ops); - if (!nfs_pageio_add_request(&desc->pgio, new)) { - nfs_list_remove_request(new); - nfs_readpage_release(new, desc->pgio.pg_error); - } + + error = readpage_async_filler(desc, page); + if (error) + goto out; + nfs_pageio_complete(&desc->pgio); /* It doesn't make sense to do mirrored reads! */ @@ -153,6 +144,9 @@ int nfs_readpage_async(void *data, struct inode *inode, NFS_I(inode)->read_io += pgm->pg_bytes_written; return desc->pgio.pg_error < 0 ? desc->pgio.pg_error : 0; + +out: + return error; } static void nfs_page_group_set_uptodate(struct nfs_page *req) -- cgit v1.2.3 From 1e83b173b2663b7d357309584678cf24787f0713 Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Thu, 28 Jan 2021 09:55:03 -0500 Subject: NFS: Add nfs_pageio_complete_read() and remove nfs_readpage_async() Add nfs_pageio_complete_read() and call this from both nfs_readpage() and nfs_readpages(), since the submission and accounting is the same for both functions. Signed-off-by: Dave Wysochanski Signed-off-by: Anna Schumaker --- fs/nfs/fscache.c | 4 -- fs/nfs/read.c | 137 ++++++++++++++++++++++--------------------------- include/linux/nfs_fs.h | 1 - 3 files changed, 61 insertions(+), 81 deletions(-) (limited to 'fs') diff --git a/fs/nfs/fscache.c b/fs/nfs/fscache.c index a60df88efc40..c4c021c6ebbd 100644 --- a/fs/nfs/fscache.c +++ b/fs/nfs/fscache.c @@ -390,10 +390,6 @@ static void nfs_readpage_from_fscache_complete(struct page *page, if (!error) { SetPageUptodate(page); unlock_page(page); - } else { - error = nfs_readpage_async(context, page->mapping->host, page); - if (error) - unlock_page(page); } } diff --git a/fs/nfs/read.c b/fs/nfs/read.c index 0ed79e6bc486..d2b6dce1f99f 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -74,6 +74,24 @@ void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio, } EXPORT_SYMBOL_GPL(nfs_pageio_init_read); +static void nfs_pageio_complete_read(struct nfs_pageio_descriptor *pgio, + struct inode *inode) +{ + struct nfs_pgio_mirror *pgm; + unsigned long npages; + + nfs_pageio_complete(pgio); + + /* It doesn't make sense to do mirrored reads! */ + WARN_ON_ONCE(pgio->pg_mirror_count != 1); + + pgm = &pgio->pg_mirrors[0]; + NFS_I(inode)->read_io += pgm->pg_bytes_written; + npages = (pgm->pg_bytes_written + PAGE_SIZE - 1) >> PAGE_SHIFT; + nfs_add_stats(inode, NFSIOS_READPAGES, npages); +} + + void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio) { struct nfs_pgio_mirror *mirror; @@ -119,36 +137,6 @@ struct nfs_readdesc { struct nfs_open_context *ctx; }; -static int readpage_async_filler(void *data, struct page *page); - -int nfs_readpage_async(void *data, struct inode *inode, - struct page *page) -{ - struct nfs_readdesc *desc = data; - struct nfs_pgio_mirror *pgm; - int error; - - nfs_pageio_init_read(&desc->pgio, inode, false, - &nfs_async_read_completion_ops); - - error = readpage_async_filler(desc, page); - if (error) - goto out; - - nfs_pageio_complete(&desc->pgio); - - /* It doesn't make sense to do mirrored reads! */ - WARN_ON_ONCE(desc->pgio.pg_mirror_count != 1); - - pgm = &desc->pgio.pg_mirrors[0]; - NFS_I(inode)->read_io += pgm->pg_bytes_written; - - return desc->pgio.pg_error < 0 ? desc->pgio.pg_error : 0; - -out: - return error; -} - static void nfs_page_group_set_uptodate(struct nfs_page *req) { if (nfs_page_group_sync_on_bit(req, PG_UPTODATE)) @@ -170,8 +158,7 @@ static void nfs_read_completion(struct nfs_pgio_header *hdr) if (test_bit(NFS_IOHDR_EOF, &hdr->flags)) { /* note: regions of the page not covered by a - * request are zeroed in nfs_readpage_async / - * readpage_async_filler */ + * request are zeroed in readpage_async_filler */ if (bytes > hdr->good_bytes) { /* nothing in this request was good, so zero * the full extent of the request */ @@ -303,6 +290,38 @@ static void nfs_readpage_result(struct rpc_task *task, nfs_readpage_retry(task, hdr); } +static int +readpage_async_filler(void *data, struct page *page) +{ + struct nfs_readdesc *desc = data; + struct nfs_page *new; + unsigned int len; + int error; + + len = nfs_page_length(page); + if (len == 0) + return nfs_return_empty_page(page); + + new = nfs_create_request(desc->ctx, page, 0, len); + if (IS_ERR(new)) + goto out_error; + + if (len < PAGE_SIZE) + zero_user_segment(page, len, PAGE_SIZE); + if (!nfs_pageio_add_request(&desc->pgio, new)) { + nfs_list_remove_request(new); + error = desc->pgio.pg_error; + nfs_readpage_release(new, error); + goto out; + } + return 0; +out_error: + error = PTR_ERR(new); + unlock_page(page); +out: + return error; +} + /* * Read a page over NFS. * We read the page synchronously in the following case: @@ -351,13 +370,20 @@ int nfs_readpage(struct file *file, struct page *page) } xchg(&desc.ctx->error, 0); - ret = nfs_readpage_async(&desc, inode, page); + nfs_pageio_init_read(&desc.pgio, inode, false, + &nfs_async_read_completion_ops); + + ret = readpage_async_filler(&desc, page); + + if (!ret) + nfs_pageio_complete_read(&desc.pgio, inode); + + ret = desc.pgio.pg_error < 0 ? desc.pgio.pg_error : 0; if (!ret) { ret = wait_on_page_locked_killable(page); if (!PageUptodate(page) && !ret) ret = xchg(&desc.ctx->error, 0); } - nfs_add_stats(inode, NFSIOS_READPAGES, 1); out: put_nfs_open_context(desc.ctx); return ret; @@ -366,45 +392,11 @@ out_unlock: return ret; } -static int -readpage_async_filler(void *data, struct page *page) -{ - struct nfs_readdesc *desc = (struct nfs_readdesc *)data; - struct nfs_page *new; - unsigned int len; - int error; - - len = nfs_page_length(page); - if (len == 0) - return nfs_return_empty_page(page); - - new = nfs_create_request(desc->ctx, page, 0, len); - if (IS_ERR(new)) - goto out_error; - - if (len < PAGE_SIZE) - zero_user_segment(page, len, PAGE_SIZE); - if (!nfs_pageio_add_request(&desc->pgio, new)) { - nfs_list_remove_request(new); - error = desc->pgio.pg_error; - nfs_readpage_release(new, error); - goto out; - } - return 0; -out_error: - error = PTR_ERR(new); - unlock_page(page); -out: - return error; -} - int nfs_readpages(struct file *file, struct address_space *mapping, struct list_head *pages, unsigned nr_pages) { - struct nfs_pgio_mirror *pgm; struct nfs_readdesc desc; struct inode *inode = mapping->host; - unsigned long npages; int ret; dprintk("NFS: nfs_readpages (%s/%Lu %d)\n", @@ -437,16 +429,9 @@ int nfs_readpages(struct file *file, struct address_space *mapping, &nfs_async_read_completion_ops); ret = read_cache_pages(mapping, pages, readpage_async_filler, &desc); - nfs_pageio_complete(&desc.pgio); - /* It doesn't make sense to do mirrored reads! */ - WARN_ON_ONCE(desc.pgio.pg_mirror_count != 1); + nfs_pageio_complete_read(&desc.pgio, inode); - pgm = &desc.pgio.pg_mirrors[0]; - NFS_I(inode)->read_io += pgm->pg_bytes_written; - npages = (pgm->pg_bytes_written + PAGE_SIZE - 1) >> - PAGE_SHIFT; - nfs_add_stats(inode, NFSIOS_READPAGES, npages); read_complete: put_nfs_open_context(desc.ctx); out: diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index cb0248a34518..3cfcf219e96b 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -570,7 +570,6 @@ nfs_have_writebacks(struct inode *inode) extern int nfs_readpage(struct file *, struct page *); extern int nfs_readpages(struct file *, struct address_space *, struct list_head *, unsigned); -extern int nfs_readpage_async(void *, struct inode *, struct page *); /* * inline functions -- cgit v1.2.3 From 13770a71ed35512cc73c6b350297a797f0b27880 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 1 Feb 2021 15:23:42 +0300 Subject: io_uring: Fix NULL dereference in error in io_sqe_files_register() If we hit a "goto out_free;" before the "ctx->file_data" pointer has been assigned then it leads to a NULL derefence when we call: free_fixed_rsrc_data(ctx->file_data); We can fix this by moving the assignment earlier. Fixes: 1ad555c6ae6e ("io_uring: create common fixed_rsrc_data allocation routines") Signed-off-by: Dan Carpenter Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index a8bf867b6cf2..6711200ece22 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7865,6 +7865,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, file_data = alloc_fixed_rsrc_data(ctx); if (!file_data) return -ENOMEM; + ctx->file_data = file_data; nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE); file_data->table = kcalloc(nr_tables, sizeof(*file_data->table), @@ -7874,7 +7875,6 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, if (io_sqe_alloc_file_tables(file_data, nr_tables, nr_args)) goto out_free; - ctx->file_data = file_data; for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { struct fixed_rsrc_table *table; -- cgit v1.2.3 From 9ae1f8dd372e0e4c020b345cf9e09f519265e981 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:51 +0000 Subject: io_uring: fix inconsistent lock state WARNING: inconsistent lock state inconsistent {HARDIRQ-ON-W} -> {IN-HARDIRQ-W} usage. syz-executor217/8450 [HC1[1]:SC0[0]:HE0:SE1] takes: ffff888023d6e620 (&fs->lock){?.+.}-{2:2}, at: spin_lock include/linux/spinlock.h:354 [inline] ffff888023d6e620 (&fs->lock){?.+.}-{2:2}, at: io_req_clean_work fs/io_uring.c:1398 [inline] ffff888023d6e620 (&fs->lock){?.+.}-{2:2}, at: io_dismantle_req+0x66f/0xf60 fs/io_uring.c:2029 other info that might help us debug this: Possible unsafe locking scenario: CPU0 ---- lock(&fs->lock); lock(&fs->lock); *** DEADLOCK *** 1 lock held by syz-executor217/8450: #0: ffff88802417c3e8 (&ctx->uring_lock){+.+.}-{3:3}, at: __do_sys_io_uring_enter+0x1071/0x1f30 fs/io_uring.c:9442 stack backtrace: CPU: 1 PID: 8450 Comm: syz-executor217 Not tainted 5.11.0-rc5-next-20210129-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: [...] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:354 [inline] io_req_clean_work fs/io_uring.c:1398 [inline] io_dismantle_req+0x66f/0xf60 fs/io_uring.c:2029 __io_free_req+0x3d/0x2e0 fs/io_uring.c:2046 io_free_req fs/io_uring.c:2269 [inline] io_double_put_req fs/io_uring.c:2392 [inline] io_put_req+0xf9/0x570 fs/io_uring.c:2388 io_link_timeout_fn+0x30c/0x480 fs/io_uring.c:6497 __run_hrtimer kernel/time/hrtimer.c:1519 [inline] __hrtimer_run_queues+0x609/0xe40 kernel/time/hrtimer.c:1583 hrtimer_interrupt+0x334/0x940 kernel/time/hrtimer.c:1645 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1085 [inline] __sysvec_apic_timer_interrupt+0x146/0x540 arch/x86/kernel/apic/apic.c:1102 asm_call_irq_on_stack+0xf/0x20 __run_sysvec_on_irqstack arch/x86/include/asm/irq_stack.h:37 [inline] run_sysvec_on_irqstack_cond arch/x86/include/asm/irq_stack.h:89 [inline] sysvec_apic_timer_interrupt+0xbd/0x100 arch/x86/kernel/apic/apic.c:1096 asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:629 RIP: 0010:__raw_spin_unlock_irq include/linux/spinlock_api_smp.h:169 [inline] RIP: 0010:_raw_spin_unlock_irq+0x25/0x40 kernel/locking/spinlock.c:199 spin_unlock_irq include/linux/spinlock.h:404 [inline] io_queue_linked_timeout+0x194/0x1f0 fs/io_uring.c:6525 __io_queue_sqe+0x328/0x1290 fs/io_uring.c:6594 io_queue_sqe+0x631/0x10d0 fs/io_uring.c:6639 io_queue_link_head fs/io_uring.c:6650 [inline] io_submit_sqe fs/io_uring.c:6697 [inline] io_submit_sqes+0x19b5/0x2720 fs/io_uring.c:6960 __do_sys_io_uring_enter+0x107d/0x1f30 fs/io_uring.c:9443 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9 Don't free requests from under hrtimer context (softirq) as it may sleep or take spinlocks improperly (e.g. non-irq versions). Cc: stable@vger.kernel.org # 5.6+ Reported-by: syzbot+81d17233a2b02eafba33@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6711200ece22..1310c074f4cc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1886,8 +1886,8 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) __io_cqring_fill_event(req, res, 0); } -static void io_req_complete_nostate(struct io_kiocb *req, long res, - unsigned int cflags) +static void io_req_complete_post(struct io_kiocb *req, long res, + unsigned int cflags) { struct io_ring_ctx *ctx = req->ctx; unsigned long flags; @@ -1898,6 +1898,12 @@ static void io_req_complete_nostate(struct io_kiocb *req, long res, spin_unlock_irqrestore(&ctx->completion_lock, flags); io_cqring_ev_posted(ctx); +} + +static inline void io_req_complete_nostate(struct io_kiocb *req, long res, + unsigned int cflags) +{ + io_req_complete_post(req, res, cflags); io_put_req(req); } @@ -6489,9 +6495,10 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) if (prev) { req_set_fail_links(prev); io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME); - io_put_req(prev); + io_put_req_deferred(prev, 1); } else { - io_req_complete(req, -ETIME); + io_req_complete_post(req, -ETIME, 0); + io_put_req_deferred(req, 1); } return HRTIMER_NORESTART; } -- cgit v1.2.3 From ba13e23f37c795bdd993523a6749d7afbf5ff7fb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:52 +0000 Subject: io_uring: kill not used needs_file_no_error We have no request types left using needs_file_no_error, remove it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 1310c074f4cc..66d9f3f4e43b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -788,8 +788,6 @@ struct io_submit_state { struct io_op_def { /* needs req->file assigned */ unsigned needs_file : 1; - /* don't fail if file grab fails */ - unsigned needs_file_no_error : 1; /* hash wq insertion if file is a regular file */ unsigned hash_reg_file : 1; /* unbound wq insertion if file is a non-regular file */ @@ -6896,8 +6894,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, bool fixed = req->flags & REQ_F_FIXED_FILE; req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed); - if (unlikely(!req->file && - !io_op_defs[req->opcode].needs_file_no_error)) + if (unlikely(!req->file)) ret = -EBADF; } -- cgit v1.2.3 From 34e08fed2c1cc67df88d85fedde1d05fec62e5ca Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:53 +0000 Subject: io_uring: inline io_req_drop_files() req->files now have same lifetime as all other iowq-work resources, inline io_req_drop_files() for consistency. Moreover, since REQ_F_INFLIGHT is no more files specific, the function name became very confusing. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 66d9f3f4e43b..9354e61243d9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1036,7 +1036,6 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, const struct iovec *fast_iov, struct iov_iter *iter, bool force); -static void io_req_drop_files(struct io_kiocb *req); static void io_req_task_queue(struct io_kiocb *req); static struct kmem_cache *req_cachep; @@ -1402,8 +1401,23 @@ static void io_req_clean_work(struct io_kiocb *req) free_fs_struct(fs); req->work.flags &= ~IO_WQ_WORK_FS; } - if (req->flags & REQ_F_INFLIGHT) - io_req_drop_files(req); + if (req->work.flags & IO_WQ_WORK_FILES) { + put_files_struct(req->work.identity->files); + put_nsproxy(req->work.identity->nsproxy); + req->work.flags &= ~IO_WQ_WORK_FILES; + } + if (req->flags & REQ_F_INFLIGHT) { + struct io_ring_ctx *ctx = req->ctx; + struct io_uring_task *tctx = req->task->io_uring; + unsigned long flags; + + spin_lock_irqsave(&ctx->inflight_lock, flags); + list_del(&req->inflight_entry); + spin_unlock_irqrestore(&ctx->inflight_lock, flags); + req->flags &= ~REQ_F_INFLIGHT; + if (atomic_read(&tctx->in_idle)) + wake_up(&tctx->wait); + } io_put_identity(req->task->io_uring, req); } @@ -6164,25 +6178,6 @@ static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EIOCBQUEUED; } -static void io_req_drop_files(struct io_kiocb *req) -{ - struct io_ring_ctx *ctx = req->ctx; - struct io_uring_task *tctx = req->task->io_uring; - unsigned long flags; - - if (req->work.flags & IO_WQ_WORK_FILES) { - put_files_struct(req->work.identity->files); - put_nsproxy(req->work.identity->nsproxy); - } - spin_lock_irqsave(&ctx->inflight_lock, flags); - list_del(&req->inflight_entry); - spin_unlock_irqrestore(&ctx->inflight_lock, flags); - req->flags &= ~REQ_F_INFLIGHT; - req->work.flags &= ~IO_WQ_WORK_FILES; - if (atomic_read(&tctx->in_idle)) - wake_up(&tctx->wait); -} - static void __io_clean_op(struct io_kiocb *req) { if (req->flags & REQ_F_BUFFER_SELECTED) { -- cgit v1.2.3 From e86d004729ae9ce7d16ff3fad3708e1601eec0d2 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:54 +0000 Subject: io_uring: remove work flags after cleanup Shouldn't be a problem now, but it's better to clean REQ_F_WORK_INITIALIZED and work->flags only after relevant resources are killed, so cancellation see them. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 9354e61243d9..9b1f919b05c9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1374,22 +1374,14 @@ static void io_req_clean_work(struct io_kiocb *req) if (!(req->flags & REQ_F_WORK_INITIALIZED)) return; - req->flags &= ~REQ_F_WORK_INITIALIZED; - - if (req->work.flags & IO_WQ_WORK_MM) { + if (req->work.flags & IO_WQ_WORK_MM) mmdrop(req->work.identity->mm); - req->work.flags &= ~IO_WQ_WORK_MM; - } #ifdef CONFIG_BLK_CGROUP - if (req->work.flags & IO_WQ_WORK_BLKCG) { + if (req->work.flags & IO_WQ_WORK_BLKCG) css_put(req->work.identity->blkcg_css); - req->work.flags &= ~IO_WQ_WORK_BLKCG; - } #endif - if (req->work.flags & IO_WQ_WORK_CREDS) { + if (req->work.flags & IO_WQ_WORK_CREDS) put_cred(req->work.identity->creds); - req->work.flags &= ~IO_WQ_WORK_CREDS; - } if (req->work.flags & IO_WQ_WORK_FS) { struct fs_struct *fs = req->work.identity->fs; @@ -1399,12 +1391,10 @@ static void io_req_clean_work(struct io_kiocb *req) spin_unlock(&req->work.identity->fs->lock); if (fs) free_fs_struct(fs); - req->work.flags &= ~IO_WQ_WORK_FS; } if (req->work.flags & IO_WQ_WORK_FILES) { put_files_struct(req->work.identity->files); put_nsproxy(req->work.identity->nsproxy); - req->work.flags &= ~IO_WQ_WORK_FILES; } if (req->flags & REQ_F_INFLIGHT) { struct io_ring_ctx *ctx = req->ctx; @@ -1419,6 +1409,9 @@ static void io_req_clean_work(struct io_kiocb *req) wake_up(&tctx->wait); } + req->flags &= ~REQ_F_WORK_INITIALIZED; + req->work.flags &= ~(IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | IO_WQ_WORK_FS | + IO_WQ_WORK_CREDS | IO_WQ_WORK_FILES); io_put_identity(req->task->io_uring, req); } -- cgit v1.2.3 From ce3d5aae331fa0eb1e88199e0380f517ed0c58f6 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:55 +0000 Subject: io_uring: deduplicate adding to REQ_F_INFLIGHT We don't know for how long REQ_F_INFLIGHT is going to stay, cleaner to extract a helper for marking requests as so. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 9b1f919b05c9..77878274fcb1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1460,11 +1460,24 @@ static bool io_identity_cow(struct io_kiocb *req) return true; } +static void io_req_track_inflight(struct io_kiocb *req) +{ + struct io_ring_ctx *ctx = req->ctx; + + if (!(req->flags & REQ_F_INFLIGHT)) { + io_req_init_async(req); + req->flags |= REQ_F_INFLIGHT; + + spin_lock_irq(&ctx->inflight_lock); + list_add(&req->inflight_entry, &ctx->inflight_list); + spin_unlock_irq(&ctx->inflight_lock); + } +} + static bool io_grab_identity(struct io_kiocb *req) { const struct io_op_def *def = &io_op_defs[req->opcode]; struct io_identity *id = req->work.identity; - struct io_ring_ctx *ctx = req->ctx; if (def->work_flags & IO_WQ_WORK_FSIZE) { if (id->fsize != rlimit(RLIMIT_FSIZE)) @@ -1520,15 +1533,8 @@ static bool io_grab_identity(struct io_kiocb *req) return false; atomic_inc(&id->files->count); get_nsproxy(id->nsproxy); - - if (!(req->flags & REQ_F_INFLIGHT)) { - req->flags |= REQ_F_INFLIGHT; - - spin_lock_irq(&ctx->inflight_lock); - list_add(&req->inflight_entry, &ctx->inflight_list); - spin_unlock_irq(&ctx->inflight_lock); - } req->work.flags |= IO_WQ_WORK_FILES; + io_req_track_inflight(req); } if (!(req->work.flags & IO_WQ_WORK_MM) && (def->work_flags & IO_WQ_WORK_MM)) { @@ -6443,16 +6449,8 @@ static struct file *io_file_get(struct io_submit_state *state, file = __io_file_get(state, fd); } - if (file && file->f_op == &io_uring_fops && - !(req->flags & REQ_F_INFLIGHT)) { - io_req_init_async(req); - req->flags |= REQ_F_INFLIGHT; - - spin_lock_irq(&ctx->inflight_lock); - list_add(&req->inflight_entry, &ctx->inflight_list); - spin_unlock_irq(&ctx->inflight_lock); - } - + if (file && unlikely(file->f_op == &io_uring_fops)) + io_req_track_inflight(req); return file; } -- cgit v1.2.3 From 57cd657b8272a66277c139e7bbdc8b86057cb415 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Feb 2021 18:59:56 +0000 Subject: io_uring: simplify do_read return parsing do_read() returning 0 bytes read (not -EAGAIN/etc.) is not an important enough of a case to prioritise it. Fold it into ret < 0 check, so we get rid of an extra if and make it a bit more readable. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 77878274fcb1..24ad36d71289 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3526,7 +3526,6 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, else kiocb->ki_flags |= IOCB_NOWAIT; - /* If the file doesn't support async, just async punt */ no_async = force_nonblock && !io_file_supports_async(req->file, READ); if (no_async) @@ -3538,9 +3537,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, ret = io_iter_do_read(req, iter); - if (!ret) { - goto done; - } else if (ret == -EIOCBQUEUED) { + if (ret == -EIOCBQUEUED) { ret = 0; goto out_free; } else if (ret == -EAGAIN) { @@ -3554,7 +3551,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = 0; goto copy_iov; - } else if (ret < 0) { + } else if (ret <= 0) { /* make sure -ERESTARTSYS -> -EINTR is done */ goto done; } -- cgit v1.2.3 From b0ff4fe746fd028eef920ddc8c7b0361c1ede6ec Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Tue, 26 Jan 2021 17:00:42 -0800 Subject: f2fs: flush data when enabling checkpoint back During checkpoint=disable period, f2fs bypasses all the synchronous IOs such as sync and fsync. So, when enabling it back, we must flush all of them in order to keep the data persistent. Otherwise, suddern power-cut right after enabling checkpoint will cause data loss. Fixes: 4354994f097d ("f2fs: checkpoint disabling") Cc: stable@vger.kernel.org Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/super.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index fc343243799c..429bc00af440 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -1892,6 +1892,9 @@ restore_flag: static void f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) { + /* we should flush all the data to keep data consistency */ + sync_inodes_sb(sbi->sb); + down_write(&sbi->gc_lock); f2fs_dirty_to_prefree(sbi); -- cgit v1.2.3 From 25fb04dbce6a0e165d28fd1fa8a1d7018c637fe8 Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Thu, 28 Jan 2021 17:02:56 +0800 Subject: f2fs: fix to avoid inconsistent quota data Occasionally, quota data may be corrupted detected by fsck: Info: checkpoint state = 45 : crc compacted_summary unmount [QUOTA WARNING] Usage inconsistent for ID 0:actual (1543036928, 762) != expected (1543032832, 762) [ASSERT] (fsck_chk_quota_files:1986) --> Quota file is missing or invalid quota file content found. [QUOTA WARNING] Usage inconsistent for ID 0:actual (1352478720, 344) != expected (1352474624, 344) [ASSERT] (fsck_chk_quota_files:1986) --> Quota file is missing or invalid quota file content found. [FSCK] Unreachable nat entries [Ok..] [0x0] [FSCK] SIT valid block bitmap checking [Ok..] [FSCK] Hard link checking for regular file [Ok..] [0x0] [FSCK] valid_block_count matching with CP [Ok..] [0xdf299] [FSCK] valid_node_count matcing with CP (de lookup) [Ok..] [0x2b01] [FSCK] valid_node_count matcing with CP (nat lookup) [Ok..] [0x2b01] [FSCK] valid_inode_count matched with CP [Ok..] [0x2665] [FSCK] free segment_count matched with CP [Ok..] [0xcb04] [FSCK] next block offset is free [Ok..] [FSCK] fixing SIT types [FSCK] other corrupted bugs [Fail] The root cause is: If we open file w/ readonly flag, disk quota info won't be initialized for this file, however, following mmap() will force to convert inline inode via f2fs_convert_inline_inode(), which may increase block usage for this inode w/o updating quota data, it causes inconsistent disk quota info. The issue will happen in following stack: open(file, O_RDONLY) mmap(file) - f2fs_convert_inline_inode - f2fs_convert_inline_page - f2fs_reserve_block - f2fs_reserve_new_block - f2fs_reserve_new_blocks - f2fs_i_blocks_write - dquot_claim_block inode->i_blocks increase, but the dqb_curspace keep the size for the dquots is NULL. To fix this issue, let's call dquot_initialize() anyway in both f2fs_truncate() and f2fs_convert_inline_inode() functions to avoid potential inconsistent quota data issue. Fixes: 0abd675e97e6 ("f2fs: support plain user/group quota") Signed-off-by: Daiyue Zhang Signed-off-by: Dehe Gu Signed-off-by: Junchao Jiang Signed-off-by: Ge Qiu Signed-off-by: Yi Chen Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 4 ++++ fs/f2fs/inline.c | 4 ++++ 2 files changed, 8 insertions(+) (limited to 'fs') diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 7db27c81d034..00b2ce47fa37 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -771,6 +771,10 @@ int f2fs_truncate(struct inode *inode) return -EIO; } + err = dquot_initialize(inode); + if (err) + return err; + /* we should check inline_data size */ if (!f2fs_may_inline_data(inode)) { err = f2fs_convert_inline_inode(inode); diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 806ebabf5870..993caefcd2bb 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -192,6 +192,10 @@ int f2fs_convert_inline_inode(struct inode *inode) f2fs_hw_is_readonly(sbi) || f2fs_readonly(sbi->sb)) return 0; + err = dquot_initialize(inode); + if (err) + return err; + page = f2fs_grab_cache_page(inode->i_mapping, 0, false); if (!page) return -ENOMEM; -- cgit v1.2.3 From 2e0cd472a0dd9b9a35699502570015af15d7c70f Mon Sep 17 00:00:00 2001 From: Liu Song Date: Sun, 31 Jan 2021 20:26:05 +0800 Subject: f2fs: remove unnecessary initialization in xattr.c These variables will be explicitly assigned before use, so there is no need to initialize. Signed-off-by: Liu Song Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/xattr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index 2086bef6c154..8159fae74b9a 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -327,7 +327,7 @@ static int lookup_all_xattrs(struct inode *inode, struct page *ipage, void *last_addr = NULL; nid_t xnid = F2FS_I(inode)->i_xattr_nid; unsigned int inline_size = inline_xattr_size(inode); - int err = 0; + int err; if (!xnid && !inline_size) return -ENODATA; @@ -515,7 +515,7 @@ int f2fs_getxattr(struct inode *inode, int index, const char *name, void *buffer, size_t buffer_size, struct page *ipage) { struct f2fs_xattr_entry *entry = NULL; - int error = 0; + int error; unsigned int size, len; void *base_addr = NULL; int base_size; @@ -562,7 +562,7 @@ ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) struct inode *inode = d_inode(dentry); struct f2fs_xattr_entry *entry; void *base_addr, *last_base_addr; - int error = 0; + int error; size_t rest = buffer_size; down_read(&F2FS_I(inode)->i_xattr_sem); @@ -632,7 +632,7 @@ static int __f2fs_setxattr(struct inode *inode, int index, int found, newsize; size_t len; __u32 new_hsize; - int error = 0; + int error; if (name == NULL) return -EINVAL; -- cgit v1.2.3 From 8d8d1dbefc423d42d626cf5b81aac214870ebaab Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 1 Feb 2021 20:36:54 -0600 Subject: smb3: Fix out-of-bounds bug in SMB2_negotiate() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While addressing some warnings generated by -Warray-bounds, I found this bug that was introduced back in 2017: CC [M] fs/cifs/smb2pdu.o fs/cifs/smb2pdu.c: In function ‘SMB2_negotiate’: fs/cifs/smb2pdu.c:822:16: warning: array subscript 1 is above array bounds of ‘__le16[1]’ {aka ‘short unsigned int[1]’} [-Warray-bounds] 822 | req->Dialects[1] = cpu_to_le16(SMB30_PROT_ID); | ~~~~~~~~~~~~~^~~ fs/cifs/smb2pdu.c:823:16: warning: array subscript 2 is above array bounds of ‘__le16[1]’ {aka ‘short unsigned int[1]’} [-Warray-bounds] 823 | req->Dialects[2] = cpu_to_le16(SMB302_PROT_ID); | ~~~~~~~~~~~~~^~~ fs/cifs/smb2pdu.c:824:16: warning: array subscript 3 is above array bounds of ‘__le16[1]’ {aka ‘short unsigned int[1]’} [-Warray-bounds] 824 | req->Dialects[3] = cpu_to_le16(SMB311_PROT_ID); | ~~~~~~~~~~~~~^~~ fs/cifs/smb2pdu.c:816:16: warning: array subscript 1 is above array bounds of ‘__le16[1]’ {aka ‘short unsigned int[1]’} [-Warray-bounds] 816 | req->Dialects[1] = cpu_to_le16(SMB302_PROT_ID); | ~~~~~~~~~~~~~^~~ At the time, the size of array _Dialects_ was changed from 1 to 3 in struct validate_negotiate_info_req, and then in 2019 it was changed from 3 to 4, but those changes were never made in struct smb2_negotiate_req, which has led to a 3 and a half years old out-of-bounds bug in function SMB2_negotiate() (fs/cifs/smb2pdu.c). Fix this by increasing the size of array _Dialects_ in struct smb2_negotiate_req to 4. Fixes: 9764c02fcbad ("SMB3: Add support for multidialect negotiate (SMB2.1 and later)") Fixes: d5c7076b772a ("smb3: add smb3.1.1 to default dialect list") Cc: stable@vger.kernel.org Signed-off-by: Gustavo A. R. Silva Signed-off-by: Steve French --- fs/cifs/smb2pdu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/cifs/smb2pdu.h b/fs/cifs/smb2pdu.h index d85edf5d1429..a5a9e33c0d73 100644 --- a/fs/cifs/smb2pdu.h +++ b/fs/cifs/smb2pdu.h @@ -286,7 +286,7 @@ struct smb2_negotiate_req { __le32 NegotiateContextOffset; /* SMB3.1.1 only. MBZ earlier */ __le16 NegotiateContextCount; /* SMB3.1.1 only. MBZ earlier */ __le16 Reserved2; - __le16 Dialects[1]; /* One dialect (vers=) at a time for now */ + __le16 Dialects[4]; /* BB expand this if autonegotiate > 4 dialects */ } __packed; /* Dialects */ -- cgit v1.2.3 From 39f71b7e40e21805d6b15fc7750bdd9cab6a5010 Mon Sep 17 00:00:00 2001 From: Dehe Gu Date: Tue, 2 Feb 2021 17:39:22 +0800 Subject: f2fs: fix a wrong condition in __submit_bio We should use !F2FS_IO_ALIGNED() to check and submit_io directly. Fixes: 8223ecc456d0 ("f2fs: fix to add missing F2FS_IO_ALIGNED() condition") Reviewed-by: Chao Yu Signed-off-by: Dehe Gu Signed-off-by: Ge Qiu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 38476d0d3916..1ee966a63df9 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -470,7 +470,7 @@ static inline void __submit_bio(struct f2fs_sb_info *sbi, if (f2fs_lfs_mode(sbi) && current->plug) blk_finish_plug(current->plug); - if (F2FS_IO_ALIGNED(sbi)) + if (!F2FS_IO_ALIGNED(sbi)) goto submit_io; start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS; -- cgit v1.2.3 From c8e43d55b1aa05d175daac25d228c7c1c71c7b11 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 2 Feb 2021 16:03:11 +0800 Subject: f2fs: relocate inline conversion from mmap() to mkwrite() If there is page fault only for read case on inline inode, we don't need to convert inline inode, instead, let's do conversion for write case. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 00b2ce47fa37..8e53f8898688 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -72,6 +72,10 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) goto err; } + err = f2fs_convert_inline_inode(inode); + if (err) + goto err; + #ifdef CONFIG_F2FS_FS_COMPRESSION if (f2fs_compressed_file(inode)) { int ret = f2fs_is_compressed_cluster(inode, page->index); @@ -506,7 +510,6 @@ static loff_t f2fs_llseek(struct file *file, loff_t offset, int whence) static int f2fs_file_mmap(struct file *file, struct vm_area_struct *vma) { struct inode *inode = file_inode(file); - int err; if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) return -EIO; @@ -514,11 +517,6 @@ static int f2fs_file_mmap(struct file *file, struct vm_area_struct *vma) if (!f2fs_is_compress_backend_ready(inode)) return -EOPNOTSUPP; - /* we don't need to use inline_data strictly */ - err = f2fs_convert_inline_inode(inode); - if (err) - return err; - file_accessed(file); vma->vm_ops = &f2fs_file_vm_ops; set_inode_flag(inode, FI_MMAP_FILE); -- cgit v1.2.3 From 59ebc7fd74506367b109497eeef36034e648c943 Mon Sep 17 00:00:00 2001 From: Zheng Yongjun Date: Thu, 24 Dec 2020 21:22:44 +0800 Subject: ext4: use DEFINE_MUTEX() for mutex lock mutex lock can be initialized automatically with DEFINE_MUTEX() rather than explicitly calling mutex_init(). Signed-off-by: Zheng Yongjun Link: https://lore.kernel.org/r/20201224132244.30907-1-zhengyongjun3@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/super.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 9a6f9875aa34..f361f9fb40d1 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -59,7 +59,7 @@ #include static struct ext4_lazy_init *ext4_li_info; -static struct mutex ext4_li_mtx; +static DEFINE_MUTEX(ext4_li_mtx); static struct ratelimit_state ext4_mount_msg_ratelimit; static int ext4_load_journal(struct super_block *, struct ext4_super_block *, @@ -6667,7 +6667,6 @@ static int __init ext4_init_fs(void) ratelimit_state_init(&ext4_mount_msg_ratelimit, 30 * HZ, 64); ext4_li_info = NULL; - mutex_init(&ext4_li_mtx); /* Build-time check for flags consistency */ ext4_check_flag_values(); -- cgit v1.2.3 From 027f14f5357279655c3ebc6d14daff8368d4f53f Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Thu, 21 Jan 2021 12:33:20 -0500 Subject: ext4: don't try to processed freed blocks until mballoc is initialized If we try to make any changes via the journal between when the journal is initialized, but before the multi-block allocated is initialized, we will end up deferencing a NULL pointer when the journal commit callback function calls ext4_process_freed_data(). The proximate cause of this failure was commit 2d01ddc86606 ("ext4: save error info to sb through journal if available") since file system corruption problems detected before the call to ext4_mb_init() would result in a journal commit before we aborted the mount of the file system.... and we would then trigger the NULL pointer deref. Link: https://lore.kernel.org/r/YAm8qH/0oo2ofSMR@mit.edu Reported-by: Murphy Zhou Reviewed-by: Jan Kara Signed-off-by: Theodore Ts'o --- fs/ext4/super.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ext4/super.c b/fs/ext4/super.c index f361f9fb40d1..071d131fadd8 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -4875,7 +4875,6 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) set_task_ioprio(sbi->s_journal->j_task, journal_ioprio); - sbi->s_journal->j_commit_callback = ext4_journal_commit_callback; sbi->s_journal->j_submit_inode_data_buffers = ext4_journal_submit_inode_data_buffers; sbi->s_journal->j_finish_inode_data_buffers = @@ -4987,6 +4986,14 @@ no_journal: goto failed_mount5; } + /* + * We can only set up the journal commit callback once + * mballoc is initialized + */ + if (sbi->s_journal) + sbi->s_journal->j_commit_callback = + ext4_journal_commit_callback; + block = ext4_count_free_clusters(sb); ext4_free_blocks_count_set(sbi->s_es, EXT4_C2B(sbi, block)); -- cgit v1.2.3 From 96e7c02d0bbca9f33fb4175e5f302dab1de6f985 Mon Sep 17 00:00:00 2001 From: Daejun Park Date: Mon, 11 Jan 2021 10:37:26 +0900 Subject: ext4: Change list_for_each* to list_for_each_entry* In the fast_commit.c, list_for_each* + list_entry can be changed to list_for_each_entry*. It reduces number of variables and lines. Signed-off-by: Daejun Park Link: https://lore.kernel.org/r/20210111013726epcms2p4579ae56040d7043db785bf0d0a785dc7@epcms2p4 Signed-off-by: Theodore Ts'o --- fs/ext4/fast_commit.c | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) (limited to 'fs') diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 0a14a7c87bf8..619412134bbf 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -915,13 +915,11 @@ static int ext4_fc_submit_inode_data_all(journal_t *journal) struct super_block *sb = (struct super_block *)(journal->j_private); struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *ei; - struct list_head *pos; int ret = 0; spin_lock(&sbi->s_fc_lock); ext4_set_mount_flag(sb, EXT4_MF_FC_COMMITTING); - list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) { - ei = list_entry(pos, struct ext4_inode_info, i_fc_list); + list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); while (atomic_read(&ei->i_fc_updates)) { DEFINE_WAIT(wait); @@ -978,17 +976,15 @@ __releases(&sbi->s_fc_lock) { struct super_block *sb = (struct super_block *)(journal->j_private); struct ext4_sb_info *sbi = EXT4_SB(sb); - struct ext4_fc_dentry_update *fc_dentry; + struct ext4_fc_dentry_update *fc_dentry, *fc_dentry_n; struct inode *inode; - struct list_head *pos, *n, *fcd_pos, *fcd_n; - struct ext4_inode_info *ei; + struct ext4_inode_info *ei, *ei_n; int ret; if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) return 0; - list_for_each_safe(fcd_pos, fcd_n, &sbi->s_fc_dentry_q[FC_Q_MAIN]) { - fc_dentry = list_entry(fcd_pos, struct ext4_fc_dentry_update, - fcd_list); + list_for_each_entry_safe(fc_dentry, fc_dentry_n, + &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) { spin_unlock(&sbi->s_fc_lock); if (!ext4_fc_add_dentry_tlv( @@ -1004,8 +1000,8 @@ __releases(&sbi->s_fc_lock) } inode = NULL; - list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) { - ei = list_entry(pos, struct ext4_inode_info, i_fc_list); + list_for_each_entry_safe(ei, ei_n, &sbi->s_fc_q[FC_Q_MAIN], + i_fc_list) { if (ei->vfs_inode.i_ino == fc_dentry->fcd_ino) { inode = &ei->vfs_inode; break; @@ -1057,7 +1053,6 @@ static int ext4_fc_perform_commit(journal_t *journal) struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *iter; struct ext4_fc_head head; - struct list_head *pos; struct inode *inode; struct blk_plug plug; int ret = 0; @@ -1099,8 +1094,7 @@ static int ext4_fc_perform_commit(journal_t *journal) goto out; } - list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) { - iter = list_entry(pos, struct ext4_inode_info, i_fc_list); + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { inode = &iter->vfs_inode; if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) continue; @@ -1226,9 +1220,8 @@ static void ext4_fc_cleanup(journal_t *journal, int full) { struct super_block *sb = journal->j_private; struct ext4_sb_info *sbi = EXT4_SB(sb); - struct ext4_inode_info *iter; + struct ext4_inode_info *iter, *iter_n; struct ext4_fc_dentry_update *fc_dentry; - struct list_head *pos, *n; if (full && sbi->s_fc_bh) sbi->s_fc_bh = NULL; @@ -1236,8 +1229,8 @@ static void ext4_fc_cleanup(journal_t *journal, int full) jbd2_fc_release_bufs(journal); spin_lock(&sbi->s_fc_lock); - list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) { - iter = list_entry(pos, struct ext4_inode_info, i_fc_list); + list_for_each_entry_safe(iter, iter_n, &sbi->s_fc_q[FC_Q_MAIN], + i_fc_list) { list_del_init(&iter->i_fc_list); ext4_clear_inode_state(&iter->vfs_inode, EXT4_STATE_FC_COMMITTING); -- cgit v1.2.3 From c6c818e50d1982ebac4dcd3ae18c1b49a66ebacb Mon Sep 17 00:00:00 2001 From: Vinicius Tinti Date: Tue, 2 Feb 2021 16:28:37 +0000 Subject: ext4: factor out htree rep invariant check This patch moves some debugging code which is used to validate the hash tree node when doing a binary search of an htree node into a separate function, which is disabled by default (since it is only used by developers when they are modifying the htree code paths). In addition to cleaning up the code to make it more maintainable, it silences a Clang compiler warning when -Wunreachable-code-aggressive is enabled. (There is no plan to enable this warning by default, since there it has far too many false positives; nevertheless, this commit reduces one of the many false positives by one.) Signed-off-by: Vinicius Tinti Link: https://lore.kernel.org/r/20210202162837.129631-1-viniciustinti@gmail.com Signed-off-by: Theodore Ts'o --- fs/ext4/namei.c | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index cf652ba3e74d..a6e28b4b5a95 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -731,6 +731,29 @@ struct stats dx_show_entries(struct dx_hash_info *hinfo, struct inode *dir, (space/bcount)*100/blocksize); return (struct stats) { names, space, bcount}; } + +/* + * Linear search cross check + */ +static inline void htree_rep_invariant_check(struct dx_entry *at, + struct dx_entry *target, + u32 hash, unsigned int n) +{ + while (n--) { + dxtrace(printk(KERN_CONT ",")); + if (dx_get_hash(++at) > hash) { + at--; + break; + } + } + ASSERT(at == target - 1); +} +#else /* DX_DEBUG */ +static inline void htree_rep_invariant_check(struct dx_entry *at, + struct dx_entry *target, + u32 hash, unsigned int n) +{ +} #endif /* DX_DEBUG */ /* @@ -827,20 +850,7 @@ dx_probe(struct ext4_filename *fname, struct inode *dir, p = m + 1; } - if (0) { // linear search cross check - unsigned n = count - 1; - at = entries; - while (n--) - { - dxtrace(printk(KERN_CONT ",")); - if (dx_get_hash(++at) > hash) - { - at--; - break; - } - } - ASSERT(at == p - 1); - } + htree_rep_invariant_check(entries, p, hash, count - 1); at = p - 1; dxtrace(printk(KERN_CONT " %x->%u\n", -- cgit v1.2.3 From 15e20a301ab06575482c7ab3b442a6830cec928e Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Wed, 3 Feb 2021 16:15:27 +0100 Subject: gfs2: Use sb_start_intwrite in gfs2_ail_empty_gl Commit 2e60d7683c8d ("GFS2: update freeze code to use freeze/thaw_super on all nodes") optimized away the sb_start_intwrite ... sb_end_intwrite protection for the on-stack transactions in gfs2_ail_empty_gl with no explanation. I can't think of a valid reason for doing that, so revert that change. This simplifies the next commit. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/glops.c | 5 ++++- fs/gfs2/trans.c | 8 +++----- 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c index 3faa421568b0..fd1f52fff170 100644 --- a/fs/gfs2/glops.c +++ b/fs/gfs2/glops.c @@ -127,9 +127,12 @@ static int gfs2_ail_empty_gl(struct gfs2_glock *gl) * on the stack */ tr.tr_reserved = 1 + gfs2_struct2blk(sdp, tr.tr_revokes); tr.tr_ip = _RET_IP_; + sb_start_intwrite(sdp->sd_vfs); ret = gfs2_log_reserve(sdp, tr.tr_reserved); - if (ret < 0) + if (ret < 0) { + sb_end_intwrite(sdp->sd_vfs); return ret; + } WARN_ON_ONCE(current->journal_info); current->journal_info = &tr; diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index 7705f04621f4..ae040b570868 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -98,10 +98,9 @@ void gfs2_trans_end(struct gfs2_sbd *sdp) if (!test_bit(TR_TOUCHED, &tr->tr_flags)) { gfs2_log_release(sdp, tr->tr_reserved); - if (alloced) { + if (alloced) gfs2_trans_free(sdp, tr); - sb_end_intwrite(sdp->sd_vfs); - } + sb_end_intwrite(sdp->sd_vfs); return; } @@ -121,8 +120,7 @@ void gfs2_trans_end(struct gfs2_sbd *sdp) if (sdp->sd_vfs->s_flags & SB_SYNCHRONOUS) gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_TRANS_END); - if (alloced) - sb_end_intwrite(sdp->sd_vfs); + sb_end_intwrite(sdp->sd_vfs); } static struct gfs2_bufdata *gfs2_alloc_bufdata(struct gfs2_glock *gl, -- cgit v1.2.3 From 1aecf3734a95f3c167d1495550ca57556d33f7ec Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 29 Jan 2021 19:06:10 -0800 Subject: xfs: fix chown leaking delalloc quota blocks when fssetxattr fails While refactoring the quota code to create a function to allocate inode change transactions, I noticed that xfs_qm_vop_chown_reserve does more than just make reservations: it also *modifies* the incore counts directly to handle the owner id change for the delalloc blocks. I then observed that the fssetxattr code continues validating input arguments after making the quota reservation but before dirtying the transaction. If the routine decides to error out, it fails to undo the accounting switch! This leads to incorrect quota reservation and failure down the line. We can fix this by making the reservation function do only that -- for the new dquot, it reserves ondisk and delalloc blocks to the transaction, and the old dquot hangs on to its incore reservation for now. Once we actually switch the dquots, we can then update the incore reservations because we've dirtied the transaction and it's too late to turn back now. No fixes tag because this has been broken since the start of git. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_qm.c | 92 ++++++++++++++++++++++----------------------------------- 1 file changed, 35 insertions(+), 57 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index c134eb4aeaa8..c2e4d3a27469 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -1785,6 +1785,29 @@ xfs_qm_vop_chown( xfs_trans_mod_dquot(tp, newdq, bfield, ip->i_d.di_nblocks); xfs_trans_mod_dquot(tp, newdq, XFS_TRANS_DQ_ICOUNT, 1); + /* + * Back when we made quota reservations for the chown, we reserved the + * ondisk blocks + delalloc blocks with the new dquot. Now that we've + * switched the dquots, decrease the new dquot's block reservation + * (having already bumped up the real counter) so that we don't have + * any reservation to give back when we commit. + */ + xfs_trans_mod_dquot(tp, newdq, XFS_TRANS_DQ_RES_BLKS, + -ip->i_delayed_blks); + + /* + * Give the incore reservation for delalloc blocks back to the old + * dquot. We don't normally handle delalloc quota reservations + * transactionally, so just lock the dquot and subtract from the + * reservation. Dirty the transaction because it's too late to turn + * back now. + */ + tp->t_flags |= XFS_TRANS_DIRTY; + xfs_dqlock(prevdq); + ASSERT(prevdq->q_blk.reserved >= ip->i_delayed_blks); + prevdq->q_blk.reserved -= ip->i_delayed_blks; + xfs_dqunlock(prevdq); + /* * Take an extra reference, because the inode is going to keep * this dquot pointer even after the trans_commit. @@ -1807,84 +1830,39 @@ xfs_qm_vop_chown_reserve( uint flags) { struct xfs_mount *mp = ip->i_mount; - uint64_t delblks; unsigned int blkflags; - struct xfs_dquot *udq_unres = NULL; - struct xfs_dquot *gdq_unres = NULL; - struct xfs_dquot *pdq_unres = NULL; struct xfs_dquot *udq_delblks = NULL; struct xfs_dquot *gdq_delblks = NULL; struct xfs_dquot *pdq_delblks = NULL; - int error; - ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL|XFS_ILOCK_SHARED)); ASSERT(XFS_IS_QUOTA_RUNNING(mp)); - delblks = ip->i_delayed_blks; blkflags = XFS_IS_REALTIME_INODE(ip) ? XFS_QMOPT_RES_RTBLKS : XFS_QMOPT_RES_REGBLKS; if (XFS_IS_UQUOTA_ON(mp) && udqp && - i_uid_read(VFS_I(ip)) != udqp->q_id) { + i_uid_read(VFS_I(ip)) != udqp->q_id) udq_delblks = udqp; - /* - * If there are delayed allocation blocks, then we have to - * unreserve those from the old dquot, and add them to the - * new dquot. - */ - if (delblks) { - ASSERT(ip->i_udquot); - udq_unres = ip->i_udquot; - } - } + if (XFS_IS_GQUOTA_ON(ip->i_mount) && gdqp && - i_gid_read(VFS_I(ip)) != gdqp->q_id) { + i_gid_read(VFS_I(ip)) != gdqp->q_id) gdq_delblks = gdqp; - if (delblks) { - ASSERT(ip->i_gdquot); - gdq_unres = ip->i_gdquot; - } - } if (XFS_IS_PQUOTA_ON(ip->i_mount) && pdqp && - ip->i_d.di_projid != pdqp->q_id) { + ip->i_d.di_projid != pdqp->q_id) pdq_delblks = pdqp; - if (delblks) { - ASSERT(ip->i_pdquot); - pdq_unres = ip->i_pdquot; - } - } - - error = xfs_trans_reserve_quota_bydquots(tp, ip->i_mount, - udq_delblks, gdq_delblks, pdq_delblks, - ip->i_d.di_nblocks, 1, flags | blkflags); - if (error) - return error; /* - * Do the delayed blks reservations/unreservations now. Since, these - * are done without the help of a transaction, if a reservation fails - * its previous reservations won't be automatically undone by trans - * code. So, we have to do it manually here. + * Reserve enough quota to handle blocks on disk and reserved for a + * delayed allocation. We'll actually transfer the delalloc + * reservation between dquots at chown time, even though that part is + * only semi-transactional. */ - if (delblks) { - /* - * Do the reservations first. Unreservation can't fail. - */ - ASSERT(udq_delblks || gdq_delblks || pdq_delblks); - ASSERT(udq_unres || gdq_unres || pdq_unres); - error = xfs_trans_reserve_quota_bydquots(NULL, ip->i_mount, - udq_delblks, gdq_delblks, pdq_delblks, - (xfs_qcnt_t)delblks, 0, flags | blkflags); - if (error) - return error; - xfs_trans_reserve_quota_bydquots(NULL, ip->i_mount, - udq_unres, gdq_unres, pdq_unres, - -((xfs_qcnt_t)delblks), 0, blkflags); - } - - return 0; + return xfs_trans_reserve_quota_bydquots(tp, ip->i_mount, udq_delblks, + gdq_delblks, pdq_delblks, + ip->i_d.di_nblocks + ip->i_delayed_blks, + 1, blkflags | flags); } int -- cgit v1.2.3 From b8055ed6779d675e30f019ba3b7141848a4d6558 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 28 Jan 2021 10:56:38 -0800 Subject: xfs: reduce quota reservation when doing a dax unwritten extent conversion In commit 3b0fe47805802, we reduced the free space requirement to perform a pre-write unwritten extent conversion on an S_DAX file. Since we're not actually allocating any space, the logic goes, we only need enough reservation to handle shape changes in the bmbt. The same logic should have been applied to quota -- we're not allocating any space, so we only need to reserve enough quota to handle the bmbt shape changes. Fixes: 3b0fe4780580 ("xfs: Don't use reserved blocks for data blocks with DAX") Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_iomap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 8f4b27cded20..b05cfeb09301 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -236,7 +236,7 @@ xfs_iomap_write_direct( bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO; if (imap->br_state == XFS_EXT_UNWRITTEN) { tflags |= XFS_TRANS_RESERVE; - resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1; + resblks = qblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1; } } error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, resrtextents, -- cgit v1.2.3 From 4abe21ad67a7b9dc6844f55e91a6e3ef81879d42 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:33 -0800 Subject: xfs: clean up quota reservation callsites Convert a few xfs_trans_*reserve* callsites that are open-coding other convenience functions. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/libxfs/xfs_bmap.c | 3 +-- fs/xfs/xfs_bmap_util.c | 4 ++-- fs/xfs/xfs_reflink.c | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 7ea1dbbe3d0b..c730288b5981 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4937,8 +4937,7 @@ xfs_bmap_del_extent_delay( * sb counters as we might have to borrow some blocks for the * indirect block accounting. */ - error = xfs_trans_reserve_quota_nblks(NULL, ip, - -((long)del->br_blockcount), 0, + error = xfs_trans_unreserve_quota_nblks(NULL, ip, del->br_blockcount, 0, isrt ? XFS_QMOPT_RES_RTBLKS : XFS_QMOPT_RES_REGBLKS); if (error) return error; diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index f3f8c48ff5bf..792809debaaa 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -884,8 +884,8 @@ xfs_unmap_extent( } xfs_ilock(ip, XFS_ILOCK_EXCL); - error = xfs_trans_reserve_quota(tp, mp, ip->i_udquot, ip->i_gdquot, - ip->i_pdquot, resblks, 0, XFS_QMOPT_RES_REGBLKS); + error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, + XFS_QMOPT_RES_REGBLKS); if (error) goto out_trans_cancel; diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index e1c98dbf79e4..183142fd0961 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -508,8 +508,8 @@ xfs_reflink_cancel_cow_blocks( xfs_bmap_del_extent_cow(ip, &icur, &got, &del); /* Remove the quota reservation */ - error = xfs_trans_reserve_quota_nblks(NULL, ip, - -(long)del.br_blockcount, 0, + error = xfs_trans_unreserve_quota_nblks(NULL, ip, + del.br_blockcount, 0, XFS_QMOPT_RES_REGBLKS); if (error) break; -- cgit v1.2.3 From 8554650003b8a66f3dd357692ab73101d088d938 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:34 -0800 Subject: xfs: create convenience wrappers for incore quota block reservations Create a couple of convenience wrappers for creating and deleting quota block reservations against future changes. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/libxfs/xfs_bmap.c | 10 ++++------ fs/xfs/xfs_quota.h | 19 +++++++++++++++++++ fs/xfs/xfs_reflink.c | 5 ++--- 3 files changed, 25 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index c730288b5981..94d582a9587d 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4097,8 +4097,7 @@ xfs_bmapi_reserve_delalloc( * blocks. This number gets adjusted later. We return if we haven't * allocated blocks already inside this loop. */ - error = xfs_trans_reserve_quota_nblks(NULL, ip, (long)alen, 0, - XFS_QMOPT_RES_REGBLKS); + error = xfs_quota_reserve_blkres(ip, alen); if (error) return error; @@ -4144,8 +4143,7 @@ out_unreserve_blocks: xfs_mod_fdblocks(mp, alen, false); out_unreserve_quota: if (XFS_IS_QUOTA_ON(mp)) - xfs_trans_unreserve_quota_nblks(NULL, ip, (long)alen, 0, - XFS_QMOPT_RES_REGBLKS); + xfs_quota_unreserve_blkres(ip, alen); return error; } @@ -4937,8 +4935,8 @@ xfs_bmap_del_extent_delay( * sb counters as we might have to borrow some blocks for the * indirect block accounting. */ - error = xfs_trans_unreserve_quota_nblks(NULL, ip, del->br_blockcount, 0, - isrt ? XFS_QMOPT_RES_RTBLKS : XFS_QMOPT_RES_REGBLKS); + ASSERT(!isrt); + error = xfs_quota_unreserve_blkres(ip, del->br_blockcount); if (error) return error; ip->i_delayed_blks -= del->br_blockcount; diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h index 5a62398940d0..1d1a1634ea29 100644 --- a/fs/xfs/xfs_quota.h +++ b/fs/xfs/xfs_quota.h @@ -108,6 +108,12 @@ extern void xfs_qm_mount_quotas(struct xfs_mount *); extern void xfs_qm_unmount(struct xfs_mount *); extern void xfs_qm_unmount_quotas(struct xfs_mount *); +static inline int +xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) +{ + return xfs_trans_reserve_quota_nblks(NULL, ip, blocks, 0, + XFS_QMOPT_RES_REGBLKS); +} #else static inline int xfs_qm_vop_dqalloc(struct xfs_inode *ip, kuid_t kuid, kgid_t kgid, @@ -136,6 +142,13 @@ static inline int xfs_trans_reserve_quota_bydquots(struct xfs_trans *tp, { return 0; } + +static inline int +xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) +{ + return 0; +} + #define xfs_qm_vop_create_dqattach(tp, ip, u, g, p) #define xfs_qm_vop_rename_dqattach(it) (0) #define xfs_qm_vop_chown(tp, ip, old, new) (NULL) @@ -157,6 +170,12 @@ static inline int xfs_trans_reserve_quota_bydquots(struct xfs_trans *tp, xfs_trans_reserve_quota_bydquots(tp, mp, ud, gd, pd, nb, ni, \ f | XFS_QMOPT_RES_REGBLKS) +static inline int +xfs_quota_unreserve_blkres(struct xfs_inode *ip, int64_t blocks) +{ + return xfs_quota_reserve_blkres(ip, -blocks); +} + extern int xfs_mount_reset_sbqflags(struct xfs_mount *); #endif /* __XFS_QUOTA_H__ */ diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 183142fd0961..bea64ed5a57f 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -508,9 +508,8 @@ xfs_reflink_cancel_cow_blocks( xfs_bmap_del_extent_cow(ip, &icur, &got, &del); /* Remove the quota reservation */ - error = xfs_trans_unreserve_quota_nblks(NULL, ip, - del.br_blockcount, 0, - XFS_QMOPT_RES_REGBLKS); + error = xfs_quota_unreserve_blkres(ip, + del.br_blockcount); if (error) break; } else { -- cgit v1.2.3 From 35b1101099e85af74a46b8e36f4d1fdac0367ffd Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 26 Jan 2021 17:23:30 -0800 Subject: xfs: remove xfs_trans_unreserve_quota_nblks completely xfs_trans_cancel will release all the quota resources that were reserved on behalf of the transaction, so get rid of the explicit unreserve step. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_bmap_util.c | 11 ++++------- fs/xfs/xfs_iomap.c | 6 ++---- fs/xfs/xfs_quota.h | 2 -- fs/xfs/xfs_reflink.c | 5 +---- 4 files changed, 7 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 792809debaaa..ae2d98af693c 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -820,12 +820,12 @@ xfs_alloc_file_space( error = xfs_trans_reserve_quota_nblks(tp, ip, qblocks, 0, quota_flag); if (error) - goto error1; + goto error; error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, XFS_IEXT_ADD_NOSPLIT_CNT); if (error) - goto error0; + goto error; xfs_trans_ijoin(tp, ip, 0); @@ -833,7 +833,7 @@ xfs_alloc_file_space( allocatesize_fsb, alloc_type, 0, imapp, &nimaps); if (error) - goto error0; + goto error; /* * Complete the transaction @@ -856,10 +856,7 @@ xfs_alloc_file_space( return error; -error0: /* unlock inode, unreserve quota blocks, cancel trans */ - xfs_trans_unreserve_quota_nblks(tp, ip, (long)qblocks, 0, quota_flag); - -error1: /* Just cancel transaction */ +error: xfs_trans_cancel(tp); xfs_iunlock(ip, XFS_ILOCK_EXCL); return error; diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index b05cfeb09301..b046eadd3b21 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -253,7 +253,7 @@ xfs_iomap_write_direct( error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, XFS_IEXT_ADD_NOSPLIT_CNT); if (error) - goto out_res_cancel; + goto out_trans_cancel; xfs_trans_ijoin(tp, ip, 0); @@ -265,7 +265,7 @@ xfs_iomap_write_direct( error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, bmapi_flags, 0, imap, &nimaps); if (error) - goto out_res_cancel; + goto out_trans_cancel; /* * Complete the transaction @@ -289,8 +289,6 @@ out_unlock: xfs_iunlock(ip, XFS_ILOCK_EXCL); return error; -out_res_cancel: - xfs_trans_unreserve_quota_nblks(tp, ip, (long)qblocks, 0, quota_flag); out_trans_cancel: xfs_trans_cancel(tp); goto out_unlock; diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h index 1d1a1634ea29..31d0de899cc4 100644 --- a/fs/xfs/xfs_quota.h +++ b/fs/xfs/xfs_quota.h @@ -164,8 +164,6 @@ xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) #define xfs_qm_unmount_quotas(mp) #endif /* CONFIG_XFS_QUOTA */ -#define xfs_trans_unreserve_quota_nblks(tp, ip, nblks, ninos, flags) \ - xfs_trans_reserve_quota_nblks(tp, ip, -(nblks), -(ninos), flags) #define xfs_trans_reserve_quota(tp, mp, ud, gd, pd, nb, ni, f) \ xfs_trans_reserve_quota_bydquots(tp, mp, ud, gd, pd, nb, ni, \ f | XFS_QMOPT_RES_REGBLKS) diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index bea64ed5a57f..15435229bc1f 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -411,7 +411,7 @@ xfs_reflink_allocate_cow( XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, 0, cmap, &nimaps); if (error) - goto out_unreserve; + goto out_trans_cancel; xfs_inode_set_cowblocks_tag(ip); error = xfs_trans_commit(tp); @@ -436,9 +436,6 @@ convert: trace_xfs_reflink_convert_cow(ip, cmap); return xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb); -out_unreserve: - xfs_trans_unreserve_quota_nblks(tp, ip, (long)resblks, 0, - XFS_QMOPT_RES_REGBLKS); out_trans_cancel: xfs_trans_cancel(tp); return error; -- cgit v1.2.3 From ad4a74739708e193c21245dae908ff50f72ff207 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:34 -0800 Subject: xfs: clean up icreate quota reservation calls Create a proper helper so that inode creation calls can reserve quota with a dedicated function. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Chaitanya Kulkarni --- fs/xfs/xfs_inode.c | 6 ++---- fs/xfs/xfs_quota.h | 14 ++++++++++---- fs/xfs/xfs_symlink.c | 3 +-- fs/xfs/xfs_trans_dquot.c | 18 ++++++++++++++++++ 4 files changed, 31 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index e2a1db4cee43..4bbd2fb628f7 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -1037,8 +1037,7 @@ xfs_create( /* * Reserve disk quota and the inode. */ - error = xfs_trans_reserve_quota(tp, mp, udqp, gdqp, - pdqp, resblks, 1, 0); + error = xfs_trans_reserve_quota_icreate(tp, udqp, gdqp, pdqp, resblks); if (error) goto out_trans_cancel; @@ -1169,8 +1168,7 @@ xfs_create_tmpfile( if (error) goto out_release_inode; - error = xfs_trans_reserve_quota(tp, mp, udqp, gdqp, - pdqp, resblks, 1, 0); + error = xfs_trans_reserve_quota_icreate(tp, udqp, gdqp, pdqp, resblks); if (error) goto out_trans_cancel; diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h index 31d0de899cc4..919c3a924821 100644 --- a/fs/xfs/xfs_quota.h +++ b/fs/xfs/xfs_quota.h @@ -86,6 +86,9 @@ extern int xfs_trans_reserve_quota_nblks(struct xfs_trans *, extern int xfs_trans_reserve_quota_bydquots(struct xfs_trans *, struct xfs_mount *, struct xfs_dquot *, struct xfs_dquot *, struct xfs_dquot *, int64_t, long, uint); +int xfs_trans_reserve_quota_icreate(struct xfs_trans *tp, + struct xfs_dquot *udqp, struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, int64_t dblocks); extern int xfs_qm_vop_dqalloc(struct xfs_inode *, kuid_t, kgid_t, prid_t, uint, struct xfs_dquot **, struct xfs_dquot **, @@ -149,6 +152,13 @@ xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) return 0; } +static inline int +xfs_trans_reserve_quota_icreate(struct xfs_trans *tp, struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, int64_t dblocks) +{ + return 0; +} + #define xfs_qm_vop_create_dqattach(tp, ip, u, g, p) #define xfs_qm_vop_rename_dqattach(it) (0) #define xfs_qm_vop_chown(tp, ip, old, new) (NULL) @@ -164,10 +174,6 @@ xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) #define xfs_qm_unmount_quotas(mp) #endif /* CONFIG_XFS_QUOTA */ -#define xfs_trans_reserve_quota(tp, mp, ud, gd, pd, nb, ni, f) \ - xfs_trans_reserve_quota_bydquots(tp, mp, ud, gd, pd, nb, ni, \ - f | XFS_QMOPT_RES_REGBLKS) - static inline int xfs_quota_unreserve_blkres(struct xfs_inode *ip, int64_t blocks) { diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 7f96649e918a..d5dee8f409b2 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -215,8 +215,7 @@ xfs_symlink( /* * Reserve disk quota : blocks and inode. */ - error = xfs_trans_reserve_quota(tp, mp, udqp, gdqp, - pdqp, resblks, 1, 0); + error = xfs_trans_reserve_quota_icreate(tp, udqp, gdqp, pdqp, resblks); if (error) goto out_trans_cancel; diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index 28b8ac701919..22aa875b84f7 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -804,6 +804,24 @@ xfs_trans_reserve_quota_nblks( nblks, ninos, flags); } +/* Change the quota reservations for an inode creation activity. */ +int +xfs_trans_reserve_quota_icreate( + struct xfs_trans *tp, + struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, + int64_t dblocks) +{ + struct xfs_mount *mp = tp->t_mountp; + + if (!XFS_IS_QUOTA_RUNNING(mp) || !XFS_IS_QUOTA_ON(mp)) + return 0; + + return xfs_trans_reserve_quota_bydquots(tp, mp, udqp, gdqp, pdqp, + dblocks, 1, XFS_QMOPT_RES_REGBLKS); +} + /* * This routine is called to allocate a quotaoff log item. */ -- cgit v1.2.3 From 7ac6eb46c9f32d3e6ae37943191cd744ffa1ef33 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 25 Jan 2021 11:18:00 -0800 Subject: xfs: fix up build warnings when quotas are disabled Fix some build warnings on gcc 10.2 when quotas are disabled. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_quota.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h index 919c3a924821..03235c184aab 100644 --- a/fs/xfs/xfs_quota.h +++ b/fs/xfs/xfs_quota.h @@ -130,7 +130,7 @@ xfs_qm_vop_dqalloc(struct xfs_inode *ip, kuid_t kuid, kgid_t kgid, } #define xfs_trans_dup_dqinfo(tp, tp2) #define xfs_trans_free_dqinfo(tp) -#define xfs_trans_mod_dquot_byino(tp, ip, fields, delta) +#define xfs_trans_mod_dquot_byino(tp, ip, fields, delta) do { } while (0) #define xfs_trans_apply_dquot_deltas(tp) #define xfs_trans_unreserve_and_mod_dquots(tp) static inline int xfs_trans_reserve_quota_nblks(struct xfs_trans *tp, @@ -166,8 +166,8 @@ xfs_trans_reserve_quota_icreate(struct xfs_trans *tp, struct xfs_dquot *udqp, #define xfs_qm_dqattach(ip) (0) #define xfs_qm_dqattach_locked(ip, fl) (0) #define xfs_qm_dqdetach(ip) -#define xfs_qm_dqrele(d) -#define xfs_qm_statvfs(ip, s) +#define xfs_qm_dqrele(d) do { (d) = (d); } while(0) +#define xfs_qm_statvfs(ip, s) do { } while(0) #define xfs_qm_newmount(mp, a, b) (0) #define xfs_qm_mount_quotas(mp) #define xfs_qm_unmount(mp) -- cgit v1.2.3 From 02b7ee4eb613240d2bb3f6a67723f94ceda19eb6 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 26 Jan 2021 17:20:42 -0800 Subject: xfs: reserve data and rt quota at the same time Modify xfs_trans_reserve_quota_nblks so that we can reserve data and realtime blocks from the dquot at the same time. This change has the theoretical side effect that for allocations to realtime files we will reserve from the dquot both the number of rtblocks being allocated and the number of bmbt blocks that might be needed to add the mapping. However, since the mount code disables quota if it finds a realtime device, this should not result in any behavior changes. Now that we've moved the inode creation callers away from using the _nblks function, we can repurpose the (now unused) ninos argument for realtime blocks, so make that change. This also replaces the flags argument with a boolean parameter to force the reservation since we don't need to distinguish between data and rt quota reservations any more, and the only flag being passed in was FORCE_RES. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/libxfs/xfs_attr.c | 6 +----- fs/xfs/libxfs/xfs_bmap.c | 4 +--- fs/xfs/xfs_bmap_util.c | 24 +++++++++++------------- fs/xfs/xfs_iomap.c | 26 +++++++++++++------------- fs/xfs/xfs_quota.h | 10 +++++----- fs/xfs/xfs_reflink.c | 6 ++---- fs/xfs/xfs_trans_dquot.c | 40 +++++++++++++++++++++++++++------------- 7 files changed, 60 insertions(+), 56 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index be51e7068dcd..e05dc0bc4a8f 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -474,12 +474,8 @@ xfs_attr_set( } if (args->value) { - unsigned int quota_flags = XFS_QMOPT_RES_REGBLKS; - - if (rsvd) - quota_flags |= XFS_QMOPT_FORCE_RES; error = xfs_trans_reserve_quota_nblks(args->trans, dp, - args->total, 0, quota_flags); + args->total, 0, rsvd); if (error) goto out_trans_cancel; diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 94d582a9587d..6e6734398f0d 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -1085,9 +1085,7 @@ xfs_bmap_add_attrfork( return error; xfs_ilock(ip, XFS_ILOCK_EXCL); - error = xfs_trans_reserve_quota_nblks(tp, ip, blks, 0, rsvd ? - XFS_QMOPT_RES_REGBLKS | XFS_QMOPT_FORCE_RES : - XFS_QMOPT_RES_REGBLKS); + error = xfs_trans_reserve_quota_nblks(tp, ip, blks, 0, rsvd); if (error) goto trans_cancel; if (XFS_IFORK_Q(ip)) diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index ae2d98af693c..ef8f7055af77 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -727,11 +727,10 @@ xfs_alloc_file_space( xfs_fileoff_t startoffset_fsb; xfs_fileoff_t endoffset_fsb; int nimaps; - int quota_flag; int rt; xfs_trans_t *tp; xfs_bmbt_irec_t imaps[1], *imapp; - uint qblocks, resblks, resrtextents; + uint resblks, resrtextents; int error; trace_xfs_alloc_file_space(ip); @@ -761,6 +760,7 @@ xfs_alloc_file_space( */ while (allocatesize_fsb && !error) { xfs_fileoff_t s, e; + unsigned int dblocks, rblocks; /* * Determine space reservations for data/realtime. @@ -790,20 +790,19 @@ xfs_alloc_file_space( */ resblks = min_t(xfs_fileoff_t, (e - s), (MAXEXTLEN * nimaps)); if (unlikely(rt)) { - resrtextents = qblocks = resblks; + resrtextents = resblks; resrtextents /= mp->m_sb.sb_rextsize; - resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0); - quota_flag = XFS_QMOPT_RES_RTBLKS; + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0); + rblocks = resblks; } else { - resrtextents = 0; - resblks = qblocks = XFS_DIOSTRAT_SPACE_RES(mp, resblks); - quota_flag = XFS_QMOPT_RES_REGBLKS; + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resblks); + rblocks = 0; } /* * Allocate and setup the transaction. */ - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, dblocks, resrtextents, 0, &tp); /* @@ -817,8 +816,8 @@ xfs_alloc_file_space( break; } xfs_ilock(ip, XFS_ILOCK_EXCL); - error = xfs_trans_reserve_quota_nblks(tp, ip, qblocks, - 0, quota_flag); + error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, rblocks, + false); if (error) goto error; @@ -881,8 +880,7 @@ xfs_unmap_extent( } xfs_ilock(ip, XFS_ILOCK_EXCL); - error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, - XFS_QMOPT_RES_REGBLKS); + error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, false); if (error) goto out_trans_cancel; diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index b046eadd3b21..fba7303d8ed6 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -194,25 +194,25 @@ xfs_iomap_write_direct( struct xfs_trans *tp; xfs_filblks_t resaligned; int nimaps; - int quota_flag; - uint qblocks, resblks; + unsigned int dblocks, rblocks; unsigned int resrtextents = 0; int error; int bmapi_flags = XFS_BMAPI_PREALLOC; - uint tflags = 0; + int tflags = 0; + bool force = false; ASSERT(count_fsb > 0); resaligned = xfs_aligned_fsb_count(offset_fsb, count_fsb, xfs_get_extsz_hint(ip)); if (unlikely(XFS_IS_REALTIME_INODE(ip))) { - resrtextents = qblocks = resaligned; + resrtextents = resaligned; resrtextents /= mp->m_sb.sb_rextsize; - resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0); - quota_flag = XFS_QMOPT_RES_RTBLKS; + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0); + rblocks = resaligned; } else { - resblks = qblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); - quota_flag = XFS_QMOPT_RES_REGBLKS; + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); + rblocks = 0; } error = xfs_qm_dqattach(ip); @@ -235,18 +235,19 @@ xfs_iomap_write_direct( if (IS_DAX(VFS_I(ip))) { bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO; if (imap->br_state == XFS_EXT_UNWRITTEN) { + force = true; tflags |= XFS_TRANS_RESERVE; - resblks = qblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1; + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1; } } - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, resrtextents, + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, dblocks, resrtextents, tflags, &tp); if (error) return error; xfs_ilock(ip, XFS_ILOCK_EXCL); - error = xfs_trans_reserve_quota_nblks(tp, ip, qblocks, 0, quota_flag); + error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, rblocks, force); if (error) goto out_trans_cancel; @@ -559,8 +560,7 @@ xfs_iomap_write_unwritten( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, 0); - error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, - XFS_QMOPT_RES_REGBLKS | XFS_QMOPT_FORCE_RES); + error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, true); if (error) goto error_on_bmapi_transaction; diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h index 03235c184aab..6ddc4b358ede 100644 --- a/fs/xfs/xfs_quota.h +++ b/fs/xfs/xfs_quota.h @@ -81,8 +81,8 @@ extern void xfs_trans_mod_dquot_byino(struct xfs_trans *, struct xfs_inode *, uint, int64_t); extern void xfs_trans_apply_dquot_deltas(struct xfs_trans *); extern void xfs_trans_unreserve_and_mod_dquots(struct xfs_trans *); -extern int xfs_trans_reserve_quota_nblks(struct xfs_trans *, - struct xfs_inode *, int64_t, long, uint); +int xfs_trans_reserve_quota_nblks(struct xfs_trans *tp, struct xfs_inode *ip, + int64_t dblocks, int64_t rblocks, bool force); extern int xfs_trans_reserve_quota_bydquots(struct xfs_trans *, struct xfs_mount *, struct xfs_dquot *, struct xfs_dquot *, struct xfs_dquot *, int64_t, long, uint); @@ -114,8 +114,7 @@ extern void xfs_qm_unmount_quotas(struct xfs_mount *); static inline int xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) { - return xfs_trans_reserve_quota_nblks(NULL, ip, blocks, 0, - XFS_QMOPT_RES_REGBLKS); + return xfs_trans_reserve_quota_nblks(NULL, ip, blocks, 0, false); } #else static inline int @@ -134,7 +133,8 @@ xfs_qm_vop_dqalloc(struct xfs_inode *ip, kuid_t kuid, kgid_t kgid, #define xfs_trans_apply_dquot_deltas(tp) #define xfs_trans_unreserve_and_mod_dquots(tp) static inline int xfs_trans_reserve_quota_nblks(struct xfs_trans *tp, - struct xfs_inode *ip, int64_t nblks, long ninos, uint flags) + struct xfs_inode *ip, int64_t dblocks, int64_t rblocks, + bool force) { return 0; } diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 15435229bc1f..0778b5810c26 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -398,8 +398,7 @@ xfs_reflink_allocate_cow( goto convert; } - error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, - XFS_QMOPT_RES_REGBLKS); + error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, false); if (error) goto out_trans_cancel; @@ -1090,8 +1089,7 @@ xfs_reflink_remap_extent( if (!smap_real && dmap_written) qres += dmap->br_blockcount; if (qres > 0) { - error = xfs_trans_reserve_quota_nblks(tp, ip, qres, 0, - XFS_QMOPT_RES_REGBLKS); + error = xfs_trans_reserve_quota_nblks(tp, ip, qres, 0, false); if (error) goto out_cancel; } diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index 22aa875b84f7..a1a72b7900c5 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -780,28 +780,42 @@ int xfs_trans_reserve_quota_nblks( struct xfs_trans *tp, struct xfs_inode *ip, - int64_t nblks, - long ninos, - uint flags) + int64_t dblocks, + int64_t rblocks, + bool force) { struct xfs_mount *mp = ip->i_mount; + unsigned int qflags = 0; + int error; if (!XFS_IS_QUOTA_RUNNING(mp) || !XFS_IS_QUOTA_ON(mp)) return 0; ASSERT(!xfs_is_quota_inode(&mp->m_sb, ip->i_ino)); - ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); - ASSERT((flags & ~(XFS_QMOPT_FORCE_RES)) == XFS_TRANS_DQ_RES_RTBLKS || - (flags & ~(XFS_QMOPT_FORCE_RES)) == XFS_TRANS_DQ_RES_BLKS); - /* - * Reserve nblks against these dquots, with trans as the mediator. - */ - return xfs_trans_reserve_quota_bydquots(tp, mp, - ip->i_udquot, ip->i_gdquot, - ip->i_pdquot, - nblks, ninos, flags); + if (force) + qflags |= XFS_QMOPT_FORCE_RES; + + /* Reserve data device quota against the inode's dquots. */ + error = xfs_trans_reserve_quota_bydquots(tp, mp, ip->i_udquot, + ip->i_gdquot, ip->i_pdquot, dblocks, 0, + XFS_QMOPT_RES_REGBLKS | qflags); + if (error) + return error; + + /* Do the same but for realtime blocks. */ + error = xfs_trans_reserve_quota_bydquots(tp, mp, ip->i_udquot, + ip->i_gdquot, ip->i_pdquot, rblocks, 0, + XFS_QMOPT_RES_RTBLKS | qflags); + if (error) { + xfs_trans_reserve_quota_bydquots(tp, mp, ip->i_udquot, + ip->i_gdquot, ip->i_pdquot, -dblocks, 0, + XFS_QMOPT_RES_REGBLKS); + return error; + } + + return 0; } /* Change the quota reservations for an inode creation activity. */ -- cgit v1.2.3 From 3a1af6c317d0a55524f39079183be107be4c1f39 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 26 Jan 2021 16:33:29 -0800 Subject: xfs: refactor common transaction/inode/quota allocation idiom Create a new helper xfs_trans_alloc_inode that allocates a transaction, locks and joins an inode to it, and then reserves the appropriate amount of quota against that transction. Then replace all the open-coded idioms with a single call to this helper. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/libxfs/xfs_attr.c | 11 +---------- fs/xfs/libxfs/xfs_bmap.c | 10 ++-------- fs/xfs/xfs_bmap_util.c | 14 +++----------- fs/xfs/xfs_iomap.c | 11 ++--------- fs/xfs/xfs_trans.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_trans.h | 3 +++ 6 files changed, 59 insertions(+), 38 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index e05dc0bc4a8f..cb95bc77fe59 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -458,14 +458,10 @@ xfs_attr_set( * Root fork attributes can use reserved data blocks for this * operation if necessary */ - error = xfs_trans_alloc(mp, &tres, total, 0, - rsvd ? XFS_TRANS_RESERVE : 0, &args->trans); + error = xfs_trans_alloc_inode(dp, &tres, total, rsvd, &args->trans); if (error) return error; - xfs_ilock(dp, XFS_ILOCK_EXCL); - xfs_trans_ijoin(args->trans, dp, 0); - if (args->value || xfs_inode_hasattr(dp)) { error = xfs_iext_count_may_overflow(dp, XFS_ATTR_FORK, XFS_IEXT_ATTR_MANIP_CNT(rmt_blks)); @@ -474,11 +470,6 @@ xfs_attr_set( } if (args->value) { - error = xfs_trans_reserve_quota_nblks(args->trans, dp, - args->total, 0, rsvd); - if (error) - goto out_trans_cancel; - error = xfs_has_attr(args); if (error == -EEXIST && (args->attr_flags & XATTR_CREATE)) goto out_trans_cancel; diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 6e6734398f0d..be6661645b59 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -1079,19 +1079,13 @@ xfs_bmap_add_attrfork( blks = XFS_ADDAFORK_SPACE_RES(mp); - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_addafork, blks, 0, - rsvd ? XFS_TRANS_RESERVE : 0, &tp); + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_addafork, blks, + rsvd, &tp); if (error) return error; - - xfs_ilock(ip, XFS_ILOCK_EXCL); - error = xfs_trans_reserve_quota_nblks(tp, ip, blks, 0, rsvd); - if (error) - goto trans_cancel; if (XFS_IFORK_Q(ip)) goto trans_cancel; - xfs_trans_ijoin(tp, ip, 0); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); error = xfs_bmap_set_attrforkoff(ip, size, &version); if (error) diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index ef8f7055af77..c5687ae437dc 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -873,18 +873,10 @@ xfs_unmap_extent( uint resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0); int error; - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); - if (error) { - ASSERT(error == -ENOSPC || XFS_FORCED_SHUTDOWN(mp)); - return error; - } - - xfs_ilock(ip, XFS_ILOCK_EXCL); - error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, false); + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, + false, &tp); if (error) - goto out_trans_cancel; - - xfs_trans_ijoin(tp, ip, 0); + return error; error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, XFS_IEXT_PUNCH_HOLE_CNT); diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index fba7303d8ed6..ac91c971342d 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -552,18 +552,11 @@ xfs_iomap_write_unwritten( * here as we might be asked to write out the same inode that we * complete here and might deadlock on the iolock. */ - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, - XFS_TRANS_RESERVE, &tp); + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, + true, &tp); if (error) return error; - xfs_ilock(ip, XFS_ILOCK_EXCL); - xfs_trans_ijoin(tp, ip, 0); - - error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, true); - if (error) - goto error_on_bmapi_transaction; - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, XFS_IEXT_WRITE_UNWRITTEN_CNT); if (error) diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index e72730f85af1..156b9ed8534f 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -20,6 +20,7 @@ #include "xfs_trace.h" #include "xfs_error.h" #include "xfs_defer.h" +#include "xfs_inode.h" kmem_zone_t *xfs_trans_zone; @@ -1024,3 +1025,50 @@ xfs_trans_roll( tres.tr_logflags = XFS_TRANS_PERM_LOG_RES; return xfs_trans_reserve(*tpp, &tres, 0, 0); } + +/* + * Allocate an transaction, lock and join the inode to it, and reserve quota. + * + * The caller must ensure that the on-disk dquots attached to this inode have + * already been allocated and initialized. The caller is responsible for + * releasing ILOCK_EXCL if a new transaction is returned. + */ +int +xfs_trans_alloc_inode( + struct xfs_inode *ip, + struct xfs_trans_res *resv, + unsigned int dblocks, + bool force, + struct xfs_trans **tpp) +{ + struct xfs_trans *tp; + struct xfs_mount *mp = ip->i_mount; + int error; + + error = xfs_trans_alloc(mp, resv, dblocks, 0, + force ? XFS_TRANS_RESERVE : 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + error = xfs_qm_dqattach_locked(ip, false); + if (error) { + /* Caller should have allocated the dquots! */ + ASSERT(error != -ENOENT); + goto out_cancel; + } + + error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, 0, force); + if (error) + goto out_cancel; + + *tpp = tp; + return 0; + +out_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h index 084658946cc8..aa50be244432 100644 --- a/fs/xfs/xfs_trans.h +++ b/fs/xfs/xfs_trans.h @@ -268,4 +268,7 @@ xfs_trans_item_relog( return lip->li_ops->iop_relog(lip, tp); } +int xfs_trans_alloc_inode(struct xfs_inode *ip, struct xfs_trans_res *resv, + unsigned int dblocks, bool force, struct xfs_trans **tpp); + #endif /* __XFS_TRANS_H__ */ -- cgit v1.2.3 From 3de4eb106fcc97f086b78bd17a0c3529691e8259 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 26 Jan 2021 16:44:07 -0800 Subject: xfs: allow reservation of rtblocks with xfs_trans_alloc_inode Make it so that we can reserve rt blocks with the xfs_trans_alloc_inode wrapper function, then convert a few more callsites. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/libxfs/xfs_attr.c | 2 +- fs/xfs/libxfs/xfs_bmap.c | 2 +- fs/xfs/xfs_bmap_util.c | 29 +++++------------------------ fs/xfs/xfs_iomap.c | 22 +++++----------------- fs/xfs/xfs_trans.c | 6 ++++-- fs/xfs/xfs_trans.h | 3 ++- 6 files changed, 18 insertions(+), 46 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index cb95bc77fe59..472b3039eabb 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -458,7 +458,7 @@ xfs_attr_set( * Root fork attributes can use reserved data blocks for this * operation if necessary */ - error = xfs_trans_alloc_inode(dp, &tres, total, rsvd, &args->trans); + error = xfs_trans_alloc_inode(dp, &tres, total, 0, rsvd, &args->trans); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index be6661645b59..e0905ad171f0 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -1079,7 +1079,7 @@ xfs_bmap_add_attrfork( blks = XFS_ADDAFORK_SPACE_RES(mp); - error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_addafork, blks, + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_addafork, blks, 0, rsvd, &tp); if (error) return error; diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index c5687ae437dc..e7d68318e6a5 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -730,7 +730,6 @@ xfs_alloc_file_space( int rt; xfs_trans_t *tp; xfs_bmbt_irec_t imaps[1], *imapp; - uint resblks, resrtextents; int error; trace_xfs_alloc_file_space(ip); @@ -760,7 +759,7 @@ xfs_alloc_file_space( */ while (allocatesize_fsb && !error) { xfs_fileoff_t s, e; - unsigned int dblocks, rblocks; + unsigned int dblocks, rblocks, resblks; /* * Determine space reservations for data/realtime. @@ -790,8 +789,6 @@ xfs_alloc_file_space( */ resblks = min_t(xfs_fileoff_t, (e - s), (MAXEXTLEN * nimaps)); if (unlikely(rt)) { - resrtextents = resblks; - resrtextents /= mp->m_sb.sb_rextsize; dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0); rblocks = resblks; } else { @@ -802,32 +799,16 @@ xfs_alloc_file_space( /* * Allocate and setup the transaction. */ - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, dblocks, - resrtextents, 0, &tp); - - /* - * Check for running out of space - */ - if (error) { - /* - * Free the transaction structure. - */ - ASSERT(error == -ENOSPC || XFS_FORCED_SHUTDOWN(mp)); - break; - } - xfs_ilock(ip, XFS_ILOCK_EXCL); - error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, rblocks, - false); + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, + dblocks, rblocks, false, &tp); if (error) - goto error; + break; error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, XFS_IEXT_ADD_NOSPLIT_CNT); if (error) goto error; - xfs_trans_ijoin(tp, ip, 0); - error = xfs_bmapi_write(tp, ip, startoffset_fsb, allocatesize_fsb, alloc_type, 0, imapp, &nimaps); @@ -873,7 +854,7 @@ xfs_unmap_extent( uint resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0); int error; - error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, 0, false, &tp); if (error) return error; diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index ac91c971342d..fe2bbd9b6fdb 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -195,19 +195,15 @@ xfs_iomap_write_direct( xfs_filblks_t resaligned; int nimaps; unsigned int dblocks, rblocks; - unsigned int resrtextents = 0; + bool force = false; int error; int bmapi_flags = XFS_BMAPI_PREALLOC; - int tflags = 0; - bool force = false; ASSERT(count_fsb > 0); resaligned = xfs_aligned_fsb_count(offset_fsb, count_fsb, xfs_get_extsz_hint(ip)); if (unlikely(XFS_IS_REALTIME_INODE(ip))) { - resrtextents = resaligned; - resrtextents /= mp->m_sb.sb_rextsize; dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0); rblocks = resaligned; } else { @@ -236,28 +232,20 @@ xfs_iomap_write_direct( bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO; if (imap->br_state == XFS_EXT_UNWRITTEN) { force = true; - tflags |= XFS_TRANS_RESERVE; dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1; } } - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, dblocks, resrtextents, - tflags, &tp); - if (error) - return error; - xfs_ilock(ip, XFS_ILOCK_EXCL); - - error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, rblocks, force); + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, dblocks, + rblocks, force, &tp); if (error) - goto out_trans_cancel; + return error; error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, XFS_IEXT_ADD_NOSPLIT_CNT); if (error) goto out_trans_cancel; - xfs_trans_ijoin(tp, ip, 0); - /* * From this point onwards we overwrite the imap pointer that the * caller gave to us. @@ -553,7 +541,7 @@ xfs_iomap_write_unwritten( * complete here and might deadlock on the iolock. */ error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, - true, &tp); + 0, true, &tp); if (error) return error; diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 156b9ed8534f..151f274eee43 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -1038,6 +1038,7 @@ xfs_trans_alloc_inode( struct xfs_inode *ip, struct xfs_trans_res *resv, unsigned int dblocks, + unsigned int rblocks, bool force, struct xfs_trans **tpp) { @@ -1045,7 +1046,8 @@ xfs_trans_alloc_inode( struct xfs_mount *mp = ip->i_mount; int error; - error = xfs_trans_alloc(mp, resv, dblocks, 0, + error = xfs_trans_alloc(mp, resv, dblocks, + rblocks / mp->m_sb.sb_rextsize, force ? XFS_TRANS_RESERVE : 0, &tp); if (error) return error; @@ -1060,7 +1062,7 @@ xfs_trans_alloc_inode( goto out_cancel; } - error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, 0, force); + error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, rblocks, force); if (error) goto out_cancel; diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h index aa50be244432..52bbd7e6a552 100644 --- a/fs/xfs/xfs_trans.h +++ b/fs/xfs/xfs_trans.h @@ -269,6 +269,7 @@ xfs_trans_item_relog( } int xfs_trans_alloc_inode(struct xfs_inode *ip, struct xfs_trans_res *resv, - unsigned int dblocks, bool force, struct xfs_trans **tpp); + unsigned int dblocks, unsigned int rblocks, bool force, + struct xfs_trans **tpp); #endif /* __XFS_TRANS_H__ */ -- cgit v1.2.3 From f273387b048543f2b8b2d809cc65fca28e7788a1 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Wed, 27 Jan 2021 10:07:27 -0800 Subject: xfs: refactor reflink functions to use xfs_trans_alloc_inode The two remaining callers of xfs_trans_reserve_quota_nblks are in the reflink code. These conversions aren't as uniform as the previous conversions, so call that out in a separate patch. Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_iomap.c | 3 ++- fs/xfs/xfs_reflink.c | 53 +++++++++++++++++++++------------------------------- 2 files changed, 23 insertions(+), 33 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index fe2bbd9b6fdb..70c341658c01 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -831,7 +831,8 @@ out_found_cow: return xfs_bmbt_to_iomap(ip, iomap, &cmap, IOMAP_F_SHARED); out_unlock: - xfs_iunlock(ip, lockmode); + if (lockmode) + xfs_iunlock(ip, lockmode); return error; } diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 0778b5810c26..27f875fa7a0d 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -376,16 +376,14 @@ xfs_reflink_allocate_cow( resblks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); xfs_iunlock(ip, *lockmode); - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); - *lockmode = XFS_ILOCK_EXCL; - xfs_ilock(ip, *lockmode); + *lockmode = 0; + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, 0, + false, &tp); if (error) return error; - error = xfs_qm_dqattach_locked(ip, false); - if (error) - goto out_trans_cancel; + *lockmode = XFS_ILOCK_EXCL; /* * Check for an overlapping extent again now that we dropped the ilock. @@ -398,12 +396,6 @@ xfs_reflink_allocate_cow( goto convert; } - error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, false); - if (error) - goto out_trans_cancel; - - xfs_trans_ijoin(tp, ip, 0); - /* Allocate the entire reservation as unwritten blocks. */ nimaps = 1; error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount, @@ -997,7 +989,7 @@ xfs_reflink_remap_extent( struct xfs_mount *mp = ip->i_mount; struct xfs_trans *tp; xfs_off_t newlen; - int64_t qres, qdelta; + int64_t qdelta = 0; unsigned int resblks; bool smap_real; bool dmap_written = xfs_bmap_is_written_extent(dmap); @@ -1005,15 +997,22 @@ xfs_reflink_remap_extent( int nimaps; int error; - /* Start a rolling transaction to switch the mappings */ + /* + * Start a rolling transaction to switch the mappings. + * + * Adding a written extent to the extent map can cause a bmbt split, + * and removing a mapped extent from the extent can cause a bmbt split. + * The two operations cannot both cause a split since they operate on + * the same index in the bmap btree, so we only need a reservation for + * one bmbt split if either thing is happening. However, we haven't + * locked the inode yet, so we reserve assuming this is the case. + */ resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp); + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, 0, + false, &tp); if (error) goto out; - xfs_ilock(ip, XFS_ILOCK_EXCL); - xfs_trans_ijoin(tp, ip, 0); - /* * Read what's currently mapped in the destination file into smap. * If smap isn't a hole, we will have to remove it before we can add @@ -1061,15 +1060,9 @@ xfs_reflink_remap_extent( } /* - * Compute quota reservation if we think the quota block counter for + * Increase quota reservation if we think the quota block counter for * this file could increase. * - * Adding a written extent to the extent map can cause a bmbt split, - * and removing a mapped extent from the extent can cause a bmbt split. - * The two operations cannot both cause a split since they operate on - * the same index in the bmap btree, so we only need a reservation for - * one bmbt split if either thing is happening. - * * If we are mapping a written extent into the file, we need to have * enough quota block count reservation to handle the blocks in that * extent. We log only the delta to the quota block counts, so if the @@ -1083,13 +1076,9 @@ xfs_reflink_remap_extent( * before we started. That should have removed all the delalloc * reservations, but we code defensively. */ - qres = qdelta = 0; - if (smap_real || dmap_written) - qres = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); - if (!smap_real && dmap_written) - qres += dmap->br_blockcount; - if (qres > 0) { - error = xfs_trans_reserve_quota_nblks(tp, ip, qres, 0, false); + if (!smap_real && dmap_written) { + error = xfs_trans_reserve_quota_nblks(tp, ip, + dmap->br_blockcount, 0, false); if (error) goto out_cancel; } -- cgit v1.2.3 From f2f7b9ff62a28928f6fe2bd55cdb4d4b02ab7477 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Wed, 27 Jan 2021 12:07:57 -0800 Subject: xfs: refactor inode creation transaction/inode/quota allocation idiom For file creation, create a new helper xfs_trans_alloc_icreate that allocates a transaction and reserves the appropriate amount of quota against that transction. Replace all the open-coded idioms with a single call to this helper so that we can contain the retry loops in the next patchset. This changes the locking behavior for non-tempfile creation slightly, in that we now make the quota reservation without holding the directory ILOCK. While the dquots chosen for inode creation are based on the directory state at a given point in time, the directory ILOCK was released as soon as the dquot references are picked up. Hence it was never necessary to hold the directory ILOCK for the quota reservation. Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_inode.c | 28 ++++++++++------------------ fs/xfs/xfs_symlink.c | 14 ++++---------- fs/xfs/xfs_trans.c | 33 +++++++++++++++++++++++++++++++++ fs/xfs/xfs_trans.h | 6 ++++++ 4 files changed, 53 insertions(+), 28 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 4bbd2fb628f7..636ac13b1df2 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -1022,25 +1022,20 @@ xfs_create( * the case we'll drop the one we have and get a more * appropriate transaction later. */ - error = xfs_trans_alloc(mp, tres, resblks, 0, 0, &tp); + error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp, resblks, + &tp); if (error == -ENOSPC) { /* flush outstanding delalloc blocks and retry */ xfs_flush_inodes(mp); - error = xfs_trans_alloc(mp, tres, resblks, 0, 0, &tp); + error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp, + resblks, &tp); } if (error) - goto out_release_inode; + goto out_release_dquots; xfs_ilock(dp, XFS_ILOCK_EXCL | XFS_ILOCK_PARENT); unlock_dp_on_error = true; - /* - * Reserve disk quota and the inode. - */ - error = xfs_trans_reserve_quota_icreate(tp, udqp, gdqp, pdqp, resblks); - if (error) - goto out_trans_cancel; - error = xfs_iext_count_may_overflow(dp, XFS_DATA_FORK, XFS_IEXT_DIR_MANIP_CNT(mp)); if (error) @@ -1120,7 +1115,7 @@ xfs_create( xfs_finish_inode_setup(ip); xfs_irele(ip); } - + out_release_dquots: xfs_qm_dqrele(udqp); xfs_qm_dqrele(gdqp); xfs_qm_dqrele(pdqp); @@ -1164,13 +1159,10 @@ xfs_create_tmpfile( resblks = XFS_IALLOC_SPACE_RES(mp); tres = &M_RES(mp)->tr_create_tmpfile; - error = xfs_trans_alloc(mp, tres, resblks, 0, 0, &tp); - if (error) - goto out_release_inode; - - error = xfs_trans_reserve_quota_icreate(tp, udqp, gdqp, pdqp, resblks); + error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp, resblks, + &tp); if (error) - goto out_trans_cancel; + goto out_release_dquots; error = xfs_dir_ialloc(&tp, dp, mode, 0, 0, prid, &ip); if (error) @@ -1213,7 +1205,7 @@ xfs_create_tmpfile( xfs_finish_inode_setup(ip); xfs_irele(ip); } - + out_release_dquots: xfs_qm_dqrele(udqp); xfs_qm_dqrele(gdqp); xfs_qm_dqrele(pdqp); diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index d5dee8f409b2..8565663b16cd 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -197,9 +197,10 @@ xfs_symlink( fs_blocks = xfs_symlink_blocks(mp, pathlen); resblks = XFS_SYMLINK_SPACE_RES(mp, link_name->len, fs_blocks); - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_symlink, resblks, 0, 0, &tp); + error = xfs_trans_alloc_icreate(mp, &M_RES(mp)->tr_symlink, udqp, gdqp, + pdqp, resblks, &tp); if (error) - goto out_release_inode; + goto out_release_dquots; xfs_ilock(dp, XFS_ILOCK_EXCL | XFS_ILOCK_PARENT); unlock_dp_on_error = true; @@ -212,13 +213,6 @@ xfs_symlink( goto out_trans_cancel; } - /* - * Reserve disk quota : blocks and inode. - */ - error = xfs_trans_reserve_quota_icreate(tp, udqp, gdqp, pdqp, resblks); - if (error) - goto out_trans_cancel; - error = xfs_iext_count_may_overflow(dp, XFS_DATA_FORK, XFS_IEXT_DIR_MANIP_CNT(mp)); if (error) @@ -347,7 +341,7 @@ out_release_inode: xfs_finish_inode_setup(ip); xfs_irele(ip); } - +out_release_dquots: xfs_qm_dqrele(udqp); xfs_qm_dqrele(gdqp); xfs_qm_dqrele(pdqp); diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 151f274eee43..6c68635cc6ac 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -21,6 +21,8 @@ #include "xfs_error.h" #include "xfs_defer.h" #include "xfs_inode.h" +#include "xfs_dquot_item.h" +#include "xfs_dquot.h" kmem_zone_t *xfs_trans_zone; @@ -1074,3 +1076,34 @@ out_cancel: xfs_iunlock(ip, XFS_ILOCK_EXCL); return error; } + +/* + * Allocate an transaction in preparation for inode creation by reserving quota + * against the given dquots. Callers are not required to hold any inode locks. + */ +int +xfs_trans_alloc_icreate( + struct xfs_mount *mp, + struct xfs_trans_res *resv, + struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, + unsigned int dblocks, + struct xfs_trans **tpp) +{ + struct xfs_trans *tp; + int error; + + error = xfs_trans_alloc(mp, resv, dblocks, 0, 0, &tp); + if (error) + return error; + + error = xfs_trans_reserve_quota_icreate(tp, udqp, gdqp, pdqp, dblocks); + if (error) { + xfs_trans_cancel(tp); + return error; + } + + *tpp = tp; + return 0; +} diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h index 52bbd7e6a552..04c132c55e9b 100644 --- a/fs/xfs/xfs_trans.h +++ b/fs/xfs/xfs_trans.h @@ -268,8 +268,14 @@ xfs_trans_item_relog( return lip->li_ops->iop_relog(lip, tp); } +struct xfs_dquot; + int xfs_trans_alloc_inode(struct xfs_inode *ip, struct xfs_trans_res *resv, unsigned int dblocks, unsigned int rblocks, bool force, struct xfs_trans **tpp); +int xfs_trans_alloc_icreate(struct xfs_mount *mp, struct xfs_trans_res *resv, + struct xfs_dquot *udqp, struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, unsigned int dblocks, + struct xfs_trans **tpp); #endif /* __XFS_TRANS_H__ */ -- cgit v1.2.3 From 7317a03df703f7cdae3ae9e9635a0ef45849fe09 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 29 Jan 2021 11:32:09 -0800 Subject: xfs: refactor inode ownership change transaction/inode/quota allocation idiom For file ownership (uid, gid, prid) changes, create a new helper xfs_trans_alloc_ichange that allocates a transaction and reserves the appropriate amount of quota against that transction in preparation for a change of user, group, or project id. Replace all the open-coded idioms with a single call to this helper so that we can contain the retry loops in the next patchset. This changes the locking behavior for ichange transactions slightly. Since tr_ichange does not have a permanent reservation and cannot roll, we pass XFS_ILOCK_EXCL to ijoin so that the inode will be unlocked automatically at commit time. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_ioctl.c | 29 +++++++++---------------- fs/xfs/xfs_iops.c | 26 ++--------------------- fs/xfs/xfs_trans.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_trans.h | 3 +++ 4 files changed, 77 insertions(+), 43 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 3fbd98f61ea5..78ee201eb7cb 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1275,24 +1275,23 @@ xfs_ioctl_setattr_prepare_dax( */ static struct xfs_trans * xfs_ioctl_setattr_get_trans( - struct xfs_inode *ip) + struct xfs_inode *ip, + struct xfs_dquot *pdqp) { struct xfs_mount *mp = ip->i_mount; struct xfs_trans *tp; int error = -EROFS; if (mp->m_flags & XFS_MOUNT_RDONLY) - goto out_unlock; + goto out_error; error = -EIO; if (XFS_FORCED_SHUTDOWN(mp)) - goto out_unlock; + goto out_error; - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); + error = xfs_trans_alloc_ichange(ip, NULL, NULL, pdqp, + capable(CAP_FOWNER), &tp); if (error) - goto out_unlock; - - xfs_ilock(ip, XFS_ILOCK_EXCL); - xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + goto out_error; /* * CAP_FOWNER overrides the following restrictions: @@ -1312,7 +1311,7 @@ xfs_ioctl_setattr_get_trans( out_cancel: xfs_trans_cancel(tp); -out_unlock: +out_error: return ERR_PTR(error); } @@ -1462,20 +1461,12 @@ xfs_ioctl_setattr( xfs_ioctl_setattr_prepare_dax(ip, fa); - tp = xfs_ioctl_setattr_get_trans(ip); + tp = xfs_ioctl_setattr_get_trans(ip, pdqp); if (IS_ERR(tp)) { code = PTR_ERR(tp); goto error_free_dquots; } - if (XFS_IS_QUOTA_RUNNING(mp) && XFS_IS_PQUOTA_ON(mp) && - ip->i_d.di_projid != fa->fsx_projid) { - code = xfs_qm_vop_chown_reserve(tp, ip, NULL, NULL, pdqp, - capable(CAP_FOWNER) ? XFS_QMOPT_FORCE_RES : 0); - if (code) /* out of quota */ - goto error_trans_cancel; - } - xfs_fill_fsxattr(ip, false, &old_fa); code = vfs_ioc_fssetxattr_check(VFS_I(ip), &old_fa, fa); if (code) @@ -1608,7 +1599,7 @@ xfs_ioc_setxflags( xfs_ioctl_setattr_prepare_dax(ip, &fa); - tp = xfs_ioctl_setattr_get_trans(ip); + tp = xfs_ioctl_setattr_get_trans(ip, NULL); if (IS_ERR(tp)) { error = PTR_ERR(tp); goto out_drop_write; diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index f1e21b6cfa48..00369502fe25 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -700,13 +700,11 @@ xfs_setattr_nonsize( return error; } - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); + error = xfs_trans_alloc_ichange(ip, udqp, gdqp, NULL, + capable(CAP_FOWNER), &tp); if (error) goto out_dqrele; - xfs_ilock(ip, XFS_ILOCK_EXCL); - xfs_trans_ijoin(tp, ip, 0); - /* * Change file ownership. Must be the owner or privileged. */ @@ -722,21 +720,6 @@ xfs_setattr_nonsize( gid = (mask & ATTR_GID) ? iattr->ia_gid : igid; uid = (mask & ATTR_UID) ? iattr->ia_uid : iuid; - /* - * Do a quota reservation only if uid/gid is actually - * going to change. - */ - if (XFS_IS_QUOTA_RUNNING(mp) && - ((XFS_IS_UQUOTA_ON(mp) && !uid_eq(iuid, uid)) || - (XFS_IS_GQUOTA_ON(mp) && !gid_eq(igid, gid)))) { - ASSERT(tp); - error = xfs_qm_vop_chown_reserve(tp, ip, udqp, gdqp, - NULL, capable(CAP_FOWNER) ? - XFS_QMOPT_FORCE_RES : 0); - if (error) /* out of quota */ - goto out_cancel; - } - /* * CAP_FSETID overrides the following restrictions: * @@ -786,8 +769,6 @@ xfs_setattr_nonsize( xfs_trans_set_sync(tp); error = xfs_trans_commit(tp); - xfs_iunlock(ip, XFS_ILOCK_EXCL); - /* * Release any dquot(s) the inode had kept before chown. */ @@ -814,9 +795,6 @@ xfs_setattr_nonsize( return 0; -out_cancel: - xfs_trans_cancel(tp); - xfs_iunlock(ip, XFS_ILOCK_EXCL); out_dqrele: xfs_qm_dqrele(udqp); xfs_qm_dqrele(gdqp); diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 6c68635cc6ac..60672b5545c9 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -1107,3 +1107,65 @@ xfs_trans_alloc_icreate( *tpp = tp; return 0; } + +/* + * Allocate an transaction, lock and join the inode to it, and reserve quota + * in preparation for inode attribute changes that include uid, gid, or prid + * changes. + * + * The caller must ensure that the on-disk dquots attached to this inode have + * already been allocated and initialized. The ILOCK will be dropped when the + * transaction is committed or cancelled. + */ +int +xfs_trans_alloc_ichange( + struct xfs_inode *ip, + struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, + bool force, + struct xfs_trans **tpp) +{ + struct xfs_trans *tp; + struct xfs_mount *mp = ip->i_mount; + int error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + + error = xfs_qm_dqattach_locked(ip, false); + if (error) { + /* Caller should have allocated the dquots! */ + ASSERT(error != -ENOENT); + goto out_cancel; + } + + /* + * For each quota type, skip quota reservations if the inode's dquots + * now match the ones that came from the caller, or the caller didn't + * pass one in. + */ + if (udqp == ip->i_udquot) + udqp = NULL; + if (gdqp == ip->i_gdquot) + gdqp = NULL; + if (pdqp == ip->i_pdquot) + pdqp = NULL; + if (udqp || gdqp || pdqp) { + error = xfs_qm_vop_chown_reserve(tp, ip, udqp, gdqp, pdqp, + force ? XFS_QMOPT_FORCE_RES : 0); + if (error) + goto out_cancel; + } + + *tpp = tp; + return 0; + +out_cancel: + xfs_trans_cancel(tp); + return error; +} diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h index 04c132c55e9b..8b03fbfe9a1b 100644 --- a/fs/xfs/xfs_trans.h +++ b/fs/xfs/xfs_trans.h @@ -277,5 +277,8 @@ int xfs_trans_alloc_icreate(struct xfs_mount *mp, struct xfs_trans_res *resv, struct xfs_dquot *udqp, struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, unsigned int dblocks, struct xfs_trans **tpp); +int xfs_trans_alloc_ichange(struct xfs_inode *ip, struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, bool force, + struct xfs_trans **tpp); #endif /* __XFS_TRANS_H__ */ -- cgit v1.2.3 From 5c615f0feb9a559abd08da0842d6fcfee105b7e3 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 1 Feb 2021 10:38:51 -0800 Subject: xfs: remove xfs_qm_vop_chown_reserve Now that the only caller of this function is xfs_trans_alloc_ichange, just open-code the meat of _chown_reserve in that caller. Drop the (redundant) [ugp]id checks because xfs has a 1:1 relationship between quota ids and incore dquots. Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_qm.c | 48 ------------------------------------------------ fs/xfs/xfs_quota.h | 4 ---- fs/xfs/xfs_trans.c | 16 ++++++++++++++-- 3 files changed, 14 insertions(+), 54 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index c2e4d3a27469..742d1413e2d0 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -1817,54 +1817,6 @@ xfs_qm_vop_chown( return prevdq; } -/* - * Quota reservations for setattr(AT_UID|AT_GID|AT_PROJID). - */ -int -xfs_qm_vop_chown_reserve( - struct xfs_trans *tp, - struct xfs_inode *ip, - struct xfs_dquot *udqp, - struct xfs_dquot *gdqp, - struct xfs_dquot *pdqp, - uint flags) -{ - struct xfs_mount *mp = ip->i_mount; - unsigned int blkflags; - struct xfs_dquot *udq_delblks = NULL; - struct xfs_dquot *gdq_delblks = NULL; - struct xfs_dquot *pdq_delblks = NULL; - - ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL|XFS_ILOCK_SHARED)); - ASSERT(XFS_IS_QUOTA_RUNNING(mp)); - - blkflags = XFS_IS_REALTIME_INODE(ip) ? - XFS_QMOPT_RES_RTBLKS : XFS_QMOPT_RES_REGBLKS; - - if (XFS_IS_UQUOTA_ON(mp) && udqp && - i_uid_read(VFS_I(ip)) != udqp->q_id) - udq_delblks = udqp; - - if (XFS_IS_GQUOTA_ON(ip->i_mount) && gdqp && - i_gid_read(VFS_I(ip)) != gdqp->q_id) - gdq_delblks = gdqp; - - if (XFS_IS_PQUOTA_ON(ip->i_mount) && pdqp && - ip->i_d.di_projid != pdqp->q_id) - pdq_delblks = pdqp; - - /* - * Reserve enough quota to handle blocks on disk and reserved for a - * delayed allocation. We'll actually transfer the delalloc - * reservation between dquots at chown time, even though that part is - * only semi-transactional. - */ - return xfs_trans_reserve_quota_bydquots(tp, ip->i_mount, udq_delblks, - gdq_delblks, pdq_delblks, - ip->i_d.di_nblocks + ip->i_delayed_blks, - 1, blkflags | flags); -} - int xfs_qm_vop_rename_dqattach( struct xfs_inode **i_tab) diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h index 6ddc4b358ede..d00d01302545 100644 --- a/fs/xfs/xfs_quota.h +++ b/fs/xfs/xfs_quota.h @@ -98,9 +98,6 @@ extern void xfs_qm_vop_create_dqattach(struct xfs_trans *, struct xfs_inode *, extern int xfs_qm_vop_rename_dqattach(struct xfs_inode **); extern struct xfs_dquot *xfs_qm_vop_chown(struct xfs_trans *, struct xfs_inode *, struct xfs_dquot **, struct xfs_dquot *); -extern int xfs_qm_vop_chown_reserve(struct xfs_trans *, struct xfs_inode *, - struct xfs_dquot *, struct xfs_dquot *, - struct xfs_dquot *, uint); extern int xfs_qm_dqattach(struct xfs_inode *); extern int xfs_qm_dqattach_locked(struct xfs_inode *ip, bool doalloc); extern void xfs_qm_dqdetach(struct xfs_inode *); @@ -162,7 +159,6 @@ xfs_trans_reserve_quota_icreate(struct xfs_trans *tp, struct xfs_dquot *udqp, #define xfs_qm_vop_create_dqattach(tp, ip, u, g, p) #define xfs_qm_vop_rename_dqattach(it) (0) #define xfs_qm_vop_chown(tp, ip, old, new) (NULL) -#define xfs_qm_vop_chown_reserve(tp, ip, u, g, p, fl) (0) #define xfs_qm_dqattach(ip) (0) #define xfs_qm_dqattach_locked(ip, fl) (0) #define xfs_qm_dqdetach(ip) diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 60672b5545c9..29dca1bc4c1a 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -1156,8 +1156,20 @@ xfs_trans_alloc_ichange( if (pdqp == ip->i_pdquot) pdqp = NULL; if (udqp || gdqp || pdqp) { - error = xfs_qm_vop_chown_reserve(tp, ip, udqp, gdqp, pdqp, - force ? XFS_QMOPT_FORCE_RES : 0); + unsigned int qflags = XFS_QMOPT_RES_REGBLKS; + + if (force) + qflags |= XFS_QMOPT_FORCE_RES; + + /* + * Reserve enough quota to handle blocks on disk and reserved + * for a delayed allocation. We'll actually transfer the + * delalloc reservation between dquots at chown time, even + * though that part is only semi-transactional. + */ + error = xfs_trans_reserve_quota_bydquots(tp, mp, udqp, gdqp, + pdqp, ip->i_d.di_nblocks + ip->i_delayed_blks, + 1, qflags); if (error) goto out_cancel; } -- cgit v1.2.3 From fea7aae6cecfed1b6a520cc527d297df8801b999 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 29 Jan 2021 17:14:33 -0800 Subject: xfs: rename code to error in xfs_ioctl_setattr Rename the 'code' variable to 'error' to follow the naming convention of most other functions in xfs. Signed-off-by: Darrick J. Wong Reviewed-by: Chaitanya Kulkarni Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_ioctl.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 78ee201eb7cb..38ee66d999d8 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1435,13 +1435,13 @@ xfs_ioctl_setattr( struct xfs_trans *tp; struct xfs_dquot *pdqp = NULL; struct xfs_dquot *olddquot = NULL; - int code; + int error; trace_xfs_ioctl_setattr(ip); - code = xfs_ioctl_setattr_check_projid(ip, fa); - if (code) - return code; + error = xfs_ioctl_setattr_check_projid(ip, fa); + if (error) + return error; /* * If disk quotas is on, we make sure that the dquots do exist on disk, @@ -1452,36 +1452,36 @@ xfs_ioctl_setattr( * because the i_*dquot fields will get updated anyway. */ if (XFS_IS_QUOTA_ON(mp)) { - code = xfs_qm_vop_dqalloc(ip, VFS_I(ip)->i_uid, + error = xfs_qm_vop_dqalloc(ip, VFS_I(ip)->i_uid, VFS_I(ip)->i_gid, fa->fsx_projid, XFS_QMOPT_PQUOTA, NULL, NULL, &pdqp); - if (code) - return code; + if (error) + return error; } xfs_ioctl_setattr_prepare_dax(ip, fa); tp = xfs_ioctl_setattr_get_trans(ip, pdqp); if (IS_ERR(tp)) { - code = PTR_ERR(tp); + error = PTR_ERR(tp); goto error_free_dquots; } xfs_fill_fsxattr(ip, false, &old_fa); - code = vfs_ioc_fssetxattr_check(VFS_I(ip), &old_fa, fa); - if (code) + error = vfs_ioc_fssetxattr_check(VFS_I(ip), &old_fa, fa); + if (error) goto error_trans_cancel; - code = xfs_ioctl_setattr_check_extsize(ip, fa); - if (code) + error = xfs_ioctl_setattr_check_extsize(ip, fa); + if (error) goto error_trans_cancel; - code = xfs_ioctl_setattr_check_cowextsize(ip, fa); - if (code) + error = xfs_ioctl_setattr_check_cowextsize(ip, fa); + if (error) goto error_trans_cancel; - code = xfs_ioctl_setattr_xflags(tp, ip, fa); - if (code) + error = xfs_ioctl_setattr_xflags(tp, ip, fa); + if (error) goto error_trans_cancel; /* @@ -1521,7 +1521,7 @@ xfs_ioctl_setattr( else ip->i_d.di_cowextsize = 0; - code = xfs_trans_commit(tp); + error = xfs_trans_commit(tp); /* * Release any dquot(s) the inode had kept before chown. @@ -1529,13 +1529,13 @@ xfs_ioctl_setattr( xfs_qm_dqrele(olddquot); xfs_qm_dqrele(pdqp); - return code; + return error; error_trans_cancel: xfs_trans_cancel(tp); error_free_dquots: xfs_qm_dqrele(pdqp); - return code; + return error; } STATIC int -- cgit v1.2.3 From 2a4bdfa8558ca2904dc17b83497dc82aa7fc05e9 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 29 Jan 2021 15:44:32 -0800 Subject: xfs: shut down the filesystem if we screw up quota reservation If we ever screw up the quota reservations enough to trip the assertions, something's wrong with the quota code. Shut down the filesystem when this happens, because this is corruption. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_trans_dquot.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index a1a72b7900c5..48e09ea30ee5 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -16,6 +16,7 @@ #include "xfs_quota.h" #include "xfs_qm.h" #include "xfs_trace.h" +#include "xfs_error.h" STATIC void xfs_trans_alloc_dqinfo(xfs_trans_t *); @@ -691,9 +692,11 @@ xfs_trans_dqresv( nblks); xfs_trans_mod_dquot(tp, dqp, XFS_TRANS_DQ_RES_INOS, ninos); } - ASSERT(dqp->q_blk.reserved >= dqp->q_blk.count); - ASSERT(dqp->q_rtb.reserved >= dqp->q_rtb.count); - ASSERT(dqp->q_ino.reserved >= dqp->q_ino.count); + + if (XFS_IS_CORRUPT(mp, dqp->q_blk.reserved < dqp->q_blk.count) || + XFS_IS_CORRUPT(mp, dqp->q_rtb.reserved < dqp->q_rtb.count) || + XFS_IS_CORRUPT(mp, dqp->q_ino.reserved < dqp->q_ino.count)) + goto error_corrupt; xfs_dqunlock(dqp); return 0; @@ -703,6 +706,10 @@ error_return: if (xfs_dquot_type(dqp) == XFS_DQTYPE_PROJ) return -ENOSPC; return -EDQUOT; +error_corrupt: + xfs_dqunlock(dqp); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + return -EFSCORRUPTED; } -- cgit v1.2.3 From a636b1d1cf73804e385990c975e33cf06c032b64 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:34 -0800 Subject: xfs: trigger all block gc scans when low on quota space The functions to run an eof/cowblocks scan to try to reduce quota usage are kind of a mess -- the logic repeatedly initializes an eofb structure and there are logic bugs in the code that result in the cowblocks scan never actually happening. Replace all three functions with a single function that fills out an eofb and runs both eof and cowblocks scans. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_file.c | 15 ++++++--------- fs/xfs/xfs_icache.c | 46 ++++++++++++++++------------------------------ fs/xfs/xfs_icache.h | 4 ++-- 3 files changed, 24 insertions(+), 41 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 48f6f89a38a9..b55f1eb8f1c8 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -672,7 +672,7 @@ xfs_file_buffered_aio_write( struct inode *inode = mapping->host; struct xfs_inode *ip = XFS_I(inode); ssize_t ret; - int enospc = 0; + bool cleared_space = false; int iolock; if (iocb->ki_flags & IOCB_NOWAIT) @@ -704,19 +704,16 @@ write_retry: * also behaves as a filter to prevent too many eofblocks scans from * running at the same time. */ - if (ret == -EDQUOT && !enospc) { + if (ret == -EDQUOT && !cleared_space) { xfs_iunlock(ip, iolock); - enospc = xfs_inode_free_quota_eofblocks(ip); - if (enospc) - goto write_retry; - enospc = xfs_inode_free_quota_cowblocks(ip); - if (enospc) + cleared_space = xfs_inode_free_quota_blocks(ip); + if (cleared_space) goto write_retry; iolock = 0; - } else if (ret == -ENOSPC && !enospc) { + } else if (ret == -ENOSPC && !cleared_space) { struct xfs_eofblocks eofb = {0}; - enospc = 1; + cleared_space = true; xfs_flush_inodes(ip->i_mount); xfs_iunlock(ip, iolock); diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index deb99300d171..c71eb15e3835 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1397,33 +1397,31 @@ xfs_icache_free_eofblocks( } /* - * Run eofblocks scans on the quotas applicable to the inode. For inodes with - * multiple quotas, we don't know exactly which quota caused an allocation + * Run cow/eofblocks scans on the quotas applicable to the inode. For inodes + * with multiple quotas, we don't know exactly which quota caused an allocation * failure. We make a best effort by including each quota under low free space * conditions (less than 1% free space) in the scan. */ -static int -__xfs_inode_free_quota_eofblocks( - struct xfs_inode *ip, - int (*execute)(struct xfs_mount *mp, - struct xfs_eofblocks *eofb)) +bool +xfs_inode_free_quota_blocks( + struct xfs_inode *ip) { - int scan = 0; - struct xfs_eofblocks eofb = {0}; - struct xfs_dquot *dq; + struct xfs_eofblocks eofb = {0}; + struct xfs_dquot *dq; + bool do_work = false; /* * Run a sync scan to increase effectiveness and use the union filter to * cover all applicable quotas in a single scan. */ - eofb.eof_flags = XFS_EOF_FLAGS_UNION|XFS_EOF_FLAGS_SYNC; + eofb.eof_flags = XFS_EOF_FLAGS_UNION | XFS_EOF_FLAGS_SYNC; if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) { dq = xfs_inode_dquot(ip, XFS_DQTYPE_USER); if (dq && xfs_dquot_lowsp(dq)) { eofb.eof_uid = VFS_I(ip)->i_uid; eofb.eof_flags |= XFS_EOF_FLAGS_UID; - scan = 1; + do_work = true; } } @@ -1432,21 +1430,16 @@ __xfs_inode_free_quota_eofblocks( if (dq && xfs_dquot_lowsp(dq)) { eofb.eof_gid = VFS_I(ip)->i_gid; eofb.eof_flags |= XFS_EOF_FLAGS_GID; - scan = 1; + do_work = true; } } - if (scan) - execute(ip->i_mount, &eofb); - - return scan; -} + if (!do_work) + return false; -int -xfs_inode_free_quota_eofblocks( - struct xfs_inode *ip) -{ - return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_eofblocks); + xfs_icache_free_eofblocks(ip->i_mount, &eofb); + xfs_icache_free_cowblocks(ip->i_mount, &eofb); + return true; } static inline unsigned long @@ -1646,13 +1639,6 @@ xfs_icache_free_cowblocks( XFS_ICI_COWBLOCKS_TAG); } -int -xfs_inode_free_quota_cowblocks( - struct xfs_inode *ip) -{ - return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_cowblocks); -} - void xfs_inode_set_cowblocks_tag( xfs_inode_t *ip) diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index 3a4c8b382cd0..3f7ddbca8638 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -54,17 +54,17 @@ long xfs_reclaim_inodes_nr(struct xfs_mount *mp, int nr_to_scan); void xfs_inode_set_reclaim_tag(struct xfs_inode *ip); +bool xfs_inode_free_quota_blocks(struct xfs_inode *ip); + void xfs_inode_set_eofblocks_tag(struct xfs_inode *ip); void xfs_inode_clear_eofblocks_tag(struct xfs_inode *ip); int xfs_icache_free_eofblocks(struct xfs_mount *, struct xfs_eofblocks *); -int xfs_inode_free_quota_eofblocks(struct xfs_inode *ip); void xfs_eofblocks_worker(struct work_struct *); void xfs_queue_eofblocks(struct xfs_mount *); void xfs_inode_set_cowblocks_tag(struct xfs_inode *ip); void xfs_inode_clear_cowblocks_tag(struct xfs_inode *ip); int xfs_icache_free_cowblocks(struct xfs_mount *, struct xfs_eofblocks *); -int xfs_inode_free_quota_cowblocks(struct xfs_inode *ip); void xfs_cowblocks_worker(struct work_struct *); void xfs_queue_cowblocks(struct xfs_mount *); -- cgit v1.2.3 From f41a0716f4b08678a73173d71ff3f409b996df2d Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:35 -0800 Subject: xfs: don't stall cowblocks scan if we can't take locks Don't stall the cowblocks scan on a locked inode if we possibly can. We'd much rather the background scanner keep moving. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index c71eb15e3835..89f9e692fde7 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1605,17 +1605,31 @@ xfs_inode_free_cowblocks( void *args) { struct xfs_eofblocks *eofb = args; + bool wait; int ret = 0; + wait = eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC); + if (!xfs_prep_free_cowblocks(ip)) return 0; if (!xfs_inode_matches_eofb(ip, eofb)) return 0; - /* Free the CoW blocks */ - xfs_ilock(ip, XFS_IOLOCK_EXCL); - xfs_ilock(ip, XFS_MMAPLOCK_EXCL); + /* + * If the caller is waiting, return -EAGAIN to keep the background + * scanner moving and revisit the inode in a subsequent pass. + */ + if (!xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) { + if (wait) + return -EAGAIN; + return 0; + } + if (!xfs_ilock_nowait(ip, XFS_MMAPLOCK_EXCL)) { + if (wait) + ret = -EAGAIN; + goto out_iolock; + } /* * Check again, nobody else should be able to dirty blocks or change @@ -1625,6 +1639,7 @@ xfs_inode_free_cowblocks( ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, false); xfs_iunlock(ip, XFS_MMAPLOCK_EXCL); +out_iolock: xfs_iunlock(ip, XFS_IOLOCK_EXCL); return ret; -- cgit v1.2.3 From 9a537de3b009d95cfb048b7cbfe9bdb0f655596e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:35 -0800 Subject: xfs: xfs_inode_free_quota_blocks should scan project quota Buffered writers who have run out of quota reservation call xfs_inode_free_quota_blocks to try to free any space reservations that might reduce the quota usage. Unfortunately, the buffered write path treats "out of project quota" the same as "out of overall space" so this function has never supported scanning for space that might ease an "out of project quota" condition. We're about to start using this function for cases where we actually /can/ tell if we're out of project quota, so add in this functionality. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 89f9e692fde7..10c1a0dee17d 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1434,6 +1434,15 @@ xfs_inode_free_quota_blocks( } } + if (XFS_IS_PQUOTA_ENFORCED(ip->i_mount)) { + dq = xfs_inode_dquot(ip, XFS_DQTYPE_PROJ); + if (dq && xfs_dquot_lowsp(dq)) { + eofb.eof_prid = ip->i_d.di_projid; + eofb.eof_flags |= XFS_EOF_FLAGS_PRID; + do_work = true; + } + } + if (!do_work) return false; -- cgit v1.2.3 From 3d4feec00673d34fbbfe0277d2e0ed1f51d20cb2 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:36 -0800 Subject: xfs: move and rename xfs_inode_free_quota_blocks to avoid conflicts Move this function further down in the file so that later cleanups won't have to declare static functions. Change the name because we're about to rework all the code that performs garbage collection of speculatively allocated file blocks. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_file.c | 2 +- fs/xfs/xfs_icache.c | 110 ++++++++++++++++++++++++++-------------------------- fs/xfs/xfs_icache.h | 2 +- 3 files changed, 57 insertions(+), 57 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index b55f1eb8f1c8..eade63d53be5 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -706,7 +706,7 @@ write_retry: */ if (ret == -EDQUOT && !cleared_space) { xfs_iunlock(ip, iolock); - cleared_space = xfs_inode_free_quota_blocks(ip); + cleared_space = xfs_blockgc_free_quota(ip); if (cleared_space) goto write_retry; iolock = 0; diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 10c1a0dee17d..aba901d5637b 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1396,61 +1396,6 @@ xfs_icache_free_eofblocks( XFS_ICI_EOFBLOCKS_TAG); } -/* - * Run cow/eofblocks scans on the quotas applicable to the inode. For inodes - * with multiple quotas, we don't know exactly which quota caused an allocation - * failure. We make a best effort by including each quota under low free space - * conditions (less than 1% free space) in the scan. - */ -bool -xfs_inode_free_quota_blocks( - struct xfs_inode *ip) -{ - struct xfs_eofblocks eofb = {0}; - struct xfs_dquot *dq; - bool do_work = false; - - /* - * Run a sync scan to increase effectiveness and use the union filter to - * cover all applicable quotas in a single scan. - */ - eofb.eof_flags = XFS_EOF_FLAGS_UNION | XFS_EOF_FLAGS_SYNC; - - if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) { - dq = xfs_inode_dquot(ip, XFS_DQTYPE_USER); - if (dq && xfs_dquot_lowsp(dq)) { - eofb.eof_uid = VFS_I(ip)->i_uid; - eofb.eof_flags |= XFS_EOF_FLAGS_UID; - do_work = true; - } - } - - if (XFS_IS_GQUOTA_ENFORCED(ip->i_mount)) { - dq = xfs_inode_dquot(ip, XFS_DQTYPE_GROUP); - if (dq && xfs_dquot_lowsp(dq)) { - eofb.eof_gid = VFS_I(ip)->i_gid; - eofb.eof_flags |= XFS_EOF_FLAGS_GID; - do_work = true; - } - } - - if (XFS_IS_PQUOTA_ENFORCED(ip->i_mount)) { - dq = xfs_inode_dquot(ip, XFS_DQTYPE_PROJ); - if (dq && xfs_dquot_lowsp(dq)) { - eofb.eof_prid = ip->i_d.di_projid; - eofb.eof_flags |= XFS_EOF_FLAGS_PRID; - do_work = true; - } - } - - if (!do_work) - return false; - - xfs_icache_free_eofblocks(ip->i_mount, &eofb); - xfs_icache_free_cowblocks(ip->i_mount, &eofb); - return true; -} - static inline unsigned long xfs_iflag_for_tag( int tag) @@ -1699,3 +1644,58 @@ xfs_start_block_reaping( xfs_queue_eofblocks(mp); xfs_queue_cowblocks(mp); } + +/* + * Run cow/eofblocks scans on the quotas applicable to the inode. For inodes + * with multiple quotas, we don't know exactly which quota caused an allocation + * failure. We make a best effort by including each quota under low free space + * conditions (less than 1% free space) in the scan. + */ +bool +xfs_blockgc_free_quota( + struct xfs_inode *ip) +{ + struct xfs_eofblocks eofb = {0}; + struct xfs_dquot *dq; + bool do_work = false; + + /* + * Run a sync scan to increase effectiveness and use the union filter to + * cover all applicable quotas in a single scan. + */ + eofb.eof_flags = XFS_EOF_FLAGS_UNION | XFS_EOF_FLAGS_SYNC; + + if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) { + dq = xfs_inode_dquot(ip, XFS_DQTYPE_USER); + if (dq && xfs_dquot_lowsp(dq)) { + eofb.eof_uid = VFS_I(ip)->i_uid; + eofb.eof_flags |= XFS_EOF_FLAGS_UID; + do_work = true; + } + } + + if (XFS_IS_GQUOTA_ENFORCED(ip->i_mount)) { + dq = xfs_inode_dquot(ip, XFS_DQTYPE_GROUP); + if (dq && xfs_dquot_lowsp(dq)) { + eofb.eof_gid = VFS_I(ip)->i_gid; + eofb.eof_flags |= XFS_EOF_FLAGS_GID; + do_work = true; + } + } + + if (XFS_IS_PQUOTA_ENFORCED(ip->i_mount)) { + dq = xfs_inode_dquot(ip, XFS_DQTYPE_PROJ); + if (dq && xfs_dquot_lowsp(dq)) { + eofb.eof_prid = ip->i_d.di_projid; + eofb.eof_flags |= XFS_EOF_FLAGS_PRID; + do_work = true; + } + } + + if (!do_work) + return false; + + xfs_icache_free_eofblocks(ip->i_mount, &eofb); + xfs_icache_free_cowblocks(ip->i_mount, &eofb); + return true; +} diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index 3f7ddbca8638..21b726a05b0d 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -54,7 +54,7 @@ long xfs_reclaim_inodes_nr(struct xfs_mount *mp, int nr_to_scan); void xfs_inode_set_reclaim_tag(struct xfs_inode *ip); -bool xfs_inode_free_quota_blocks(struct xfs_inode *ip); +bool xfs_blockgc_free_quota(struct xfs_inode *ip); void xfs_inode_set_eofblocks_tag(struct xfs_inode *ip); void xfs_inode_clear_eofblocks_tag(struct xfs_inode *ip); -- cgit v1.2.3 From 111068f80eac00173816c2e822c52c316b650df3 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:36 -0800 Subject: xfs: pass flags and return gc errors from xfs_blockgc_free_quota Change the signature of xfs_blockgc_free_quota in preparation for the next few patches. Callers can now pass EOF_FLAGS into the function to control scan parameters; and the function will now pass back any corruption errors seen while scanning, though for our retry loops we'll just try again unconditionally. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_file.c | 10 +++++----- fs/xfs/xfs_icache.c | 26 +++++++++++++++++--------- fs/xfs/xfs_icache.h | 2 +- 3 files changed, 23 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index eade63d53be5..8546dbf6c5ac 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -702,14 +702,14 @@ write_retry: * metadata space. This reduces the chances that the eofblocks scan * waits on dirty mappings. Since xfs_flush_inodes() is serialized, this * also behaves as a filter to prevent too many eofblocks scans from - * running at the same time. + * running at the same time. Use a synchronous scan to increase the + * effectiveness of the scan. */ if (ret == -EDQUOT && !cleared_space) { xfs_iunlock(ip, iolock); - cleared_space = xfs_blockgc_free_quota(ip); - if (cleared_space) - goto write_retry; - iolock = 0; + xfs_blockgc_free_quota(ip, XFS_EOF_FLAGS_SYNC); + cleared_space = true; + goto write_retry; } else if (ret == -ENOSPC && !cleared_space) { struct xfs_eofblocks eofb = {0}; diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index aba901d5637b..4a074aa12b52 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1650,20 +1650,26 @@ xfs_start_block_reaping( * with multiple quotas, we don't know exactly which quota caused an allocation * failure. We make a best effort by including each quota under low free space * conditions (less than 1% free space) in the scan. + * + * Callers must not hold any inode's ILOCK. If requesting a synchronous scan + * (XFS_EOF_FLAGS_SYNC), the caller also must not hold any inode's IOLOCK or + * MMAPLOCK. */ -bool +int xfs_blockgc_free_quota( - struct xfs_inode *ip) + struct xfs_inode *ip, + unsigned int eof_flags) { struct xfs_eofblocks eofb = {0}; struct xfs_dquot *dq; bool do_work = false; + int error; /* - * Run a sync scan to increase effectiveness and use the union filter to - * cover all applicable quotas in a single scan. + * Run a scan to free blocks using the union filter to cover all + * applicable quotas in a single scan. */ - eofb.eof_flags = XFS_EOF_FLAGS_UNION | XFS_EOF_FLAGS_SYNC; + eofb.eof_flags = XFS_EOF_FLAGS_UNION | eof_flags; if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) { dq = xfs_inode_dquot(ip, XFS_DQTYPE_USER); @@ -1693,9 +1699,11 @@ xfs_blockgc_free_quota( } if (!do_work) - return false; + return 0; - xfs_icache_free_eofblocks(ip->i_mount, &eofb); - xfs_icache_free_cowblocks(ip->i_mount, &eofb); - return true; + error = xfs_icache_free_eofblocks(ip->i_mount, &eofb); + if (error) + return error; + + return xfs_icache_free_cowblocks(ip->i_mount, &eofb); } diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index 21b726a05b0d..d64ea8f5c589 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -54,7 +54,7 @@ long xfs_reclaim_inodes_nr(struct xfs_mount *mp, int nr_to_scan); void xfs_inode_set_reclaim_tag(struct xfs_inode *ip); -bool xfs_blockgc_free_quota(struct xfs_inode *ip); +int xfs_blockgc_free_quota(struct xfs_inode *ip, unsigned int eof_flags); void xfs_inode_set_eofblocks_tag(struct xfs_inode *ip); void xfs_inode_clear_eofblocks_tag(struct xfs_inode *ip); -- cgit v1.2.3 From 4ca74205685ee3a72ab7fe475f51cc26dea36509 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Wed, 27 Jan 2021 10:40:00 -0800 Subject: xfs: try worst case space reservation upfront in xfs_reflink_remap_extent Now that we've converted xfs_reflink_remap_extent to use the new xfs_trans_alloc_inode API, we can focus on its slightly unusual behavior with regard to quota reservations. Since it's valid to remap written blocks into a hole, we must be able to increase the quota count by the number of blocks in the mapping. However, the incore space reservation process requires us to supply an asymptotic guess before we can gain exclusive access to resources. We'd like to reserve all the quota we need up front, but we also don't want to fail a written -> allocated remap operation unnecessarily. The solution is to make the remap_extents function call the transaction allocation function twice. The first time we ask to reserve enough space and quota to handle the absolute worst case situation, but if that fails, we can fall back to the old strategy: ask for the bare minimum space reservation upfront and increase the quota reservation later if we need to. Later in this patchset we change the transaction and quota code to try to reclaim space if we cannot reserve free space or quota. Restructuring the remap_extent function in this manner means that if the fallback increase fails, we can pass that back to the caller knowing that the transaction allocation already tried freeing space. Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_reflink.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 27f875fa7a0d..086866f6e71f 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -991,6 +991,7 @@ xfs_reflink_remap_extent( xfs_off_t newlen; int64_t qdelta = 0; unsigned int resblks; + bool quota_reserved = true; bool smap_real; bool dmap_written = xfs_bmap_is_written_extent(dmap); int iext_delta = 0; @@ -1006,10 +1007,26 @@ xfs_reflink_remap_extent( * the same index in the bmap btree, so we only need a reservation for * one bmbt split if either thing is happening. However, we haven't * locked the inode yet, so we reserve assuming this is the case. + * + * The first allocation call tries to reserve enough space to handle + * mapping dmap into a sparse part of the file plus the bmbt split. We + * haven't locked the inode or read the existing mapping yet, so we do + * not know for sure that we need the space. This should succeed most + * of the time. + * + * If the first attempt fails, try again but reserving only enough + * space to handle a bmbt split. This is the hard minimum requirement, + * and we revisit quota reservations later when we know more about what + * we're remapping. */ resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); - error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, 0, - false, &tp); + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, + resblks + dmap->br_blockcount, 0, false, &tp); + if (error == -EDQUOT || error == -ENOSPC) { + quota_reserved = false; + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, + resblks, 0, false, &tp); + } if (error) goto out; @@ -1076,7 +1093,7 @@ xfs_reflink_remap_extent( * before we started. That should have removed all the delalloc * reservations, but we code defensively. */ - if (!smap_real && dmap_written) { + if (!quota_reserved && !smap_real && dmap_written) { error = xfs_trans_reserve_quota_nblks(tp, ip, dmap->br_blockcount, 0, false); if (error) -- cgit v1.2.3 From 766aabd59929cd05fc1a249f376e4395bed93d30 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:37 -0800 Subject: xfs: flush eof/cowblocks if we can't reserve quota for file blocks If a fs modification (data write, reflink, xattr set, fallocate, etc.) is unable to reserve enough quota to handle the modification, try clearing whatever space the filesystem might have been hanging onto in the hopes of speeding up the filesystem. The flushing behavior will become particularly important when we add deferred inode inactivation because that will increase the amount of space that isn't actively tied to user data. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_reflink.c | 5 +++++ fs/xfs/xfs_trans.c | 10 ++++++++++ 2 files changed, 15 insertions(+) (limited to 'fs') diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 086866f6e71f..725c7d8e4438 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -1092,6 +1092,11 @@ xfs_reflink_remap_extent( * count. This is suboptimal, but the VFS flushed the dest range * before we started. That should have removed all the delalloc * reservations, but we code defensively. + * + * xfs_trans_alloc_inode above already tried to grab an even larger + * quota reservation, and kicked off a blockgc scan if it couldn't. + * If we can't get a potentially smaller quota reservation now, we're + * done. */ if (!quota_reserved && !smap_real && dmap_written) { error = xfs_trans_reserve_quota_nblks(tp, ip, diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 29dca1bc4c1a..4071bbed2d48 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -23,6 +23,7 @@ #include "xfs_inode.h" #include "xfs_dquot_item.h" #include "xfs_dquot.h" +#include "xfs_icache.h" kmem_zone_t *xfs_trans_zone; @@ -1046,8 +1047,10 @@ xfs_trans_alloc_inode( { struct xfs_trans *tp; struct xfs_mount *mp = ip->i_mount; + bool retried = false; int error; +retry: error = xfs_trans_alloc(mp, resv, dblocks, rblocks / mp->m_sb.sb_rextsize, force ? XFS_TRANS_RESERVE : 0, &tp); @@ -1065,6 +1068,13 @@ xfs_trans_alloc_inode( } error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, rblocks, force); + if ((error == -EDQUOT || error == -ENOSPC) && !retried) { + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_blockgc_free_quota(ip, 0); + retried = true; + goto retry; + } if (error) goto out_cancel; -- cgit v1.2.3 From c237dd7c709432611a7642ca10c2a0c8c48ea313 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:37 -0800 Subject: xfs: flush eof/cowblocks if we can't reserve quota for inode creation If an inode creation is unable to reserve enough quota to handle the modification, try clearing whatever space the filesystem might have been hanging onto in the hopes of speeding up the filesystem. The flushing behavior will become particularly important when we add deferred inode inactivation because that will increase the amount of space that isn't actively tied to user data. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 68 ++++++++++++++++++++++++++++++----------------------- fs/xfs/xfs_icache.h | 3 +++ fs/xfs/xfs_trans.c | 8 +++++++ 3 files changed, 49 insertions(+), 30 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 4a074aa12b52..df9533d6bc16 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1646,64 +1646,72 @@ xfs_start_block_reaping( } /* - * Run cow/eofblocks scans on the quotas applicable to the inode. For inodes - * with multiple quotas, we don't know exactly which quota caused an allocation - * failure. We make a best effort by including each quota under low free space - * conditions (less than 1% free space) in the scan. + * Run cow/eofblocks scans on the supplied dquots. We don't know exactly which + * quota caused an allocation failure, so we make a best effort by including + * each quota under low free space conditions (less than 1% free space) in the + * scan. * * Callers must not hold any inode's ILOCK. If requesting a synchronous scan * (XFS_EOF_FLAGS_SYNC), the caller also must not hold any inode's IOLOCK or * MMAPLOCK. */ int -xfs_blockgc_free_quota( - struct xfs_inode *ip, +xfs_blockgc_free_dquots( + struct xfs_mount *mp, + struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, unsigned int eof_flags) { struct xfs_eofblocks eofb = {0}; - struct xfs_dquot *dq; bool do_work = false; int error; + if (!udqp && !gdqp && !pdqp) + return 0; + /* * Run a scan to free blocks using the union filter to cover all * applicable quotas in a single scan. */ eofb.eof_flags = XFS_EOF_FLAGS_UNION | eof_flags; - if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) { - dq = xfs_inode_dquot(ip, XFS_DQTYPE_USER); - if (dq && xfs_dquot_lowsp(dq)) { - eofb.eof_uid = VFS_I(ip)->i_uid; - eofb.eof_flags |= XFS_EOF_FLAGS_UID; - do_work = true; - } + if (XFS_IS_UQUOTA_ENFORCED(mp) && udqp && xfs_dquot_lowsp(udqp)) { + eofb.eof_uid = make_kuid(mp->m_super->s_user_ns, udqp->q_id); + eofb.eof_flags |= XFS_EOF_FLAGS_UID; + do_work = true; } - if (XFS_IS_GQUOTA_ENFORCED(ip->i_mount)) { - dq = xfs_inode_dquot(ip, XFS_DQTYPE_GROUP); - if (dq && xfs_dquot_lowsp(dq)) { - eofb.eof_gid = VFS_I(ip)->i_gid; - eofb.eof_flags |= XFS_EOF_FLAGS_GID; - do_work = true; - } + if (XFS_IS_UQUOTA_ENFORCED(mp) && gdqp && xfs_dquot_lowsp(gdqp)) { + eofb.eof_gid = make_kgid(mp->m_super->s_user_ns, gdqp->q_id); + eofb.eof_flags |= XFS_EOF_FLAGS_GID; + do_work = true; } - if (XFS_IS_PQUOTA_ENFORCED(ip->i_mount)) { - dq = xfs_inode_dquot(ip, XFS_DQTYPE_PROJ); - if (dq && xfs_dquot_lowsp(dq)) { - eofb.eof_prid = ip->i_d.di_projid; - eofb.eof_flags |= XFS_EOF_FLAGS_PRID; - do_work = true; - } + if (XFS_IS_PQUOTA_ENFORCED(mp) && pdqp && xfs_dquot_lowsp(pdqp)) { + eofb.eof_prid = pdqp->q_id; + eofb.eof_flags |= XFS_EOF_FLAGS_PRID; + do_work = true; } if (!do_work) return 0; - error = xfs_icache_free_eofblocks(ip->i_mount, &eofb); + error = xfs_icache_free_eofblocks(mp, &eofb); if (error) return error; - return xfs_icache_free_cowblocks(ip->i_mount, &eofb); + return xfs_icache_free_cowblocks(mp, &eofb); +} + +/* Run cow/eofblocks scans on the quotas attached to the inode. */ +int +xfs_blockgc_free_quota( + struct xfs_inode *ip, + unsigned int eof_flags) +{ + return xfs_blockgc_free_dquots(ip->i_mount, + xfs_inode_dquot(ip, XFS_DQTYPE_USER), + xfs_inode_dquot(ip, XFS_DQTYPE_GROUP), + xfs_inode_dquot(ip, XFS_DQTYPE_PROJ), eof_flags); } diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index d64ea8f5c589..5f7d7c192d1e 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -54,6 +54,9 @@ long xfs_reclaim_inodes_nr(struct xfs_mount *mp, int nr_to_scan); void xfs_inode_set_reclaim_tag(struct xfs_inode *ip); +int xfs_blockgc_free_dquots(struct xfs_mount *mp, struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, + unsigned int eof_flags); int xfs_blockgc_free_quota(struct xfs_inode *ip, unsigned int eof_flags); void xfs_inode_set_eofblocks_tag(struct xfs_inode *ip); diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 4071bbed2d48..b29434199079 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -1102,13 +1102,21 @@ xfs_trans_alloc_icreate( struct xfs_trans **tpp) { struct xfs_trans *tp; + bool retried = false; int error; +retry: error = xfs_trans_alloc(mp, resv, dblocks, 0, 0, &tp); if (error) return error; error = xfs_trans_reserve_quota_icreate(tp, udqp, gdqp, pdqp, dblocks); + if ((error == -EDQUOT || error == -ENOSPC) && !retried) { + xfs_trans_cancel(tp); + xfs_blockgc_free_dquots(mp, udqp, gdqp, pdqp, 0); + retried = true; + goto retry; + } if (error) { xfs_trans_cancel(tp); return error; -- cgit v1.2.3 From 758303d1449965819661048e9e31f32d61888f70 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:38 -0800 Subject: xfs: flush eof/cowblocks if we can't reserve quota for chown If a file user, group, or project change is unable to reserve enough quota to handle the modification, try clearing whatever space the filesystem might have been hanging onto in the hopes of speeding up the filesystem. The flushing behavior will become particularly important when we add deferred inode inactivation because that will increase the amount of space that isn't actively tied to user data. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_trans.c | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index b29434199079..a23400f59e7d 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -1138,16 +1138,21 @@ retry: int xfs_trans_alloc_ichange( struct xfs_inode *ip, - struct xfs_dquot *udqp, - struct xfs_dquot *gdqp, - struct xfs_dquot *pdqp, + struct xfs_dquot *new_udqp, + struct xfs_dquot *new_gdqp, + struct xfs_dquot *new_pdqp, bool force, struct xfs_trans **tpp) { struct xfs_trans *tp; struct xfs_mount *mp = ip->i_mount; + struct xfs_dquot *udqp; + struct xfs_dquot *gdqp; + struct xfs_dquot *pdqp; + bool retried = false; int error; +retry: error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); if (error) return error; @@ -1165,14 +1170,12 @@ xfs_trans_alloc_ichange( /* * For each quota type, skip quota reservations if the inode's dquots * now match the ones that came from the caller, or the caller didn't - * pass one in. + * pass one in. The inode's dquots can change if we drop the ILOCK to + * perform a blockgc scan, so we must preserve the caller's arguments. */ - if (udqp == ip->i_udquot) - udqp = NULL; - if (gdqp == ip->i_gdquot) - gdqp = NULL; - if (pdqp == ip->i_pdquot) - pdqp = NULL; + udqp = (new_udqp != ip->i_udquot) ? new_udqp : NULL; + gdqp = (new_gdqp != ip->i_gdquot) ? new_gdqp : NULL; + pdqp = (new_pdqp != ip->i_pdquot) ? new_pdqp : NULL; if (udqp || gdqp || pdqp) { unsigned int qflags = XFS_QMOPT_RES_REGBLKS; @@ -1188,6 +1191,12 @@ xfs_trans_alloc_ichange( error = xfs_trans_reserve_quota_bydquots(tp, mp, udqp, gdqp, pdqp, ip->i_d.di_nblocks + ip->i_delayed_blks, 1, qflags); + if ((error == -EDQUOT || error == -ENOSPC) && !retried) { + xfs_trans_cancel(tp); + xfs_blockgc_free_dquots(mp, udqp, gdqp, pdqp, 0); + retried = true; + goto retry; + } if (error) goto out_cancel; } -- cgit v1.2.3 From 38899f8099945559662e6a6e355b9059088e3b34 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:38 -0800 Subject: xfs: add a tracepoint for blockgc scans Add some tracepoints so that we can observe when the speculative preallocation garbage collector runs. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_ioctl.c | 2 ++ fs/xfs/xfs_trace.c | 1 + fs/xfs/xfs_trace.h | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) (limited to 'fs') diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 38ee66d999d8..3ddefb685ac6 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -2339,6 +2339,8 @@ xfs_file_ioctl( if (error) return error; + trace_xfs_ioc_free_eofblocks(mp, &keofb, _RET_IP_); + sb_start_write(mp->m_super); error = xfs_icache_free_eofblocks(mp, &keofb); sb_end_write(mp->m_super); diff --git a/fs/xfs/xfs_trace.c b/fs/xfs/xfs_trace.c index 120398a37c2a..9b8d703dc9fd 100644 --- a/fs/xfs/xfs_trace.c +++ b/fs/xfs/xfs_trace.c @@ -29,6 +29,7 @@ #include "xfs_filestream.h" #include "xfs_fsmap.h" #include "xfs_btree_staging.h" +#include "xfs_icache.h" /* * We include this last to have the helpers above available for the trace diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index e2e0092c331d..965873026116 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -37,6 +37,7 @@ struct xfs_trans_res; struct xfs_inobt_rec_incore; union xfs_btree_ptr; struct xfs_dqtrx; +struct xfs_eofblocks; #define XFS_ATTR_FILTER_FLAGS \ { XFS_ATTR_ROOT, "ROOT" }, \ @@ -3888,6 +3889,46 @@ DEFINE_EVENT(xfs_timestamp_range_class, name, \ DEFINE_TIMESTAMP_RANGE_EVENT(xfs_inode_timestamp_range); DEFINE_TIMESTAMP_RANGE_EVENT(xfs_quota_expiry_range); +DECLARE_EVENT_CLASS(xfs_eofblocks_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_eofblocks *eofb, + unsigned long caller_ip), + TP_ARGS(mp, eofb, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(__u32, flags) + __field(uint32_t, uid) + __field(uint32_t, gid) + __field(prid_t, prid) + __field(__u64, min_file_size) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->flags = eofb ? eofb->eof_flags : 0; + __entry->uid = eofb ? from_kuid(mp->m_super->s_user_ns, + eofb->eof_uid) : 0; + __entry->gid = eofb ? from_kgid(mp->m_super->s_user_ns, + eofb->eof_gid) : 0; + __entry->prid = eofb ? eofb->eof_prid : 0; + __entry->min_file_size = eofb ? eofb->eof_min_file_size : 0; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d flags 0x%x uid %u gid %u prid %u minsize %llu caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->flags, + __entry->uid, + __entry->gid, + __entry->prid, + __entry->min_file_size, + (char *)__entry->caller_ip) +); +#define DEFINE_EOFBLOCKS_EVENT(name) \ +DEFINE_EVENT(xfs_eofblocks_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_eofblocks *eofb, \ + unsigned long caller_ip), \ + TP_ARGS(mp, eofb, caller_ip)) +DEFINE_EOFBLOCKS_EVENT(xfs_ioc_free_eofblocks); + #endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH -- cgit v1.2.3 From 85c5b27075ba0389855d9f46ff1b1d5c34a44c94 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:39 -0800 Subject: xfs: refactor xfs_icache_free_{eof,cow}blocks call sites In anticipation of more restructuring of the eof/cowblocks gc code, refactor calling of those two functions into a single internal helper function, then present a new standard interface to purge speculative block preallocations and start shifting higher level code to use that. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_file.c | 3 +-- fs/xfs/xfs_icache.c | 39 +++++++++++++++++++++++++++++++++------ fs/xfs/xfs_icache.h | 1 + fs/xfs/xfs_trace.h | 1 + 4 files changed, 36 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 8546dbf6c5ac..38528e59030e 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -718,8 +718,7 @@ write_retry: xfs_iunlock(ip, iolock); eofb.eof_flags = XFS_EOF_FLAGS_SYNC; - xfs_icache_free_eofblocks(ip->i_mount, &eofb); - xfs_icache_free_cowblocks(ip->i_mount, &eofb); + xfs_blockgc_free_space(ip->i_mount, &eofb); goto write_retry; } diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index df9533d6bc16..0d81330a0fd3 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1645,6 +1645,38 @@ xfs_start_block_reaping( xfs_queue_cowblocks(mp); } +/* Scan all incore inodes for block preallocations that we can remove. */ +static inline int +xfs_blockgc_scan( + struct xfs_mount *mp, + struct xfs_eofblocks *eofb) +{ + int error; + + error = xfs_icache_free_eofblocks(mp, eofb); + if (error) + return error; + + error = xfs_icache_free_cowblocks(mp, eofb); + if (error) + return error; + + return 0; +} + +/* + * Try to free space in the filesystem by purging eofblocks and cowblocks. + */ +int +xfs_blockgc_free_space( + struct xfs_mount *mp, + struct xfs_eofblocks *eofb) +{ + trace_xfs_blockgc_free_space(mp, eofb, _RET_IP_); + + return xfs_blockgc_scan(mp, eofb); +} + /* * Run cow/eofblocks scans on the supplied dquots. We don't know exactly which * quota caused an allocation failure, so we make a best effort by including @@ -1665,7 +1697,6 @@ xfs_blockgc_free_dquots( { struct xfs_eofblocks eofb = {0}; bool do_work = false; - int error; if (!udqp && !gdqp && !pdqp) return 0; @@ -1697,11 +1728,7 @@ xfs_blockgc_free_dquots( if (!do_work) return 0; - error = xfs_icache_free_eofblocks(mp, &eofb); - if (error) - return error; - - return xfs_icache_free_cowblocks(mp, &eofb); + return xfs_blockgc_free_space(mp, &eofb); } /* Run cow/eofblocks scans on the quotas attached to the inode. */ diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index 5f7d7c192d1e..f7dc8d1c91e5 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -58,6 +58,7 @@ int xfs_blockgc_free_dquots(struct xfs_mount *mp, struct xfs_dquot *udqp, struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, unsigned int eof_flags); int xfs_blockgc_free_quota(struct xfs_inode *ip, unsigned int eof_flags); +int xfs_blockgc_free_space(struct xfs_mount *mp, struct xfs_eofblocks *eofb); void xfs_inode_set_eofblocks_tag(struct xfs_inode *ip); void xfs_inode_clear_eofblocks_tag(struct xfs_inode *ip); diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 965873026116..404a00ea9d9e 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -3928,6 +3928,7 @@ DEFINE_EVENT(xfs_eofblocks_class, name, \ unsigned long caller_ip), \ TP_ARGS(mp, eofb, caller_ip)) DEFINE_EOFBLOCKS_EVENT(xfs_ioc_free_eofblocks); +DEFINE_EOFBLOCKS_EVENT(xfs_blockgc_free_space); #endif /* _TRACE_XFS_H */ -- cgit v1.2.3 From a1a7d05a05765eec042942a5c360e909c0dd0131 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:39 -0800 Subject: xfs: flush speculative space allocations when we run out of space If a fs modification (creation, file write, reflink, etc.) is unable to reserve enough space to handle the modification, try clearing whatever space the filesystem might have been hanging onto in the hopes of speeding up the filesystem. The flushing behavior will become particularly important when we add deferred inode inactivation because that will increase the amount of space that isn't actively tied to user data. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_trans.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'fs') diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index a23400f59e7d..44f72c09c203 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -289,6 +289,17 @@ xfs_trans_alloc( tp->t_firstblock = NULLFSBLOCK; error = xfs_trans_reserve(tp, resp, blocks, rtextents); + if (error == -ENOSPC) { + /* + * We weren't able to reserve enough space for the transaction. + * Flush the other speculative space allocations to free space. + * Do not perform a synchronous scan because callers can hold + * other locks. + */ + error = xfs_blockgc_free_space(mp, NULL); + if (!error) + error = xfs_trans_reserve(tp, resp, blocks, rtextents); + } if (error) { xfs_trans_cancel(tp); return error; -- cgit v1.2.3 From f83d436aef5def77b318effc14809fdc57092588 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:41 -0800 Subject: xfs: increase the default parallelism levels of pwork clients Increase the parallelism level for pwork clients to the workqueue defaults so that we can take advantage of computers with a lot of CPUs and a lot of hardware. On fast systems this will speed up quotacheck by a large factor, and the following posteof/cowblocks cleanup series will use the functionality presented in this patch to run garbage collection as quickly as possible. We do this by switching the pwork workqueue to unbounded, since the current user (quotacheck) runs lengthy scans for each work item and we don't care about dispatching the work on a warm cpu cache or anything like that. Also set WQ_SYSFS so that we can monitor where the wq is running. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- Documentation/admin-guide/xfs.rst | 38 ++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_iwalk.c | 5 +---- fs/xfs/xfs_pwork.c | 25 +++++-------------------- fs/xfs/xfs_pwork.h | 4 +--- 4 files changed, 45 insertions(+), 27 deletions(-) (limited to 'fs') diff --git a/Documentation/admin-guide/xfs.rst b/Documentation/admin-guide/xfs.rst index 86de8a1ad91c..b00b1eece9de 100644 --- a/Documentation/admin-guide/xfs.rst +++ b/Documentation/admin-guide/xfs.rst @@ -495,3 +495,41 @@ the class and error context. For example, the default values for "metadata/ENODEV" are "0" rather than "-1" so that this error handler defaults to "fail immediately" behaviour. This is done because ENODEV is a fatal, unrecoverable error no matter how many times the metadata IO is retried. + +Workqueue Concurrency +===================== + +XFS uses kernel workqueues to parallelize metadata update processes. This +enables it to take advantage of storage hardware that can service many IO +operations simultaneously. This interface exposes internal implementation +details of XFS, and as such is explicitly not part of any userspace API/ABI +guarantee the kernel may give userspace. These are undocumented features of +the generic workqueue implementation XFS uses for concurrency, and they are +provided here purely for diagnostic and tuning purposes and may change at any +time in the future. + +The control knobs for a filesystem's workqueues are organized by task at hand +and the short name of the data device. They all can be found in: + + /sys/bus/workqueue/devices/${task}!${device} + +================ =========== + Task Description +================ =========== + xfs_iwalk-$pid Inode scans of the entire filesystem. Currently limited to + mount time quotacheck. +================ =========== + +For example, the knobs for the quotacheck workqueue for /dev/nvme0n1 would be +found in /sys/bus/workqueue/devices/xfs_iwalk-1111!nvme0n1/. + +The interesting knobs for XFS workqueues are as follows: + +============ =========== + Knob Description +============ =========== + max_active Maximum number of background threads that can be started to + run the work. + cpumask CPUs upon which the threads are allowed to run. + nice Relative priority of scheduling the threads. These are the + same nice levels that can be applied to userspace processes. diff --git a/fs/xfs/xfs_iwalk.c b/fs/xfs/xfs_iwalk.c index eae3aff9bc97..c4a340f1f1e1 100644 --- a/fs/xfs/xfs_iwalk.c +++ b/fs/xfs/xfs_iwalk.c @@ -618,15 +618,12 @@ xfs_iwalk_threaded( { struct xfs_pwork_ctl pctl; xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino); - unsigned int nr_threads; int error; ASSERT(agno < mp->m_sb.sb_agcount); ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL)); - nr_threads = xfs_pwork_guess_datadev_parallelism(mp); - error = xfs_pwork_init(mp, &pctl, xfs_iwalk_ag_work, "xfs_iwalk", - nr_threads); + error = xfs_pwork_init(mp, &pctl, xfs_iwalk_ag_work, "xfs_iwalk"); if (error) return error; diff --git a/fs/xfs/xfs_pwork.c b/fs/xfs/xfs_pwork.c index b03333f1c84a..c283b801cc5d 100644 --- a/fs/xfs/xfs_pwork.c +++ b/fs/xfs/xfs_pwork.c @@ -61,16 +61,18 @@ xfs_pwork_init( struct xfs_mount *mp, struct xfs_pwork_ctl *pctl, xfs_pwork_work_fn work_fn, - const char *tag, - unsigned int nr_threads) + const char *tag) { + unsigned int nr_threads = 0; + #ifdef DEBUG if (xfs_globals.pwork_threads >= 0) nr_threads = xfs_globals.pwork_threads; #endif trace_xfs_pwork_init(mp, nr_threads, current->pid); - pctl->wq = alloc_workqueue("%s-%d", WQ_FREEZABLE, nr_threads, tag, + pctl->wq = alloc_workqueue("%s-%d", + WQ_UNBOUND | WQ_SYSFS | WQ_FREEZABLE, nr_threads, tag, current->pid); if (!pctl->wq) return -ENOMEM; @@ -117,20 +119,3 @@ xfs_pwork_poll( atomic_read(&pctl->nr_work) == 0, HZ) == 0) touch_softlockup_watchdog(); } - -/* - * Return the amount of parallelism that the data device can handle, or 0 for - * no limit. - */ -unsigned int -xfs_pwork_guess_datadev_parallelism( - struct xfs_mount *mp) -{ - struct xfs_buftarg *btp = mp->m_ddev_targp; - - /* - * For now we'll go with the most conservative setting possible, - * which is two threads for an SSD and 1 thread everywhere else. - */ - return blk_queue_nonrot(btp->bt_bdev->bd_disk->queue) ? 2 : 1; -} diff --git a/fs/xfs/xfs_pwork.h b/fs/xfs/xfs_pwork.h index 8133124cf3bb..c0ef81fc85dd 100644 --- a/fs/xfs/xfs_pwork.h +++ b/fs/xfs/xfs_pwork.h @@ -51,11 +51,9 @@ xfs_pwork_want_abort( } int xfs_pwork_init(struct xfs_mount *mp, struct xfs_pwork_ctl *pctl, - xfs_pwork_work_fn work_fn, const char *tag, - unsigned int nr_threads); + xfs_pwork_work_fn work_fn, const char *tag); void xfs_pwork_queue(struct xfs_pwork_ctl *pctl, struct xfs_pwork *pwork); int xfs_pwork_destroy(struct xfs_pwork_ctl *pctl); void xfs_pwork_poll(struct xfs_pwork_ctl *pctl); -unsigned int xfs_pwork_guess_datadev_parallelism(struct xfs_mount *mp); #endif /* __XFS_PWORK_H__ */ -- cgit v1.2.3 From 05a302a17062ca73dc91b508cf2a0b25724db15d Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:42 -0800 Subject: xfs: set WQ_SYSFS on all workqueues in debug mode When CONFIG_XFS_DEBUG=y, set WQ_SYSFS on all workqueues that we create so that we (developers) have a means to monitor cpu affinity and whatnot for background workers. In the next patchset we'll expose knobs for more of the workqueues publicly and document it, but not now. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner Reviewed-by: Brian Foster --- fs/xfs/xfs_log.c | 5 +++-- fs/xfs/xfs_mru_cache.c | 2 +- fs/xfs/xfs_super.c | 23 ++++++++++++++--------- fs/xfs/xfs_super.h | 6 ++++++ 4 files changed, 24 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index d8b814227734..9aa30e7cd314 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -1492,8 +1492,9 @@ xlog_alloc_log( log->l_iclog->ic_prev = prev_iclog; /* re-write 1st prev ptr */ log->l_ioend_workqueue = alloc_workqueue("xfs-log/%s", - WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_HIGHPRI, 0, - mp->m_super->s_id); + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | + WQ_HIGHPRI), + 0, mp->m_super->s_id); if (!log->l_ioend_workqueue) goto out_free_iclog; diff --git a/fs/xfs/xfs_mru_cache.c b/fs/xfs/xfs_mru_cache.c index a06661dac5be..34c3b16f834f 100644 --- a/fs/xfs/xfs_mru_cache.c +++ b/fs/xfs/xfs_mru_cache.c @@ -294,7 +294,7 @@ int xfs_mru_cache_init(void) { xfs_mru_reap_wq = alloc_workqueue("xfs_mru_cache", - WQ_MEM_RECLAIM|WQ_FREEZABLE, 1); + XFS_WQFLAGS(WQ_MEM_RECLAIM | WQ_FREEZABLE), 1); if (!xfs_mru_reap_wq) return -ENOMEM; return 0; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index aed74a3fc787..8959561351ca 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -495,33 +495,37 @@ xfs_init_mount_workqueues( struct xfs_mount *mp) { mp->m_buf_workqueue = alloc_workqueue("xfs-buf/%s", - WQ_MEM_RECLAIM|WQ_FREEZABLE, 1, mp->m_super->s_id); + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM), + 1, mp->m_super->s_id); if (!mp->m_buf_workqueue) goto out; mp->m_unwritten_workqueue = alloc_workqueue("xfs-conv/%s", - WQ_MEM_RECLAIM|WQ_FREEZABLE, 0, mp->m_super->s_id); + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM), + 0, mp->m_super->s_id); if (!mp->m_unwritten_workqueue) goto out_destroy_buf; mp->m_cil_workqueue = alloc_workqueue("xfs-cil/%s", - WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND, + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_UNBOUND), 0, mp->m_super->s_id); if (!mp->m_cil_workqueue) goto out_destroy_unwritten; mp->m_reclaim_workqueue = alloc_workqueue("xfs-reclaim/%s", - WQ_MEM_RECLAIM|WQ_FREEZABLE, 0, mp->m_super->s_id); + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM), + 0, mp->m_super->s_id); if (!mp->m_reclaim_workqueue) goto out_destroy_cil; mp->m_eofblocks_workqueue = alloc_workqueue("xfs-eofblocks/%s", - WQ_MEM_RECLAIM|WQ_FREEZABLE, 0, mp->m_super->s_id); + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM), + 0, mp->m_super->s_id); if (!mp->m_eofblocks_workqueue) goto out_destroy_reclaim; - mp->m_sync_workqueue = alloc_workqueue("xfs-sync/%s", WQ_FREEZABLE, 0, - mp->m_super->s_id); + mp->m_sync_workqueue = alloc_workqueue("xfs-sync/%s", + XFS_WQFLAGS(WQ_FREEZABLE), 0, mp->m_super->s_id); if (!mp->m_sync_workqueue) goto out_destroy_eofb; @@ -2085,11 +2089,12 @@ xfs_init_workqueues(void) * max_active value for this workqueue. */ xfs_alloc_wq = alloc_workqueue("xfsalloc", - WQ_MEM_RECLAIM|WQ_FREEZABLE, 0); + XFS_WQFLAGS(WQ_MEM_RECLAIM | WQ_FREEZABLE), 0); if (!xfs_alloc_wq) return -ENOMEM; - xfs_discard_wq = alloc_workqueue("xfsdiscard", WQ_UNBOUND, 0); + xfs_discard_wq = alloc_workqueue("xfsdiscard", XFS_WQFLAGS(WQ_UNBOUND), + 0); if (!xfs_discard_wq) goto out_free_alloc_wq; diff --git a/fs/xfs/xfs_super.h b/fs/xfs/xfs_super.h index b552cf6d3379..1ca484b8357f 100644 --- a/fs/xfs/xfs_super.h +++ b/fs/xfs/xfs_super.h @@ -75,6 +75,12 @@ extern void xfs_qm_exit(void); XFS_ASSERT_FATAL_STRING \ XFS_DBG_STRING /* DBG must be last */ +#ifdef DEBUG +# define XFS_WQFLAGS(wqflags) (WQ_SYSFS | (wqflags)) +#else +# define XFS_WQFLAGS(wqflags) (wqflags) +#endif + struct xfs_inode; struct xfs_mount; struct xfs_buftarg; -- cgit v1.2.3 From f9296569837c3fd66ae32717b0f8f5a26758b4b7 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:39 -0800 Subject: xfs: relocate the eofb/cowb workqueue functions Move the xfs_{eof,cow}blocks_worker and xfs_queue_{eof,cow}blocks functions further down in the file so that the cleanups in the next patches won't have to pre-declare static functions. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icache.c | 126 ++++++++++++++++++++++++++-------------------------- 1 file changed, 63 insertions(+), 63 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 0d81330a0fd3..ae0c14df0f9b 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -915,69 +915,6 @@ xfs_inode_walk( return last_error; } -/* - * Background scanning to trim post-EOF preallocated space. This is queued - * based on the 'speculative_prealloc_lifetime' tunable (5m by default). - */ -void -xfs_queue_eofblocks( - struct xfs_mount *mp) -{ - rcu_read_lock(); - if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_EOFBLOCKS_TAG)) - queue_delayed_work(mp->m_eofblocks_workqueue, - &mp->m_eofblocks_work, - msecs_to_jiffies(xfs_eofb_secs * 1000)); - rcu_read_unlock(); -} - -void -xfs_eofblocks_worker( - struct work_struct *work) -{ - struct xfs_mount *mp = container_of(to_delayed_work(work), - struct xfs_mount, m_eofblocks_work); - - if (!sb_start_write_trylock(mp->m_super)) - return; - xfs_icache_free_eofblocks(mp, NULL); - sb_end_write(mp->m_super); - - xfs_queue_eofblocks(mp); -} - -/* - * Background scanning to trim preallocated CoW space. This is queued - * based on the 'speculative_cow_prealloc_lifetime' tunable (5m by default). - * (We'll just piggyback on the post-EOF prealloc space workqueue.) - */ -void -xfs_queue_cowblocks( - struct xfs_mount *mp) -{ - rcu_read_lock(); - if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_COWBLOCKS_TAG)) - queue_delayed_work(mp->m_eofblocks_workqueue, - &mp->m_cowblocks_work, - msecs_to_jiffies(xfs_cowb_secs * 1000)); - rcu_read_unlock(); -} - -void -xfs_cowblocks_worker( - struct work_struct *work) -{ - struct xfs_mount *mp = container_of(to_delayed_work(work), - struct xfs_mount, m_cowblocks_work); - - if (!sb_start_write_trylock(mp->m_super)) - return; - xfs_icache_free_cowblocks(mp, NULL); - sb_end_write(mp->m_super); - - xfs_queue_cowblocks(mp); -} - /* * Grab the inode for reclaim exclusively. * @@ -1396,6 +1333,37 @@ xfs_icache_free_eofblocks( XFS_ICI_EOFBLOCKS_TAG); } +/* + * Background scanning to trim post-EOF preallocated space. This is queued + * based on the 'speculative_prealloc_lifetime' tunable (5m by default). + */ +void +xfs_queue_eofblocks( + struct xfs_mount *mp) +{ + rcu_read_lock(); + if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_EOFBLOCKS_TAG)) + queue_delayed_work(mp->m_eofblocks_workqueue, + &mp->m_eofblocks_work, + msecs_to_jiffies(xfs_eofb_secs * 1000)); + rcu_read_unlock(); +} + +void +xfs_eofblocks_worker( + struct work_struct *work) +{ + struct xfs_mount *mp = container_of(to_delayed_work(work), + struct xfs_mount, m_eofblocks_work); + + if (!sb_start_write_trylock(mp->m_super)) + return; + xfs_icache_free_eofblocks(mp, NULL); + sb_end_write(mp->m_super); + + xfs_queue_eofblocks(mp); +} + static inline unsigned long xfs_iflag_for_tag( int tag) @@ -1608,6 +1576,38 @@ xfs_icache_free_cowblocks( XFS_ICI_COWBLOCKS_TAG); } +/* + * Background scanning to trim preallocated CoW space. This is queued + * based on the 'speculative_cow_prealloc_lifetime' tunable (5m by default). + * (We'll just piggyback on the post-EOF prealloc space workqueue.) + */ +void +xfs_queue_cowblocks( + struct xfs_mount *mp) +{ + rcu_read_lock(); + if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_COWBLOCKS_TAG)) + queue_delayed_work(mp->m_eofblocks_workqueue, + &mp->m_cowblocks_work, + msecs_to_jiffies(xfs_cowb_secs * 1000)); + rcu_read_unlock(); +} + +void +xfs_cowblocks_worker( + struct work_struct *work) +{ + struct xfs_mount *mp = container_of(to_delayed_work(work), + struct xfs_mount, m_cowblocks_work); + + if (!sb_start_write_trylock(mp->m_super)) + return; + xfs_icache_free_cowblocks(mp, NULL); + sb_end_write(mp->m_super); + + xfs_queue_cowblocks(mp); +} + void xfs_inode_set_cowblocks_tag( xfs_inode_t *ip) -- cgit v1.2.3 From 0461a320e33a16405ac3c165463837e028a42680 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:40 -0800 Subject: xfs: hide xfs_icache_free_eofblocks Change the one remaining caller of xfs_icache_free_eofblocks to use our new combined blockgc scan function instead, since we will soon be combining the two scans. This introduces a slight behavior change, since the XFS_IOC_FREE_EOFBLOCKS now clears out speculative CoW reservations in addition to post-eof blocks. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icache.c | 2 +- fs/xfs/xfs_icache.h | 1 - fs/xfs/xfs_ioctl.c | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index ae0c14df0f9b..d1a849053e68 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1324,7 +1324,7 @@ xfs_inode_free_eofblocks( return ret; } -int +static int xfs_icache_free_eofblocks( struct xfs_mount *mp, struct xfs_eofblocks *eofb) diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index f7dc8d1c91e5..b8c2f23aa829 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -62,7 +62,6 @@ int xfs_blockgc_free_space(struct xfs_mount *mp, struct xfs_eofblocks *eofb); void xfs_inode_set_eofblocks_tag(struct xfs_inode *ip); void xfs_inode_clear_eofblocks_tag(struct xfs_inode *ip); -int xfs_icache_free_eofblocks(struct xfs_mount *, struct xfs_eofblocks *); void xfs_eofblocks_worker(struct work_struct *); void xfs_queue_eofblocks(struct xfs_mount *); diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 3ddefb685ac6..78f965425fa6 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -2342,7 +2342,7 @@ xfs_file_ioctl( trace_xfs_ioc_free_eofblocks(mp, &keofb, _RET_IP_); sb_start_write(mp->m_super); - error = xfs_icache_free_eofblocks(mp, &keofb); + error = xfs_blockgc_free_space(mp, &keofb); sb_end_write(mp->m_super); return error; } -- cgit v1.2.3 From b943c0cd5615233ae4cea66666725a9bf2edccca Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:40 -0800 Subject: xfs: hide xfs_icache_free_cowblocks Change the one remaining caller of xfs_icache_free_cowblocks to use our new combined blockgc scan function instead, since we will soon be combining the two scans. This introduces a slight behavior change, since a readonly remount now clears out post-EOF preallocations and not just CoW staging extents. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icache.c | 2 +- fs/xfs/xfs_icache.h | 1 - fs/xfs/xfs_super.c | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index d1a849053e68..45146e016975 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1567,7 +1567,7 @@ out_iolock: return ret; } -int +static int xfs_icache_free_cowblocks( struct xfs_mount *mp, struct xfs_eofblocks *eofb) diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index b8c2f23aa829..de7604576e9e 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -67,7 +67,6 @@ void xfs_queue_eofblocks(struct xfs_mount *); void xfs_inode_set_cowblocks_tag(struct xfs_inode *ip); void xfs_inode_clear_cowblocks_tag(struct xfs_inode *ip); -int xfs_icache_free_cowblocks(struct xfs_mount *, struct xfs_eofblocks *); void xfs_cowblocks_worker(struct work_struct *); void xfs_queue_cowblocks(struct xfs_mount *); diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 8959561351ca..e8f71714d737 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -1713,7 +1713,7 @@ xfs_remount_ro( xfs_stop_block_reaping(mp); /* Get rid of any leftover CoW reservations... */ - error = xfs_icache_free_cowblocks(mp, NULL); + error = xfs_blockgc_free_space(mp, NULL); if (error) { xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); return error; -- cgit v1.2.3 From 865ac8e253c97423c41e22ce615615eb006fc52e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:41 -0800 Subject: xfs: remove trivial eof/cowblocks functions Get rid of these trivial helpers. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icache.c | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 45146e016975..705f16193596 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1324,15 +1324,6 @@ xfs_inode_free_eofblocks( return ret; } -static int -xfs_icache_free_eofblocks( - struct xfs_mount *mp, - struct xfs_eofblocks *eofb) -{ - return xfs_inode_walk(mp, 0, xfs_inode_free_eofblocks, eofb, - XFS_ICI_EOFBLOCKS_TAG); -} - /* * Background scanning to trim post-EOF preallocated space. This is queued * based on the 'speculative_prealloc_lifetime' tunable (5m by default). @@ -1358,7 +1349,8 @@ xfs_eofblocks_worker( if (!sb_start_write_trylock(mp->m_super)) return; - xfs_icache_free_eofblocks(mp, NULL); + xfs_inode_walk(mp, 0, xfs_inode_free_eofblocks, NULL, + XFS_ICI_EOFBLOCKS_TAG); sb_end_write(mp->m_super); xfs_queue_eofblocks(mp); @@ -1567,15 +1559,6 @@ out_iolock: return ret; } -static int -xfs_icache_free_cowblocks( - struct xfs_mount *mp, - struct xfs_eofblocks *eofb) -{ - return xfs_inode_walk(mp, 0, xfs_inode_free_cowblocks, eofb, - XFS_ICI_COWBLOCKS_TAG); -} - /* * Background scanning to trim preallocated CoW space. This is queued * based on the 'speculative_cow_prealloc_lifetime' tunable (5m by default). @@ -1602,7 +1585,8 @@ xfs_cowblocks_worker( if (!sb_start_write_trylock(mp->m_super)) return; - xfs_icache_free_cowblocks(mp, NULL); + xfs_inode_walk(mp, 0, xfs_inode_free_cowblocks, NULL, + XFS_ICI_COWBLOCKS_TAG); sb_end_write(mp->m_super); xfs_queue_cowblocks(mp); @@ -1653,11 +1637,13 @@ xfs_blockgc_scan( { int error; - error = xfs_icache_free_eofblocks(mp, eofb); + error = xfs_inode_walk(mp, 0, xfs_inode_free_eofblocks, eofb, + XFS_ICI_EOFBLOCKS_TAG); if (error) return error; - error = xfs_icache_free_cowblocks(mp, eofb); + error = xfs_inode_walk(mp, 0, xfs_inode_free_cowblocks, eofb, + XFS_ICI_COWBLOCKS_TAG); if (error) return error; -- cgit v1.2.3 From ce2d3bbe06473fa76eb9dad21529f9cc48408000 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:43 -0800 Subject: xfs: consolidate incore inode radix tree posteof/cowblocks tags The clearing of posteof blocks and cowblocks serve the same purpose: removing speculative block preallocations from inactive files. We don't need to burn two radix tree tags on this, so combine them into one. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icache.c | 114 +++++++++++++++++++++++++--------------------------- fs/xfs/xfs_icache.h | 4 +- fs/xfs/xfs_trace.h | 6 +-- 3 files changed, 58 insertions(+), 66 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 705f16193596..34cc84fc7391 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1291,6 +1291,9 @@ xfs_inode_free_eofblocks( wait = eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC); + if (!xfs_iflags_test(ip, XFS_IEOFBLOCKS)) + return 0; + if (!xfs_can_free_eofblocks(ip, false)) { /* inode could be preallocated or append-only */ trace_xfs_inode_free_eofblocks_invalid(ip); @@ -1333,7 +1336,7 @@ xfs_queue_eofblocks( struct xfs_mount *mp) { rcu_read_lock(); - if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_EOFBLOCKS_TAG)) + if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_BLOCKGC_TAG)) queue_delayed_work(mp->m_eofblocks_workqueue, &mp->m_eofblocks_work, msecs_to_jiffies(xfs_eofb_secs * 1000)); @@ -1350,67 +1353,54 @@ xfs_eofblocks_worker( if (!sb_start_write_trylock(mp->m_super)) return; xfs_inode_walk(mp, 0, xfs_inode_free_eofblocks, NULL, - XFS_ICI_EOFBLOCKS_TAG); + XFS_ICI_BLOCKGC_TAG); sb_end_write(mp->m_super); xfs_queue_eofblocks(mp); } -static inline unsigned long -xfs_iflag_for_tag( - int tag) -{ - switch (tag) { - case XFS_ICI_EOFBLOCKS_TAG: - return XFS_IEOFBLOCKS; - case XFS_ICI_COWBLOCKS_TAG: - return XFS_ICOWBLOCKS; - default: - ASSERT(0); - return 0; - } -} - static void -__xfs_inode_set_blocks_tag( - xfs_inode_t *ip, - void (*execute)(struct xfs_mount *mp), - void (*set_tp)(struct xfs_mount *mp, xfs_agnumber_t agno, - int error, unsigned long caller_ip), - int tag) +xfs_blockgc_set_iflag( + struct xfs_inode *ip, + void (*execute)(struct xfs_mount *mp), + unsigned long iflag) { - struct xfs_mount *mp = ip->i_mount; - struct xfs_perag *pag; - int tagged; + struct xfs_mount *mp = ip->i_mount; + struct xfs_perag *pag; + int tagged; + + ASSERT((iflag & ~(XFS_IEOFBLOCKS | XFS_ICOWBLOCKS)) == 0); /* * Don't bother locking the AG and looking up in the radix trees * if we already know that we have the tag set. */ - if (ip->i_flags & xfs_iflag_for_tag(tag)) + if (ip->i_flags & iflag) return; spin_lock(&ip->i_flags_lock); - ip->i_flags |= xfs_iflag_for_tag(tag); + ip->i_flags |= iflag; spin_unlock(&ip->i_flags_lock); pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); spin_lock(&pag->pag_ici_lock); - tagged = radix_tree_tagged(&pag->pag_ici_root, tag); + tagged = radix_tree_tagged(&pag->pag_ici_root, XFS_ICI_BLOCKGC_TAG); radix_tree_tag_set(&pag->pag_ici_root, - XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag); + XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), + XFS_ICI_BLOCKGC_TAG); if (!tagged) { - /* propagate the eofblocks tag up into the perag radix tree */ + /* propagate the blockgc tag up into the perag radix tree */ spin_lock(&ip->i_mount->m_perag_lock); radix_tree_tag_set(&ip->i_mount->m_perag_tree, XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino), - tag); + XFS_ICI_BLOCKGC_TAG); spin_unlock(&ip->i_mount->m_perag_lock); /* kick off background trimming */ execute(ip->i_mount); - set_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_); + trace_xfs_perag_set_blockgc(ip->i_mount, pag->pag_agno, -1, + _RET_IP_); } spin_unlock(&pag->pag_ici_lock); @@ -1422,38 +1412,43 @@ xfs_inode_set_eofblocks_tag( xfs_inode_t *ip) { trace_xfs_inode_set_eofblocks_tag(ip); - return __xfs_inode_set_blocks_tag(ip, xfs_queue_eofblocks, - trace_xfs_perag_set_eofblocks, - XFS_ICI_EOFBLOCKS_TAG); + return xfs_blockgc_set_iflag(ip, xfs_queue_eofblocks, XFS_IEOFBLOCKS); } static void -__xfs_inode_clear_blocks_tag( - xfs_inode_t *ip, - void (*clear_tp)(struct xfs_mount *mp, xfs_agnumber_t agno, - int error, unsigned long caller_ip), - int tag) +xfs_blockgc_clear_iflag( + struct xfs_inode *ip, + unsigned long iflag) { - struct xfs_mount *mp = ip->i_mount; - struct xfs_perag *pag; + struct xfs_mount *mp = ip->i_mount; + struct xfs_perag *pag; + bool clear_tag; + + ASSERT((iflag & ~(XFS_IEOFBLOCKS | XFS_ICOWBLOCKS)) == 0); spin_lock(&ip->i_flags_lock); - ip->i_flags &= ~xfs_iflag_for_tag(tag); + ip->i_flags &= ~iflag; + clear_tag = (ip->i_flags & (XFS_IEOFBLOCKS | XFS_ICOWBLOCKS)) == 0; spin_unlock(&ip->i_flags_lock); + if (!clear_tag) + return; + pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); spin_lock(&pag->pag_ici_lock); radix_tree_tag_clear(&pag->pag_ici_root, - XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag); - if (!radix_tree_tagged(&pag->pag_ici_root, tag)) { - /* clear the eofblocks tag from the perag radix tree */ + XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), + XFS_ICI_BLOCKGC_TAG); + if (!radix_tree_tagged(&pag->pag_ici_root, XFS_ICI_BLOCKGC_TAG)) { + /* clear the blockgc tag from the perag radix tree */ spin_lock(&ip->i_mount->m_perag_lock); radix_tree_tag_clear(&ip->i_mount->m_perag_tree, XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino), - tag); + XFS_ICI_BLOCKGC_TAG); spin_unlock(&ip->i_mount->m_perag_lock); - clear_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_); + trace_xfs_perag_clear_blockgc(ip->i_mount, pag->pag_agno, -1, + _RET_IP_); } spin_unlock(&pag->pag_ici_lock); @@ -1465,8 +1460,7 @@ xfs_inode_clear_eofblocks_tag( xfs_inode_t *ip) { trace_xfs_inode_clear_eofblocks_tag(ip); - return __xfs_inode_clear_blocks_tag(ip, - trace_xfs_perag_clear_eofblocks, XFS_ICI_EOFBLOCKS_TAG); + return xfs_blockgc_clear_iflag(ip, XFS_IEOFBLOCKS); } /* @@ -1524,6 +1518,9 @@ xfs_inode_free_cowblocks( wait = eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC); + if (!xfs_iflags_test(ip, XFS_ICOWBLOCKS)) + return 0; + if (!xfs_prep_free_cowblocks(ip)) return 0; @@ -1569,7 +1566,7 @@ xfs_queue_cowblocks( struct xfs_mount *mp) { rcu_read_lock(); - if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_COWBLOCKS_TAG)) + if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_BLOCKGC_TAG)) queue_delayed_work(mp->m_eofblocks_workqueue, &mp->m_cowblocks_work, msecs_to_jiffies(xfs_cowb_secs * 1000)); @@ -1586,7 +1583,7 @@ xfs_cowblocks_worker( if (!sb_start_write_trylock(mp->m_super)) return; xfs_inode_walk(mp, 0, xfs_inode_free_cowblocks, NULL, - XFS_ICI_COWBLOCKS_TAG); + XFS_ICI_BLOCKGC_TAG); sb_end_write(mp->m_super); xfs_queue_cowblocks(mp); @@ -1597,9 +1594,7 @@ xfs_inode_set_cowblocks_tag( xfs_inode_t *ip) { trace_xfs_inode_set_cowblocks_tag(ip); - return __xfs_inode_set_blocks_tag(ip, xfs_queue_cowblocks, - trace_xfs_perag_set_cowblocks, - XFS_ICI_COWBLOCKS_TAG); + return xfs_blockgc_set_iflag(ip, xfs_queue_cowblocks, XFS_ICOWBLOCKS); } void @@ -1607,8 +1602,7 @@ xfs_inode_clear_cowblocks_tag( xfs_inode_t *ip) { trace_xfs_inode_clear_cowblocks_tag(ip); - return __xfs_inode_clear_blocks_tag(ip, - trace_xfs_perag_clear_cowblocks, XFS_ICI_COWBLOCKS_TAG); + return xfs_blockgc_clear_iflag(ip, XFS_ICOWBLOCKS); } /* Disable post-EOF and CoW block auto-reclamation. */ @@ -1638,12 +1632,12 @@ xfs_blockgc_scan( int error; error = xfs_inode_walk(mp, 0, xfs_inode_free_eofblocks, eofb, - XFS_ICI_EOFBLOCKS_TAG); + XFS_ICI_BLOCKGC_TAG); if (error) return error; error = xfs_inode_walk(mp, 0, xfs_inode_free_cowblocks, eofb, - XFS_ICI_COWBLOCKS_TAG); + XFS_ICI_BLOCKGC_TAG); if (error) return error; diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index de7604576e9e..ab7107370be4 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -23,8 +23,8 @@ struct xfs_eofblocks { #define XFS_ICI_NO_TAG (-1) /* special flag for an untagged lookup in xfs_inode_walk */ #define XFS_ICI_RECLAIM_TAG 0 /* inode is to be reclaimed */ -#define XFS_ICI_EOFBLOCKS_TAG 1 /* inode has blocks beyond EOF */ -#define XFS_ICI_COWBLOCKS_TAG 2 /* inode can have cow blocks to gc */ +/* Inode has speculative preallocations (posteof or cow) to clean. */ +#define XFS_ICI_BLOCKGC_TAG 1 /* * Flags for xfs_iget() diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 404a00ea9d9e..63ecbc6b1a76 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -155,10 +155,8 @@ DEFINE_PERAG_REF_EVENT(xfs_perag_get_tag); DEFINE_PERAG_REF_EVENT(xfs_perag_put); DEFINE_PERAG_REF_EVENT(xfs_perag_set_reclaim); DEFINE_PERAG_REF_EVENT(xfs_perag_clear_reclaim); -DEFINE_PERAG_REF_EVENT(xfs_perag_set_eofblocks); -DEFINE_PERAG_REF_EVENT(xfs_perag_clear_eofblocks); -DEFINE_PERAG_REF_EVENT(xfs_perag_set_cowblocks); -DEFINE_PERAG_REF_EVENT(xfs_perag_clear_cowblocks); +DEFINE_PERAG_REF_EVENT(xfs_perag_set_blockgc); +DEFINE_PERAG_REF_EVENT(xfs_perag_clear_blockgc); DECLARE_EVENT_CLASS(xfs_ag_class, TP_PROTO(struct xfs_mount *mp, xfs_agnumber_t agno), -- cgit v1.2.3 From 9669f51de5c0c93e79257f690d1feaf16ebc179b Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:43 -0800 Subject: xfs: consolidate the eofblocks and cowblocks workers Remove the separate cowblocks work items and knob so that we can control and run everything from a single blockgc work queue. Note that the speculative_prealloc_lifetime sysfs knob retains its historical name even though the functions move to prefix xfs_blockgc_*. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_globals.c | 7 ++-- fs/xfs/xfs_icache.c | 96 +++++++++++++++++----------------------------------- fs/xfs/xfs_icache.h | 6 ++-- fs/xfs/xfs_linux.h | 3 +- fs/xfs/xfs_mount.h | 6 ++-- fs/xfs/xfs_super.c | 11 +++--- fs/xfs/xfs_sysctl.c | 15 ++------ fs/xfs/xfs_sysctl.h | 3 +- 8 files changed, 48 insertions(+), 99 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_globals.c b/fs/xfs/xfs_globals.c index fa55ab8b8d80..f62fa652c2fd 100644 --- a/fs/xfs/xfs_globals.c +++ b/fs/xfs/xfs_globals.c @@ -8,8 +8,8 @@ /* * Tunable XFS parameters. xfs_params is required even when CONFIG_SYSCTL=n, * other XFS code uses these values. Times are measured in centisecs (i.e. - * 100ths of a second) with the exception of eofb_timer and cowb_timer, which - * are measured in seconds. + * 100ths of a second) with the exception of blockgc_timer, which is measured + * in seconds. */ xfs_param_t xfs_params = { /* MIN DFLT MAX */ @@ -28,8 +28,7 @@ xfs_param_t xfs_params = { .rotorstep = { 1, 1, 255 }, .inherit_nodfrg = { 0, 1, 1 }, .fstrm_timer = { 1, 30*100, 3600*100}, - .eofb_timer = { 1, 300, 3600*24}, - .cowb_timer = { 1, 1800, 3600*24}, + .blockgc_timer = { 1, 300, 3600*24}, }; struct xfs_globals xfs_globals = { diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 34cc84fc7391..c6ef4d14fb8d 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1328,41 +1328,24 @@ xfs_inode_free_eofblocks( } /* - * Background scanning to trim post-EOF preallocated space. This is queued - * based on the 'speculative_prealloc_lifetime' tunable (5m by default). + * Background scanning to trim preallocated space. This is queued based on the + * 'speculative_prealloc_lifetime' tunable (5m by default). */ -void -xfs_queue_eofblocks( - struct xfs_mount *mp) +static inline void +xfs_blockgc_queue( + struct xfs_mount *mp) { rcu_read_lock(); if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_BLOCKGC_TAG)) - queue_delayed_work(mp->m_eofblocks_workqueue, - &mp->m_eofblocks_work, - msecs_to_jiffies(xfs_eofb_secs * 1000)); + queue_delayed_work(mp->m_blockgc_workqueue, + &mp->m_blockgc_work, + msecs_to_jiffies(xfs_blockgc_secs * 1000)); rcu_read_unlock(); } -void -xfs_eofblocks_worker( - struct work_struct *work) -{ - struct xfs_mount *mp = container_of(to_delayed_work(work), - struct xfs_mount, m_eofblocks_work); - - if (!sb_start_write_trylock(mp->m_super)) - return; - xfs_inode_walk(mp, 0, xfs_inode_free_eofblocks, NULL, - XFS_ICI_BLOCKGC_TAG); - sb_end_write(mp->m_super); - - xfs_queue_eofblocks(mp); -} - static void xfs_blockgc_set_iflag( struct xfs_inode *ip, - void (*execute)(struct xfs_mount *mp), unsigned long iflag) { struct xfs_mount *mp = ip->i_mount; @@ -1397,7 +1380,7 @@ xfs_blockgc_set_iflag( spin_unlock(&ip->i_mount->m_perag_lock); /* kick off background trimming */ - execute(ip->i_mount); + xfs_blockgc_queue(ip->i_mount); trace_xfs_perag_set_blockgc(ip->i_mount, pag->pag_agno, -1, _RET_IP_); @@ -1412,7 +1395,7 @@ xfs_inode_set_eofblocks_tag( xfs_inode_t *ip) { trace_xfs_inode_set_eofblocks_tag(ip); - return xfs_blockgc_set_iflag(ip, xfs_queue_eofblocks, XFS_IEOFBLOCKS); + return xfs_blockgc_set_iflag(ip, XFS_IEOFBLOCKS); } static void @@ -1556,45 +1539,12 @@ out_iolock: return ret; } -/* - * Background scanning to trim preallocated CoW space. This is queued - * based on the 'speculative_cow_prealloc_lifetime' tunable (5m by default). - * (We'll just piggyback on the post-EOF prealloc space workqueue.) - */ -void -xfs_queue_cowblocks( - struct xfs_mount *mp) -{ - rcu_read_lock(); - if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_BLOCKGC_TAG)) - queue_delayed_work(mp->m_eofblocks_workqueue, - &mp->m_cowblocks_work, - msecs_to_jiffies(xfs_cowb_secs * 1000)); - rcu_read_unlock(); -} - -void -xfs_cowblocks_worker( - struct work_struct *work) -{ - struct xfs_mount *mp = container_of(to_delayed_work(work), - struct xfs_mount, m_cowblocks_work); - - if (!sb_start_write_trylock(mp->m_super)) - return; - xfs_inode_walk(mp, 0, xfs_inode_free_cowblocks, NULL, - XFS_ICI_BLOCKGC_TAG); - sb_end_write(mp->m_super); - - xfs_queue_cowblocks(mp); -} - void xfs_inode_set_cowblocks_tag( xfs_inode_t *ip) { trace_xfs_inode_set_cowblocks_tag(ip); - return xfs_blockgc_set_iflag(ip, xfs_queue_cowblocks, XFS_ICOWBLOCKS); + return xfs_blockgc_set_iflag(ip, XFS_ICOWBLOCKS); } void @@ -1610,8 +1560,7 @@ void xfs_stop_block_reaping( struct xfs_mount *mp) { - cancel_delayed_work_sync(&mp->m_eofblocks_work); - cancel_delayed_work_sync(&mp->m_cowblocks_work); + cancel_delayed_work_sync(&mp->m_blockgc_work); } /* Enable post-EOF and CoW block auto-reclamation. */ @@ -1619,8 +1568,7 @@ void xfs_start_block_reaping( struct xfs_mount *mp) { - xfs_queue_eofblocks(mp); - xfs_queue_cowblocks(mp); + xfs_blockgc_queue(mp); } /* Scan all incore inodes for block preallocations that we can remove. */ @@ -1644,6 +1592,24 @@ xfs_blockgc_scan( return 0; } +/* Background worker that trims preallocated space. */ +void +xfs_blockgc_worker( + struct work_struct *work) +{ + struct xfs_mount *mp = container_of(to_delayed_work(work), + struct xfs_mount, m_blockgc_work); + int error; + + if (!sb_start_write_trylock(mp->m_super)) + return; + error = xfs_blockgc_scan(mp, NULL); + if (error) + xfs_info(mp, "preallocation gc worker failed, err=%d", error); + sb_end_write(mp->m_super); + xfs_blockgc_queue(mp); +} + /* * Try to free space in the filesystem by purging eofblocks and cowblocks. */ diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index ab7107370be4..5c57476287f6 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -62,13 +62,11 @@ int xfs_blockgc_free_space(struct xfs_mount *mp, struct xfs_eofblocks *eofb); void xfs_inode_set_eofblocks_tag(struct xfs_inode *ip); void xfs_inode_clear_eofblocks_tag(struct xfs_inode *ip); -void xfs_eofblocks_worker(struct work_struct *); -void xfs_queue_eofblocks(struct xfs_mount *); void xfs_inode_set_cowblocks_tag(struct xfs_inode *ip); void xfs_inode_clear_cowblocks_tag(struct xfs_inode *ip); -void xfs_cowblocks_worker(struct work_struct *); -void xfs_queue_cowblocks(struct xfs_mount *); + +void xfs_blockgc_worker(struct work_struct *work); int xfs_inode_walk(struct xfs_mount *mp, int iter_flags, int (*execute)(struct xfs_inode *ip, void *args), diff --git a/fs/xfs/xfs_linux.h b/fs/xfs/xfs_linux.h index 5b7a1e201559..af6be9b9ccdf 100644 --- a/fs/xfs/xfs_linux.h +++ b/fs/xfs/xfs_linux.h @@ -98,8 +98,7 @@ typedef __u32 xfs_nlink_t; #define xfs_rotorstep xfs_params.rotorstep.val #define xfs_inherit_nodefrag xfs_params.inherit_nodfrg.val #define xfs_fstrm_centisecs xfs_params.fstrm_timer.val -#define xfs_eofb_secs xfs_params.eofb_timer.val -#define xfs_cowb_secs xfs_params.cowb_timer.val +#define xfs_blockgc_secs xfs_params.blockgc_timer.val #define current_cpu() (raw_smp_processor_id()) #define current_set_flags_nested(sp, f) \ diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index 452ca7654dc5..316e0d79cc40 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -93,7 +93,7 @@ typedef struct xfs_mount { struct workqueue_struct *m_unwritten_workqueue; struct workqueue_struct *m_cil_workqueue; struct workqueue_struct *m_reclaim_workqueue; - struct workqueue_struct *m_eofblocks_workqueue; + struct workqueue_struct *m_blockgc_workqueue; struct workqueue_struct *m_sync_workqueue; int m_bsize; /* fs logical block size */ @@ -177,9 +177,7 @@ typedef struct xfs_mount { uint64_t m_resblks_avail;/* available reserved blocks */ uint64_t m_resblks_save; /* reserved blks @ remount,ro */ struct delayed_work m_reclaim_work; /* background inode reclaim */ - struct delayed_work m_eofblocks_work; /* background eof blocks - trimming */ - struct delayed_work m_cowblocks_work; /* background cow blocks + struct delayed_work m_blockgc_work; /* background prealloc blocks trimming */ struct xfs_kobj m_kobj; struct xfs_kobj m_error_kobj; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index e8f71714d737..471592e8dba6 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -518,10 +518,10 @@ xfs_init_mount_workqueues( if (!mp->m_reclaim_workqueue) goto out_destroy_cil; - mp->m_eofblocks_workqueue = alloc_workqueue("xfs-eofblocks/%s", + mp->m_blockgc_workqueue = alloc_workqueue("xfs-blockgc/%s", XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM), 0, mp->m_super->s_id); - if (!mp->m_eofblocks_workqueue) + if (!mp->m_blockgc_workqueue) goto out_destroy_reclaim; mp->m_sync_workqueue = alloc_workqueue("xfs-sync/%s", @@ -532,7 +532,7 @@ xfs_init_mount_workqueues( return 0; out_destroy_eofb: - destroy_workqueue(mp->m_eofblocks_workqueue); + destroy_workqueue(mp->m_blockgc_workqueue); out_destroy_reclaim: destroy_workqueue(mp->m_reclaim_workqueue); out_destroy_cil: @@ -550,7 +550,7 @@ xfs_destroy_mount_workqueues( struct xfs_mount *mp) { destroy_workqueue(mp->m_sync_workqueue); - destroy_workqueue(mp->m_eofblocks_workqueue); + destroy_workqueue(mp->m_blockgc_workqueue); destroy_workqueue(mp->m_reclaim_workqueue); destroy_workqueue(mp->m_cil_workqueue); destroy_workqueue(mp->m_unwritten_workqueue); @@ -1842,8 +1842,7 @@ static int xfs_init_fs_context( mutex_init(&mp->m_growlock); INIT_WORK(&mp->m_flush_inodes_work, xfs_flush_inodes_worker); INIT_DELAYED_WORK(&mp->m_reclaim_work, xfs_reclaim_worker); - INIT_DELAYED_WORK(&mp->m_eofblocks_work, xfs_eofblocks_worker); - INIT_DELAYED_WORK(&mp->m_cowblocks_work, xfs_cowblocks_worker); + INIT_DELAYED_WORK(&mp->m_blockgc_work, xfs_blockgc_worker); mp->m_kobj.kobject.kset = xfs_kset; /* * We don't create the finobt per-ag space reservation until after log diff --git a/fs/xfs/xfs_sysctl.c b/fs/xfs/xfs_sysctl.c index fac9de7ee6d0..145e06c47744 100644 --- a/fs/xfs/xfs_sysctl.c +++ b/fs/xfs/xfs_sysctl.c @@ -194,21 +194,12 @@ static struct ctl_table xfs_table[] = { }, { .procname = "speculative_prealloc_lifetime", - .data = &xfs_params.eofb_timer.val, + .data = &xfs_params.blockgc_timer.val, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec_minmax, - .extra1 = &xfs_params.eofb_timer.min, - .extra2 = &xfs_params.eofb_timer.max, - }, - { - .procname = "speculative_cow_prealloc_lifetime", - .data = &xfs_params.cowb_timer.val, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &xfs_params.cowb_timer.min, - .extra2 = &xfs_params.cowb_timer.max, + .extra1 = &xfs_params.blockgc_timer.min, + .extra2 = &xfs_params.blockgc_timer.max, }, /* please keep this the last entry */ #ifdef CONFIG_PROC_FS diff --git a/fs/xfs/xfs_sysctl.h b/fs/xfs/xfs_sysctl.h index 8abf4640f1d5..7692e76ead33 100644 --- a/fs/xfs/xfs_sysctl.h +++ b/fs/xfs/xfs_sysctl.h @@ -35,8 +35,7 @@ typedef struct xfs_param { xfs_sysctl_val_t rotorstep; /* inode32 AG rotoring control knob */ xfs_sysctl_val_t inherit_nodfrg;/* Inherit the "nodefrag" inode flag. */ xfs_sysctl_val_t fstrm_timer; /* Filestream dir-AG assoc'n timeout. */ - xfs_sysctl_val_t eofb_timer; /* Interval between eofb scan wakeups */ - xfs_sysctl_val_t cowb_timer; /* Interval between cowb scan wakeups */ + xfs_sysctl_val_t blockgc_timer; /* Interval between blockgc scans */ } xfs_param_t; /* -- cgit v1.2.3 From 419567534e16eb553e7c19eecaa4d03cbc6693be Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:43 -0800 Subject: xfs: only walk the incore inode tree once per blockgc scan Perform background block preallocation gc scans more efficiently by walking the incore inode tree once. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icache.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index c6ef4d14fb8d..0c0d5779fea0 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1571,21 +1571,19 @@ xfs_start_block_reaping( xfs_blockgc_queue(mp); } -/* Scan all incore inodes for block preallocations that we can remove. */ -static inline int -xfs_blockgc_scan( - struct xfs_mount *mp, - struct xfs_eofblocks *eofb) +/* Scan one incore inode for block preallocations that we can remove. */ +static int +xfs_blockgc_scan_inode( + struct xfs_inode *ip, + void *args) { int error; - error = xfs_inode_walk(mp, 0, xfs_inode_free_eofblocks, eofb, - XFS_ICI_BLOCKGC_TAG); + error = xfs_inode_free_eofblocks(ip, args); if (error) return error; - error = xfs_inode_walk(mp, 0, xfs_inode_free_cowblocks, eofb, - XFS_ICI_BLOCKGC_TAG); + error = xfs_inode_free_cowblocks(ip, args); if (error) return error; @@ -1603,7 +1601,8 @@ xfs_blockgc_worker( if (!sb_start_write_trylock(mp->m_super)) return; - error = xfs_blockgc_scan(mp, NULL); + error = xfs_inode_walk(mp, 0, xfs_blockgc_scan_inode, NULL, + XFS_ICI_BLOCKGC_TAG); if (error) xfs_info(mp, "preallocation gc worker failed, err=%d", error); sb_end_write(mp->m_super); @@ -1620,7 +1619,8 @@ xfs_blockgc_free_space( { trace_xfs_blockgc_free_space(mp, eofb, _RET_IP_); - return xfs_blockgc_scan(mp, eofb); + return xfs_inode_walk(mp, 0, xfs_blockgc_scan_inode, eofb, + XFS_ICI_BLOCKGC_TAG); } /* -- cgit v1.2.3 From c9a6526fe7ae64528d924c6f255af15312211432 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:44 -0800 Subject: xfs: rename block gc start and stop functions Shorten the names of the two functions that start and stop block preallocation garbage collection and move them up to the other blockgc functions. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/common.c | 4 ++-- fs/xfs/xfs_icache.c | 4 ++-- fs/xfs/xfs_icache.h | 4 ++-- fs/xfs/xfs_mount.c | 2 +- fs/xfs/xfs_super.c | 8 ++++---- 5 files changed, 11 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index 8ea6d4aa3f55..53456f3de881 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -888,7 +888,7 @@ xchk_stop_reaping( struct xfs_scrub *sc) { sc->flags |= XCHK_REAPING_DISABLED; - xfs_stop_block_reaping(sc->mp); + xfs_blockgc_stop(sc->mp); } /* Restart background reaping of resources. */ @@ -896,6 +896,6 @@ void xchk_start_reaping( struct xfs_scrub *sc) { - xfs_start_block_reaping(sc->mp); + xfs_blockgc_start(sc->mp); sc->flags &= ~XCHK_REAPING_DISABLED; } diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 0c0d5779fea0..35c735514565 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1557,7 +1557,7 @@ xfs_inode_clear_cowblocks_tag( /* Disable post-EOF and CoW block auto-reclamation. */ void -xfs_stop_block_reaping( +xfs_blockgc_stop( struct xfs_mount *mp) { cancel_delayed_work_sync(&mp->m_blockgc_work); @@ -1565,7 +1565,7 @@ xfs_stop_block_reaping( /* Enable post-EOF and CoW block auto-reclamation. */ void -xfs_start_block_reaping( +xfs_blockgc_start( struct xfs_mount *mp) { xfs_blockgc_queue(mp); diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index 5c57476287f6..d1fddb152420 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -75,7 +75,7 @@ int xfs_inode_walk(struct xfs_mount *mp, int iter_flags, int xfs_icache_inode_is_allocated(struct xfs_mount *mp, struct xfs_trans *tp, xfs_ino_t ino, bool *inuse); -void xfs_stop_block_reaping(struct xfs_mount *mp); -void xfs_start_block_reaping(struct xfs_mount *mp); +void xfs_blockgc_stop(struct xfs_mount *mp); +void xfs_blockgc_start(struct xfs_mount *mp); #endif diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index 53b8ccab7235..be9ce114527f 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1054,7 +1054,7 @@ xfs_unmountfs( uint64_t resblks; int error; - xfs_stop_block_reaping(mp); + xfs_blockgc_stop(mp); xfs_fs_unreserve_ag_blocks(mp); xfs_qm_unmount_quotas(mp); xfs_rtunmount_inodes(mp); diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 471592e8dba6..ea942089d074 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -891,7 +891,7 @@ xfs_fs_freeze( * set a GFP_NOFS context here to avoid recursion deadlocks. */ flags = memalloc_nofs_save(); - xfs_stop_block_reaping(mp); + xfs_blockgc_stop(mp); xfs_save_resvblks(mp); ret = xfs_log_quiesce(mp); memalloc_nofs_restore(flags); @@ -906,7 +906,7 @@ xfs_fs_unfreeze( xfs_restore_resvblks(mp); xfs_log_work_queue(mp); - xfs_start_block_reaping(mp); + xfs_blockgc_start(mp); return 0; } @@ -1690,7 +1690,7 @@ xfs_remount_rw( xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); return error; } - xfs_start_block_reaping(mp); + xfs_blockgc_start(mp); /* Create the per-AG metadata reservation pool .*/ error = xfs_fs_reserve_ag_blocks(mp); @@ -1710,7 +1710,7 @@ xfs_remount_ro( * Cancel background eofb scanning so it cannot race with the final * log force+buftarg wait and deadlock the remount. */ - xfs_stop_block_reaping(mp); + xfs_blockgc_stop(mp); /* Get rid of any leftover CoW reservations... */ error = xfs_blockgc_free_space(mp, NULL); -- cgit v1.2.3 From 894ecacf0f27fd1701c34f2946148b7f017bf984 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 22 Jan 2021 16:48:44 -0800 Subject: xfs: parallelize block preallocation garbage collection Split the block preallocation garbage collection work into per-AG work items so that we can take advantage of parallelization. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icache.c | 42 ++++++++++++++++++++++++++++++------------ fs/xfs/xfs_mount.c | 3 +++ fs/xfs/xfs_mount.h | 5 +++-- fs/xfs/xfs_super.c | 4 ++-- 4 files changed, 38 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 35c735514565..460fa7b3a31c 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1333,12 +1333,12 @@ xfs_inode_free_eofblocks( */ static inline void xfs_blockgc_queue( - struct xfs_mount *mp) + struct xfs_perag *pag) { rcu_read_lock(); - if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_BLOCKGC_TAG)) - queue_delayed_work(mp->m_blockgc_workqueue, - &mp->m_blockgc_work, + if (radix_tree_tagged(&pag->pag_ici_root, XFS_ICI_BLOCKGC_TAG)) + queue_delayed_work(pag->pag_mount->m_blockgc_workqueue, + &pag->pag_blockgc_work, msecs_to_jiffies(xfs_blockgc_secs * 1000)); rcu_read_unlock(); } @@ -1380,7 +1380,7 @@ xfs_blockgc_set_iflag( spin_unlock(&ip->i_mount->m_perag_lock); /* kick off background trimming */ - xfs_blockgc_queue(ip->i_mount); + xfs_blockgc_queue(pag); trace_xfs_perag_set_blockgc(ip->i_mount, pag->pag_agno, -1, _RET_IP_); @@ -1555,12 +1555,24 @@ xfs_inode_clear_cowblocks_tag( return xfs_blockgc_clear_iflag(ip, XFS_ICOWBLOCKS); } +#define for_each_perag_tag(mp, next_agno, pag, tag) \ + for ((next_agno) = 0, (pag) = xfs_perag_get_tag((mp), 0, (tag)); \ + (pag) != NULL; \ + (next_agno) = (pag)->pag_agno + 1, \ + xfs_perag_put(pag), \ + (pag) = xfs_perag_get_tag((mp), (next_agno), (tag))) + + /* Disable post-EOF and CoW block auto-reclamation. */ void xfs_blockgc_stop( struct xfs_mount *mp) { - cancel_delayed_work_sync(&mp->m_blockgc_work); + struct xfs_perag *pag; + xfs_agnumber_t agno; + + for_each_perag_tag(mp, agno, pag, XFS_ICI_BLOCKGC_TAG) + cancel_delayed_work_sync(&pag->pag_blockgc_work); } /* Enable post-EOF and CoW block auto-reclamation. */ @@ -1568,7 +1580,11 @@ void xfs_blockgc_start( struct xfs_mount *mp) { - xfs_blockgc_queue(mp); + struct xfs_perag *pag; + xfs_agnumber_t agno; + + for_each_perag_tag(mp, agno, pag, XFS_ICI_BLOCKGC_TAG) + xfs_blockgc_queue(pag); } /* Scan one incore inode for block preallocations that we can remove. */ @@ -1595,18 +1611,20 @@ void xfs_blockgc_worker( struct work_struct *work) { - struct xfs_mount *mp = container_of(to_delayed_work(work), - struct xfs_mount, m_blockgc_work); + struct xfs_perag *pag = container_of(to_delayed_work(work), + struct xfs_perag, pag_blockgc_work); + struct xfs_mount *mp = pag->pag_mount; int error; if (!sb_start_write_trylock(mp->m_super)) return; - error = xfs_inode_walk(mp, 0, xfs_blockgc_scan_inode, NULL, + error = xfs_inode_walk_ag(pag, 0, xfs_blockgc_scan_inode, NULL, XFS_ICI_BLOCKGC_TAG); if (error) - xfs_info(mp, "preallocation gc worker failed, err=%d", error); + xfs_info(mp, "AG %u preallocation gc worker failed, err=%d", + pag->pag_agno, error); sb_end_write(mp->m_super); - xfs_blockgc_queue(mp); + xfs_blockgc_queue(pag); } /* diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index be9ce114527f..52370d0a3f43 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -126,6 +126,7 @@ __xfs_free_perag( { struct xfs_perag *pag = container_of(head, struct xfs_perag, rcu_head); + ASSERT(!delayed_work_pending(&pag->pag_blockgc_work)); ASSERT(atomic_read(&pag->pag_ref) == 0); kmem_free(pag); } @@ -146,6 +147,7 @@ xfs_free_perag( spin_unlock(&mp->m_perag_lock); ASSERT(pag); ASSERT(atomic_read(&pag->pag_ref) == 0); + cancel_delayed_work_sync(&pag->pag_blockgc_work); xfs_iunlink_destroy(pag); xfs_buf_hash_destroy(pag); call_rcu(&pag->rcu_head, __xfs_free_perag); @@ -201,6 +203,7 @@ xfs_initialize_perag( pag->pag_agno = index; pag->pag_mount = mp; spin_lock_init(&pag->pag_ici_lock); + INIT_DELAYED_WORK(&pag->pag_blockgc_work, xfs_blockgc_worker); INIT_RADIX_TREE(&pag->pag_ici_root, GFP_ATOMIC); error = xfs_buf_hash_init(pag); diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index 316e0d79cc40..659ad95fe3e0 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -177,8 +177,6 @@ typedef struct xfs_mount { uint64_t m_resblks_avail;/* available reserved blocks */ uint64_t m_resblks_save; /* reserved blks @ remount,ro */ struct delayed_work m_reclaim_work; /* background inode reclaim */ - struct delayed_work m_blockgc_work; /* background prealloc blocks - trimming */ struct xfs_kobj m_kobj; struct xfs_kobj m_error_kobj; struct xfs_kobj m_error_meta_kobj; @@ -367,6 +365,9 @@ typedef struct xfs_perag { /* Blocks reserved for the reverse mapping btree. */ struct xfs_ag_resv pag_rmapbt_resv; + /* background prealloc block trimming */ + struct delayed_work pag_blockgc_work; + /* reference count */ uint8_t pagf_refcount_level; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index ea942089d074..2b04818627e9 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -35,6 +35,7 @@ #include "xfs_refcount_item.h" #include "xfs_bmap_item.h" #include "xfs_reflink.h" +#include "xfs_pwork.h" #include #include @@ -519,7 +520,7 @@ xfs_init_mount_workqueues( goto out_destroy_cil; mp->m_blockgc_workqueue = alloc_workqueue("xfs-blockgc/%s", - XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM), + XFS_WQFLAGS(WQ_UNBOUND | WQ_FREEZABLE | WQ_MEM_RECLAIM), 0, mp->m_super->s_id); if (!mp->m_blockgc_workqueue) goto out_destroy_reclaim; @@ -1842,7 +1843,6 @@ static int xfs_init_fs_context( mutex_init(&mp->m_growlock); INIT_WORK(&mp->m_flush_inodes_work, xfs_flush_inodes_worker); INIT_DELAYED_WORK(&mp->m_reclaim_work, xfs_reclaim_worker); - INIT_DELAYED_WORK(&mp->m_blockgc_work, xfs_blockgc_worker); mp->m_kobj.kobject.kset = xfs_kset; /* * We don't create the finobt per-ag space reservation until after log -- cgit v1.2.3 From 47bd6d3457fb96d287278027aed8a78d14f1d32d Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 25 Jan 2021 16:39:01 -0800 Subject: xfs: expose the blockgc workqueue knobs publicly Expose the workqueue sysfs knobs for the speculative preallocation gc workers on all kernels, and update the sysadmin information. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- Documentation/admin-guide/xfs.rst | 3 +++ fs/xfs/xfs_super.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/Documentation/admin-guide/xfs.rst b/Documentation/admin-guide/xfs.rst index b00b1eece9de..d2064a52811b 100644 --- a/Documentation/admin-guide/xfs.rst +++ b/Documentation/admin-guide/xfs.rst @@ -518,6 +518,9 @@ and the short name of the data device. They all can be found in: ================ =========== xfs_iwalk-$pid Inode scans of the entire filesystem. Currently limited to mount time quotacheck. + xfs-blockgc Background garbage collection of disk space that have been + speculatively allocated beyond EOF or for staging copy on + write operations. ================ =========== For example, the knobs for the quotacheck workqueue for /dev/nvme0n1 would be diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 2b04818627e9..21b1d034aca3 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -520,7 +520,7 @@ xfs_init_mount_workqueues( goto out_destroy_cil; mp->m_blockgc_workqueue = alloc_workqueue("xfs-blockgc/%s", - XFS_WQFLAGS(WQ_UNBOUND | WQ_FREEZABLE | WQ_MEM_RECLAIM), + WQ_SYSFS | WQ_UNBOUND | WQ_FREEZABLE | WQ_MEM_RECLAIM, 0, mp->m_super->s_id); if (!mp->m_blockgc_workqueue) goto out_destroy_reclaim; -- cgit v1.2.3 From 0fa4a10a2f5f96a06373ea81f8cd5f97c5cc264f Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 25 Jan 2021 21:09:49 -0800 Subject: xfs: don't bounce the iolock between free_{eof,cow}blocks Since xfs_inode_free_eofblocks and xfs_inode_free_cowblocks are now internal static functions, we can save ourselves a cycling of the iolock by passing the lock state out to xfs_blockgc_scan_inode and letting it do all the unlocking. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icache.c | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 460fa7b3a31c..1d7720a0c068 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1283,11 +1283,11 @@ xfs_reclaim_worker( STATIC int xfs_inode_free_eofblocks( struct xfs_inode *ip, - void *args) + void *args, + unsigned int *lockflags) { struct xfs_eofblocks *eofb = args; bool wait; - int ret; wait = eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC); @@ -1320,11 +1320,9 @@ xfs_inode_free_eofblocks( return -EAGAIN; return 0; } + *lockflags |= XFS_IOLOCK_EXCL; - ret = xfs_free_eofblocks(ip); - xfs_iunlock(ip, XFS_IOLOCK_EXCL); - - return ret; + return xfs_free_eofblocks(ip); } /* @@ -1493,7 +1491,8 @@ xfs_prep_free_cowblocks( STATIC int xfs_inode_free_cowblocks( struct xfs_inode *ip, - void *args) + void *args, + unsigned int *lockflags) { struct xfs_eofblocks *eofb = args; bool wait; @@ -1514,16 +1513,20 @@ xfs_inode_free_cowblocks( * If the caller is waiting, return -EAGAIN to keep the background * scanner moving and revisit the inode in a subsequent pass. */ - if (!xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) { + if (!(*lockflags & XFS_IOLOCK_EXCL) && + !xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) { if (wait) return -EAGAIN; return 0; } + *lockflags |= XFS_IOLOCK_EXCL; + if (!xfs_ilock_nowait(ip, XFS_MMAPLOCK_EXCL)) { if (wait) - ret = -EAGAIN; - goto out_iolock; + return -EAGAIN; + return 0; } + *lockflags |= XFS_MMAPLOCK_EXCL; /* * Check again, nobody else should be able to dirty blocks or change @@ -1531,11 +1534,6 @@ xfs_inode_free_cowblocks( */ if (xfs_prep_free_cowblocks(ip)) ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, false); - - xfs_iunlock(ip, XFS_MMAPLOCK_EXCL); -out_iolock: - xfs_iunlock(ip, XFS_IOLOCK_EXCL); - return ret; } @@ -1593,17 +1591,18 @@ xfs_blockgc_scan_inode( struct xfs_inode *ip, void *args) { + unsigned int lockflags = 0; int error; - error = xfs_inode_free_eofblocks(ip, args); + error = xfs_inode_free_eofblocks(ip, args, &lockflags); if (error) - return error; + goto unlock; - error = xfs_inode_free_cowblocks(ip, args); - if (error) - return error; - - return 0; + error = xfs_inode_free_cowblocks(ip, args, &lockflags); +unlock: + if (lockflags) + xfs_iunlock(ip, lockflags); + return error; } /* Background worker that trims preallocated space. */ -- cgit v1.2.3 From bc41fa5321f93ecbabec177f888451cfc17ad66d Mon Sep 17 00:00:00 2001 From: Zorro Lang Date: Tue, 2 Feb 2021 12:11:53 -0800 Subject: libxfs: expose inobtcount in xfs geometry As xfs supports the feature of inode btree block counters now, expose this feature flag in xfs geometry, for userspace can check if the inobtcnt is enabled or not. Signed-off-by: Zorro Lang Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_fs.h | 1 + fs/xfs/libxfs/xfs_sb.c | 2 ++ 2 files changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index 2a2e3cfd94f0..6fad140d4c8e 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -250,6 +250,7 @@ typedef struct xfs_fsop_resblks { #define XFS_FSOP_GEOM_FLAGS_RMAPBT (1 << 19) /* reverse mapping btree */ #define XFS_FSOP_GEOM_FLAGS_REFLINK (1 << 20) /* files can share blocks */ #define XFS_FSOP_GEOM_FLAGS_BIGTIME (1 << 21) /* 64-bit nsec timestamps */ +#define XFS_FSOP_GEOM_FLAGS_INOBTCNT (1 << 22) /* inobt btree counter */ /* * Minimum and maximum sizes need for growth checks. diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c index bbda117e5d85..60e6d255e5e2 100644 --- a/fs/xfs/libxfs/xfs_sb.c +++ b/fs/xfs/libxfs/xfs_sb.c @@ -1138,6 +1138,8 @@ xfs_fs_geometry( geo->flags |= XFS_FSOP_GEOM_FLAGS_REFLINK; if (xfs_sb_version_hasbigtime(sbp)) geo->flags |= XFS_FSOP_GEOM_FLAGS_BIGTIME; + if (xfs_sb_version_hasinobtcounts(sbp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_INOBTCNT; if (xfs_sb_version_hassector(sbp)) geo->logsectsize = sbp->sb_logsectsize; else -- cgit v1.2.3 From ce5e1062e2539c7f7d311548494ea2705184c784 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Tue, 2 Feb 2021 18:24:06 -0800 Subject: xfs: rename `new' to `delta' in xfs_growfs_data_private() It actually means the delta block count of growfs. Rename it in order to make it clear. Also introduce nb_div to avoid reusing `delta`. Reviewed-by: Darrick J. Wong Signed-off-by: Gao Xiang Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_fsops.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c index 959ce91a3755..62600d78bbf1 100644 --- a/fs/xfs/xfs_fsops.c +++ b/fs/xfs/xfs_fsops.c @@ -32,8 +32,8 @@ xfs_growfs_data_private( int error; xfs_agnumber_t nagcount; xfs_agnumber_t nagimax = 0; - xfs_rfsblock_t nb, nb_mod; - xfs_rfsblock_t new; + xfs_rfsblock_t nb, nb_div, nb_mod; + xfs_rfsblock_t delta; xfs_agnumber_t oagcount; xfs_trans_t *tp; struct aghdr_init_data id = {}; @@ -50,16 +50,16 @@ xfs_growfs_data_private( return error; xfs_buf_relse(bp); - new = nb; /* use new as a temporary here */ - nb_mod = do_div(new, mp->m_sb.sb_agblocks); - nagcount = new + (nb_mod != 0); + nb_div = nb; + nb_mod = do_div(nb_div, mp->m_sb.sb_agblocks); + nagcount = nb_div + (nb_mod != 0); if (nb_mod && nb_mod < XFS_MIN_AG_BLOCKS) { nagcount--; nb = (xfs_rfsblock_t)nagcount * mp->m_sb.sb_agblocks; if (nb < mp->m_sb.sb_dblocks) return -EINVAL; } - new = nb - mp->m_sb.sb_dblocks; + delta = nb - mp->m_sb.sb_dblocks; oagcount = mp->m_sb.sb_agcount; /* allocate the new per-ag structures */ @@ -89,7 +89,7 @@ xfs_growfs_data_private( INIT_LIST_HEAD(&id.buffer_list); for (id.agno = nagcount - 1; id.agno >= oagcount; - id.agno--, new -= id.agsize) { + id.agno--, delta -= id.agsize) { if (id.agno == nagcount - 1) id.agsize = nb - @@ -110,8 +110,8 @@ xfs_growfs_data_private( xfs_trans_agblocks_delta(tp, id.nfree); /* If there are new blocks in the old last AG, extend it. */ - if (new) { - error = xfs_ag_extend_space(mp, tp, &id, new); + if (delta) { + error = xfs_ag_extend_space(mp, tp, &id, delta); if (error) goto out_trans_cancel; } @@ -143,7 +143,7 @@ xfs_growfs_data_private( * If we expanded the last AG, free the per-AG reservation * so we can reinitialize it with the new size. */ - if (new) { + if (delta) { struct xfs_perag *pag; pag = xfs_perag_get(mp, id.agno); -- cgit v1.2.3 From 07aabd9c4a881276cf9b7b2d3a7f1d14dd832ed0 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Tue, 2 Feb 2021 18:24:06 -0800 Subject: xfs: get rid of xfs_growfs_{data,log}_t Such usage isn't encouraged by the kernel coding style. Leave the definitions alone in case of userspace users. Reviewed-by: Darrick J. Wong Reviewed-by: Eric Sandeen Signed-off-by: Gao Xiang Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_fsops.c | 12 ++++++------ fs/xfs/xfs_fsops.h | 4 ++-- fs/xfs/xfs_ioctl.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c index 62600d78bbf1..a2a407039227 100644 --- a/fs/xfs/xfs_fsops.c +++ b/fs/xfs/xfs_fsops.c @@ -25,8 +25,8 @@ */ static int xfs_growfs_data_private( - xfs_mount_t *mp, /* mount point for filesystem */ - xfs_growfs_data_t *in) /* growfs data input struct */ + struct xfs_mount *mp, /* mount point for filesystem */ + struct xfs_growfs_data *in) /* growfs data input struct */ { struct xfs_buf *bp; int error; @@ -35,7 +35,7 @@ xfs_growfs_data_private( xfs_rfsblock_t nb, nb_div, nb_mod; xfs_rfsblock_t delta; xfs_agnumber_t oagcount; - xfs_trans_t *tp; + struct xfs_trans *tp; struct aghdr_init_data id = {}; nb = in->newblocks; @@ -170,8 +170,8 @@ out_trans_cancel: static int xfs_growfs_log_private( - xfs_mount_t *mp, /* mount point for filesystem */ - xfs_growfs_log_t *in) /* growfs log input struct */ + struct xfs_mount *mp, /* mount point for filesystem */ + struct xfs_growfs_log *in) /* growfs log input struct */ { xfs_extlen_t nb; @@ -268,7 +268,7 @@ out_error: int xfs_growfs_log( xfs_mount_t *mp, - xfs_growfs_log_t *in) + struct xfs_growfs_log *in) { int error; diff --git a/fs/xfs/xfs_fsops.h b/fs/xfs/xfs_fsops.h index 92869f6ec8d3..2cffe51a31e8 100644 --- a/fs/xfs/xfs_fsops.h +++ b/fs/xfs/xfs_fsops.h @@ -6,8 +6,8 @@ #ifndef __XFS_FSOPS_H__ #define __XFS_FSOPS_H__ -extern int xfs_growfs_data(xfs_mount_t *mp, xfs_growfs_data_t *in); -extern int xfs_growfs_log(xfs_mount_t *mp, xfs_growfs_log_t *in); +extern int xfs_growfs_data(struct xfs_mount *mp, struct xfs_growfs_data *in); +extern int xfs_growfs_log(struct xfs_mount *mp, struct xfs_growfs_log *in); extern void xfs_fs_counts(xfs_mount_t *mp, xfs_fsop_counts_t *cnt); extern int xfs_reserve_blocks(xfs_mount_t *mp, uint64_t *inval, xfs_fsop_resblks_t *outval); diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 78f965425fa6..248083ea0276 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -2251,7 +2251,7 @@ xfs_file_ioctl( } case XFS_IOC_FSGROWFSDATA: { - xfs_growfs_data_t in; + struct xfs_growfs_data in; if (copy_from_user(&in, arg, sizeof(in))) return -EFAULT; @@ -2265,7 +2265,7 @@ xfs_file_ioctl( } case XFS_IOC_FSGROWFSLOG: { - xfs_growfs_log_t in; + struct xfs_growfs_log in; if (copy_from_user(&in, arg, sizeof(in))) return -EFAULT; -- cgit v1.2.3 From c968f5788bc91fe4f86df1a68f0d6471396b4d78 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Fri, 29 Jan 2021 16:45:33 +0100 Subject: gfs2: Clean up on-stack transactions Replace the TR_ALLOCED flag by its inverse, TR_ONSTACK: that way, the flag only needs to be set in the exceptional case of on-stack transactions. Split off __gfs2_trans_begin from gfs2_trans_begin and use it to replace the open-coded version in gfs2_ail_empty_gl. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/glops.c | 33 ++++++++++----------------------- fs/gfs2/incore.h | 2 +- fs/gfs2/log.c | 2 +- fs/gfs2/trans.c | 40 +++++++++++++++++++++++----------------- fs/gfs2/trans.h | 3 +++ 5 files changed, 38 insertions(+), 42 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c index fd1f52fff170..a067924341e3 100644 --- a/fs/gfs2/glops.c +++ b/fs/gfs2/glops.c @@ -86,16 +86,12 @@ static int gfs2_ail_empty_gl(struct gfs2_glock *gl) { struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; struct gfs2_trans tr; + unsigned int revokes; int ret; - memset(&tr, 0, sizeof(tr)); - INIT_LIST_HEAD(&tr.tr_buf); - INIT_LIST_HEAD(&tr.tr_databuf); - INIT_LIST_HEAD(&tr.tr_ail1_list); - INIT_LIST_HEAD(&tr.tr_ail2_list); - tr.tr_revokes = atomic_read(&gl->gl_ail_count); + revokes = atomic_read(&gl->gl_ail_count); - if (!tr.tr_revokes) { + if (!revokes) { bool have_revokes; bool log_in_flight; @@ -122,23 +118,14 @@ static int gfs2_ail_empty_gl(struct gfs2_glock *gl) return 0; } - /* A shortened, inline version of gfs2_trans_begin() - * tr->alloced is not set since the transaction structure is - * on the stack */ - tr.tr_reserved = 1 + gfs2_struct2blk(sdp, tr.tr_revokes); - tr.tr_ip = _RET_IP_; - sb_start_intwrite(sdp->sd_vfs); - ret = gfs2_log_reserve(sdp, tr.tr_reserved); - if (ret < 0) { - sb_end_intwrite(sdp->sd_vfs); - return ret; - } - WARN_ON_ONCE(current->journal_info); - current->journal_info = &tr; - - __gfs2_ail_flush(gl, 0, tr.tr_revokes); - + memset(&tr, 0, sizeof(tr)); + set_bit(TR_ONSTACK, &tr.tr_flags); + ret = __gfs2_trans_begin(&tr, sdp, 0, revokes, _RET_IP_); + if (ret) + goto flush; + __gfs2_ail_flush(gl, 0, revokes); gfs2_trans_end(sdp); + flush: gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_AIL_EMPTY_GL); diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index 8e1ab8ed4abc..c3f6dd378b10 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -490,7 +490,7 @@ struct gfs2_quota_data { enum { TR_TOUCHED = 1, TR_ATTACHED = 2, - TR_ALLOCED = 3, + TR_ONSTACK = 3, }; struct gfs2_trans { diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index e4dc23a24569..0fceb60907a2 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -1114,7 +1114,7 @@ static void log_refund(struct gfs2_sbd *sdp, struct gfs2_trans *tr) if (sdp->sd_log_tr) { gfs2_merge_trans(sdp, tr); } else if (tr->tr_num_buf_new || tr->tr_num_databuf_new) { - gfs2_assert_withdraw(sdp, test_bit(TR_ALLOCED, &tr->tr_flags)); + gfs2_assert_withdraw(sdp, !test_bit(TR_ONSTACK, &tr->tr_flags)); sdp->sd_log_tr = tr; set_bit(TR_ATTACHED, &tr->tr_flags); } diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index ae040b570868..db29ca253853 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -37,10 +37,10 @@ static void gfs2_print_trans(struct gfs2_sbd *sdp, const struct gfs2_trans *tr) tr->tr_num_revoke, tr->tr_num_revoke_rm); } -int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, - unsigned int revokes) +int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, + unsigned int blocks, unsigned int revokes, + unsigned long ip) { - struct gfs2_trans *tr; int error; if (current->journal_info) { @@ -52,15 +52,10 @@ int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, if (!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) return -EROFS; - tr = kmem_cache_zalloc(gfs2_trans_cachep, GFP_NOFS); - if (!tr) - return -ENOMEM; - - tr->tr_ip = _RET_IP_; + tr->tr_ip = ip; tr->tr_blocks = blocks; tr->tr_revokes = revokes; tr->tr_reserved = 1; - set_bit(TR_ALLOCED, &tr->tr_flags); if (blocks) tr->tr_reserved += 6 + blocks; if (revokes) @@ -74,17 +69,28 @@ int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, sb_start_intwrite(sdp->sd_vfs); error = gfs2_log_reserve(sdp, tr->tr_reserved); - if (error) - goto fail; + if (error) { + sb_end_intwrite(sdp->sd_vfs); + return error; + } current->journal_info = tr; return 0; +} -fail: - sb_end_intwrite(sdp->sd_vfs); - kmem_cache_free(gfs2_trans_cachep, tr); +int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, + unsigned int revokes) +{ + struct gfs2_trans *tr; + int error; + tr = kmem_cache_zalloc(gfs2_trans_cachep, GFP_NOFS); + if (!tr) + return -ENOMEM; + error = __gfs2_trans_begin(tr, sdp, blocks, revokes, _RET_IP_); + if (error) + kmem_cache_free(gfs2_trans_cachep, tr); return error; } @@ -92,13 +98,12 @@ void gfs2_trans_end(struct gfs2_sbd *sdp) { struct gfs2_trans *tr = current->journal_info; s64 nbuf; - int alloced = test_bit(TR_ALLOCED, &tr->tr_flags); current->journal_info = NULL; if (!test_bit(TR_TOUCHED, &tr->tr_flags)) { gfs2_log_release(sdp, tr->tr_reserved); - if (alloced) + if (!test_bit(TR_ONSTACK, &tr->tr_flags)) gfs2_trans_free(sdp, tr); sb_end_intwrite(sdp->sd_vfs); return; @@ -113,7 +118,8 @@ void gfs2_trans_end(struct gfs2_sbd *sdp) gfs2_print_trans(sdp, tr); gfs2_log_commit(sdp, tr); - if (alloced && !test_bit(TR_ATTACHED, &tr->tr_flags)) + if (!test_bit(TR_ONSTACK, &tr->tr_flags) && + !test_bit(TR_ATTACHED, &tr->tr_flags)) gfs2_trans_free(sdp, tr); up_read(&sdp->sd_log_flush_lock); diff --git a/fs/gfs2/trans.h b/fs/gfs2/trans.h index 83199ce5a5c5..55f253015cf8 100644 --- a/fs/gfs2/trans.h +++ b/fs/gfs2/trans.h @@ -34,6 +34,9 @@ static inline unsigned int gfs2_rg_blocks(const struct gfs2_inode *ip, unsigned return rgd->rd_length; } +extern int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, + unsigned int blocks, unsigned int revokes, + unsigned long ip); extern int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, unsigned int revokes); -- cgit v1.2.3 From f3708fb59f6c2498e8ec4f29010375f600b68642 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sun, 13 Dec 2020 09:21:34 +0100 Subject: gfs2: Get rid of sd_reserving_log This counter and the associated wait queue are only used so that gfs2_make_fs_ro can efficiently wait for all pending log space allocations to fail after setting the filesystem to read-only. This comes at the cost of waking up that wait queue very frequently. Instead, when gfs2_log_reserve fails because the filesystem has become read-only, Wake up sd_log_waitq. In gfs2_make_fs_ro, set the file system read-only and then wait until all the log space has been released. Give up and report the problem after a while. With that, sd_reserving_log and sd_reserving_log_wait can be removed. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/incore.h | 3 --- fs/gfs2/log.c | 17 ++++++++++------- fs/gfs2/log.h | 1 + fs/gfs2/ops_fstype.c | 2 -- fs/gfs2/super.c | 12 ++++++------ fs/gfs2/trans.c | 2 ++ 6 files changed, 19 insertions(+), 18 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index c3f6dd378b10..8f8676cf72ed 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -849,9 +849,6 @@ struct gfs2_sbd { int sd_log_error; /* First log error */ wait_queue_head_t sd_withdraw_wait; - atomic_t sd_reserving_log; - wait_queue_head_t sd_reserving_log_wait; - unsigned int sd_log_flush_head; spinlock_t sd_ail_lock; diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 0fceb60907a2..0da05492e8b8 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -397,6 +397,15 @@ static void ail2_empty(struct gfs2_sbd *sdp, unsigned int new_tail) spin_unlock(&sdp->sd_ail_lock); } +/** + * gfs2_log_is_empty - Check if the log is empty + * @sdp: The GFS2 superblock + */ + +bool gfs2_log_is_empty(struct gfs2_sbd *sdp) { + return atomic_read(&sdp->sd_log_blks_free) == sdp->sd_jdesc->jd_blocks; +} + /** * gfs2_log_release - Release a given number of log blocks * @sdp: The GFS2 superblock @@ -461,13 +470,9 @@ retry: } while(free_blocks <= wanted); finish_wait(&sdp->sd_log_waitq, &wait); } - atomic_inc(&sdp->sd_reserving_log); if (atomic_cmpxchg(&sdp->sd_log_blks_free, free_blocks, - free_blocks - blks) != free_blocks) { - if (atomic_dec_and_test(&sdp->sd_reserving_log)) - wake_up(&sdp->sd_reserving_log_wait); + free_blocks - blks) != free_blocks) goto retry; - } atomic_sub(blks, &sdp->sd_log_blks_needed); trace_gfs2_log_blocks(sdp, -blks); @@ -483,8 +488,6 @@ retry: gfs2_log_release(sdp, blks); ret = -EROFS; } - if (atomic_dec_and_test(&sdp->sd_reserving_log)) - wake_up(&sdp->sd_reserving_log_wait); return ret; } diff --git a/fs/gfs2/log.h b/fs/gfs2/log.h index a9cdbc990edf..16efbe614279 100644 --- a/fs/gfs2/log.h +++ b/fs/gfs2/log.h @@ -64,6 +64,7 @@ static inline void gfs2_ordered_add_inode(struct gfs2_inode *ip) extern void gfs2_ordered_del_inode(struct gfs2_inode *ip); extern unsigned int gfs2_struct2blk(struct gfs2_sbd *sdp, unsigned int nstruct); extern void gfs2_remove_from_ail(struct gfs2_bufdata *bd); +extern bool gfs2_log_is_empty(struct gfs2_sbd *sdp); extern void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks); extern int gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks); extern void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index 61fce59cb4d3..986dc2ebebf0 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -136,8 +136,6 @@ static struct gfs2_sbd *init_sbd(struct super_block *sb) init_rwsem(&sdp->sd_log_flush_lock); atomic_set(&sdp->sd_log_in_flight, 0); - atomic_set(&sdp->sd_reserving_log, 0); - init_waitqueue_head(&sdp->sd_reserving_log_wait); init_waitqueue_head(&sdp->sd_log_flush_wait); atomic_set(&sdp->sd_freeze_state, SFS_UNFROZEN); mutex_init(&sdp->sd_freeze_mutex); diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index ed7a829e9ffe..f188277f7d48 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -645,13 +645,13 @@ int gfs2_make_fs_ro(struct gfs2_sbd *sdp) gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_SHUTDOWN | GFS2_LFC_MAKE_FS_RO); - wait_event(sdp->sd_reserving_log_wait, - atomic_read(&sdp->sd_reserving_log) == 0); - gfs2_assert_warn(sdp, atomic_read(&sdp->sd_log_blks_free) == - sdp->sd_jdesc->jd_blocks); + wait_event_timeout(sdp->sd_log_waitq, + gfs2_log_is_empty(sdp), + HZ * 5); + gfs2_assert_warn(sdp, gfs2_log_is_empty(sdp)); } else { - wait_event_timeout(sdp->sd_reserving_log_wait, - atomic_read(&sdp->sd_reserving_log) == 0, + wait_event_timeout(sdp->sd_log_waitq, + gfs2_log_is_empty(sdp), HZ * 5); } if (gfs2_holder_initialized(&freeze_gh)) diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index db29ca253853..aefe450e009e 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -71,6 +71,8 @@ int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, error = gfs2_log_reserve(sdp, tr->tr_reserved); if (error) { sb_end_intwrite(sdp->sd_vfs); + if (error == -EROFS) + wake_up(&sdp->sd_log_waitq); return error; } -- cgit v1.2.3 From c1eba1b0bca59316f34aa6f70fe5004abba8082d Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sat, 12 Dec 2020 23:30:22 +0100 Subject: gfs2: Move lock flush locking to gfs2_trans_{begin,end} Move the read locking of sd_log_flush_lock from gfs2_log_reserve to gfs2_trans_begin, and its unlocking from gfs2_log_release to gfs2_trans_end. Use gfs2_log_release in two places in which it was open coded before. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 28 +++------------------------- fs/gfs2/log.h | 2 +- fs/gfs2/trans.c | 19 ++++++++++++------- 3 files changed, 16 insertions(+), 33 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 0da05492e8b8..e7183c84ffc0 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -420,7 +420,6 @@ void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks) trace_gfs2_log_blocks(sdp, blks); gfs2_assert_withdraw(sdp, atomic_read(&sdp->sd_log_blks_free) <= sdp->sd_jdesc->jd_blocks); - up_read(&sdp->sd_log_flush_lock); } /** @@ -439,22 +438,16 @@ void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks) * with queued waiters, we use an exclusive wait. This means that when we * get woken with enough journal space to get our reservation, we need to * wake the next waiter on the list. - * - * Returns: errno */ -int gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks) +void gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks) { - int ret = 0; unsigned reserved_blks = 7 * (4096 / sdp->sd_vfs->s_blocksize); unsigned wanted = blks + reserved_blks; DEFINE_WAIT(wait); int did_wait = 0; unsigned int free_blocks; - if (gfs2_assert_warn(sdp, blks) || - gfs2_assert_warn(sdp, blks <= sdp->sd_jdesc->jd_blocks)) - return -EINVAL; atomic_add(blks, &sdp->sd_log_blks_needed); retry: free_blocks = atomic_read(&sdp->sd_log_blks_free); @@ -482,13 +475,6 @@ retry: */ if (unlikely(did_wait)) wake_up(&sdp->sd_log_waitq); - - down_read(&sdp->sd_log_flush_lock); - if (unlikely(!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags))) { - gfs2_log_release(sdp, blks); - ret = -EROFS; - } - return ret; } /** @@ -585,12 +571,7 @@ static void log_pull_tail(struct gfs2_sbd *sdp, unsigned int new_tail) unsigned int dist = log_distance(sdp, new_tail, sdp->sd_log_tail); ail2_empty(sdp, new_tail); - - atomic_add(dist, &sdp->sd_log_blks_free); - trace_gfs2_log_blocks(sdp, dist); - gfs2_assert_withdraw(sdp, atomic_read(&sdp->sd_log_blks_free) <= - sdp->sd_jdesc->jd_blocks); - + gfs2_log_release(sdp, dist); sdp->sd_log_tail = new_tail; } @@ -1127,10 +1108,7 @@ static void log_refund(struct gfs2_sbd *sdp, struct gfs2_trans *tr) maxres = sdp->sd_log_blks_reserved + tr->tr_reserved; gfs2_assert_withdraw(sdp, maxres >= reserved); unused = maxres - reserved; - atomic_add(unused, &sdp->sd_log_blks_free); - trace_gfs2_log_blocks(sdp, unused); - gfs2_assert_withdraw(sdp, atomic_read(&sdp->sd_log_blks_free) <= - sdp->sd_jdesc->jd_blocks); + gfs2_log_release(sdp, unused); sdp->sd_log_blks_reserved = reserved; gfs2_log_unlock(sdp); diff --git a/fs/gfs2/log.h b/fs/gfs2/log.h index 16efbe614279..cbc097ca9244 100644 --- a/fs/gfs2/log.h +++ b/fs/gfs2/log.h @@ -66,7 +66,7 @@ extern unsigned int gfs2_struct2blk(struct gfs2_sbd *sdp, unsigned int nstruct); extern void gfs2_remove_from_ail(struct gfs2_bufdata *bd); extern bool gfs2_log_is_empty(struct gfs2_sbd *sdp); extern void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks); -extern int gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks); +extern void gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks); extern void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, u64 seq, u32 tail, u32 lblock, u32 flags, int op_flags); diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index aefe450e009e..2269aa7ad69d 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -41,8 +41,6 @@ int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, unsigned int blocks, unsigned int revokes, unsigned long ip) { - int error; - if (current->journal_info) { gfs2_print_trans(sdp, current->journal_info); BUG(); @@ -66,14 +64,20 @@ int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, INIT_LIST_HEAD(&tr->tr_ail1_list); INIT_LIST_HEAD(&tr->tr_ail2_list); + if (gfs2_assert_warn(sdp, tr->tr_reserved <= sdp->sd_jdesc->jd_blocks)) + return -EINVAL; + sb_start_intwrite(sdp->sd_vfs); - error = gfs2_log_reserve(sdp, tr->tr_reserved); - if (error) { + gfs2_log_reserve(sdp, tr->tr_reserved); + + down_read(&sdp->sd_log_flush_lock); + if (unlikely(!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags))) { + gfs2_log_release(sdp, tr->tr_reserved); + up_read(&sdp->sd_log_flush_lock); sb_end_intwrite(sdp->sd_vfs); - if (error == -EROFS) - wake_up(&sdp->sd_log_waitq); - return error; + wake_up(&sdp->sd_log_waitq); + return -EROFS; } current->journal_info = tr; @@ -105,6 +109,7 @@ void gfs2_trans_end(struct gfs2_sbd *sdp) if (!test_bit(TR_TOUCHED, &tr->tr_flags)) { gfs2_log_release(sdp, tr->tr_reserved); + up_read(&sdp->sd_log_flush_lock); if (!test_bit(TR_ONSTACK, &tr->tr_flags)) gfs2_trans_free(sdp, tr); sb_end_intwrite(sdp->sd_vfs); -- cgit v1.2.3 From 4a3d049db42b42a36ae84eb8b59d2f5119737253 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Fri, 11 Dec 2020 01:36:25 +0100 Subject: gfs2: Don't wait for journal flush in clean_journal Commit 588bff95c94e added gfs2_write_log_header() and started using it in clean_journal(), with an additional call to log_flush_wait() at the end of gfs2_write_log_header() which is unnecessary for clean_journal(). Move that call out of gfs2_write_log_header() to restore the previous behavior. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index e7183c84ffc0..cff95889b6f4 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -757,7 +757,7 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, u64 dblock; if (gfs2_withdrawn(sdp)) - goto out; + return; page = mempool_alloc(gfs2_page_pool, GFP_NOIO); lh = page_address(page); @@ -812,8 +812,6 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, gfs2_log_write(sdp, page, sb->s_blocksize, 0, dblock); gfs2_log_submit_bio(&sdp->sd_log_bio, REQ_OP_WRITE | op_flags); -out: - log_flush_wait(sdp); } /** @@ -842,6 +840,7 @@ static void log_write_header(struct gfs2_sbd *sdp, u32 flags) gfs2_write_log_header(sdp, sdp->sd_jdesc, sdp->sd_log_sequence++, tail, sdp->sd_log_flush_head, flags, op_flags); gfs2_log_incr_head(sdp); + log_flush_wait(sdp); if (sdp->sd_log_tail != tail) log_pull_tail(sdp, tail); -- cgit v1.2.3 From 5ae8fff8d031b5728f4c0e36e971bba42bb78bea Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sun, 13 Dec 2020 11:37:17 +0100 Subject: gfs2: Clean up gfs2_log_reserve Wake up log waiters in gfs2_log_release when log space has actually become available. This is a much better place for the wakeup than gfs2_logd. Check if enough log space is immeditely available before anything else. If there isn't, use io_wait_event to wait instead of open-coding it. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 58 +++++++++++++++++++++++++-------------------------------- fs/gfs2/trans.c | 3 +-- 2 files changed, 26 insertions(+), 35 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index cff95889b6f4..ca9fa481913d 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -415,11 +415,12 @@ bool gfs2_log_is_empty(struct gfs2_sbd *sdp) { void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks) { - atomic_add(blks, &sdp->sd_log_blks_free); trace_gfs2_log_blocks(sdp, blks); gfs2_assert_withdraw(sdp, atomic_read(&sdp->sd_log_blks_free) <= sdp->sd_jdesc->jd_blocks); + if (atomic_read(&sdp->sd_log_blks_needed)) + wake_up(&sdp->sd_log_waitq); } /** @@ -444,36 +445,33 @@ void gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks) { unsigned reserved_blks = 7 * (4096 / sdp->sd_vfs->s_blocksize); unsigned wanted = blks + reserved_blks; - DEFINE_WAIT(wait); - int did_wait = 0; unsigned int free_blocks; - atomic_add(blks, &sdp->sd_log_blks_needed); -retry: free_blocks = atomic_read(&sdp->sd_log_blks_free); - if (unlikely(free_blocks <= wanted)) { - do { - prepare_to_wait_exclusive(&sdp->sd_log_waitq, &wait, - TASK_UNINTERRUPTIBLE); + while (free_blocks >= wanted) { + if (atomic_try_cmpxchg(&sdp->sd_log_blks_free, &free_blocks, + free_blocks - blks)) + return; + } + + atomic_add(blks, &sdp->sd_log_blks_needed); + for (;;) { + if (current != sdp->sd_logd_process) wake_up(&sdp->sd_logd_waitq); - did_wait = 1; - if (atomic_read(&sdp->sd_log_blks_free) <= wanted) - io_schedule(); - free_blocks = atomic_read(&sdp->sd_log_blks_free); - } while(free_blocks <= wanted); - finish_wait(&sdp->sd_log_waitq, &wait); + io_wait_event(sdp->sd_log_waitq, + (free_blocks = atomic_read(&sdp->sd_log_blks_free), + free_blocks >= wanted)); + do { + if (atomic_try_cmpxchg(&sdp->sd_log_blks_free, + &free_blocks, + free_blocks - blks)) + goto reserved; + } while (free_blocks >= wanted); } - if (atomic_cmpxchg(&sdp->sd_log_blks_free, free_blocks, - free_blocks - blks) != free_blocks) - goto retry; - atomic_sub(blks, &sdp->sd_log_blks_needed); - trace_gfs2_log_blocks(sdp, -blks); - /* - * If we waited, then so might others, wake them up _after_ we get - * our share of the log. - */ - if (unlikely(did_wait)) +reserved: + trace_gfs2_log_blocks(sdp, -blks); + if (atomic_sub_return(blks, &sdp->sd_log_blks_needed)) wake_up(&sdp->sd_log_waitq); } @@ -1107,7 +1105,8 @@ static void log_refund(struct gfs2_sbd *sdp, struct gfs2_trans *tr) maxres = sdp->sd_log_blks_reserved + tr->tr_reserved; gfs2_assert_withdraw(sdp, maxres >= reserved); unused = maxres - reserved; - gfs2_log_release(sdp, unused); + if (unused) + gfs2_log_release(sdp, unused); sdp->sd_log_blks_reserved = reserved; gfs2_log_unlock(sdp); @@ -1192,7 +1191,6 @@ int gfs2_logd(void *data) struct gfs2_sbd *sdp = data; unsigned long t = 1; DEFINE_WAIT(wait); - bool did_flush; while (!kthread_should_stop()) { @@ -1211,12 +1209,10 @@ int gfs2_logd(void *data) continue; } - did_flush = false; if (gfs2_jrnl_flush_reqd(sdp) || t == 0) { gfs2_ail1_empty(sdp, 0); gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_LOGD_JFLUSH_REQD); - did_flush = true; } if (gfs2_ail_flush_reqd(sdp)) { @@ -1225,12 +1221,8 @@ int gfs2_logd(void *data) gfs2_ail1_empty(sdp, 0); gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_LOGD_AIL_FLUSH_REQD); - did_flush = true; } - if (!gfs2_ail_flush_reqd(sdp) || did_flush) - wake_up(&sdp->sd_log_waitq); - t = gfs2_tune_get(sdp, gt_logd_secs) * HZ; try_to_freeze(); diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index 2269aa7ad69d..cac93b2004cf 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -73,10 +73,9 @@ int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, down_read(&sdp->sd_log_flush_lock); if (unlikely(!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags))) { - gfs2_log_release(sdp, tr->tr_reserved); up_read(&sdp->sd_log_flush_lock); + gfs2_log_release(sdp, tr->tr_reserved); sb_end_intwrite(sdp->sd_vfs); - wake_up(&sdp->sd_log_waitq); return -EROFS; } -- cgit v1.2.3 From 297de3180dd7ecbb3798f32e58691168587a8f85 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sun, 6 Dec 2020 14:04:36 +0100 Subject: gfs2: Use a tighter bound in gfs2_trans_begin Use a tighter bound for the number of blocks required by transactions in gfs2_trans_begin: in the worst case, we'll have mixed data and metadata, so we'll need a log desciptor for each type. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/trans.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index cac93b2004cf..f73d6b8f3b53 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -54,8 +54,14 @@ int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, tr->tr_blocks = blocks; tr->tr_revokes = revokes; tr->tr_reserved = 1; - if (blocks) - tr->tr_reserved += 6 + blocks; + if (blocks) { + /* + * The reserved blocks are either used for data or metadata. + * We can have mixed data and metadata, each with its own log + * descriptor block; see calc_reserved(). + */ + tr->tr_reserved += blocks + 1 + DIV_ROUND_UP(blocks - 1, databuf_limit(sdp)); + } if (revokes) tr->tr_reserved += gfs2_struct2blk(sdp, revokes); INIT_LIST_HEAD(&tr->tr_databuf); -- cgit v1.2.3 From 5cb738b5fbd2f3ebe9dec0e428577a4f2128adbe Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sat, 19 Dec 2020 07:15:17 +0100 Subject: gfs2: Get rid of current_tail() Keep the current value of the updated log tail in the super block as sb_log_flush_tail instead of computing it on the fly. This avoids unnecessary sd_ail_lock taking and cleans up the code. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/incore.h | 5 ++-- fs/gfs2/log.c | 72 ++++++++++++++++++++++++++++++-------------------------- fs/gfs2/log.h | 4 +++- 3 files changed, 44 insertions(+), 37 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index 8f8676cf72ed..51656b053170 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -838,8 +838,6 @@ struct gfs2_sbd { wait_queue_head_t sd_logd_waitq; u64 sd_log_sequence; - unsigned int sd_log_head; - unsigned int sd_log_tail; int sd_log_idle; struct rw_semaphore sd_log_flush_lock; @@ -849,6 +847,9 @@ struct gfs2_sbd { int sd_log_error; /* First log error */ wait_queue_head_t sd_withdraw_wait; + unsigned int sd_log_tail; + unsigned int sd_log_flush_tail; + unsigned int sd_log_head; unsigned int sd_log_flush_head; spinlock_t sd_ail_lock; diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index ca9fa481913d..6b5584fd8ff7 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -242,6 +242,28 @@ static void gfs2_ail1_start(struct gfs2_sbd *sdp) return gfs2_ail1_flush(sdp, &wbc); } +static void gfs2_log_update_flush_tail(struct gfs2_sbd *sdp) +{ + unsigned int new_flush_tail = sdp->sd_log_head; + struct gfs2_trans *tr; + + if (!list_empty(&sdp->sd_ail1_list)) { + tr = list_last_entry(&sdp->sd_ail1_list, + struct gfs2_trans, tr_list); + new_flush_tail = tr->tr_first; + } + sdp->sd_log_flush_tail = new_flush_tail; +} + +static void gfs2_log_update_head(struct gfs2_sbd *sdp) +{ + unsigned int new_head = sdp->sd_log_flush_head; + + if (sdp->sd_log_flush_tail == sdp->sd_log_head) + sdp->sd_log_flush_tail = new_head; + sdp->sd_log_head = new_head; +} + /** * gfs2_ail1_empty_one - Check whether or not a trans in the AIL has been synced * @sdp: the filesystem @@ -317,6 +339,7 @@ static int gfs2_ail1_empty(struct gfs2_sbd *sdp, int max_revokes) else oldest_tr = 0; } + gfs2_log_update_flush_tail(sdp); ret = list_empty(&sdp->sd_ail1_list); spin_unlock(&sdp->sd_ail_lock); @@ -544,30 +567,14 @@ static unsigned int calc_reserved(struct gfs2_sbd *sdp) return reserved; } -static unsigned int current_tail(struct gfs2_sbd *sdp) -{ - struct gfs2_trans *tr; - unsigned int tail; - - spin_lock(&sdp->sd_ail_lock); - - if (list_empty(&sdp->sd_ail1_list)) { - tail = sdp->sd_log_head; - } else { - tr = list_last_entry(&sdp->sd_ail1_list, struct gfs2_trans, - tr_list); - tail = tr->tr_first; - } - - spin_unlock(&sdp->sd_ail_lock); - - return tail; -} - -static void log_pull_tail(struct gfs2_sbd *sdp, unsigned int new_tail) +static void log_pull_tail(struct gfs2_sbd *sdp) { - unsigned int dist = log_distance(sdp, new_tail, sdp->sd_log_tail); + unsigned int new_tail = sdp->sd_log_flush_tail; + unsigned int dist; + if (new_tail == sdp->sd_log_tail) + return; + dist = log_distance(sdp, new_tail, sdp->sd_log_tail); ail2_empty(sdp, new_tail); gfs2_log_release(sdp, dist); sdp->sd_log_tail = new_tail; @@ -822,26 +829,23 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, static void log_write_header(struct gfs2_sbd *sdp, u32 flags) { - unsigned int tail; int op_flags = REQ_PREFLUSH | REQ_FUA | REQ_META | REQ_SYNC; enum gfs2_freeze_state state = atomic_read(&sdp->sd_freeze_state); gfs2_assert_withdraw(sdp, (state != SFS_FROZEN)); - tail = current_tail(sdp); if (test_bit(SDF_NOBARRIERS, &sdp->sd_flags)) { gfs2_ordered_wait(sdp); log_flush_wait(sdp); op_flags = REQ_SYNC | REQ_META | REQ_PRIO; } - sdp->sd_log_idle = (tail == sdp->sd_log_flush_head); - gfs2_write_log_header(sdp, sdp->sd_jdesc, sdp->sd_log_sequence++, tail, - sdp->sd_log_flush_head, flags, op_flags); + sdp->sd_log_idle = (sdp->sd_log_flush_tail == sdp->sd_log_flush_head); + gfs2_write_log_header(sdp, sdp->sd_jdesc, sdp->sd_log_sequence++, + sdp->sd_log_flush_tail, sdp->sd_log_flush_head, + flags, op_flags); gfs2_log_incr_head(sdp); log_flush_wait(sdp); - - if (sdp->sd_log_tail != tail) - log_pull_tail(sdp, tail); + log_pull_tail(sdp); } /** @@ -991,7 +995,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) if (sdp->sd_log_head != sdp->sd_log_flush_head) { log_flush_wait(sdp); log_write_header(sdp, flags); - } else if (sdp->sd_log_tail != current_tail(sdp) && !sdp->sd_log_idle) { + } else if (sdp->sd_log_tail != sdp->sd_log_flush_tail && !sdp->sd_log_idle) { atomic_dec(&sdp->sd_log_blks_free); /* Adjust for unreserved buffer */ trace_gfs2_log_blocks(sdp, -1); log_write_header(sdp, flags); @@ -1001,7 +1005,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) lops_after_commit(sdp, tr); gfs2_log_lock(sdp); - sdp->sd_log_head = sdp->sd_log_flush_head; + gfs2_log_update_head(sdp); sdp->sd_log_blks_reserved = 0; sdp->sd_log_committed_revoke = 0; @@ -1021,7 +1025,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) atomic_dec(&sdp->sd_log_blks_free); /* Adjust for unreserved buffer */ trace_gfs2_log_blocks(sdp, -1); log_write_header(sdp, flags); - sdp->sd_log_head = sdp->sd_log_flush_head; + gfs2_log_update_head(sdp); } if (flags & (GFS2_LOG_HEAD_FLUSH_SHUTDOWN | GFS2_LOG_HEAD_FLUSH_FREEZE)) @@ -1156,7 +1160,7 @@ static void gfs2_log_shutdown(struct gfs2_sbd *sdp) gfs2_assert_warn(sdp, sdp->sd_log_head == sdp->sd_log_tail); gfs2_assert_warn(sdp, list_empty(&sdp->sd_ail2_list)); - sdp->sd_log_head = sdp->sd_log_flush_head; + gfs2_log_update_head(sdp); sdp->sd_log_tail = sdp->sd_log_head; } diff --git a/fs/gfs2/log.h b/fs/gfs2/log.h index cbc097ca9244..b36a3539f352 100644 --- a/fs/gfs2/log.h +++ b/fs/gfs2/log.h @@ -43,7 +43,9 @@ static inline void gfs2_log_pointers_init(struct gfs2_sbd *sdp, if (++value == sdp->sd_jdesc->jd_blocks) { value = 0; } - sdp->sd_log_head = sdp->sd_log_tail = value; + sdp->sd_log_tail = value; + sdp->sd_log_flush_tail = value; + sdp->sd_log_head = value; } static inline void gfs2_ordered_add_inode(struct gfs2_inode *ip) -- cgit v1.2.3 From 76fce6548961a0c6246c4796e71800cdc63d5851 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Sat, 19 Dec 2020 10:54:51 +0100 Subject: gfs2: Move function gfs2_ail_empty_tr Move this function further up in log.c so that we can use it in the next patch. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 6b5584fd8ff7..6fd4ded1e357 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -264,6 +264,23 @@ static void gfs2_log_update_head(struct gfs2_sbd *sdp) sdp->sd_log_head = new_head; } +/** + * gfs2_ail_empty_tr - empty one of the ail lists of a transaction + */ + +static void gfs2_ail_empty_tr(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + struct list_head *head) +{ + struct gfs2_bufdata *bd; + + while (!list_empty(head)) { + bd = list_first_entry(head, struct gfs2_bufdata, + bd_ail_st_list); + gfs2_assert(sdp, bd->bd_tr == tr); + gfs2_remove_from_ail(bd); + } +} + /** * gfs2_ail1_empty_one - Check whether or not a trans in the AIL has been synced * @sdp: the filesystem @@ -373,23 +390,6 @@ static void gfs2_ail1_wait(struct gfs2_sbd *sdp) spin_unlock(&sdp->sd_ail_lock); } -/** - * gfs2_ail_empty_tr - empty one of the ail lists for a transaction - */ - -static void gfs2_ail_empty_tr(struct gfs2_sbd *sdp, struct gfs2_trans *tr, - struct list_head *head) -{ - struct gfs2_bufdata *bd; - - while (!list_empty(head)) { - bd = list_first_entry(head, struct gfs2_bufdata, - bd_ail_st_list); - gfs2_assert(sdp, bd->bd_tr == tr); - gfs2_remove_from_ail(bd); - } -} - static void __ail2_empty(struct gfs2_sbd *sdp, struct gfs2_trans *tr) { gfs2_ail_empty_tr(sdp, tr, &tr->tr_ail2_list); -- cgit v1.2.3 From a0b3cb71a1f16942eb7be7483b2568c7b0fe0231 Mon Sep 17 00:00:00 2001 From: BingJing Chang Date: Fri, 29 Jan 2021 12:53:15 +0800 Subject: isofs: handle large user and group ID If uid or gid of mount options is larger than INT_MAX, isofs_fill_super will return -EINVAL. The problem can be encountered by a domain user or reproduced via: mount -o loop,uid=2147483648 ubuntu-16.04.6-server-amd64.iso /mnt This can be fixed as commit 233a01fa9c4c ("fuse: handle large user and group ID"). Link: https://lore.kernel.org/r/20210129045315.10375-1-bingjingc@synology.com Reviewed-by: Robbie Ko Reviewed-by: Chung-Chiang Cheng Signed-off-by: BingJing Chang Signed-off-by: Jan Kara --- fs/isofs/inode.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c index ec90773527ee..21edc423b79f 100644 --- a/fs/isofs/inode.c +++ b/fs/isofs/inode.c @@ -339,6 +339,7 @@ static int parse_options(char *options, struct iso9660_options *popt) { char *p; int option; + unsigned int uv; popt->map = 'n'; popt->rock = 1; @@ -434,17 +435,17 @@ static int parse_options(char *options, struct iso9660_options *popt) case Opt_ignore: break; case Opt_uid: - if (match_int(&args[0], &option)) + if (match_uint(&args[0], &uv)) return 0; - popt->uid = make_kuid(current_user_ns(), option); + popt->uid = make_kuid(current_user_ns(), uv); if (!uid_valid(popt->uid)) return 0; popt->uid_set = 1; break; case Opt_gid: - if (match_int(&args[0], &option)) + if (match_uint(&args[0], &uv)) return 0; - popt->gid = make_kgid(current_user_ns(), option); + popt->gid = make_kgid(current_user_ns(), uv); if (!gid_valid(popt->gid)) return 0; popt->gid_set = 1; -- cgit v1.2.3 From 3a9a3aa805af842f7a20adbf7f1006caddb87d31 Mon Sep 17 00:00:00 2001 From: BingJing Chang Date: Fri, 29 Jan 2021 12:55:02 +0800 Subject: udf: handle large user and group ID If uid or gid of mount options is larger than INT_MAX, udf_fill_super will return -EINVAL. The problem can be encountered by a domain user or reproduced via: mount -o loop,uid=2147483648 something-in-udf-format.iso /mnt This can be fixed as commit 233a01fa9c4c ("fuse: handle large user and group ID"). Link: https://lore.kernel.org/r/20210129045502.10546-1-bingjingc@synology.com Reviewed-by: Robbie Ko Reviewed-by: Chung-Chiang Cheng Signed-off-by: BingJing Chang Signed-off-by: Jan Kara --- fs/udf/super.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/udf/super.c b/fs/udf/super.c index 5bef3a68395d..3c45117280b0 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -459,6 +459,7 @@ static int udf_parse_options(char *options, struct udf_options *uopt, { char *p; int option; + unsigned int uv; uopt->novrs = 0; uopt->session = 0xFFFFFFFF; @@ -508,17 +509,17 @@ static int udf_parse_options(char *options, struct udf_options *uopt, uopt->flags &= ~(1 << UDF_FLAG_USE_SHORT_AD); break; case Opt_gid: - if (match_int(args, &option)) + if (match_uint(args, &uv)) return 0; - uopt->gid = make_kgid(current_user_ns(), option); + uopt->gid = make_kgid(current_user_ns(), uv); if (!gid_valid(uopt->gid)) return 0; uopt->flags |= (1 << UDF_FLAG_GID_SET); break; case Opt_uid: - if (match_int(args, &option)) + if (match_uint(args, &uv)) return 0; - uopt->uid = make_kuid(current_user_ns(), option); + uopt->uid = make_kuid(current_user_ns(), uv); if (!uid_valid(uopt->uid)) return 0; uopt->flags |= (1 << UDF_FLAG_UID_SET); -- cgit v1.2.3 From f69e8091c4a2ae291e1f55225e8116fef05dc156 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 3 Feb 2021 18:30:10 +0100 Subject: xfs: remove the possibly unused mp variable in xfs_file_compat_ioctl The mp variable in xfs_file_compat_ioctl is only used when BROKEN_X86_ALIGNMENT is define. Remove it and just open code the dereference in a few places. Link: https://lore.kernel.org/r/20210203173009.462205-1-christian.brauner@ubuntu.com Fixes: f736d93d76d3 ("xfs: support idmapped mounts") Reviewed-by: Darrick J. Wong Acked-by: Christian Brauner Signed-off-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/xfs/xfs_ioctl32.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_ioctl32.c b/fs/xfs/xfs_ioctl32.c index 926427b19573..33c09ec8e6c0 100644 --- a/fs/xfs/xfs_ioctl32.c +++ b/fs/xfs/xfs_ioctl32.c @@ -438,7 +438,6 @@ xfs_file_compat_ioctl( { struct inode *inode = file_inode(filp); struct xfs_inode *ip = XFS_I(inode); - struct xfs_mount *mp = ip->i_mount; void __user *arg = compat_ptr(p); int error; @@ -458,7 +457,7 @@ xfs_file_compat_ioctl( return xfs_ioc_space(filp, &bf); } case XFS_IOC_FSGEOMETRY_V1_32: - return xfs_compat_ioc_fsgeometry_v1(mp, arg); + return xfs_compat_ioc_fsgeometry_v1(ip->i_mount, arg); case XFS_IOC_FSGROWFSDATA_32: { struct xfs_growfs_data in; @@ -467,7 +466,7 @@ xfs_file_compat_ioctl( error = mnt_want_write_file(filp); if (error) return error; - error = xfs_growfs_data(mp, &in); + error = xfs_growfs_data(ip->i_mount, &in); mnt_drop_write_file(filp); return error; } @@ -479,7 +478,7 @@ xfs_file_compat_ioctl( error = mnt_want_write_file(filp); if (error) return error; - error = xfs_growfs_rt(mp, &in); + error = xfs_growfs_rt(ip->i_mount, &in); mnt_drop_write_file(filp); return error; } -- cgit v1.2.3 From 261eeb9c1585de4515a770b48a3c89672c08ae7f Mon Sep 17 00:00:00 2001 From: Daeho Jeong Date: Tue, 19 Jan 2021 09:00:42 +0900 Subject: f2fs: introduce checkpoint_merge mount option We've added a new mount options, "checkpoint_merge" and "nocheckpoint_merge", which creates a kernel daemon and makes it to merge concurrent checkpoint requests as much as possible to eliminate redundant checkpoint issues. Plus, we can eliminate the sluggish issue caused by slow checkpoint operation when the checkpoint is done in a process context in a cgroup having low i/o budget and cpu shares. To make this do better, we set the default i/o priority of the kernel daemon to "3", to give one higher priority than other kernel threads. The below verification result explains this. The basic idea has come from https://opensource.samsung.com. [Verification] Android Pixel Device(ARM64, 7GB RAM, 256GB UFS) Create two I/O cgroups (fg w/ weight 100, bg w/ wight 20) Set "strict_guarantees" to "1" in BFQ tunables In "fg" cgroup, - thread A => trigger 1000 checkpoint operations "for i in `seq 1 1000`; do touch test_dir1/file; fsync test_dir1; done" - thread B => gererating async. I/O "fio --rw=write --numjobs=1 --bs=128k --runtime=3600 --time_based=1 --filename=test_img --name=test" In "bg" cgroup, - thread C => trigger repeated checkpoint operations "echo $$ > /dev/blkio/bg/tasks; while true; do touch test_dir2/file; fsync test_dir2; done" We've measured thread A's execution time. [ w/o patch ] Elapsed Time: Avg. 68 seconds [ w/ patch ] Elapsed Time: Avg. 48 seconds Reported-by: kernel test robot Reported-by: Dan Carpenter [Jaegeuk Kim: fix the return value in f2fs_start_ckpt_thread, reported by Dan] Signed-off-by: Daeho Jeong Signed-off-by: Sungjong Seo Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- Documentation/filesystems/f2fs.rst | 11 +++ fs/f2fs/checkpoint.c | 177 +++++++++++++++++++++++++++++++++++++ fs/f2fs/debug.c | 12 +++ fs/f2fs/f2fs.h | 27 ++++++ fs/f2fs/super.c | 58 ++++++++++-- 5 files changed, 277 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index 5eff4009e77e..f75ec244762f 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -247,6 +247,17 @@ checkpoint=%s[:%u[%]] Set to "disable" to turn off checkpointing. Set to "enabl hide up to all remaining free space. The actual space that would be unusable can be viewed at /sys/fs/f2fs//unusable This space is reclaimed once checkpoint=enable. +checkpoint_merge When checkpoint is enabled, this can be used to create a kernel + daemon and make it to merge concurrent checkpoint requests as + much as possible to eliminate redundant checkpoint issues. Plus, + we can eliminate the sluggish issue caused by slow checkpoint + operation when the checkpoint is done in a process context in + a cgroup having low i/o budget and cpu shares. To make this + do better, we set the default i/o priority of the kernel daemon + to "3", to give one higher priority than other kernel threads. + This is the same way to give a I/O priority to the jbd2 + journaling thread of ext4 filesystem. +nocheckpoint_merge Disable checkpoint merge feature. compress_algorithm=%s Control compress algorithm, currently f2fs supports "lzo", "lz4", "zstd" and "lzo-rle" algorithm. compress_algorithm=%s:%d Control compress algorithm and its compress level, now, only diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 8c79ba0566b1..6f6ecba8f920 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -13,12 +13,15 @@ #include #include #include +#include #include "f2fs.h" #include "node.h" #include "segment.h" #include +#define DEFAULT_CHECKPOINT_IOPRIO (IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, 3)) + static struct kmem_cache *ino_entry_slab; struct kmem_cache *f2fs_inode_entry_slab; @@ -1704,3 +1707,177 @@ void f2fs_destroy_checkpoint_caches(void) kmem_cache_destroy(ino_entry_slab); kmem_cache_destroy(f2fs_inode_entry_slab); } + +static int __write_checkpoint_sync(struct f2fs_sb_info *sbi) +{ + struct cp_control cpc = { .reason = CP_SYNC, }; + int err; + + down_write(&sbi->gc_lock); + err = f2fs_write_checkpoint(sbi, &cpc); + up_write(&sbi->gc_lock); + + return err; +} + +static void __checkpoint_and_complete_reqs(struct f2fs_sb_info *sbi) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + struct ckpt_req *req, *next; + struct llist_node *dispatch_list; + u64 sum_diff = 0, diff, count = 0; + int ret; + + dispatch_list = llist_del_all(&cprc->issue_list); + if (!dispatch_list) + return; + dispatch_list = llist_reverse_order(dispatch_list); + + ret = __write_checkpoint_sync(sbi); + atomic_inc(&cprc->issued_ckpt); + + llist_for_each_entry_safe(req, next, dispatch_list, llnode) { + diff = (u64)ktime_ms_delta(ktime_get(), req->queue_time); + req->ret = ret; + complete(&req->wait); + + sum_diff += diff; + count++; + } + atomic_sub(count, &cprc->queued_ckpt); + atomic_add(count, &cprc->total_ckpt); + + spin_lock(&cprc->stat_lock); + cprc->cur_time = (unsigned int)div64_u64(sum_diff, count); + if (cprc->peak_time < cprc->cur_time) + cprc->peak_time = cprc->cur_time; + spin_unlock(&cprc->stat_lock); +} + +static int issue_checkpoint_thread(void *data) +{ + struct f2fs_sb_info *sbi = data; + struct ckpt_req_control *cprc = &sbi->cprc_info; + wait_queue_head_t *q = &cprc->ckpt_wait_queue; +repeat: + if (kthread_should_stop()) + return 0; + + sb_start_intwrite(sbi->sb); + + if (!llist_empty(&cprc->issue_list)) + __checkpoint_and_complete_reqs(sbi); + + sb_end_intwrite(sbi->sb); + + wait_event_interruptible(*q, + kthread_should_stop() || !llist_empty(&cprc->issue_list)); + goto repeat; +} + +static void flush_remained_ckpt_reqs(struct f2fs_sb_info *sbi, + struct ckpt_req *wait_req) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + + if (!llist_empty(&cprc->issue_list)) { + __checkpoint_and_complete_reqs(sbi); + } else { + /* already dispatched by issue_checkpoint_thread */ + if (wait_req) + wait_for_completion(&wait_req->wait); + } +} + +static void init_ckpt_req(struct ckpt_req *req) +{ + memset(req, 0, sizeof(struct ckpt_req)); + + init_completion(&req->wait); + req->queue_time = ktime_get(); +} + +int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + struct ckpt_req req; + struct cp_control cpc; + + cpc.reason = __get_cp_reason(sbi); + if (!test_opt(sbi, MERGE_CHECKPOINT) || cpc.reason != CP_SYNC) { + int ret; + + down_write(&sbi->gc_lock); + ret = f2fs_write_checkpoint(sbi, &cpc); + up_write(&sbi->gc_lock); + + return ret; + } + + if (!cprc->f2fs_issue_ckpt) + return __write_checkpoint_sync(sbi); + + init_ckpt_req(&req); + + llist_add(&req.llnode, &cprc->issue_list); + atomic_inc(&cprc->queued_ckpt); + + /* update issue_list before we wake up issue_checkpoint thread */ + smp_mb(); + + if (waitqueue_active(&cprc->ckpt_wait_queue)) + wake_up(&cprc->ckpt_wait_queue); + + if (cprc->f2fs_issue_ckpt) + wait_for_completion(&req.wait); + else + flush_remained_ckpt_reqs(sbi, &req); + + return req.ret; +} + +int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi) +{ + dev_t dev = sbi->sb->s_bdev->bd_dev; + struct ckpt_req_control *cprc = &sbi->cprc_info; + + if (cprc->f2fs_issue_ckpt) + return 0; + + cprc->f2fs_issue_ckpt = kthread_run(issue_checkpoint_thread, sbi, + "f2fs_ckpt-%u:%u", MAJOR(dev), MINOR(dev)); + if (IS_ERR(cprc->f2fs_issue_ckpt)) { + cprc->f2fs_issue_ckpt = NULL; + return -ENOMEM; + } + + set_task_ioprio(cprc->f2fs_issue_ckpt, DEFAULT_CHECKPOINT_IOPRIO); + + return 0; +} + +void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + + if (cprc->f2fs_issue_ckpt) { + struct task_struct *ckpt_task = cprc->f2fs_issue_ckpt; + + cprc->f2fs_issue_ckpt = NULL; + kthread_stop(ckpt_task); + + flush_remained_ckpt_reqs(sbi, NULL); + } +} + +void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + + atomic_set(&cprc->issued_ckpt, 0); + atomic_set(&cprc->total_ckpt, 0); + atomic_set(&cprc->queued_ckpt, 0); + init_waitqueue_head(&cprc->ckpt_wait_queue); + init_llist_head(&cprc->issue_list); + spin_lock_init(&cprc->stat_lock); +} diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index 197c914119da..91855d5721cd 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -120,6 +120,13 @@ static void update_general_status(struct f2fs_sb_info *sbi) atomic_read(&SM_I(sbi)->dcc_info->discard_cmd_cnt); si->undiscard_blks = SM_I(sbi)->dcc_info->undiscard_blks; } + si->nr_issued_ckpt = atomic_read(&sbi->cprc_info.issued_ckpt); + si->nr_total_ckpt = atomic_read(&sbi->cprc_info.total_ckpt); + si->nr_queued_ckpt = atomic_read(&sbi->cprc_info.queued_ckpt); + spin_lock(&sbi->cprc_info.stat_lock); + si->cur_ckpt_time = sbi->cprc_info.cur_time; + si->peak_ckpt_time = sbi->cprc_info.peak_time; + spin_unlock(&sbi->cprc_info.stat_lock); si->total_count = (int)sbi->user_block_count / sbi->blocks_per_seg; si->rsvd_segs = reserved_segments(sbi); si->overp_segs = overprovision_segments(sbi); @@ -417,6 +424,11 @@ static int stat_show(struct seq_file *s, void *v) si->meta_count[META_NAT]); seq_printf(s, " - ssa blocks : %u\n", si->meta_count[META_SSA]); + seq_printf(s, "CP merge (Queued: %4d, Issued: %4d, Total: %4d, " + "Cur time: %4d(ms), Peak time: %4d(ms))\n", + si->nr_queued_ckpt, si->nr_issued_ckpt, + si->nr_total_ckpt, si->cur_ckpt_time, + si->peak_ckpt_time); seq_printf(s, "GC calls: %d (BG: %d)\n", si->call_count, si->bg_gc); seq_printf(s, " - data segments : %d (%d)\n", diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index a2e520a13630..f7536aca8a31 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -97,6 +97,7 @@ extern const char *f2fs_fault_name[FAULT_MAX]; #define F2FS_MOUNT_DISABLE_CHECKPOINT 0x02000000 #define F2FS_MOUNT_NORECOVERY 0x04000000 #define F2FS_MOUNT_ATGC 0x08000000 +#define F2FS_MOUNT_MERGE_CHECKPOINT 0x10000000 #define F2FS_OPTION(sbi) ((sbi)->mount_opt) #define clear_opt(sbi, option) (F2FS_OPTION(sbi).opt &= ~F2FS_MOUNT_##option) @@ -267,6 +268,25 @@ struct fsync_node_entry { unsigned int seq_id; /* sequence id */ }; +struct ckpt_req { + struct completion wait; /* completion for checkpoint done */ + struct llist_node llnode; /* llist_node to be linked in wait queue */ + int ret; /* return code of checkpoint */ + ktime_t queue_time; /* request queued time */ +}; + +struct ckpt_req_control { + struct task_struct *f2fs_issue_ckpt; /* checkpoint task */ + wait_queue_head_t ckpt_wait_queue; /* waiting queue for wake-up */ + atomic_t issued_ckpt; /* # of actually issued ckpts */ + atomic_t total_ckpt; /* # of total ckpts */ + atomic_t queued_ckpt; /* # of queued ckpts */ + struct llist_head issue_list; /* list for command issue */ + spinlock_t stat_lock; /* lock for below checkpoint time stats */ + unsigned int cur_time; /* cur wait time in msec for currently issued checkpoint */ + unsigned int peak_time; /* peak wait time in msec until now */ +}; + /* for the bitmap indicate blocks to be discarded */ struct discard_entry { struct list_head list; /* list head */ @@ -1433,6 +1453,7 @@ struct f2fs_sb_info { wait_queue_head_t cp_wait; unsigned long last_time[MAX_TIME]; /* to store time in jiffies */ long interval_time[MAX_TIME]; /* to store thresholds */ + struct ckpt_req_control cprc_info; /* for checkpoint request control */ struct inode_management im[MAX_INO_ENTRY]; /* manage inode cache */ @@ -3450,6 +3471,10 @@ int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc); void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi); int __init f2fs_create_checkpoint_caches(void); void f2fs_destroy_checkpoint_caches(void); +int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi); +int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi); +void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi); +void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi); /* * data.c @@ -3562,6 +3587,8 @@ struct f2fs_stat_info { int nr_discarding, nr_discarded; int nr_discard_cmd; unsigned int undiscard_blks; + int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt; + unsigned int cur_ckpt_time, peak_ckpt_time; int inline_xattr, inline_inode, inline_dir, append, update, orphans; int compr_inode; unsigned long long compr_blocks; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 429bc00af440..1000d21120ca 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -144,6 +144,8 @@ enum { Opt_checkpoint_disable_cap, Opt_checkpoint_disable_cap_perc, Opt_checkpoint_enable, + Opt_checkpoint_merge, + Opt_nocheckpoint_merge, Opt_compress_algorithm, Opt_compress_log_size, Opt_compress_extension, @@ -214,6 +216,8 @@ static match_table_t f2fs_tokens = { {Opt_checkpoint_disable_cap, "checkpoint=disable:%u"}, {Opt_checkpoint_disable_cap_perc, "checkpoint=disable:%u%%"}, {Opt_checkpoint_enable, "checkpoint=enable"}, + {Opt_checkpoint_merge, "checkpoint_merge"}, + {Opt_nocheckpoint_merge, "nocheckpoint_merge"}, {Opt_compress_algorithm, "compress_algorithm=%s"}, {Opt_compress_log_size, "compress_log_size=%u"}, {Opt_compress_extension, "compress_extension=%s"}, @@ -941,6 +945,12 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) case Opt_checkpoint_enable: clear_opt(sbi, DISABLE_CHECKPOINT); break; + case Opt_checkpoint_merge: + set_opt(sbi, MERGE_CHECKPOINT); + break; + case Opt_nocheckpoint_merge: + clear_opt(sbi, MERGE_CHECKPOINT); + break; #ifdef CONFIG_F2FS_FS_COMPRESSION case Opt_compress_algorithm: if (!f2fs_sb_has_compression(sbi)) { @@ -1340,6 +1350,12 @@ static void f2fs_put_super(struct super_block *sb) /* prevent remaining shrinker jobs */ mutex_lock(&sbi->umount_mutex); + /* + * flush all issued checkpoints and stop checkpoint issue thread. + * after then, all checkpoints should be done by each process context. + */ + f2fs_stop_ckpt_thread(sbi); + /* * We don't need to do checkpoint when superblock is clean. * But, the previous checkpoint was not done by umount, it needs to do @@ -1438,15 +1454,9 @@ int f2fs_sync_fs(struct super_block *sb, int sync) if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) return -EAGAIN; - if (sync) { - struct cp_control cpc; - - cpc.reason = __get_cp_reason(sbi); + if (sync) + err = f2fs_issue_checkpoint(sbi); - down_write(&sbi->gc_lock); - err = f2fs_write_checkpoint(sbi, &cpc); - up_write(&sbi->gc_lock); - } return err; } @@ -1770,6 +1780,10 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) if (test_opt(sbi, DISABLE_CHECKPOINT)) seq_printf(seq, ",checkpoint=disable:%u", F2FS_OPTION(sbi).unusable_cap); + if (test_opt(sbi, MERGE_CHECKPOINT)) + seq_puts(seq, ",checkpoint_merge"); + else + seq_puts(seq, ",nocheckpoint_merge"); if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_POSIX) seq_printf(seq, ",fsync_mode=%s", "posix"); else if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT) @@ -2053,6 +2067,19 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) } } + if (!test_opt(sbi, DISABLE_CHECKPOINT) && + test_opt(sbi, MERGE_CHECKPOINT)) { + err = f2fs_start_ckpt_thread(sbi); + if (err) { + f2fs_err(sbi, + "Failed to start F2FS issue_checkpoint_thread (%d)", + err); + goto restore_gc; + } + } else { + f2fs_stop_ckpt_thread(sbi); + } + /* * We stop issue flush thread if FS is mounted as RO * or if flush_merge is not passed in mount option. @@ -3804,6 +3831,19 @@ try_onemore: f2fs_init_fsync_node_info(sbi); + /* setup checkpoint request control and start checkpoint issue thread */ + f2fs_init_ckpt_req_control(sbi); + if (!test_opt(sbi, DISABLE_CHECKPOINT) && + test_opt(sbi, MERGE_CHECKPOINT)) { + err = f2fs_start_ckpt_thread(sbi); + if (err) { + f2fs_err(sbi, + "Failed to start F2FS issue_checkpoint_thread (%d)", + err); + goto stop_ckpt_thread; + } + } + /* setup f2fs internal modules */ err = f2fs_build_segment_manager(sbi); if (err) { @@ -4013,6 +4053,8 @@ free_nm: free_sm: f2fs_destroy_segment_manager(sbi); f2fs_destroy_post_read_wq(sbi); +stop_ckpt_thread: + f2fs_stop_ckpt_thread(sbi); free_devices: destroy_device_list(sbi); kvfree(sbi->ckpt); -- cgit v1.2.3 From e65920661708b7c0f3db45c9cd5d0095034ee37f Mon Sep 17 00:00:00 2001 From: Daeho Jeong Date: Thu, 21 Jan 2021 22:45:29 +0900 Subject: f2fs: add ckpt_thread_ioprio sysfs node Added "ckpt_thread_ioprio" sysfs node to give a way to change checkpoint merge daemon's io priority. Its default value is "be,3", which means "BE" I/O class and I/O priority "3". We can select the class between "rt" and "be", and set the I/O priority within valid range of it. "," delimiter is necessary in between I/O class and priority number. Signed-off-by: Daeho Jeong Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- Documentation/ABI/testing/sysfs-fs-f2fs | 9 ++++++ fs/f2fs/checkpoint.c | 3 +- fs/f2fs/f2fs.h | 1 + fs/f2fs/sysfs.c | 55 +++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index 362803901614..cbeac1bebe2f 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -400,3 +400,12 @@ Description: Show status of f2fs superblock in real time. 0x1000 SBI_QUOTA_NEED_REPAIR quota file may be corrupted 0x2000 SBI_IS_RESIZEFS resizefs is in process ====== ===================== ================================= + +What: /sys/fs/f2fs//ckpt_thread_ioprio +Date: January 2021 +Contact: "Daeho Jeong" +Description: Give a way to change checkpoint merge daemon's io priority. + Its default value is "be,3", which means "BE" I/O class and + I/O priority "3". We can select the class between "rt" and "be", + and set the I/O priority within valid range of it. "," delimiter + is necessary in between I/O class and priority number. diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 6f6ecba8f920..579b9c3603cc 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -1851,7 +1851,7 @@ int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi) return -ENOMEM; } - set_task_ioprio(cprc->f2fs_issue_ckpt, DEFAULT_CHECKPOINT_IOPRIO); + set_task_ioprio(cprc->f2fs_issue_ckpt, cprc->ckpt_thread_ioprio); return 0; } @@ -1877,6 +1877,7 @@ void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi) atomic_set(&cprc->issued_ckpt, 0); atomic_set(&cprc->total_ckpt, 0); atomic_set(&cprc->queued_ckpt, 0); + cprc->ckpt_thread_ioprio = DEFAULT_CHECKPOINT_IOPRIO; init_waitqueue_head(&cprc->ckpt_wait_queue); init_llist_head(&cprc->issue_list); spin_lock_init(&cprc->stat_lock); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index f7536aca8a31..2860003a09ed 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -277,6 +277,7 @@ struct ckpt_req { struct ckpt_req_control { struct task_struct *f2fs_issue_ckpt; /* checkpoint task */ + int ckpt_thread_ioprio; /* checkpoint merge thread ioprio */ wait_queue_head_t ckpt_wait_queue; /* waiting queue for wake-up */ atomic_t issued_ckpt; /* # of actually issued ckpts */ atomic_t total_ckpt; /* # of total ckpts */ diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index f39874d512ea..e38a7f6921dd 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "f2fs.h" #include "segment.h" @@ -34,6 +35,7 @@ enum { FAULT_INFO_TYPE, /* struct f2fs_fault_info */ #endif RESERVED_BLOCKS, /* struct f2fs_sb_info */ + CPRC_INFO, /* struct ckpt_req_control */ }; struct f2fs_attr { @@ -70,6 +72,8 @@ static unsigned char *__struct_ptr(struct f2fs_sb_info *sbi, int struct_type) else if (struct_type == STAT_INFO) return (unsigned char *)F2FS_STAT(sbi); #endif + else if (struct_type == CPRC_INFO) + return (unsigned char *)&sbi->cprc_info; return NULL; } @@ -261,6 +265,23 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, return len; } + if (!strcmp(a->attr.name, "ckpt_thread_ioprio")) { + struct ckpt_req_control *cprc = &sbi->cprc_info; + int len = 0; + int class = IOPRIO_PRIO_CLASS(cprc->ckpt_thread_ioprio); + int data = IOPRIO_PRIO_DATA(cprc->ckpt_thread_ioprio); + + if (class == IOPRIO_CLASS_RT) + len += scnprintf(buf + len, PAGE_SIZE - len, "rt,"); + else if (class == IOPRIO_CLASS_BE) + len += scnprintf(buf + len, PAGE_SIZE - len, "be,"); + else + return -EINVAL; + + len += scnprintf(buf + len, PAGE_SIZE - len, "%d\n", data); + return len; + } + ui = (unsigned int *)(ptr + a->offset); return sprintf(buf, "%u\n", *ui); @@ -314,6 +335,38 @@ out: return ret ? ret : count; } + if (!strcmp(a->attr.name, "ckpt_thread_ioprio")) { + const char *name = strim((char *)buf); + struct ckpt_req_control *cprc = &sbi->cprc_info; + int class; + long data; + int ret; + + if (!strncmp(name, "rt,", 3)) + class = IOPRIO_CLASS_RT; + else if (!strncmp(name, "be,", 3)) + class = IOPRIO_CLASS_BE; + else + return -EINVAL; + + name += 3; + ret = kstrtol(name, 10, &data); + if (ret) + return ret; + if (data >= IOPRIO_BE_NR || data < 0) + return -EINVAL; + + cprc->ckpt_thread_ioprio = IOPRIO_PRIO_VALUE(class, data); + if (test_opt(sbi, MERGE_CHECKPOINT)) { + ret = set_task_ioprio(cprc->f2fs_issue_ckpt, + cprc->ckpt_thread_ioprio); + if (ret) + return ret; + } + + return count; + } + ui = (unsigned int *)(ptr + a->offset); ret = kstrtoul(skip_spaces(buf), 0, &t); @@ -573,6 +626,7 @@ F2FS_RW_ATTR(FAULT_INFO_TYPE, f2fs_fault_info, inject_type, inject_type); #endif F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, data_io_flag, data_io_flag); F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, node_io_flag, node_io_flag); +F2FS_RW_ATTR(CPRC_INFO, ckpt_req_control, ckpt_thread_ioprio, ckpt_thread_ioprio); F2FS_GENERAL_RO_ATTR(dirty_segments); F2FS_GENERAL_RO_ATTR(free_segments); F2FS_GENERAL_RO_ATTR(lifetime_write_kbytes); @@ -658,6 +712,7 @@ static struct attribute *f2fs_attrs[] = { #endif ATTR_LIST(data_io_flag), ATTR_LIST(node_io_flag), + ATTR_LIST(ckpt_thread_ioprio), ATTR_LIST(dirty_segments), ATTR_LIST(free_segments), ATTR_LIST(unusable), -- cgit v1.2.3 From b5776e7524afbd4569978ff790864755c438bba7 Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Thu, 4 Feb 2021 00:05:20 -0500 Subject: ext4: fix potential htree index checksum corruption In the case where we need to do an interior node split, and immediately afterwards, we are unable to allocate a new directory leaf block due to ENOSPC, the directory index checksum's will not be filled in correctly (and indeed, will not be correctly journalled). This looks like a bug that was introduced when we added largedir support. The original code doesn't make any sense (and should have been caught in code review), but it was hidden because most of the time, the index node checksum will be set by do_split(). But if do_split bails out due to ENOSPC, then ext4_handle_dirty_dx_node() won't get called, and so the directory index checksum field will not get set, leading to: EXT4-fs error (device sdb): dx_probe:858: inode #6635543: block 4022: comm nfsd: Directory index failed checksum Google-Bug-Id: 176345532 Fixes: e08ac99fa2a2 ("ext4: add largedir feature") Cc: Artem Blagodarenko Signed-off-by: Theodore Ts'o --- fs/ext4/namei.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index a6e28b4b5a95..115762180801 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -2411,11 +2411,10 @@ again: (frame - 1)->bh); if (err) goto journal_error; - if (restart) { - err = ext4_handle_dirty_dx_node(handle, dir, - frame->bh); + err = ext4_handle_dirty_dx_node(handle, dir, + frame->bh); + if (err) goto journal_error; - } } else { struct dx_root *dxroot; memcpy((char *) entries2, (char *) entries, -- cgit v1.2.3 From d7e10d47691d1702db1cd1edcc689d3031eefc67 Mon Sep 17 00:00:00 2001 From: Xiaoguang Wang Date: Thu, 4 Feb 2021 17:20:56 +0800 Subject: io_uring: don't modify identity's files uncess identity is cowed Abaci Robot reported following panic: BUG: kernel NULL pointer dereference, address: 0000000000000000 PGD 800000010ef3f067 P4D 800000010ef3f067 PUD 10d9df067 PMD 0 Oops: 0002 [#1] SMP PTI CPU: 0 PID: 1869 Comm: io_wqe_worker-0 Not tainted 5.11.0-rc3+ #1 Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011 RIP: 0010:put_files_struct+0x1b/0x120 Code: 24 18 c7 00 f4 ff ff ff e9 4d fd ff ff 66 90 0f 1f 44 00 00 41 57 41 56 49 89 fe 41 55 41 54 55 53 48 83 ec 08 e8 b5 6b db ff 41 ff 0e 74 13 48 83 c4 08 5b 5d 41 5c 41 5d 41 5e 41 5f e9 9c RSP: 0000:ffffc90002147d48 EFLAGS: 00010293 RAX: 0000000000000000 RBX: ffff88810d9a5300 RCX: 0000000000000000 RDX: ffff88810d87c280 RSI: ffffffff8144ba6b RDI: 0000000000000000 RBP: 0000000000000080 R08: 0000000000000001 R09: ffffffff81431500 R10: ffff8881001be000 R11: 0000000000000000 R12: ffff88810ac2f800 R13: ffff88810af38a00 R14: 0000000000000000 R15: ffff8881057130c0 FS: 0000000000000000(0000) GS:ffff88813bc00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000000000 CR3: 000000010dbaa002 CR4: 00000000003706f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: __io_clean_op+0x10c/0x2a0 io_dismantle_req+0x3c7/0x600 __io_free_req+0x34/0x280 io_put_req+0x63/0xb0 io_worker_handle_work+0x60e/0x830 ? io_wqe_worker+0x135/0x520 io_wqe_worker+0x158/0x520 ? __kthread_parkme+0x96/0xc0 ? io_worker_handle_work+0x830/0x830 kthread+0x134/0x180 ? kthread_create_worker_on_cpu+0x90/0x90 ret_from_fork+0x1f/0x30 Modules linked in: CR2: 0000000000000000 ---[ end trace c358ca86af95b1e7 ]--- I guess case below can trigger above panic: there're two threads which operates different io_uring ctxs and share same sqthread identity, and later one thread exits, io_uring_cancel_task_requests() will clear task->io_uring->identity->files to be NULL in sqpoll mode, then another ctx that uses same identity will panic. Indeed we don't need to clear task->io_uring->identity->files here, io_grab_identity() should handle identity->files changes well, if task->io_uring->identity->files is not equal to current->files, io_cow_identity() should handle this changes well. Cc: stable@vger.kernel.org # 5.5+ Reported-by: Abaci Robot Signed-off-by: Xiaoguang Wang Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 38c6cbe1ab38..5d3348d66f06 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8982,12 +8982,6 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { atomic_dec(&task->io_uring->in_idle); - /* - * If the files that are going away are the ones in the thread - * identity, clear them out. - */ - if (task->io_uring->identity->files == files) - task->io_uring->identity->files = NULL; io_sq_thread_unpark(ctx->sq_data); } } -- cgit v1.2.3 From 9936c7c2bc76a0b2276f6d19de6d1d92f03deeab Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:51:56 +0000 Subject: io_uring: deduplicate core cancellations sequence Files and task cancellations go over same steps trying to cancel requests in io-wq, poll, etc. Deduplicate it with a helper. note: new io_uring_try_cancel_requests() is former __io_uring_cancel_task_requests() with files passed as an agrument and flushing overflowed requests. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 85 ++++++++++++++++++++++++++++------------------------------- 1 file changed, 40 insertions(+), 45 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 24ad36d71289..a750c504366d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1003,9 +1003,9 @@ enum io_mem_account { ACCT_PINNED, }; -static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, - struct task_struct *task); - +static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, + struct task_struct *task, + struct files_struct *files); static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node); static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( struct io_ring_ctx *ctx); @@ -8817,7 +8817,7 @@ static void io_ring_exit_work(struct work_struct *work) * as nobody else will be looking for them. */ do { - __io_uring_cancel_task_requests(ctx, NULL); + io_uring_try_cancel_requests(ctx, NULL, NULL); } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)); io_ring_ctx_free(ctx); } @@ -8931,6 +8931,40 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx, } } +static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, + struct task_struct *task, + struct files_struct *files) +{ + struct io_task_cancel cancel = { .task = task, .files = files, }; + + while (1) { + enum io_wq_cancel cret; + bool ret = false; + + if (ctx->io_wq) { + cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, + &cancel, true); + ret |= (cret != IO_WQ_CANCEL_NOTFOUND); + } + + /* SQPOLL thread does its own polling */ + if (!(ctx->flags & IORING_SETUP_SQPOLL) && !files) { + while (!list_empty_careful(&ctx->iopoll_list)) { + io_iopoll_try_reap_events(ctx); + ret = true; + } + } + + ret |= io_poll_remove_all(ctx, task, files); + ret |= io_kill_timeouts(ctx, task, files); + ret |= io_run_task_work(); + io_cqring_overflow_flush(ctx, true, task, files); + if (!ret) + break; + cond_resched(); + } +} + static int io_uring_count_inflight(struct io_ring_ctx *ctx, struct task_struct *task, struct files_struct *files) @@ -8950,7 +8984,6 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, struct files_struct *files) { while (!list_empty_careful(&ctx->inflight_list)) { - struct io_task_cancel cancel = { .task = task, .files = files }; DEFINE_WAIT(wait); int inflight; @@ -8958,13 +8991,7 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, if (!inflight) break; - io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true); - io_poll_remove_all(ctx, task, files); - io_kill_timeouts(ctx, task, files); - io_cqring_overflow_flush(ctx, true, task, files); - /* cancellations _may_ trigger task work */ - io_run_task_work(); - + io_uring_try_cancel_requests(ctx, task, files); prepare_to_wait(&task->io_uring->wait, &wait, TASK_UNINTERRUPTIBLE); if (inflight == io_uring_count_inflight(ctx, task, files)) @@ -8973,37 +9000,6 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, } } -static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx, - struct task_struct *task) -{ - while (1) { - struct io_task_cancel cancel = { .task = task, .files = NULL, }; - enum io_wq_cancel cret; - bool ret = false; - - if (ctx->io_wq) { - cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, - &cancel, true); - ret |= (cret != IO_WQ_CANCEL_NOTFOUND); - } - - /* SQPOLL thread does its own polling */ - if (!(ctx->flags & IORING_SETUP_SQPOLL)) { - while (!list_empty_careful(&ctx->iopoll_list)) { - io_iopoll_try_reap_events(ctx); - ret = true; - } - } - - ret |= io_poll_remove_all(ctx, task, NULL); - ret |= io_kill_timeouts(ctx, task, NULL); - ret |= io_run_task_work(); - if (!ret) - break; - cond_resched(); - } -} - static void io_disable_sqo_submit(struct io_ring_ctx *ctx) { mutex_lock(&ctx->uring_lock); @@ -9033,11 +9029,10 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, } io_cancel_defer_files(ctx, task, files); - io_cqring_overflow_flush(ctx, true, task, files); io_uring_cancel_files(ctx, task, files); if (!files) - __io_uring_cancel_task_requests(ctx, task); + io_uring_try_cancel_requests(ctx, task, NULL); if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { atomic_dec(&task->io_uring->in_idle); -- cgit v1.2.3 From c1d5a224683b333ddbe278e455d639ccd4f5ca2b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:51:57 +0000 Subject: io_uring: refactor scheduling in io_cqring_wait schedule_timeout() with timeout=MAX_SCHEDULE_TIMEOUT is guaranteed to work just as schedule(), so instead of hand-coding it based on arguments always use the timeout version and simplify code. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index a750c504366d..5b735635b8f0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7213,9 +7213,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, .to_wait = min_events, }; struct io_rings *rings = ctx->rings; - struct timespec64 ts; - signed long timeout = 0; - int ret = 0; + signed long timeout = MAX_SCHEDULE_TIMEOUT; + int ret; do { io_cqring_overflow_flush(ctx, false, NULL, NULL); @@ -7239,6 +7238,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, } if (uts) { + struct timespec64 ts; + if (get_timespec64(&ts, uts)) return -EFAULT; timeout = timespec64_to_jiffies(&ts); @@ -7264,14 +7265,10 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, finish_wait(&ctx->wait, &iowq.wq); continue; } - if (uts) { - timeout = schedule_timeout(timeout); - if (timeout == 0) { - ret = -ETIME; - break; - } - } else { - schedule(); + timeout = schedule_timeout(timeout); + if (timeout == 0) { + ret = -ETIME; + break; } } while (1); finish_wait(&ctx->wait, &iowq.wq); -- cgit v1.2.3 From eeb60b9ab4000d20261973642dfc9fb0e4b5d073 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:51:58 +0000 Subject: io_uring: refactor io_cqring_wait It's easy to make a mistake in io_cqring_wait() because for all break/continue clauses we need to watch for prepare/finish_wait to be used correctly. Extract all those into a new helper io_cqring_wait_schedule(), and transforming the loop into simple series of func calls: prepare(); check_and_schedule(); finish(); Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5b735635b8f0..dcb9e937daa3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7195,6 +7195,25 @@ static int io_run_task_work_sig(void) return -EINTR; } +/* when returns >0, the caller should retry */ +static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, + struct io_wait_queue *iowq, + signed long *timeout) +{ + int ret; + + /* make sure we run task_work before checking for signals */ + ret = io_run_task_work_sig(); + if (ret || io_should_wake(iowq)) + return ret; + /* let the caller flush overflows, retry */ + if (test_bit(0, &ctx->cq_check_overflow)) + return 1; + + *timeout = schedule_timeout(*timeout); + return !*timeout ? -ETIME : 1; +} + /* * Wait until events become available, if we don't already have some. The * application must reap them itself, as they reside on the shared cq ring. @@ -7251,27 +7270,9 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, io_cqring_overflow_flush(ctx, false, NULL, NULL); prepare_to_wait_exclusive(&ctx->wait, &iowq.wq, TASK_INTERRUPTIBLE); - /* make sure we run task_work before checking for signals */ - ret = io_run_task_work_sig(); - if (ret > 0) { - finish_wait(&ctx->wait, &iowq.wq); - continue; - } - else if (ret < 0) - break; - if (io_should_wake(&iowq)) - break; - if (test_bit(0, &ctx->cq_check_overflow)) { - finish_wait(&ctx->wait, &iowq.wq); - continue; - } - timeout = schedule_timeout(timeout); - if (timeout == 0) { - ret = -ETIME; - break; - } - } while (1); - finish_wait(&ctx->wait, &iowq.wq); + ret = io_cqring_wait_schedule(ctx, &iowq, &timeout); + finish_wait(&ctx->wait, &iowq.wq); + } while (ret > 0); restore_saved_sigmask_unless(ret == -EINTR); -- cgit v1.2.3 From 6713e7a6145a4b5a61e33a37f0b4d06ca6d2c6d8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:51:59 +0000 Subject: io_uring: refactor io_read for unsupported nowait !io_file_supports_async() case of io_read() is hard to read, it jumps somewhere in the middle of the function just to do async setup and fail on a similar check. Call io_setup_async_rw() directly for this case, it's much easier to follow. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index dcb9e937daa3..866e0ea83dbe 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3506,7 +3506,6 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, struct iov_iter __iter, *iter = &__iter; struct io_async_rw *rw = req->async_data; ssize_t io_size, ret, ret2; - bool no_async; if (rw) { iter = &rw->iter; @@ -3527,9 +3526,12 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, kiocb->ki_flags |= IOCB_NOWAIT; /* If the file doesn't support async, just async punt */ - no_async = force_nonblock && !io_file_supports_async(req->file, READ); - if (no_async) - goto copy_iov; + if (force_nonblock && !io_file_supports_async(req->file, READ)) { + ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true); + if (!ret) + return -EAGAIN; + goto out_free; + } ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size); if (unlikely(ret)) @@ -3568,8 +3570,6 @@ copy_iov: ret = ret2; goto out_free; } - if (no_async) - return -EAGAIN; rw = req->async_data; /* it's copied and will be cleaned with ->io */ iovec = NULL; -- cgit v1.2.3 From 1a2cc0ce8d18c9e5592733cb6381e9ff5c23d916 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:00 +0000 Subject: io_uring: further simplify do_read error parsing First, instead of checking iov_iter_count(iter) for 0 to find out that all needed bytes were read, just compare returned code against io_size. It's more reliable and arguably cleaner. Also, place the half-read case into an else branch and delete an extra label. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 866e0ea83dbe..1d1fa1f77332 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3552,19 +3552,18 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, /* some cases will consume bytes even on error returns */ iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = 0; - goto copy_iov; - } else if (ret <= 0) { + } else if (ret <= 0 || ret == io_size) { /* make sure -ERESTARTSYS -> -EINTR is done */ goto done; - } + } else { + /* we did blocking attempt. no retry. */ + if (!force_nonblock || (req->file->f_flags & O_NONBLOCK) || + !(req->flags & REQ_F_ISREG)) + goto done; - /* read it all, or we did blocking attempt. no retry. */ - if (!iov_iter_count(iter) || !force_nonblock || - (req->file->f_flags & O_NONBLOCK) || !(req->flags & REQ_F_ISREG)) - goto done; + io_size -= ret; + } - io_size -= ret; -copy_iov: ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); if (ret2) { ret = ret2; -- cgit v1.2.3 From 6bf985dc50dd882a95fffa9c7eef0d1416f512e6 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:01 +0000 Subject: io_uring: let io_setup_async_rw take care of iovec Now we give out ownership of iovec into io_setup_async_rw(), so it either sets request's context right or frees the iovec on error itself. Makes our life a bit easier at call sites. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 1d1fa1f77332..f8492d62b6a1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2721,11 +2721,7 @@ static bool io_resubmit_prep(struct io_kiocb *req) ret = io_import_iovec(rw, req, &iovec, &iter, false); if (ret < 0) return false; - ret = io_setup_async_rw(req, iovec, inline_vecs, &iter, false); - if (!ret) - return true; - kfree(iovec); - return false; + return !io_setup_async_rw(req, iovec, inline_vecs, &iter, false); } #endif @@ -3366,8 +3362,10 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, if (!force && !io_op_defs[req->opcode].needs_async_data) return 0; if (!req->async_data) { - if (__io_alloc_async_data(req)) + if (__io_alloc_async_data(req)) { + kfree(iovec); return -ENOMEM; + } io_req_map_rw(req, iovec, fast_iov, iter); } @@ -3528,9 +3526,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, /* If the file doesn't support async, just async punt */ if (force_nonblock && !io_file_supports_async(req->file, READ)) { ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true); - if (!ret) - return -EAGAIN; - goto out_free; + return ret ?: -EAGAIN; } ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size); @@ -3565,10 +3561,9 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, } ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); - if (ret2) { - ret = ret2; - goto out_free; - } + if (ret2) + return ret2; + rw = req->async_data; /* it's copied and will be cleaned with ->io */ iovec = NULL; @@ -3703,8 +3698,7 @@ copy_iov: /* some cases will consume bytes even on error returns */ iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false); - if (!ret) - return -EAGAIN; + return ret ?: -EAGAIN; } out_free: /* it's reportedly faster than delegating the null check to kfree() */ -- cgit v1.2.3 From 7335e3bf9d0a92be09bb4f38d06ab22c40f0fead Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:02 +0000 Subject: io_uring: don't forget to adjust io_size We have invariant in io_read() of how much we're trying to read spilled into an iter and io_size variable. The last one controls decision making about whether to do read-retries. However, io_size is modified only after the first read attempt, so if we happen to go for a third retry in a single call to io_read(), we will get io_size greater than in the iterator, so may lead to various side effects up to live-locking. Modify io_size each time. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index f8492d62b6a1..25fffff27c76 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3548,16 +3548,11 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, /* some cases will consume bytes even on error returns */ iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = 0; - } else if (ret <= 0 || ret == io_size) { - /* make sure -ERESTARTSYS -> -EINTR is done */ + } else if (ret <= 0 || ret == io_size || !force_nonblock || + (req->file->f_flags & O_NONBLOCK) || + !(req->flags & REQ_F_ISREG)) { + /* read all, failed, already did sync or don't want to retry */ goto done; - } else { - /* we did blocking attempt. no retry. */ - if (!force_nonblock || (req->file->f_flags & O_NONBLOCK) || - !(req->flags & REQ_F_ISREG)) - goto done; - - io_size -= ret; } ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); @@ -3570,6 +3565,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, /* now use our persistent iterator, if we aren't already */ iter = &rw->iter; retry: + io_size -= ret; rw->bytes_done += ret; /* if we can retry, do so with the callbacks armed */ if (!io_rw_should_retry(req)) { -- cgit v1.2.3 From 5ea5dd45844d1b727ab2a76f47d6e9aa65d1e921 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:03 +0000 Subject: io_uring: inline io_read()'s iovec freeing io_read() has not the simpliest control flow with a lot of jumps and it's hard to read. One of those is a out_free: label, which frees iovec. However, from the middle of io_read() iovec is NULL'ed and so kfree(iovec) is no-op, it leaves us with two place where we can inline it and further clean up the code. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 25fffff27c76..35ad889afaec 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3530,14 +3530,18 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, } ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size); - if (unlikely(ret)) - goto out_free; + if (unlikely(ret)) { + kfree(iovec); + return ret; + } ret = io_iter_do_read(req, iter); if (ret == -EIOCBQUEUED) { - ret = 0; - goto out_free; + /* it's faster to check here then delegate to kfree */ + if (iovec) + kfree(iovec); + return 0; } else if (ret == -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) @@ -3560,8 +3564,6 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, return ret2; rw = req->async_data; - /* it's copied and will be cleaned with ->io */ - iovec = NULL; /* now use our persistent iterator, if we aren't already */ iter = &rw->iter; retry: @@ -3580,21 +3582,14 @@ retry: * do, then just retry at the new offset. */ ret = io_iter_do_read(req, iter); - if (ret == -EIOCBQUEUED) { - ret = 0; - goto out_free; - } else if (ret > 0 && ret < io_size) { - /* we got some bytes, but not all. retry. */ + if (ret == -EIOCBQUEUED) + return 0; + /* we got some bytes, but not all. retry. */ + if (ret > 0 && ret < io_size) goto retry; - } done: kiocb_done(kiocb, ret, cs); - ret = 0; -out_free: - /* it's reportedly faster than delegating the null check to kfree() */ - if (iovec) - kfree(iovec); - return ret; + return 0; } static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -- cgit v1.2.3 From b23df91bff954ebd8aee39eb22e5028f41cd9e56 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:04 +0000 Subject: io_uring: highlight read-retry loop We already have implicit do-while for read-retries but with goto in the end. Convert it to an actual do-while, it highlights it so making a bit more understandable and is cleaner in general. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 35ad889afaec..bbf8ea8370d6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3566,27 +3566,27 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, rw = req->async_data; /* now use our persistent iterator, if we aren't already */ iter = &rw->iter; -retry: - io_size -= ret; - rw->bytes_done += ret; - /* if we can retry, do so with the callbacks armed */ - if (!io_rw_should_retry(req)) { - kiocb->ki_flags &= ~IOCB_WAITQ; - return -EAGAIN; - } - /* - * Now retry read with the IOCB_WAITQ parts set in the iocb. If we - * get -EIOCBQUEUED, then we'll get a notification when the desired - * page gets unlocked. We can also get a partial read here, and if we - * do, then just retry at the new offset. - */ - ret = io_iter_do_read(req, iter); - if (ret == -EIOCBQUEUED) - return 0; - /* we got some bytes, but not all. retry. */ - if (ret > 0 && ret < io_size) - goto retry; + do { + io_size -= ret; + rw->bytes_done += ret; + /* if we can retry, do so with the callbacks armed */ + if (!io_rw_should_retry(req)) { + kiocb->ki_flags &= ~IOCB_WAITQ; + return -EAGAIN; + } + + /* + * Now retry read with the IOCB_WAITQ parts set in the iocb. If + * we get -EIOCBQUEUED, then we'll get a notification when the + * desired page gets unlocked. We can also get a partial read + * here, and if we do, then just retry at the new offset. + */ + ret = io_iter_do_read(req, iter); + if (ret == -EIOCBQUEUED) + return 0; + /* we got some bytes, but not all. retry. */ + } while (ret > 0 && ret < io_size); done: kiocb_done(kiocb, ret, cs); return 0; -- cgit v1.2.3 From 75c668cdd6ca05dd9c7138a5a080c0088d72cf51 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:05 +0000 Subject: io_uring: treat NONBLOCK and RWF_NOWAIT similarly Make decision making of whether we need to retry read/write similar for O_NONBLOCK and RWF_NOWAIT. Set REQ_F_NOWAIT when either is specified and use it for all relevant checks. Also fix resubmitting NOWAIT requests via io_rw_reissue(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index bbf8ea8370d6..ce2ea3f55f65 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2734,7 +2734,9 @@ static bool io_rw_reissue(struct io_kiocb *req, long res) if (res != -EAGAIN && res != -EOPNOTSUPP) return false; mode = file_inode(req->file)->i_mode; - if ((!S_ISBLK(mode) && !S_ISREG(mode)) || io_wq_current_is_worker()) + if (!S_ISBLK(mode) && !S_ISREG(mode)) + return false; + if ((req->flags & REQ_F_NOWAIT) || io_wq_current_is_worker()) return false; lockdep_assert_held(&req->ctx->uring_lock); @@ -2907,16 +2909,17 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_ring_ctx *ctx = req->ctx; struct kiocb *kiocb = &req->rw.kiocb; + struct file *file = req->file; unsigned ioprio; int ret; - if (S_ISREG(file_inode(req->file)->i_mode)) + if (S_ISREG(file_inode(file)->i_mode)) req->flags |= REQ_F_ISREG; kiocb->ki_pos = READ_ONCE(sqe->off); - if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) { + if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) { req->flags |= REQ_F_CUR_POS; - kiocb->ki_pos = req->file->f_pos; + kiocb->ki_pos = file->f_pos; } kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp)); kiocb->ki_flags = iocb_flags(kiocb->ki_filp); @@ -2924,6 +2927,10 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (unlikely(ret)) return ret; + /* don't allow async punt for O_NONBLOCK or RWF_NOWAIT */ + if ((kiocb->ki_flags & IOCB_NOWAIT) || (file->f_flags & O_NONBLOCK)) + req->flags |= REQ_F_NOWAIT; + ioprio = READ_ONCE(sqe->ioprio); if (ioprio) { ret = ioprio_check_cap(ioprio); @@ -2934,10 +2941,6 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) } else kiocb->ki_ioprio = get_current_ioprio(); - /* don't allow async punt if RWF_NOWAIT was requested */ - if (kiocb->ki_flags & IOCB_NOWAIT) - req->flags |= REQ_F_NOWAIT; - if (ctx->flags & IORING_SETUP_IOPOLL) { if (!(kiocb->ki_flags & IOCB_DIRECT) || !kiocb->ki_filp->f_op->iopoll) @@ -3546,15 +3549,14 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, /* IOPOLL retry should happen for io-wq threads */ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) goto done; - /* no retry on NONBLOCK marked file */ - if (req->file->f_flags & O_NONBLOCK) + /* no retry on NONBLOCK nor RWF_NOWAIT */ + if (req->flags & REQ_F_NOWAIT) goto done; /* some cases will consume bytes even on error returns */ iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = 0; } else if (ret <= 0 || ret == io_size || !force_nonblock || - (req->file->f_flags & O_NONBLOCK) || - !(req->flags & REQ_F_ISREG)) { + (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) { /* read all, failed, already did sync or don't want to retry */ goto done; } @@ -3675,8 +3677,8 @@ static int io_write(struct io_kiocb *req, bool force_nonblock, */ if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) ret2 = -EAGAIN; - /* no retry on NONBLOCK marked file */ - if (ret2 == -EAGAIN && (req->file->f_flags & O_NONBLOCK)) + /* no retry on NONBLOCK nor RWF_NOWAIT */ + if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) goto done; if (!force_nonblock || ret2 != -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ -- cgit v1.2.3 From 847595de1732a6e928f241929d24dde2e9ffaf15 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:06 +0000 Subject: io_uring: io_import_iovec return type cleanup io_import_iovec() doesn't return IO size anymore, only error code. Make it more apparent by returning int instead of ssize and clean up leftovers. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index ce2ea3f55f65..24cc00ff7155 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1030,9 +1030,8 @@ static struct file *io_file_get(struct io_submit_state *state, static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs); static void io_rsrc_put_work(struct work_struct *work); -static ssize_t io_import_iovec(int rw, struct io_kiocb *req, - struct iovec **iovec, struct iov_iter *iter, - bool needs_lock); +static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, + struct iov_iter *iter, bool needs_lock); static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, const struct iovec *fast_iov, struct iov_iter *iter, bool force); @@ -2693,9 +2692,8 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res, static bool io_resubmit_prep(struct io_kiocb *req) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; - ssize_t ret = -ECANCELED; + int rw, ret = -ECANCELED; struct iov_iter iter; - int rw; /* already prepared */ if (req->async_data) @@ -3004,8 +3002,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, io_rw_done(kiocb, ret); } -static ssize_t io_import_fixed(struct io_kiocb *req, int rw, - struct iov_iter *iter) +static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter) { struct io_ring_ctx *ctx = req->ctx; size_t len = req->rw.len; @@ -3069,7 +3066,7 @@ static ssize_t io_import_fixed(struct io_kiocb *req, int rw, } } - return len; + return 0; } static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock) @@ -3210,16 +3207,14 @@ static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, return __io_iov_buffer_select(req, iov, needs_lock); } -static ssize_t io_import_iovec(int rw, struct io_kiocb *req, - struct iovec **iovec, struct iov_iter *iter, - bool needs_lock) +static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, + struct iov_iter *iter, bool needs_lock) { void __user *buf = u64_to_user_ptr(req->rw.addr); size_t sqe_len = req->rw.len; + u8 opcode = req->opcode; ssize_t ret; - u8 opcode; - opcode = req->opcode; if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) { *iovec = NULL; return io_import_fixed(req, rw, iter); @@ -3244,10 +3239,8 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, if (req->flags & REQ_F_BUFFER_SELECT) { ret = io_iov_buffer_select(req, *iovec, needs_lock); - if (!ret) { - ret = (*iovec)->iov_len; - iov_iter_init(iter, rw, *iovec, 1, ret); - } + if (!ret) + iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len); *iovec = NULL; return ret; } @@ -3379,7 +3372,7 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw) { struct io_async_rw *iorw = req->async_data; struct iovec *iov = iorw->fast_iov; - ssize_t ret; + int ret; ret = io_import_iovec(rw, req, &iov, &iorw->iter, false); if (unlikely(ret < 0)) @@ -3518,7 +3511,6 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, } io_size = iov_iter_count(iter); req->result = io_size; - ret = 0; /* Ensure we clear previously set non-block flag */ if (!force_nonblock) -- cgit v1.2.3 From ea64ec02b31d5b05ae94ac4d57e38f8a02117c76 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:07 +0000 Subject: io_uring: deduplicate file table slot calculation Extract a helper io_fixed_file_slot() returning a place in our fixed files table, so we don't hand-code it three times in the code. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 24cc00ff7155..5ee6a9273fca 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7740,6 +7740,15 @@ static void io_rsrc_put_work(struct work_struct *work) } } +static struct file **io_fixed_file_slot(struct fixed_rsrc_data *file_data, + unsigned i) +{ + struct fixed_rsrc_table *table; + + table = &file_data->table[i >> IORING_FILE_TABLE_SHIFT]; + return &table->files[i & IORING_FILE_TABLE_MASK]; +} + static void io_rsrc_node_ref_zero(struct percpu_ref *ref) { struct fixed_rsrc_ref_node *ref_node; @@ -7808,6 +7817,7 @@ static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node) kfree(ref_node); } + static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) { @@ -7840,9 +7850,6 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, goto out_free; for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { - struct fixed_rsrc_table *table; - unsigned index; - if (copy_from_user(&fd, &fds[i], sizeof(fd))) { ret = -EFAULT; goto out_fput; @@ -7867,9 +7874,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, fput(file); goto out_fput; } - table = &file_data->table[i >> IORING_FILE_TABLE_SHIFT]; - index = i & IORING_FILE_TABLE_MASK; - table->files[index] = file; + *io_fixed_file_slot(file_data, i) = file; } ret = io_sqe_files_scm(ctx); @@ -7972,7 +7977,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, { struct fixed_rsrc_data *data = ctx->file_data; struct fixed_rsrc_ref_node *ref_node; - struct file *file; + struct file *file, **file_slot; __s32 __user *fds; int fd, i, err; __u32 done; @@ -7990,9 +7995,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, fds = u64_to_user_ptr(up->data); for (done = 0; done < nr_args; done++) { - struct fixed_rsrc_table *table; - unsigned index; - err = 0; if (copy_from_user(&fd, &fds[done], sizeof(fd))) { err = -EFAULT; @@ -8002,14 +8004,13 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, continue; i = array_index_nospec(up->offset + done, ctx->nr_user_files); - table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; - index = i & IORING_FILE_TABLE_MASK; - if (table->files[index]) { - file = table->files[index]; - err = io_queue_file_removal(data, file); + file_slot = io_fixed_file_slot(ctx->file_data, i); + + if (*file_slot) { + err = io_queue_file_removal(data, *file_slot); if (err) break; - table->files[index] = NULL; + *file_slot = NULL; needs_switch = true; } if (fd != -1) { @@ -8031,13 +8032,12 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EBADF; break; } - table->files[index] = file; err = io_sqe_file_register(ctx, file, i); if (err) { - table->files[index] = NULL; fput(file); break; } + *file_slot = file; } } @@ -9488,11 +9488,8 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1); seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files); for (i = 0; has_lock && i < ctx->nr_user_files; i++) { - struct fixed_rsrc_table *table; - struct file *f; + struct file *f = *io_fixed_file_slot(ctx->file_data, i); - table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT]; - f = table->files[i & IORING_FILE_TABLE_MASK]; if (f) seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname); else -- cgit v1.2.3 From 5280f7e530f71ba85baf90169393196976ad0e52 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 13:52:08 +0000 Subject: io_uring/io-wq: return 2-step work swap scheme Saving one lock/unlock for io-wq is not super important, but adds some ugliness in the code. More important, atomic decs not turning it to zero for some archs won't give the right ordering/barriers so the io_steal_work() may pretty easily get subtly and completely broken. Return back 2-step io-wq work exchange and clean it up. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io-wq.c | 16 ++++++---------- fs/io-wq.h | 4 ++-- fs/io_uring.c | 26 ++++---------------------- 3 files changed, 12 insertions(+), 34 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 2e2f14f42bf2..63ef195b1acb 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -555,23 +555,21 @@ get_next: /* handle a whole dependent link */ do { - struct io_wq_work *old_work, *next_hashed, *linked; + struct io_wq_work *next_hashed, *linked; unsigned int hash = io_get_work_hash(work); next_hashed = wq_next_work(work); io_impersonate_work(worker, work); + wq->do_work(work); + io_assign_current_work(worker, NULL); - old_work = work; - linked = wq->do_work(work); - + linked = wq->free_work(work); work = next_hashed; if (!work && linked && !io_wq_is_hashed(linked)) { work = linked; linked = NULL; } io_assign_current_work(worker, work); - wq->free_work(old_work); - if (linked) io_wqe_enqueue(wqe, linked); @@ -850,11 +848,9 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe) struct io_wq *wq = wqe->wq; do { - struct io_wq_work *old_work = work; - work->flags |= IO_WQ_WORK_CANCEL; - work = wq->do_work(work); - wq->free_work(old_work); + wq->do_work(work); + work = wq->free_work(work); } while (work); } diff --git a/fs/io-wq.h b/fs/io-wq.h index e1ffb80a4a1d..e37a0f217cc8 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -106,8 +106,8 @@ static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) return container_of(work->list.next, struct io_wq_work, list); } -typedef void (free_work_fn)(struct io_wq_work *); -typedef struct io_wq_work *(io_wq_work_fn)(struct io_wq_work *); +typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *); +typedef void (io_wq_work_fn)(struct io_wq_work *); struct io_wq_data { struct user_struct *user; diff --git a/fs/io_uring.c b/fs/io_uring.c index 5ee6a9273fca..b740a39110d6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2379,22 +2379,6 @@ static inline void io_put_req_deferred(struct io_kiocb *req, int refs) io_free_req_deferred(req); } -static struct io_wq_work *io_steal_work(struct io_kiocb *req) -{ - struct io_kiocb *nxt; - - /* - * A ref is owned by io-wq in which context we're. So, if that's the - * last one, it's safe to steal next work. False negatives are Ok, - * it just will be re-punted async in io_put_work() - */ - if (refcount_read(&req->refs) != 1) - return NULL; - - nxt = io_req_find_next(req); - return nxt ? &nxt->work : NULL; -} - static void io_double_put_req(struct io_kiocb *req) { /* drop both submit and complete references */ @@ -6343,7 +6327,7 @@ static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, return 0; } -static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work) +static void io_wq_submit_work(struct io_wq_work *work) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); struct io_kiocb *timeout; @@ -6394,8 +6378,6 @@ static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work) if (lock_ctx) mutex_unlock(&lock_ctx->uring_lock); } - - return io_steal_work(req); } static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, @@ -8067,12 +8049,12 @@ static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg, return __io_sqe_files_update(ctx, &up, nr_args); } -static void io_free_work(struct io_wq_work *work) +static struct io_wq_work *io_free_work(struct io_wq_work *work) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); - /* Consider that io_steal_work() relies on this ref */ - io_put_req(req); + req = io_put_req_find_next(req); + return req ? &req->work : NULL; } static int io_init_wq_offload(struct io_ring_ctx *ctx, -- cgit v1.2.3 From 45068063efb7dd0a8d115c106aa05d9ab0946257 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 2 Feb 2021 11:13:58 -0800 Subject: xfs: fix incorrect root dquot corruption error when switching group/project quota types While writing up a regression test for broken behavior when a chprojid request fails, I noticed that we were logging corruption notices about the root dquot of the group/project quota file at mount time when testing V4 filesystems. In commit afeda6000b0c, I was trying to improve ondisk dquot validation by making sure that when we load an ondisk dquot into memory on behalf of an incore dquot, the dquot id and type matches. Unfortunately, I forgot that V4 filesystems only have two quota files, and can switch that file between group and project quota types at mount time. When we perform that switch, we'll try to load the default quota limits from the root dquot prior to running quotacheck and log a corruption error when the types don't match. This is inconsequential because quotacheck will reset the second quota file as part of doing the switch, but we shouldn't leave scary messages in the kernel log. Fixes: afeda6000b0c ("xfs: validate ondisk/incore dquot flags") Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Chandan Babu R --- fs/xfs/xfs_dquot.c | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 175f544f7c45..bd8379b98374 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -506,6 +506,42 @@ xfs_dquot_alloc( return dqp; } +/* Check the ondisk dquot's id and type match what the incore dquot expects. */ +static bool +xfs_dquot_check_type( + struct xfs_dquot *dqp, + struct xfs_disk_dquot *ddqp) +{ + uint8_t ddqp_type; + uint8_t dqp_type; + + ddqp_type = ddqp->d_type & XFS_DQTYPE_REC_MASK; + dqp_type = xfs_dquot_type(dqp); + + if (be32_to_cpu(ddqp->d_id) != dqp->q_id) + return false; + + /* + * V5 filesystems always expect an exact type match. V4 filesystems + * expect an exact match for user dquots and for non-root group and + * project dquots. + */ + if (xfs_sb_version_hascrc(&dqp->q_mount->m_sb) || + dqp_type == XFS_DQTYPE_USER || dqp->q_id != 0) + return ddqp_type == dqp_type; + + /* + * V4 filesystems support either group or project quotas, but not both + * at the same time. The non-user quota file can be switched between + * group and project quota uses depending on the mount options, which + * means that we can encounter the other type when we try to load quota + * defaults. Quotacheck will soon reset the the entire quota file + * (including the root dquot) anyway, but don't log scary corruption + * reports to dmesg. + */ + return ddqp_type == XFS_DQTYPE_GROUP || ddqp_type == XFS_DQTYPE_PROJ; +} + /* Copy the in-core quota fields in from the on-disk buffer. */ STATIC int xfs_dquot_from_disk( @@ -518,8 +554,7 @@ xfs_dquot_from_disk( * Ensure that we got the type and ID we were looking for. * Everything else was checked by the dquot buffer verifier. */ - if ((ddqp->d_type & XFS_DQTYPE_REC_MASK) != xfs_dquot_type(dqp) || - be32_to_cpu(ddqp->d_id) != dqp->q_id) { + if (!xfs_dquot_check_type(dqp, ddqp)) { xfs_alert_tag(bp->b_mount, XFS_PTAG_VERIFIER_ERROR, "Metadata corruption detected at %pS, quota %u", __this_address, dqp->q_id); -- cgit v1.2.3 From aec18a57edad562d620f7d19016de1fc0cc2208c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Feb 2021 19:22:46 +0000 Subject: io_uring: drop mm/files between task_work_submit Since SQPOLL task can be shared and so task_work entries can be a mix of them, we need to drop mm and files before trying to issue next request. Cc: stable@vger.kernel.org # 5.10+ Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5d3348d66f06..1f68105a41ed 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2205,6 +2205,9 @@ static void __io_req_task_submit(struct io_kiocb *req) else __io_req_task_cancel(req, -EFAULT); mutex_unlock(&ctx->uring_lock); + + if (ctx->flags & IORING_SETUP_SQPOLL) + io_sq_thread_drop_mm_files(); } static void io_req_task_submit(struct callback_head *cb) -- cgit v1.2.3 From 91792bb8089b63b7b780251eb83939348ac58a64 Mon Sep 17 00:00:00 2001 From: Pavel Shilovsky Date: Tue, 2 Feb 2021 22:34:32 -0600 Subject: smb3: fix crediting for compounding when only one request in flight Currently we try to guess if a compound request is going to succeed waiting for credits or not based on the number of requests in flight. This approach doesn't work correctly all the time because there may be only one request in flight which is going to bring multiple credits satisfying the compound request. Change the behavior to fail a request only if there are no requests in flight at all and proceed waiting for credits otherwise. Cc: # 5.1+ Signed-off-by: Pavel Shilovsky Reviewed-by: Tom Talpey Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/transport.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c index 95ef26b555b9..4a2b836eb017 100644 --- a/fs/cifs/transport.c +++ b/fs/cifs/transport.c @@ -666,10 +666,22 @@ wait_for_compound_request(struct TCP_Server_Info *server, int num, if (*credits < num) { /* - * Return immediately if not too many requests in flight since - * we will likely be stuck on waiting for credits. + * If the server is tight on resources or just gives us less + * credits for other reasons (e.g. requests are coming out of + * order and the server delays granting more credits until it + * processes a missing mid) and we exhausted most available + * credits there may be situations when we try to send + * a compound request but we don't have enough credits. At this + * point the client needs to decide if it should wait for + * additional credits or fail the request. If at least one + * request is in flight there is a high probability that the + * server will return enough credits to satisfy this compound + * request. + * + * Return immediately if no requests in flight since we will be + * stuck on waiting for credits. */ - if (server->in_flight < num - *credits) { + if (server->in_flight == 0) { spin_unlock(&server->req_lock); trace_smb3_insufficient_credits(server->CurrentMid, server->hostname, scredits, sin_flight); -- cgit v1.2.3 From 2a7808024b195a342779fb5d7b7df1c4af45cc71 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 5 Feb 2021 00:57:58 +0000 Subject: io_uring: set msg_name on msg fixup io_setup_async_msg() should fully prepare io_async_msghdr, let it also handle assigning msg_name and don't hand code it in [send,recv]msg(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index b740a39110d6..39bc1df9bb64 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4558,6 +4558,7 @@ static int io_setup_async_msg(struct io_kiocb *req, async_msg = req->async_data; req->flags |= REQ_F_NEED_CLEANUP; memcpy(async_msg, kmsg, sizeof(*kmsg)); + async_msg->msg.msg_name = &async_msg->addr; return -EAGAIN; } @@ -4610,7 +4611,6 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, if (req->async_data) { kmsg = req->async_data; - kmsg->msg.msg_name = &kmsg->addr; /* if iov is set, it's allocated already */ if (!kmsg->iov) kmsg->iov = kmsg->fast_iov; @@ -4839,7 +4839,6 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, if (req->async_data) { kmsg = req->async_data; - kmsg->msg.msg_name = &kmsg->addr; /* if iov is set, it's allocated already */ if (!kmsg->iov) kmsg->iov = kmsg->fast_iov; -- cgit v1.2.3 From 5476dfed29ad9b19d4e187685ab71bb9c496f965 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 5 Feb 2021 00:57:59 +0000 Subject: io_uring: clean iov usage for recvmsg buf select Don't pretend we don't know that REQ_F_BUFFER_SELECT for recvmsg always uses fast_iov -- clean up confusing intermixing kmsg->iov and kmsg->fast_iov for buffer select. Also don't init iter with garbage in __io_recvmsg_copy_hdr() only for it to be set shortly after in io_recvmsg(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 39bc1df9bb64..e07a7fa15cfa 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4701,11 +4701,9 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb *req, if (req->flags & REQ_F_BUFFER_SELECT) { if (iov_len > 1) return -EINVAL; - if (copy_from_user(iomsg->iov, uiov, sizeof(*uiov))) + if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov))) return -EFAULT; - sr->len = iomsg->iov[0].iov_len; - iov_iter_init(&iomsg->msg.msg_iter, READ, iomsg->iov, 1, - sr->len); + sr->len = iomsg->fast_iov[0].iov_len; iomsg->iov = NULL; } else { ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, @@ -4748,7 +4746,6 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, if (clen < 0) return -EINVAL; sr->len = clen; - iomsg->iov[0].iov_len = clen; iomsg->iov = NULL; } else { ret = __import_iovec(READ, (struct iovec __user *)uiov, len, @@ -4855,7 +4852,8 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, if (IS_ERR(kbuf)) return PTR_ERR(kbuf); kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr); - iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov, + kmsg->fast_iov[0].iov_len = req->sr_msg.len; + iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1, req->sr_msg.len); } -- cgit v1.2.3 From 257e84a5377fbbc336ff563833a8712619acce56 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 5 Feb 2021 00:58:00 +0000 Subject: io_uring: refactor sendmsg/recvmsg iov managing Current iov handling with recvmsg/sendmsg may be confusing. First make a rule for msg->iov: either it points to an allocated iov that have to be kfree()'d later, or it's NULL and we use fast_iov. That's much better than current 3-state (also can point to fast_iov). And rename it into free_iov for uniformity with read/write. Also, instead of after struct io_async_msghdr copy fixing up of msg.msg_iter.iov has been happening in io_recvmsg()/io_sendmsg(). Move it into io_setup_async_msg(), that's the right place. Signed-off-by: Pavel Begunkov [axboe: add comment on NULL check before kfree()] Signed-off-by: Jens Axboe --- fs/io_uring.c | 57 +++++++++++++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 30 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index e07a7fa15cfa..7242cc48e97b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -594,7 +594,8 @@ struct io_async_connect { struct io_async_msghdr { struct iovec fast_iov[UIO_FASTIOV]; - struct iovec *iov; + /* points to an allocated iov, if NULL we use fast_iov instead */ + struct iovec *free_iov; struct sockaddr __user *uaddr; struct msghdr msg; struct sockaddr_storage addr; @@ -4551,24 +4552,27 @@ static int io_setup_async_msg(struct io_kiocb *req, if (async_msg) return -EAGAIN; if (io_alloc_async_data(req)) { - if (kmsg->iov != kmsg->fast_iov) - kfree(kmsg->iov); + kfree(kmsg->free_iov); return -ENOMEM; } async_msg = req->async_data; req->flags |= REQ_F_NEED_CLEANUP; memcpy(async_msg, kmsg, sizeof(*kmsg)); async_msg->msg.msg_name = &async_msg->addr; + /* if were using fast_iov, set it to the new one */ + if (!async_msg->free_iov) + async_msg->msg.msg_iter.iov = async_msg->fast_iov; + return -EAGAIN; } static int io_sendmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { - iomsg->iov = iomsg->fast_iov; iomsg->msg.msg_name = &iomsg->addr; + iomsg->free_iov = iomsg->fast_iov; return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg, - req->sr_msg.msg_flags, &iomsg->iov); + req->sr_msg.msg_flags, &iomsg->free_iov); } static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) @@ -4609,13 +4613,8 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, if (unlikely(!sock)) return -ENOTSOCK; - if (req->async_data) { - kmsg = req->async_data; - /* if iov is set, it's allocated already */ - if (!kmsg->iov) - kmsg->iov = kmsg->fast_iov; - kmsg->msg.msg_iter.iov = kmsg->iov; - } else { + kmsg = req->async_data; + if (!kmsg) { ret = io_sendmsg_copy_hdr(req, &iomsg); if (ret) return ret; @@ -4634,8 +4633,9 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, if (ret == -ERESTARTSYS) ret = -EINTR; - if (kmsg->iov != kmsg->fast_iov) - kfree(kmsg->iov); + /* fast path, check for non-NULL to avoid function call */ + if (kmsg->free_iov) + kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) req_set_fail_links(req); @@ -4704,10 +4704,11 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb *req, if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov))) return -EFAULT; sr->len = iomsg->fast_iov[0].iov_len; - iomsg->iov = NULL; + iomsg->free_iov = NULL; } else { + iomsg->free_iov = iomsg->fast_iov; ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV, - &iomsg->iov, &iomsg->msg.msg_iter, + &iomsg->free_iov, &iomsg->msg.msg_iter, false); if (ret > 0) ret = 0; @@ -4746,10 +4747,11 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, if (clen < 0) return -EINVAL; sr->len = clen; - iomsg->iov = NULL; + iomsg->free_iov = NULL; } else { + iomsg->free_iov = iomsg->fast_iov; ret = __import_iovec(READ, (struct iovec __user *)uiov, len, - UIO_FASTIOV, &iomsg->iov, + UIO_FASTIOV, &iomsg->free_iov, &iomsg->msg.msg_iter, true); if (ret < 0) return ret; @@ -4763,7 +4765,6 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { iomsg->msg.msg_name = &iomsg->addr; - iomsg->iov = iomsg->fast_iov; #ifdef CONFIG_COMPAT if (req->ctx->compat) @@ -4834,13 +4835,8 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, if (unlikely(!sock)) return -ENOTSOCK; - if (req->async_data) { - kmsg = req->async_data; - /* if iov is set, it's allocated already */ - if (!kmsg->iov) - kmsg->iov = kmsg->fast_iov; - kmsg->msg.msg_iter.iov = kmsg->iov; - } else { + kmsg = req->async_data; + if (!kmsg) { ret = io_recvmsg_copy_hdr(req, &iomsg); if (ret) return ret; @@ -4872,8 +4868,9 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_recv_kbuf(req); - if (kmsg->iov != kmsg->fast_iov) - kfree(kmsg->iov); + /* fast path, check for non-NULL to avoid function call */ + if (kmsg->free_iov) + kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) req_set_fail_links(req); @@ -6166,8 +6163,8 @@ static void __io_clean_op(struct io_kiocb *req) case IORING_OP_RECVMSG: case IORING_OP_SENDMSG: { struct io_async_msghdr *io = req->async_data; - if (io->iov != io->fast_iov) - kfree(io->iov); + + kfree(io->free_iov); break; } case IORING_OP_SPLICE: -- cgit v1.2.3 From 834ec3e1ee65029029225a86c12337a6cd385af7 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Fri, 5 Feb 2021 18:11:28 +0100 Subject: gfs2: Lock imbalance on error path in gfs2_recover_one In gfs2_recover_one, fix a sd_log_flush_lock imbalance when a recovery pass fails. Fixes: c9ebc4b73799 ("gfs2: allow journal replay to hold sd_log_flush_lock") Cc: stable@vger.kernel.org # v5.7+ Signed-off-by: Andreas Gruenbacher --- fs/gfs2/recovery.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/gfs2/recovery.c b/fs/gfs2/recovery.c index cff8b9a6612c..282173774005 100644 --- a/fs/gfs2/recovery.c +++ b/fs/gfs2/recovery.c @@ -512,8 +512,10 @@ void gfs2_recover_func(struct work_struct *work) error = foreach_descriptor(jd, head.lh_tail, head.lh_blkno, pass); lops_after_scan(jd, error, pass); - if (error) + if (error) { + up_read(&sdp->sd_log_flush_lock); goto fail_gunlock_thaw; + } } recover_local_statfs(jd, &head); -- cgit v1.2.3 From 585fc0d2871c9318c949fbf45b1f081edd489e96 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 4 Feb 2021 18:32:03 -0800 Subject: mm: hugetlbfs: fix cannot migrate the fallocated HugeTLB page If a new hugetlb page is allocated during fallocate it will not be marked as active (set_page_huge_active) which will result in a later isolate_huge_page failure when the page migration code would like to move that page. Such a failure would be unexpected and wrong. Only export set_page_huge_active, just leave clear_page_huge_active as static. Because there are no external users. Link: https://lkml.kernel.org/r/20210115124942.46403-3-songmuchun@bytedance.com Fixes: 70c3547e36f5 (hugetlbfs: add hugetlbfs_fallocate()) Signed-off-by: Muchun Song Acked-by: Michal Hocko Reviewed-by: Mike Kravetz Reviewed-by: Oscar Salvador Cc: David Hildenbrand Cc: Yang Shi Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 3 ++- include/linux/hugetlb.h | 2 ++ mm/hugetlb.c | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index b5c109703daa..21c20fd5f9ee 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -735,9 +735,10 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, mutex_unlock(&hugetlb_fault_mutex_table[hash]); + set_page_huge_active(page); /* * unlock_page because locked by add_to_page_cache() - * page_put due to reference from alloc_huge_page() + * put_page() due to reference from alloc_huge_page() */ unlock_page(page); put_page(page); diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index ebca2ef02212..b5807f23caf8 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -770,6 +770,8 @@ static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, } #endif +void set_page_huge_active(struct page *page); + #else /* CONFIG_HUGETLB_PAGE */ struct hstate {}; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 18f6ee317900..6f0e242d38ca 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1349,7 +1349,7 @@ bool page_huge_active(struct page *page) } /* never called for tail page */ -static void set_page_huge_active(struct page *page) +void set_page_huge_active(struct page *page) { VM_BUG_ON_PAGE(!PageHeadHuge(page), page); SetPagePrivate(&page[1]); -- cgit v1.2.3 From 78178ca844f0eb88f21f31c7fde969384be4c901 Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Fri, 5 Feb 2021 13:50:41 -0500 Subject: gfs2: Don't skip dlm unlock if glock has an lvb Patch fb6791d100d1 was designed to allow gfs2 to unmount quicker by skipping the step where it tells dlm to unlock glocks in EX with lvbs. This was done because when gfs2 unmounts a file system, it destroys the dlm lockspace shortly after it destroys the glocks so it doesn't need to unlock them all: the unlock is implied when the lockspace is destroyed by dlm. However, that patch introduced a use-after-free in dlm: as part of its normal dlm_recoverd process, it can call ls_recovery to recover dead locks. In so doing, it can call recover_rsbs which calls recover_lvb for any mastered rsbs. Func recover_lvb runs through the list of lkbs queued to the given rsb (if the glock is cached but unlocked, it will still be queued to the lkb, but in NL--Unlocked--mode) and if it has an lvb, copies it to the rsb, thus trying to preserve the lkb. However, when gfs2 skips the dlm unlock step, it frees the glock and its lvb, which means dlm's function recover_lvb references the now freed lvb pointer, copying the freed lvb memory to the rsb. This patch changes the check in gdlm_put_lock so that it calls dlm_unlock for all glocks that contain an lvb pointer. Fixes: fb6791d100d1 ("GFS2: skip dlm_unlock calls in unmount") Cc: stable@vger.kernel.org # v3.8+ Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/lock_dlm.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/lock_dlm.c b/fs/gfs2/lock_dlm.c index 9f2b5609f225..153272f82984 100644 --- a/fs/gfs2/lock_dlm.c +++ b/fs/gfs2/lock_dlm.c @@ -284,7 +284,6 @@ static void gdlm_put_lock(struct gfs2_glock *gl) { struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; struct lm_lockstruct *ls = &sdp->sd_lockstruct; - int lvb_needs_unlock = 0; int error; if (gl->gl_lksb.sb_lkid == 0) { @@ -297,13 +296,10 @@ static void gdlm_put_lock(struct gfs2_glock *gl) gfs2_sbstats_inc(gl, GFS2_LKS_DCOUNT); gfs2_update_request_times(gl); - /* don't want to skip dlm_unlock writing the lvb when lock is ex */ - - if (gl->gl_lksb.sb_lvbptr && (gl->gl_state == LM_ST_EXCLUSIVE)) - lvb_needs_unlock = 1; + /* don't want to skip dlm_unlock writing the lvb when lock has one */ if (test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags) && - !lvb_needs_unlock) { + !gl->gl_lksb.sb_lvbptr) { gfs2_glock_free(gl); return; } -- cgit v1.2.3 From 21b200d091826a83aafc95d847139b2b0582f6d1 Mon Sep 17 00:00:00 2001 From: Aurelien Aptel Date: Fri, 5 Feb 2021 15:42:48 +0100 Subject: cifs: report error instead of invalid when revalidating a dentry fails Assuming - //HOST/a is mounted on /mnt - //HOST/b is mounted on /mnt/b On a slow connection, running 'df' and killing it while it's processing /mnt/b can make cifs_get_inode_info() returns -ERESTARTSYS. This triggers the following chain of events: => the dentry revalidation fail => dentry is put and released => superblock associated with the dentry is put => /mnt/b is unmounted This patch makes cifs_d_revalidate() return the error instead of 0 (invalid) when cifs_revalidate_dentry() fails, except for ENOENT (file deleted) and ESTALE (file recreated). Signed-off-by: Aurelien Aptel Suggested-by: Shyam Prasad N Reviewed-by: Shyam Prasad N CC: stable@vger.kernel.org Signed-off-by: Steve French --- fs/cifs/dir.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/dir.c b/fs/cifs/dir.c index 68900f1629bf..97ac363b5df1 100644 --- a/fs/cifs/dir.c +++ b/fs/cifs/dir.c @@ -737,6 +737,7 @@ static int cifs_d_revalidate(struct dentry *direntry, unsigned int flags) { struct inode *inode; + int rc; if (flags & LOOKUP_RCU) return -ECHILD; @@ -746,8 +747,25 @@ cifs_d_revalidate(struct dentry *direntry, unsigned int flags) if ((flags & LOOKUP_REVAL) && !CIFS_CACHE_READ(CIFS_I(inode))) CIFS_I(inode)->time = 0; /* force reval */ - if (cifs_revalidate_dentry(direntry)) - return 0; + rc = cifs_revalidate_dentry(direntry); + if (rc) { + cifs_dbg(FYI, "cifs_revalidate_dentry failed with rc=%d", rc); + switch (rc) { + case -ENOENT: + case -ESTALE: + /* + * Those errors mean the dentry is invalid + * (file was deleted or recreated) + */ + return 0; + default: + /* + * Otherwise some unexpected error happened + * report it as-is to VFS layer + */ + return rc; + } + } else { /* * If the inode wasn't known to be a dfs entry when -- cgit v1.2.3 From c2c8261151b32f1956fc4ecd71c9a3e7972084b6 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 15 Jan 2021 10:18:14 -0800 Subject: fs-verity: factor out fsverity_get_descriptor() The FS_IOC_READ_VERITY_METADATA ioctl will need to return the fs-verity descriptor (and signature) to userspace. There are a few ways we could implement this: - Save a copy of the descriptor (and signature) in the fsverity_info struct that hangs off of the in-memory inode. However, this would waste memory since most of the time it wouldn't be needed. - Regenerate the descriptor from the merkle_tree_params in the fsverity_info. However, this wouldn't work for the signature, nor for the salt which the merkle_tree_params only contains indirectly as part of the 'hashstate'. It would also be error-prone. - Just get them from the filesystem again. The disadvantage is that in general we can't trust that they haven't been maliciously changed since the file has opened. However, the use cases for FS_IOC_READ_VERITY_METADATA don't require that it verifies the chain of trust. So this is okay as long as we do some basic validation. In preparation for implementing the third option, factor out a helper function fsverity_get_descriptor() which gets the descriptor (and appended signature) from the filesystem and does some basic validation. As part of this, start checking the sig_size field for overflow. Currently fsverity_verify_signature() does this. But the new ioctl will need this too, so do it earlier. Link: https://lore.kernel.org/r/20210115181819.34732-2-ebiggers@kernel.org Reviewed-by: Victor Hsieh Reviewed-by: Jaegeuk Kim Reviewed-by: Chao Yu Signed-off-by: Eric Biggers --- fs/verity/fsverity_private.h | 7 ++- fs/verity/open.c | 130 ++++++++++++++++++++++++++++--------------- 2 files changed, 91 insertions(+), 46 deletions(-) (limited to 'fs') diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index 6413d28664d6..6c9caccc0602 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -122,12 +122,17 @@ int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, const u8 *salt, size_t salt_size); struct fsverity_info *fsverity_create_info(const struct inode *inode, - void *desc, size_t desc_size); + struct fsverity_descriptor *desc, + size_t desc_size); void fsverity_set_info(struct inode *inode, struct fsverity_info *vi); void fsverity_free_info(struct fsverity_info *vi); +int fsverity_get_descriptor(struct inode *inode, + struct fsverity_descriptor **desc_ret, + size_t *desc_size_ret); + int __init fsverity_init_info_cache(void); void __init fsverity_exit_info_cache(void); diff --git a/fs/verity/open.c b/fs/verity/open.c index 228d0eca3e2e..a987bb785e9b 100644 --- a/fs/verity/open.c +++ b/fs/verity/open.c @@ -142,45 +142,17 @@ static int compute_file_digest(struct fsverity_hash_alg *hash_alg, } /* - * Validate the given fsverity_descriptor and create a new fsverity_info from - * it. The signature (if present) is also checked. + * Create a new fsverity_info from the given fsverity_descriptor (with optional + * appended signature), and check the signature if present. The + * fsverity_descriptor must have already undergone basic validation. */ struct fsverity_info *fsverity_create_info(const struct inode *inode, - void *_desc, size_t desc_size) + struct fsverity_descriptor *desc, + size_t desc_size) { - struct fsverity_descriptor *desc = _desc; struct fsverity_info *vi; int err; - if (desc_size < sizeof(*desc)) { - fsverity_err(inode, "Unrecognized descriptor size: %zu bytes", - desc_size); - return ERR_PTR(-EINVAL); - } - - if (desc->version != 1) { - fsverity_err(inode, "Unrecognized descriptor version: %u", - desc->version); - return ERR_PTR(-EINVAL); - } - - if (memchr_inv(desc->__reserved, 0, sizeof(desc->__reserved))) { - fsverity_err(inode, "Reserved bits set in descriptor"); - return ERR_PTR(-EINVAL); - } - - if (desc->salt_size > sizeof(desc->salt)) { - fsverity_err(inode, "Invalid salt_size: %u", desc->salt_size); - return ERR_PTR(-EINVAL); - } - - if (le64_to_cpu(desc->data_size) != inode->i_size) { - fsverity_err(inode, - "Wrong data_size: %llu (desc) != %lld (inode)", - le64_to_cpu(desc->data_size), inode->i_size); - return ERR_PTR(-EINVAL); - } - vi = kmem_cache_zalloc(fsverity_info_cachep, GFP_KERNEL); if (!vi) return ERR_PTR(-ENOMEM); @@ -245,15 +217,57 @@ void fsverity_free_info(struct fsverity_info *vi) kmem_cache_free(fsverity_info_cachep, vi); } -/* Ensure the inode has an ->i_verity_info */ -static int ensure_verity_info(struct inode *inode) +static bool validate_fsverity_descriptor(struct inode *inode, + const struct fsverity_descriptor *desc, + size_t desc_size) { - struct fsverity_info *vi = fsverity_get_info(inode); - struct fsverity_descriptor *desc; - int res; + if (desc_size < sizeof(*desc)) { + fsverity_err(inode, "Unrecognized descriptor size: %zu bytes", + desc_size); + return false; + } - if (vi) - return 0; + if (desc->version != 1) { + fsverity_err(inode, "Unrecognized descriptor version: %u", + desc->version); + return false; + } + + if (memchr_inv(desc->__reserved, 0, sizeof(desc->__reserved))) { + fsverity_err(inode, "Reserved bits set in descriptor"); + return false; + } + + if (desc->salt_size > sizeof(desc->salt)) { + fsverity_err(inode, "Invalid salt_size: %u", desc->salt_size); + return false; + } + + if (le64_to_cpu(desc->data_size) != inode->i_size) { + fsverity_err(inode, + "Wrong data_size: %llu (desc) != %lld (inode)", + le64_to_cpu(desc->data_size), inode->i_size); + return false; + } + + if (le32_to_cpu(desc->sig_size) > desc_size - sizeof(*desc)) { + fsverity_err(inode, "Signature overflows verity descriptor"); + return false; + } + + return true; +} + +/* + * Read the inode's fsverity_descriptor (with optional appended signature) from + * the filesystem, and do basic validation of it. + */ +int fsverity_get_descriptor(struct inode *inode, + struct fsverity_descriptor **desc_ret, + size_t *desc_size_ret) +{ + int res; + struct fsverity_descriptor *desc; res = inode->i_sb->s_vop->get_verity_descriptor(inode, NULL, 0); if (res < 0) { @@ -272,20 +286,46 @@ static int ensure_verity_info(struct inode *inode) res = inode->i_sb->s_vop->get_verity_descriptor(inode, desc, res); if (res < 0) { fsverity_err(inode, "Error %d reading verity descriptor", res); - goto out_free_desc; + kfree(desc); + return res; + } + + if (!validate_fsverity_descriptor(inode, desc, res)) { + kfree(desc); + return -EINVAL; } - vi = fsverity_create_info(inode, desc, res); + *desc_ret = desc; + *desc_size_ret = res; + return 0; +} + +/* Ensure the inode has an ->i_verity_info */ +static int ensure_verity_info(struct inode *inode) +{ + struct fsverity_info *vi = fsverity_get_info(inode); + struct fsverity_descriptor *desc; + size_t desc_size; + int err; + + if (vi) + return 0; + + err = fsverity_get_descriptor(inode, &desc, &desc_size); + if (err) + return err; + + vi = fsverity_create_info(inode, desc, desc_size); if (IS_ERR(vi)) { - res = PTR_ERR(vi); + err = PTR_ERR(vi); goto out_free_desc; } fsverity_set_info(inode, vi); - res = 0; + err = 0; out_free_desc: kfree(desc); - return res; + return err; } /** -- cgit v1.2.3 From fab634c4de4604aefaaa9dc25d0e1a2cb7a961ab Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 15 Jan 2021 10:18:15 -0800 Subject: fs-verity: don't pass whole descriptor to fsverity_verify_signature() Now that fsverity_get_descriptor() validates the sig_size field, fsverity_verify_signature() doesn't need to do it. Just change the prototype of fsverity_verify_signature() to take the signature directly rather than take a fsverity_descriptor. Link: https://lore.kernel.org/r/20210115181819.34732-3-ebiggers@kernel.org Reviewed-by: Victor Hsieh Reviewed-by: Jaegeuk Kim Reviewed-by: Amy Parker Reviewed-by: Chao Yu Signed-off-by: Eric Biggers --- fs/verity/fsverity_private.h | 6 ++---- fs/verity/open.c | 3 ++- fs/verity/signature.c | 20 ++++++-------------- 3 files changed, 10 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index 6c9caccc0602..a7920434bae5 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -140,15 +140,13 @@ void __init fsverity_exit_info_cache(void); #ifdef CONFIG_FS_VERITY_BUILTIN_SIGNATURES int fsverity_verify_signature(const struct fsverity_info *vi, - const struct fsverity_descriptor *desc, - size_t desc_size); + const u8 *signature, size_t sig_size); int __init fsverity_init_signature(void); #else /* !CONFIG_FS_VERITY_BUILTIN_SIGNATURES */ static inline int fsverity_verify_signature(const struct fsverity_info *vi, - const struct fsverity_descriptor *desc, - size_t desc_size) + const u8 *signature, size_t sig_size) { return 0; } diff --git a/fs/verity/open.c b/fs/verity/open.c index a987bb785e9b..60ff8af7219f 100644 --- a/fs/verity/open.c +++ b/fs/verity/open.c @@ -181,7 +181,8 @@ struct fsverity_info *fsverity_create_info(const struct inode *inode, vi->tree_params.hash_alg->name, vi->tree_params.digest_size, vi->file_digest); - err = fsverity_verify_signature(vi, desc, desc_size); + err = fsverity_verify_signature(vi, desc->signature, + le32_to_cpu(desc->sig_size)); out: if (err) { fsverity_free_info(vi); diff --git a/fs/verity/signature.c b/fs/verity/signature.c index 012468eda2a7..143a530a8008 100644 --- a/fs/verity/signature.c +++ b/fs/verity/signature.c @@ -29,21 +29,19 @@ static struct key *fsverity_keyring; /** * fsverity_verify_signature() - check a verity file's signature * @vi: the file's fsverity_info - * @desc: the file's fsverity_descriptor - * @desc_size: size of @desc + * @signature: the file's built-in signature + * @sig_size: size of signature in bytes, or 0 if no signature * - * If the file's fs-verity descriptor includes a signature of the file digest, - * verify it against the certificates in the fs-verity keyring. + * If the file includes a signature of its fs-verity file digest, verify it + * against the certificates in the fs-verity keyring. * * Return: 0 on success (signature valid or not required); -errno on failure */ int fsverity_verify_signature(const struct fsverity_info *vi, - const struct fsverity_descriptor *desc, - size_t desc_size) + const u8 *signature, size_t sig_size) { const struct inode *inode = vi->inode; const struct fsverity_hash_alg *hash_alg = vi->tree_params.hash_alg; - const u32 sig_size = le32_to_cpu(desc->sig_size); struct fsverity_formatted_digest *d; int err; @@ -56,11 +54,6 @@ int fsverity_verify_signature(const struct fsverity_info *vi, return 0; } - if (sig_size > desc_size - sizeof(*desc)) { - fsverity_err(inode, "Signature overflows verity descriptor"); - return -EBADMSG; - } - d = kzalloc(sizeof(*d) + hash_alg->digest_size, GFP_KERNEL); if (!d) return -ENOMEM; @@ -70,8 +63,7 @@ int fsverity_verify_signature(const struct fsverity_info *vi, memcpy(d->digest, vi->file_digest, hash_alg->digest_size); err = verify_pkcs7_signature(d, sizeof(*d) + hash_alg->digest_size, - desc->signature, sig_size, - fsverity_keyring, + signature, sig_size, fsverity_keyring, VERIFYING_UNSPECIFIED_SIGNATURE, NULL, NULL); kfree(d); -- cgit v1.2.3 From e17fe6579de023725ec22a16965e9099e4a05ac9 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 15 Jan 2021 10:18:16 -0800 Subject: fs-verity: add FS_IOC_READ_VERITY_METADATA ioctl Add an ioctl FS_IOC_READ_VERITY_METADATA which will allow reading verity metadata from a file that has fs-verity enabled, including: - The Merkle tree - The fsverity_descriptor (not including the signature if present) - The built-in signature, if present This ioctl has similar semantics to pread(). It is passed the type of metadata to read (one of the above three), and a buffer, offset, and size. It returns the number of bytes read or an error. Separate patches will add support for each of the above metadata types. This patch just adds the ioctl itself. This ioctl doesn't make any assumption about where the metadata is stored on-disk. It does assume the metadata is in a stable format, but that's basically already the case: - The Merkle tree and fsverity_descriptor are defined by how fs-verity file digests are computed; see the "File digest computation" section of Documentation/filesystems/fsverity.rst. Technically, the way in which the levels of the tree are ordered relative to each other wasn't previously specified, but it's logical to put the root level first. - The built-in signature is the value passed to FS_IOC_ENABLE_VERITY. This ioctl is useful because it allows writing a server program that takes a verity file and serves it to a client program, such that the client can do its own fs-verity compatible verification of the file. This only makes sense if the client doesn't trust the server and if the server needs to provide the storage for the client. More concretely, there is interest in using this ability in Android to export APK files (which are protected by fs-verity) to "protected VMs". This would use Protected KVM (https://lwn.net/Articles/836693), which provides an isolated execution environment without having to trust the traditional "host". A "guest" VM can boot from a signed image and perform specific tasks in a minimum trusted environment using files that have fs-verity enabled on the host, without trusting the host or requiring that the guest has its own trusted storage. Technically, it would be possible to duplicate the metadata and store it in separate files for serving. However, that would be less efficient and would require extra care in userspace to maintain file consistency. In addition to the above, the ability to read the built-in signatures is useful because it allows a system that is using the in-kernel signature verification to migrate to userspace signature verification. Link: https://lore.kernel.org/r/20210115181819.34732-4-ebiggers@kernel.org Reviewed-by: Victor Hsieh Acked-by: Jaegeuk Kim Reviewed-by: Chao Yu Signed-off-by: Eric Biggers --- Documentation/filesystems/fsverity.rst | 57 ++++++++++++++++++++++++++++++++++ fs/ext4/ioctl.c | 7 +++++ fs/f2fs/file.c | 11 +++++++ fs/verity/Makefile | 1 + fs/verity/read_metadata.c | 55 ++++++++++++++++++++++++++++++++ include/linux/fsverity.h | 12 +++++++ include/uapi/linux/fsverity.h | 10 ++++++ 7 files changed, 153 insertions(+) create mode 100644 fs/verity/read_metadata.c (limited to 'fs') diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index e0204a23e997..9ef7a7de6008 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -217,6 +217,63 @@ FS_IOC_MEASURE_VERITY can fail with the following errors: - ``EOVERFLOW``: the digest is longer than the specified ``digest_size`` bytes. Try providing a larger buffer. +FS_IOC_READ_VERITY_METADATA +--------------------------- + +The FS_IOC_READ_VERITY_METADATA ioctl reads verity metadata from a +verity file. This ioctl is available since Linux v5.12. + +This ioctl allows writing a server program that takes a verity file +and serves it to a client program, such that the client can do its own +fs-verity compatible verification of the file. This only makes sense +if the client doesn't trust the server and if the server needs to +provide the storage for the client. + +This is a fairly specialized use case, and most fs-verity users won't +need this ioctl. + +This ioctl takes in a pointer to the following structure:: + + struct fsverity_read_metadata_arg { + __u64 metadata_type; + __u64 offset; + __u64 length; + __u64 buf_ptr; + __u64 __reserved; + }; + +``metadata_type`` specifies the type of metadata to read. + +The semantics are similar to those of ``pread()``. ``offset`` +specifies the offset in bytes into the metadata item to read from, and +``length`` specifies the maximum number of bytes to read from the +metadata item. ``buf_ptr`` is the pointer to the buffer to read into, +cast to a 64-bit integer. ``__reserved`` must be 0. On success, the +number of bytes read is returned. 0 is returned at the end of the +metadata item. The returned length may be less than ``length``, for +example if the ioctl is interrupted. + +The metadata returned by FS_IOC_READ_VERITY_METADATA isn't guaranteed +to be authenticated against the file digest that would be returned by +`FS_IOC_MEASURE_VERITY`_, as the metadata is expected to be used to +implement fs-verity compatible verification anyway (though absent a +malicious disk, the metadata will indeed match). E.g. to implement +this ioctl, the filesystem is allowed to just read the Merkle tree +blocks from disk without actually verifying the path to the root node. + +FS_IOC_READ_VERITY_METADATA can fail with the following errors: + +- ``EFAULT``: the caller provided inaccessible memory +- ``EINTR``: the ioctl was interrupted before any data was read +- ``EINVAL``: reserved fields were set, or ``offset + length`` + overflowed +- ``ENODATA``: the file is not a verity file +- ``ENOTTY``: this type of filesystem does not implement fs-verity, or + this ioctl is not yet implemented on it +- ``EOPNOTSUPP``: the kernel was not configured with fs-verity + support, or the filesystem superblock has not had the 'verity' + feature enabled on it. (See `Filesystem support`_.) + FS_IOC_GETFLAGS --------------- diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index d9665d2f82db..713b1ae44c1a 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -1309,6 +1309,12 @@ out: return -EOPNOTSUPP; return fsverity_ioctl_measure(filp, (void __user *)arg); + case FS_IOC_READ_VERITY_METADATA: + if (!ext4_has_feature_verity(sb)) + return -EOPNOTSUPP; + return fsverity_ioctl_read_metadata(filp, + (const void __user *)arg); + default: return -ENOTTY; } @@ -1391,6 +1397,7 @@ long ext4_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case FS_IOC_GETFSMAP: case FS_IOC_ENABLE_VERITY: case FS_IOC_MEASURE_VERITY: + case FS_IOC_READ_VERITY_METADATA: case EXT4_IOC_CLEAR_ES_CACHE: case EXT4_IOC_GETSTATE: case EXT4_IOC_GET_ES_CACHE: diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index f585545277d7..d0aefb5b97fa 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -3357,6 +3357,14 @@ static int f2fs_ioc_measure_verity(struct file *filp, unsigned long arg) return fsverity_ioctl_measure(filp, (void __user *)arg); } +static int f2fs_ioc_read_verity_metadata(struct file *filp, unsigned long arg) +{ + if (!f2fs_sb_has_verity(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + + return fsverity_ioctl_read_metadata(filp, (const void __user *)arg); +} + static int f2fs_ioc_getfslabel(struct file *filp, unsigned long arg) { struct inode *inode = file_inode(filp); @@ -4272,6 +4280,8 @@ static long __f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return f2fs_ioc_enable_verity(filp, arg); case FS_IOC_MEASURE_VERITY: return f2fs_ioc_measure_verity(filp, arg); + case FS_IOC_READ_VERITY_METADATA: + return f2fs_ioc_read_verity_metadata(filp, arg); case FS_IOC_GETFSLABEL: return f2fs_ioc_getfslabel(filp, arg); case FS_IOC_SETFSLABEL: @@ -4523,6 +4533,7 @@ long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case F2FS_IOC_RESIZE_FS: case FS_IOC_ENABLE_VERITY: case FS_IOC_MEASURE_VERITY: + case FS_IOC_READ_VERITY_METADATA: case FS_IOC_GETFSLABEL: case FS_IOC_SETFSLABEL: case F2FS_IOC_GET_COMPRESS_BLOCKS: diff --git a/fs/verity/Makefile b/fs/verity/Makefile index 570e9136334d..435559a4fa9e 100644 --- a/fs/verity/Makefile +++ b/fs/verity/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_FS_VERITY) += enable.o \ init.o \ measure.o \ open.o \ + read_metadata.o \ verify.o obj-$(CONFIG_FS_VERITY_BUILTIN_SIGNATURES) += signature.o diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c new file mode 100644 index 000000000000..43be990fd53e --- /dev/null +++ b/fs/verity/read_metadata.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Ioctl to read verity metadata + * + * Copyright 2021 Google LLC + */ + +#include "fsverity_private.h" + +#include + +/** + * fsverity_ioctl_read_metadata() - read verity metadata from a file + * @filp: file to read the metadata from + * @uarg: user pointer to fsverity_read_metadata_arg + * + * Return: length read on success, 0 on EOF, -errno on failure + */ +int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg) +{ + struct inode *inode = file_inode(filp); + const struct fsverity_info *vi; + struct fsverity_read_metadata_arg arg; + int length; + void __user *buf; + + vi = fsverity_get_info(inode); + if (!vi) + return -ENODATA; /* not a verity file */ + /* + * Note that we don't have to explicitly check that the file is open for + * reading, since verity files can only be opened for reading. + */ + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (arg.__reserved) + return -EINVAL; + + /* offset + length must not overflow. */ + if (arg.offset + arg.length < arg.offset) + return -EINVAL; + + /* Ensure that the return value will fit in INT_MAX. */ + length = min_t(u64, arg.length, INT_MAX); + + buf = u64_to_user_ptr(arg.buf_ptr); + + switch (arg.metadata_type) { + default: + return -EINVAL; + } +} +EXPORT_SYMBOL_GPL(fsverity_ioctl_read_metadata); diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h index c1144a450392..b568b3c7d095 100644 --- a/include/linux/fsverity.h +++ b/include/linux/fsverity.h @@ -138,6 +138,10 @@ int fsverity_file_open(struct inode *inode, struct file *filp); int fsverity_prepare_setattr(struct dentry *dentry, struct iattr *attr); void fsverity_cleanup_inode(struct inode *inode); +/* read_metadata.c */ + +int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg); + /* verify.c */ bool fsverity_verify_page(struct page *page); @@ -183,6 +187,14 @@ static inline void fsverity_cleanup_inode(struct inode *inode) { } +/* read_metadata.c */ + +static inline int fsverity_ioctl_read_metadata(struct file *filp, + const void __user *uarg) +{ + return -EOPNOTSUPP; +} + /* verify.c */ static inline bool fsverity_verify_page(struct page *page) diff --git a/include/uapi/linux/fsverity.h b/include/uapi/linux/fsverity.h index 33f44156f8ea..e062751294d0 100644 --- a/include/uapi/linux/fsverity.h +++ b/include/uapi/linux/fsverity.h @@ -83,7 +83,17 @@ struct fsverity_formatted_digest { __u8 digest[]; }; +struct fsverity_read_metadata_arg { + __u64 metadata_type; + __u64 offset; + __u64 length; + __u64 buf_ptr; + __u64 __reserved; +}; + #define FS_IOC_ENABLE_VERITY _IOW('f', 133, struct fsverity_enable_arg) #define FS_IOC_MEASURE_VERITY _IOWR('f', 134, struct fsverity_digest) +#define FS_IOC_READ_VERITY_METADATA \ + _IOWR('f', 135, struct fsverity_read_metadata_arg) #endif /* _UAPI_LINUX_FSVERITY_H */ -- cgit v1.2.3 From 622699cfe6ec5578f52727002d5717ff3f092e23 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 15 Jan 2021 10:18:17 -0800 Subject: fs-verity: support reading Merkle tree with ioctl Add support for FS_VERITY_METADATA_TYPE_MERKLE_TREE to FS_IOC_READ_VERITY_METADATA. This allows a userspace server program to retrieve the Merkle tree of a verity file for serving to a client which implements fs-verity compatible verification. See the patch which introduced FS_IOC_READ_VERITY_METADATA for more details. This has been tested using a new xfstest which calls this ioctl via a new subcommand for the 'fsverity' program from fsverity-utils. Link: https://lore.kernel.org/r/20210115181819.34732-5-ebiggers@kernel.org Reviewed-by: Victor Hsieh Reviewed-by: Jaegeuk Kim Reviewed-by: Chao Yu Signed-off-by: Eric Biggers --- Documentation/filesystems/fsverity.rst | 10 ++++- fs/verity/read_metadata.c | 70 ++++++++++++++++++++++++++++++++++ include/uapi/linux/fsverity.h | 2 + 3 files changed, 81 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 9ef7a7de6008..50b47a6d9ea1 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -234,6 +234,8 @@ need this ioctl. This ioctl takes in a pointer to the following structure:: + #define FS_VERITY_METADATA_TYPE_MERKLE_TREE 1 + struct fsverity_read_metadata_arg { __u64 metadata_type; __u64 offset; @@ -242,7 +244,13 @@ This ioctl takes in a pointer to the following structure:: __u64 __reserved; }; -``metadata_type`` specifies the type of metadata to read. +``metadata_type`` specifies the type of metadata to read: + +- ``FS_VERITY_METADATA_TYPE_MERKLE_TREE`` reads the blocks of the + Merkle tree. The blocks are returned in order from the root level + to the leaf level. Within each level, the blocks are returned in + the same order that their hashes are themselves hashed. + See `Merkle tree`_ for more information. The semantics are similar to those of ``pread()``. ``offset`` specifies the offset in bytes into the metadata item to read from, and diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c index 43be990fd53e..0f8ad2991cf9 100644 --- a/fs/verity/read_metadata.c +++ b/fs/verity/read_metadata.c @@ -7,8 +7,75 @@ #include "fsverity_private.h" +#include +#include +#include #include +static int fsverity_read_merkle_tree(struct inode *inode, + const struct fsverity_info *vi, + void __user *buf, u64 offset, int length) +{ + const struct fsverity_operations *vops = inode->i_sb->s_vop; + u64 end_offset; + unsigned int offs_in_page; + pgoff_t index, last_index; + int retval = 0; + int err = 0; + + end_offset = min(offset + length, vi->tree_params.tree_size); + if (offset >= end_offset) + return 0; + offs_in_page = offset_in_page(offset); + last_index = (end_offset - 1) >> PAGE_SHIFT; + + /* + * Iterate through each Merkle tree page in the requested range and copy + * the requested portion to userspace. Note that the Merkle tree block + * size isn't important here, as we are returning a byte stream; i.e., + * we can just work with pages even if the tree block size != PAGE_SIZE. + */ + for (index = offset >> PAGE_SHIFT; index <= last_index; index++) { + unsigned long num_ra_pages = + min_t(unsigned long, last_index - index + 1, + inode->i_sb->s_bdi->io_pages); + unsigned int bytes_to_copy = min_t(u64, end_offset - offset, + PAGE_SIZE - offs_in_page); + struct page *page; + const void *virt; + + page = vops->read_merkle_tree_page(inode, index, num_ra_pages); + if (IS_ERR(page)) { + err = PTR_ERR(page); + fsverity_err(inode, + "Error %d reading Merkle tree page %lu", + err, index); + break; + } + + virt = kmap(page); + if (copy_to_user(buf, virt + offs_in_page, bytes_to_copy)) { + kunmap(page); + put_page(page); + err = -EFAULT; + break; + } + kunmap(page); + put_page(page); + + retval += bytes_to_copy; + buf += bytes_to_copy; + offset += bytes_to_copy; + + if (fatal_signal_pending(current)) { + err = -EINTR; + break; + } + cond_resched(); + offs_in_page = 0; + } + return retval ? retval : err; +} /** * fsverity_ioctl_read_metadata() - read verity metadata from a file * @filp: file to read the metadata from @@ -48,6 +115,9 @@ int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg) buf = u64_to_user_ptr(arg.buf_ptr); switch (arg.metadata_type) { + case FS_VERITY_METADATA_TYPE_MERKLE_TREE: + return fsverity_read_merkle_tree(inode, vi, buf, arg.offset, + length); default: return -EINVAL; } diff --git a/include/uapi/linux/fsverity.h b/include/uapi/linux/fsverity.h index e062751294d0..94003b153cb3 100644 --- a/include/uapi/linux/fsverity.h +++ b/include/uapi/linux/fsverity.h @@ -83,6 +83,8 @@ struct fsverity_formatted_digest { __u8 digest[]; }; +#define FS_VERITY_METADATA_TYPE_MERKLE_TREE 1 + struct fsverity_read_metadata_arg { __u64 metadata_type; __u64 offset; -- cgit v1.2.3 From 947191ac8caba85e25e0e036b0f097fee9e817f3 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 15 Jan 2021 10:18:18 -0800 Subject: fs-verity: support reading descriptor with ioctl Add support for FS_VERITY_METADATA_TYPE_DESCRIPTOR to FS_IOC_READ_VERITY_METADATA. This allows a userspace server program to retrieve the fs-verity descriptor of a file for serving to a client which implements fs-verity compatible verification. See the patch which introduced FS_IOC_READ_VERITY_METADATA for more details. "fs-verity descriptor" here means only the part that userspace cares about because it is hashed to produce the file digest. It doesn't include the signature which ext4 and f2fs append to the fsverity_descriptor struct when storing it on-disk, since that way of storing the signature is an implementation detail. The next patch adds a separate metadata_type value for retrieving the signature separately. This has been tested using a new xfstest which calls this ioctl via a new subcommand for the 'fsverity' program from fsverity-utils. Link: https://lore.kernel.org/r/20210115181819.34732-6-ebiggers@kernel.org Reviewed-by: Victor Hsieh Reviewed-by: Jaegeuk Kim Reviewed-by: Chao Yu Signed-off-by: Eric Biggers --- Documentation/filesystems/fsverity.rst | 4 ++++ fs/verity/read_metadata.c | 40 ++++++++++++++++++++++++++++++++++ include/uapi/linux/fsverity.h | 1 + 3 files changed, 45 insertions(+) (limited to 'fs') diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 50b47a6d9ea1..6dc5772037ef 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -235,6 +235,7 @@ need this ioctl. This ioctl takes in a pointer to the following structure:: #define FS_VERITY_METADATA_TYPE_MERKLE_TREE 1 + #define FS_VERITY_METADATA_TYPE_DESCRIPTOR 2 struct fsverity_read_metadata_arg { __u64 metadata_type; @@ -252,6 +253,9 @@ This ioctl takes in a pointer to the following structure:: the same order that their hashes are themselves hashed. See `Merkle tree`_ for more information. +- ``FS_VERITY_METADATA_TYPE_DESCRIPTOR`` reads the fs-verity + descriptor. See `fs-verity descriptor`_. + The semantics are similar to those of ``pread()``. ``offset`` specifies the offset in bytes into the metadata item to read from, and ``length`` specifies the maximum number of bytes to read from the diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c index 0f8ad2991cf9..2dea6dd3bb05 100644 --- a/fs/verity/read_metadata.c +++ b/fs/verity/read_metadata.c @@ -76,6 +76,44 @@ static int fsverity_read_merkle_tree(struct inode *inode, } return retval ? retval : err; } + +/* Copy the requested portion of the buffer to userspace. */ +static int fsverity_read_buffer(void __user *dst, u64 offset, int length, + const void *src, size_t src_length) +{ + if (offset >= src_length) + return 0; + src += offset; + src_length -= offset; + + length = min_t(size_t, length, src_length); + + if (copy_to_user(dst, src, length)) + return -EFAULT; + + return length; +} + +static int fsverity_read_descriptor(struct inode *inode, + void __user *buf, u64 offset, int length) +{ + struct fsverity_descriptor *desc; + size_t desc_size; + int res; + + res = fsverity_get_descriptor(inode, &desc, &desc_size); + if (res) + return res; + + /* don't include the signature */ + desc_size = offsetof(struct fsverity_descriptor, signature); + desc->sig_size = 0; + + res = fsverity_read_buffer(buf, offset, length, desc, desc_size); + + kfree(desc); + return res; +} /** * fsverity_ioctl_read_metadata() - read verity metadata from a file * @filp: file to read the metadata from @@ -118,6 +156,8 @@ int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg) case FS_VERITY_METADATA_TYPE_MERKLE_TREE: return fsverity_read_merkle_tree(inode, vi, buf, arg.offset, length); + case FS_VERITY_METADATA_TYPE_DESCRIPTOR: + return fsverity_read_descriptor(inode, buf, arg.offset, length); default: return -EINVAL; } diff --git a/include/uapi/linux/fsverity.h b/include/uapi/linux/fsverity.h index 94003b153cb3..41abc283dbcc 100644 --- a/include/uapi/linux/fsverity.h +++ b/include/uapi/linux/fsverity.h @@ -84,6 +84,7 @@ struct fsverity_formatted_digest { }; #define FS_VERITY_METADATA_TYPE_MERKLE_TREE 1 +#define FS_VERITY_METADATA_TYPE_DESCRIPTOR 2 struct fsverity_read_metadata_arg { __u64 metadata_type; -- cgit v1.2.3 From 07c99001312cbf90a357d4877a358f796eede65b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 15 Jan 2021 10:18:19 -0800 Subject: fs-verity: support reading signature with ioctl Add support for FS_VERITY_METADATA_TYPE_SIGNATURE to FS_IOC_READ_VERITY_METADATA. This allows a userspace server program to retrieve the built-in signature (if present) of a verity file for serving to a client which implements fs-verity compatible verification. See the patch which introduced FS_IOC_READ_VERITY_METADATA for more details. The ability for userspace to read the built-in signatures is also useful because it allows a system that is using the in-kernel signature verification to migrate to userspace signature verification. This has been tested using a new xfstest which calls this ioctl via a new subcommand for the 'fsverity' program from fsverity-utils. Link: https://lore.kernel.org/r/20210115181819.34732-7-ebiggers@kernel.org Reviewed-by: Victor Hsieh Reviewed-by: Jaegeuk Kim Reviewed-by: Chao Yu Signed-off-by: Eric Biggers --- Documentation/filesystems/fsverity.rst | 9 ++++++++- fs/verity/read_metadata.c | 30 ++++++++++++++++++++++++++++++ include/uapi/linux/fsverity.h | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 6dc5772037ef..1d831e3cbcb3 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -236,6 +236,7 @@ This ioctl takes in a pointer to the following structure:: #define FS_VERITY_METADATA_TYPE_MERKLE_TREE 1 #define FS_VERITY_METADATA_TYPE_DESCRIPTOR 2 + #define FS_VERITY_METADATA_TYPE_SIGNATURE 3 struct fsverity_read_metadata_arg { __u64 metadata_type; @@ -256,6 +257,10 @@ This ioctl takes in a pointer to the following structure:: - ``FS_VERITY_METADATA_TYPE_DESCRIPTOR`` reads the fs-verity descriptor. See `fs-verity descriptor`_. +- ``FS_VERITY_METADATA_TYPE_SIGNATURE`` reads the signature which was + passed to FS_IOC_ENABLE_VERITY, if any. See `Built-in signature + verification`_. + The semantics are similar to those of ``pread()``. ``offset`` specifies the offset in bytes into the metadata item to read from, and ``length`` specifies the maximum number of bytes to read from the @@ -279,7 +284,9 @@ FS_IOC_READ_VERITY_METADATA can fail with the following errors: - ``EINTR``: the ioctl was interrupted before any data was read - ``EINVAL``: reserved fields were set, or ``offset + length`` overflowed -- ``ENODATA``: the file is not a verity file +- ``ENODATA``: the file is not a verity file, or + FS_VERITY_METADATA_TYPE_SIGNATURE was requested but the file doesn't + have a built-in signature - ``ENOTTY``: this type of filesystem does not implement fs-verity, or this ioctl is not yet implemented on it - ``EOPNOTSUPP``: the kernel was not configured with fs-verity diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c index 2dea6dd3bb05..7e2d0c7bdf0d 100644 --- a/fs/verity/read_metadata.c +++ b/fs/verity/read_metadata.c @@ -114,6 +114,34 @@ static int fsverity_read_descriptor(struct inode *inode, kfree(desc); return res; } + +static int fsverity_read_signature(struct inode *inode, + void __user *buf, u64 offset, int length) +{ + struct fsverity_descriptor *desc; + size_t desc_size; + int res; + + res = fsverity_get_descriptor(inode, &desc, &desc_size); + if (res) + return res; + + if (desc->sig_size == 0) { + res = -ENODATA; + goto out; + } + + /* + * Include only the signature. Note that fsverity_get_descriptor() + * already verified that sig_size is in-bounds. + */ + res = fsverity_read_buffer(buf, offset, length, desc->signature, + le32_to_cpu(desc->sig_size)); +out: + kfree(desc); + return res; +} + /** * fsverity_ioctl_read_metadata() - read verity metadata from a file * @filp: file to read the metadata from @@ -158,6 +186,8 @@ int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg) length); case FS_VERITY_METADATA_TYPE_DESCRIPTOR: return fsverity_read_descriptor(inode, buf, arg.offset, length); + case FS_VERITY_METADATA_TYPE_SIGNATURE: + return fsverity_read_signature(inode, buf, arg.offset, length); default: return -EINVAL; } diff --git a/include/uapi/linux/fsverity.h b/include/uapi/linux/fsverity.h index 41abc283dbcc..15384e22e331 100644 --- a/include/uapi/linux/fsverity.h +++ b/include/uapi/linux/fsverity.h @@ -85,6 +85,7 @@ struct fsverity_formatted_digest { #define FS_VERITY_METADATA_TYPE_MERKLE_TREE 1 #define FS_VERITY_METADATA_TYPE_DESCRIPTOR 2 +#define FS_VERITY_METADATA_TYPE_SIGNATURE 3 struct fsverity_read_metadata_arg { __u64 metadata_type; -- cgit v1.2.3 From 47b7ec1daa511cd82cb9c31e88bfdb664b031d2a Mon Sep 17 00:00:00 2001 From: Andrew Price Date: Fri, 5 Feb 2021 17:10:17 +0000 Subject: gfs2: Enable rgrplvb for sb_fs_format 1802 Turn on rgrplvb by default for sb_fs_format > 1801. Mount options still have to override this so a new args field to differentiate between 'off' and 'not specified' is added, and the new default is applied only when it's not specified. Signed-off-by: Andrew Price Signed-off-by: Andreas Gruenbacher --- fs/gfs2/incore.h | 1 + fs/gfs2/ops_fstype.c | 13 ++++++++++--- fs/gfs2/super.h | 4 ++++ 3 files changed, 15 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index 9933af070121..ade0b85eac64 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -586,6 +586,7 @@ struct gfs2_args { unsigned int ar_errors:2; /* errors=withdraw | panic */ unsigned int ar_nobarrier:1; /* do not send barriers */ unsigned int ar_rgrplvb:1; /* use lvbs for rgrp info */ + unsigned int ar_got_rgrplvb:1; /* Was the rgrplvb opt given? */ unsigned int ar_loccookie:1; /* use location based readdir cookies */ s32 ar_commit; /* Commit interval */ diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index 6118f975030c..c6537e94f4d8 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -171,7 +171,8 @@ static int gfs2_check_sb(struct gfs2_sbd *sdp, int silent) return -EINVAL; } - if (sb->sb_fs_format != GFS2_FORMAT_FS || + if (sb->sb_fs_format < GFS2_FS_FORMAT_MIN || + sb->sb_fs_format > GFS2_FS_FORMAT_MAX || sb->sb_multihost_format != GFS2_FORMAT_MULTI) { fs_warn(sdp, "Unknown on-disk format, unable to mount\n"); return -EINVAL; @@ -1032,13 +1033,14 @@ hostdata_error: } if (lm->lm_mount == NULL) { - fs_info(sdp, "Now mounting FS...\n"); + fs_info(sdp, "Now mounting FS (format %u)...\n", sdp->sd_sb.sb_fs_format); complete_all(&sdp->sd_locking_init); return 0; } ret = lm->lm_mount(sdp, table); if (ret == 0) - fs_info(sdp, "Joined cluster. Now mounting FS...\n"); + fs_info(sdp, "Joined cluster. Now mounting FS (format %u)...\n", + sdp->sd_sb.sb_fs_format); complete_all(&sdp->sd_locking_init); return ret; } @@ -1157,6 +1159,10 @@ static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) if (error) goto fail_locking; + /* Turn rgrplvb on by default if fs format is recent enough */ + if (!sdp->sd_args.ar_got_rgrplvb && sdp->sd_sb.sb_fs_format > 1801) + sdp->sd_args.ar_rgrplvb = 1; + error = wait_on_journal(sdp); if (error) goto fail_sb; @@ -1450,6 +1456,7 @@ static int gfs2_parse_param(struct fs_context *fc, struct fs_parameter *param) break; case Opt_rgrplvb: args->ar_rgrplvb = result.boolean; + args->ar_got_rgrplvb = 1; break; case Opt_loccookie: args->ar_loccookie = result.boolean; diff --git a/fs/gfs2/super.h b/fs/gfs2/super.h index c9fb2a654181..977079693bdc 100644 --- a/fs/gfs2/super.h +++ b/fs/gfs2/super.h @@ -11,6 +11,10 @@ #include #include "incore.h" +/* Supported fs format version range */ +#define GFS2_FS_FORMAT_MIN (1801) +#define GFS2_FS_FORMAT_MAX (1802) + extern void gfs2_lm_unmount(struct gfs2_sbd *sdp); static inline unsigned int gfs2_jindex_size(struct gfs2_sbd *sdp) -- cgit v1.2.3 From 866eef48d80234e1ea3a2f78b54afc563be3ea4a Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Fri, 5 Feb 2021 17:10:18 +0000 Subject: gfs2: Add trusted xattr support Add support for an additional filesystem version (sb_fs_format = 1802). When a filesystem with the new version is mounted, the filesystem supports "trusted.*" xattrs. In addition, version 1802 filesystems implement a form of forward compatibility for xattrs: when xattrs with an unknown prefix (ea_type) are found on a version 1802 filesystem, those attributes are not shown by listxattr, and they are not accessible by getxattr, setxattr, or removexattr. This mechanism might turn out to be what we need in the future, but if not, we can always bump the filesystem version and break compatibility instead. Signed-off-by: Andreas Gruenbacher Signed-off-by: Andrew Price --- fs/gfs2/ops_fstype.c | 14 +++++++++++- fs/gfs2/super.h | 4 +++- fs/gfs2/xattr.c | 48 ++++++++++++++++++++++++++++++++++++---- include/uapi/linux/gfs2_ondisk.h | 5 +++-- 4 files changed, 63 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index c6537e94f4d8..d89764856f21 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -489,6 +489,19 @@ static int init_sb(struct gfs2_sbd *sdp, int silent) goto out; } + switch(sdp->sd_sb.sb_fs_format) { + case GFS2_FS_FORMAT_MAX: + sb->s_xattr = gfs2_xattr_handlers_max; + break; + + case GFS2_FS_FORMAT_MIN: + sb->s_xattr = gfs2_xattr_handlers_min; + break; + + default: + BUG(); + } + /* Set up the buffer cache and SB for real */ if (sdp->sd_sb.sb_bsize < bdev_logical_block_size(sb->s_bdev)) { ret = -EINVAL; @@ -1110,7 +1123,6 @@ static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_op = &gfs2_super_ops; sb->s_d_op = &gfs2_dops; sb->s_export_op = &gfs2_export_ops; - sb->s_xattr = gfs2_xattr_handlers; sb->s_qcop = &gfs2_quotactl_ops; sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP; sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS_FILE; diff --git a/fs/gfs2/super.h b/fs/gfs2/super.h index 977079693bdc..08e502dec7ec 100644 --- a/fs/gfs2/super.h +++ b/fs/gfs2/super.h @@ -58,7 +58,9 @@ extern struct file_system_type gfs2meta_fs_type; extern const struct export_operations gfs2_export_ops; extern const struct super_operations gfs2_super_ops; extern const struct dentry_operations gfs2_dops; -extern const struct xattr_handler *gfs2_xattr_handlers[]; + +extern const struct xattr_handler *gfs2_xattr_handlers_max[]; +extern const struct xattr_handler **gfs2_xattr_handlers_min; #endif /* __SUPER_DOT_H__ */ diff --git a/fs/gfs2/xattr.c b/fs/gfs2/xattr.c index 9d7667bc4292..2b520ac49d5f 100644 --- a/fs/gfs2/xattr.c +++ b/fs/gfs2/xattr.c @@ -70,6 +70,20 @@ static int ea_check_size(struct gfs2_sbd *sdp, unsigned int nsize, size_t dsize) return 0; } +static bool gfs2_eatype_valid(struct gfs2_sbd *sdp, u8 type) +{ + switch(sdp->sd_sb.sb_fs_format) { + case GFS2_FS_FORMAT_MAX: + return true; + + case GFS2_FS_FORMAT_MIN: + return type <= GFS2_EATYPE_SECURITY; + + default: + return false; + } +} + typedef int (*ea_call_t) (struct gfs2_inode *ip, struct buffer_head *bh, struct gfs2_ea_header *ea, struct gfs2_ea_header *prev, void *private); @@ -77,6 +91,7 @@ typedef int (*ea_call_t) (struct gfs2_inode *ip, struct buffer_head *bh, static int ea_foreach_i(struct gfs2_inode *ip, struct buffer_head *bh, ea_call_t ea_call, void *data) { + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); struct gfs2_ea_header *ea, *prev = NULL; int error = 0; @@ -89,9 +104,8 @@ static int ea_foreach_i(struct gfs2_inode *ip, struct buffer_head *bh, if (!(bh->b_data <= (char *)ea && (char *)GFS2_EA2NEXT(ea) <= bh->b_data + bh->b_size)) goto fail; - if (!GFS2_EATYPE_VALID(ea->ea_type)) + if (!gfs2_eatype_valid(sdp, ea->ea_type)) goto fail; - error = ea_call(ip, bh, ea, prev, data); if (error) return error; @@ -344,6 +358,7 @@ static int ea_list_i(struct gfs2_inode *ip, struct buffer_head *bh, struct gfs2_ea_header *ea, struct gfs2_ea_header *prev, void *private) { + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); struct ea_list *ei = private; struct gfs2_ea_request *er = ei->ei_er; unsigned int ea_size; @@ -353,6 +368,8 @@ static int ea_list_i(struct gfs2_inode *ip, struct buffer_head *bh, if (ea->ea_type == GFS2_EATYPE_UNUSED) return 0; + BUG_ON(ea->ea_type > GFS2_EATYPE_SECURITY && + sdp->sd_sb.sb_fs_format == GFS2_FS_FORMAT_MIN); switch (ea->ea_type) { case GFS2_EATYPE_USR: prefix = "user."; @@ -366,8 +383,12 @@ static int ea_list_i(struct gfs2_inode *ip, struct buffer_head *bh, prefix = "security."; l = 9; break; + case GFS2_EATYPE_TRUSTED: + prefix = "trusted."; + l = 8; + break; default: - BUG(); + return 0; } ea_size = l + ea->ea_name_len + 1; @@ -1463,7 +1484,25 @@ static const struct xattr_handler gfs2_xattr_security_handler = { .set = gfs2_xattr_set, }; -const struct xattr_handler *gfs2_xattr_handlers[] = { +static bool +gfs2_xattr_trusted_list(struct dentry *dentry) +{ + return capable(CAP_SYS_ADMIN); +} + +static const struct xattr_handler gfs2_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .flags = GFS2_EATYPE_TRUSTED, + .list = gfs2_xattr_trusted_list, + .get = gfs2_xattr_get, + .set = gfs2_xattr_set, +}; + +const struct xattr_handler *gfs2_xattr_handlers_max[] = { + /* GFS2_FS_FORMAT_MAX */ + &gfs2_xattr_trusted_handler, + + /* GFS2_FS_FORMAT_MIN */ &gfs2_xattr_user_handler, &gfs2_xattr_security_handler, &posix_acl_access_xattr_handler, @@ -1471,3 +1510,4 @@ const struct xattr_handler *gfs2_xattr_handlers[] = { NULL, }; +const struct xattr_handler **gfs2_xattr_handlers_min = gfs2_xattr_handlers_max + 1; diff --git a/include/uapi/linux/gfs2_ondisk.h b/include/uapi/linux/gfs2_ondisk.h index 07e508e6691b..6ec4291bcc7a 100644 --- a/include/uapi/linux/gfs2_ondisk.h +++ b/include/uapi/linux/gfs2_ondisk.h @@ -47,7 +47,7 @@ #define GFS2_FORMAT_DE 1200 #define GFS2_FORMAT_QU 1500 /* These are part of the superblock */ -#define GFS2_FORMAT_FS 1801 +#define GFS2_FORMAT_FS 1802 #define GFS2_FORMAT_MULTI 1900 /* @@ -389,8 +389,9 @@ struct gfs2_leaf { #define GFS2_EATYPE_USR 1 #define GFS2_EATYPE_SYS 2 #define GFS2_EATYPE_SECURITY 3 +#define GFS2_EATYPE_TRUSTED 4 -#define GFS2_EATYPE_LAST 3 +#define GFS2_EATYPE_LAST 4 #define GFS2_EATYPE_VALID(x) ((x) <= GFS2_EATYPE_LAST) #define GFS2_EAFLAG_LAST 0x01 /* last ea in block */ -- cgit v1.2.3 From cc4a3f885e8f2bc3c86a265972e94fef32d68f67 Mon Sep 17 00:00:00 2001 From: Pavel Tikhomirov Date: Wed, 3 Feb 2021 15:41:56 +0300 Subject: fcntl: make F_GETOWN(EX) return 0 on dead owner task Currently there is no way to differentiate the file with alive owner from the file with dead owner but pid of the owner reused. That's why CRIU can't actually know if it needs to restore file owner or not, because if it restores owner but actual owner was dead, this can introduce unexpected signals to the "false"-owner (which reused the pid). Let's change the api, so that F_GETOWN(EX) returns 0 in case actual owner is dead already. This comports with the POSIX spec, which states that a PID of 0 indicates that no signal will be sent. Cc: Jeff Layton Cc: "J. Bruce Fields" Cc: Alexander Viro Cc: linux-fsdevel@vger.kernel.org Cc: linux-kernel@vger.kernel.org Cc: Cyrill Gorcunov Cc: Andrei Vagin Signed-off-by: Pavel Tikhomirov Signed-off-by: Jeff Layton --- fs/fcntl.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/fcntl.c b/fs/fcntl.c index 05b36b28f2e8..483ef8861376 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -148,11 +148,15 @@ void f_delown(struct file *filp) pid_t f_getown(struct file *filp) { - pid_t pid; + pid_t pid = 0; read_lock(&filp->f_owner.lock); - pid = pid_vnr(filp->f_owner.pid); - if (filp->f_owner.pid_type == PIDTYPE_PGID) - pid = -pid; + rcu_read_lock(); + if (pid_task(filp->f_owner.pid, filp->f_owner.pid_type)) { + pid = pid_vnr(filp->f_owner.pid); + if (filp->f_owner.pid_type == PIDTYPE_PGID) + pid = -pid; + } + rcu_read_unlock(); read_unlock(&filp->f_owner.lock); return pid; } @@ -200,11 +204,14 @@ static int f_setown_ex(struct file *filp, unsigned long arg) static int f_getown_ex(struct file *filp, unsigned long arg) { struct f_owner_ex __user *owner_p = (void __user *)arg; - struct f_owner_ex owner; + struct f_owner_ex owner = {}; int ret = 0; read_lock(&filp->f_owner.lock); - owner.pid = pid_vnr(filp->f_owner.pid); + rcu_read_lock(); + if (pid_task(filp->f_owner.pid, filp->f_owner.pid_type)) + owner.pid = pid_vnr(filp->f_owner.pid); + rcu_read_unlock(); switch (filp->f_owner.pid_type) { case PIDTYPE_PID: owner.type = F_OWNER_TID; -- cgit v1.2.3 From 093e0687c5baacc29e4e8dd3ea205bac518e38bc Mon Sep 17 00:00:00 2001 From: Yang Li Date: Sun, 7 Feb 2021 16:32:50 +0800 Subject: jfs: turn diLog(), dataLog() and txLog() into void functions These functions always return '0' and no callers use the return value. So make it a void function. This eliminates the following coccicheck warning: ./fs/jfs/jfs_txnmgr.c:1365:5-7: Unneeded variable: "rc". Return "0" on line 1414 ./fs/jfs/jfs_txnmgr.c:1422:5-7: Unneeded variable: "rc". Return "0" on line 1527 Reported-by: Abaci Robot Signed-off-by: Yang Li Signed-off-by: Dave Kleikamp --- fs/jfs/jfs_txnmgr.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/jfs/jfs_txnmgr.c b/fs/jfs/jfs_txnmgr.c index dca8edd2378c..053295cd7bc6 100644 --- a/fs/jfs/jfs_txnmgr.c +++ b/fs/jfs/jfs_txnmgr.c @@ -148,10 +148,10 @@ static struct { /* * forward references */ -static int diLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, - struct tlock * tlck, struct commit * cd); -static int dataLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, - struct tlock * tlck); +static void diLog(struct jfs_log *log, struct tblock *tblk, struct lrd *lrd, + struct tlock *tlck, struct commit *cd); +static void dataLog(struct jfs_log *log, struct tblock *tblk, struct lrd *lrd, + struct tlock *tlck); static void dtLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, struct tlock * tlck); static void mapLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, @@ -159,8 +159,8 @@ static void mapLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, static void txAllocPMap(struct inode *ip, struct maplock * maplock, struct tblock * tblk); static void txForce(struct tblock * tblk); -static int txLog(struct jfs_log * log, struct tblock * tblk, - struct commit * cd); +static void txLog(struct jfs_log *log, struct tblock *tblk, + struct commit *cd); static void txUpdateMap(struct tblock * tblk); static void txRelease(struct tblock * tblk); static void xtLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, @@ -1256,8 +1256,7 @@ int txCommit(tid_t tid, /* transaction identifier */ * * txUpdateMap() resets XAD_NEW in XAD. */ - if ((rc = txLog(log, tblk, &cd))) - goto TheEnd; + txLog(log, tblk, &cd); /* * Ensure that inode isn't reused before @@ -1365,9 +1364,8 @@ int txCommit(tid_t tid, /* transaction identifier */ * * RETURN : */ -static int txLog(struct jfs_log * log, struct tblock * tblk, struct commit * cd) +static void txLog(struct jfs_log *log, struct tblock *tblk, struct commit *cd) { - int rc = 0; struct inode *ip; lid_t lid; struct tlock *tlck; @@ -1414,7 +1412,7 @@ static int txLog(struct jfs_log * log, struct tblock * tblk, struct commit * cd) } } - return rc; + return; } /* @@ -1422,10 +1420,9 @@ static int txLog(struct jfs_log * log, struct tblock * tblk, struct commit * cd) * * function: log inode tlock and format maplock to update bmap; */ -static int diLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, - struct tlock * tlck, struct commit * cd) +static void diLog(struct jfs_log *log, struct tblock *tblk, struct lrd *lrd, + struct tlock *tlck, struct commit *cd) { - int rc = 0; struct metapage *mp; pxd_t *pxd; struct pxd_lock *pxdlock; @@ -1527,7 +1524,7 @@ static int diLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, } #endif /* _JFS_WIP */ - return rc; + return; } /* @@ -1535,8 +1532,8 @@ static int diLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, * * function: log data tlock */ -static int dataLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, - struct tlock * tlck) +static void dataLog(struct jfs_log *log, struct tblock *tblk, struct lrd *lrd, + struct tlock *tlck) { struct metapage *mp; pxd_t *pxd; @@ -1562,7 +1559,7 @@ static int dataLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, metapage_homeok(mp); discard_metapage(mp); tlck->mp = NULL; - return 0; + return; } PXDaddress(pxd, mp->index); @@ -1573,7 +1570,7 @@ static int dataLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, /* mark page as homeward bound */ tlck->flag |= tlckWRITEPAGE; - return 0; + return; } /* -- cgit v1.2.3 From 0e9ddb39b7d964d716cddd6e6bd1aab3f800066e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 7 Feb 2021 22:34:26 +0000 Subject: io_uring: cleanup up cancel SQPOLL reqs across exec For SQPOLL rings tctx_inflight() always returns zero, so it might skip doing full cancelation. It's fine because we jam all sqpoll submissions in any case and do go through files cancel for them, but not nice. Do the intended full cancellation, by mimicking __io_uring_task_cancel() waiting but impersonating SQPOLL task. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 57 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 7242cc48e97b..9c77fbc0c395 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9083,29 +9083,39 @@ void __io_uring_files_cancel(struct files_struct *files) static s64 tctx_inflight(struct io_uring_task *tctx) { - unsigned long index; - struct file *file; - s64 inflight; - - inflight = percpu_counter_sum(&tctx->inflight); - if (!tctx->sqpoll) - return inflight; + return percpu_counter_sum(&tctx->inflight); +} - /* - * If we have SQPOLL rings, then we need to iterate and find them, and - * add the pending count for those. - */ - xa_for_each(&tctx->xa, index, file) { - struct io_ring_ctx *ctx = file->private_data; +static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) +{ + struct io_uring_task *tctx; + s64 inflight; + DEFINE_WAIT(wait); - if (ctx->flags & IORING_SETUP_SQPOLL) { - struct io_uring_task *__tctx = ctx->sqo_task->io_uring; + if (!ctx->sq_data) + return; + tctx = ctx->sq_data->thread->io_uring; + io_disable_sqo_submit(ctx); - inflight += percpu_counter_sum(&__tctx->inflight); - } - } + atomic_inc(&tctx->in_idle); + do { + /* read completions before cancelations */ + inflight = tctx_inflight(tctx); + if (!inflight) + break; + io_uring_cancel_task_requests(ctx, NULL); - return inflight; + prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); + /* + * If we've seen completions, retry without waiting. This + * avoids a race where a completion comes in before we did + * prepare_to_wait(). + */ + if (inflight == tctx_inflight(tctx)) + schedule(); + finish_wait(&tctx->wait, &wait); + } while (1); + atomic_dec(&tctx->in_idle); } /* @@ -9122,8 +9132,13 @@ void __io_uring_task_cancel(void) atomic_inc(&tctx->in_idle); /* trigger io_disable_sqo_submit() */ - if (tctx->sqpoll) - __io_uring_files_cancel(NULL); + if (tctx->sqpoll) { + struct file *file; + unsigned long index; + + xa_for_each(&tctx->xa, index, file) + io_uring_cancel_sqpoll(file->private_data); + } do { /* read completions before cancelations */ -- cgit v1.2.3 From 45901a231723a5a513ff08477983f3a274a6a910 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 8 Feb 2021 08:49:32 -0500 Subject: NFSv4: Fixes for nfs4_bitmask_adjust() We don't want to ask for the ACL in a WRITE reply, since we don't have a preallocated buffer. Instead of checking NFS_INO_INVALID_ACCESS, which is really about managing the access cache, we should look at the value of NFS_INO_INVALID_OTHER. Also ensure we assign the mode, owner and owner_group flags to the correct bit mask. Finally, fix up the check for NFS_INO_INVALID_CTIME to retrieve the ctime, and add a check for NFS_INO_INVALID_CHANGE. Fixes: 76bd5c016ef4 ("NFSv4: make cache consistency bitmask dynamic") Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/nfs4proc.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 34570a7785fa..8eb9c716010f 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -5435,15 +5435,16 @@ static void nfs4_bitmask_adjust(__u32 *bitmask, struct inode *inode, if (cache_validity & NFS_INO_INVALID_ATIME) bitmask[1] |= FATTR4_WORD1_TIME_ACCESS; - if (cache_validity & NFS_INO_INVALID_ACCESS) - bitmask[0] |= FATTR4_WORD1_MODE | FATTR4_WORD1_OWNER | - FATTR4_WORD1_OWNER_GROUP; - if (cache_validity & NFS_INO_INVALID_ACL) - bitmask[0] |= FATTR4_WORD0_ACL; - if (cache_validity & NFS_INO_INVALID_LABEL) + if (cache_validity & NFS_INO_INVALID_OTHER) + bitmask[1] |= FATTR4_WORD1_MODE | FATTR4_WORD1_OWNER | + FATTR4_WORD1_OWNER_GROUP | + FATTR4_WORD1_NUMLINKS; + if (label && label->len && cache_validity & NFS_INO_INVALID_LABEL) bitmask[2] |= FATTR4_WORD2_SECURITY_LABEL; - if (cache_validity & NFS_INO_INVALID_CTIME) + if (cache_validity & NFS_INO_INVALID_CHANGE) bitmask[0] |= FATTR4_WORD0_CHANGE; + if (cache_validity & NFS_INO_INVALID_CTIME) + bitmask[1] |= FATTR4_WORD1_TIME_METADATA; if (cache_validity & NFS_INO_INVALID_MTIME) bitmask[1] |= FATTR4_WORD1_TIME_MODIFY; if (cache_validity & NFS_INO_INVALID_SIZE) -- cgit v1.2.3 From 37eaeed1a57e92d9db200ba7b4851a09c55eef5a Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 8 Feb 2021 08:55:45 -0500 Subject: NFS: Fix documenting comment for nfs_revalidate_file_size() Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/nfs/file.c b/fs/nfs/file.c index 63940a7a70be..d02a63af9c15 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -89,7 +89,7 @@ nfs_file_release(struct inode *inode, struct file *filp) EXPORT_SYMBOL_GPL(nfs_file_release); /** - * nfs_revalidate_size - Revalidate the file size + * nfs_revalidate_file_size - Revalidate the file size * @inode: pointer to inode struct * @filp: pointer to struct file * -- cgit v1.2.3 From fc9dc401899ab280fe1849a0ca5800384726a793 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 8 Feb 2021 08:55:46 -0500 Subject: NFS: Optimise sparse writes past the end of file If we're doing a write, and the entire page lies beyond the end-of-file, then we can assume the write can be extended to cover the beginning of the page, since we know the data in that region will be all zeros. Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/file.c | 4 +--- fs/nfs/write.c | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/nfs/file.c b/fs/nfs/file.c index d02a63af9c15..02795a01c7ef 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -626,13 +626,11 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from) /* * O_APPEND implies that we must revalidate the file length. */ - if (iocb->ki_flags & IOCB_APPEND) { + if (iocb->ki_flags & IOCB_APPEND || iocb->ki_pos > i_size_read(inode)) { result = nfs_revalidate_file_size(inode, file); if (result) goto out; } - if (iocb->ki_pos > i_size_read(inode)) - nfs_revalidate_mapping(inode, file->f_mapping); since = filemap_sample_wb_err(file->f_mapping); nfs_start_io_write(inode); diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 639c34fec04a..6193350356a8 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1278,19 +1278,21 @@ bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode) * the PageUptodate() flag. In this case, we will need to turn off * write optimisations that depend on the page contents being correct. */ -static bool nfs_write_pageuptodate(struct page *page, struct inode *inode) +static bool nfs_write_pageuptodate(struct page *page, struct inode *inode, + unsigned int pagelen) { struct nfs_inode *nfsi = NFS_I(inode); if (nfs_have_delegated_attributes(inode)) goto out; - if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE) + if (nfsi->cache_validity & + (NFS_INO_REVAL_PAGECACHE | NFS_INO_INVALID_SIZE)) return false; smp_rmb(); - if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags)) + if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags) && pagelen != 0) return false; out: - if (nfsi->cache_validity & NFS_INO_INVALID_DATA) + if (nfsi->cache_validity & NFS_INO_INVALID_DATA && pagelen != 0) return false; return PageUptodate(page) != 0; } @@ -1310,7 +1312,8 @@ is_whole_file_wrlock(struct file_lock *fl) * If the file is opened for synchronous writes then we can just skip the rest * of the checks. */ -static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode) +static int nfs_can_extend_write(struct file *file, struct page *page, + struct inode *inode, unsigned int pagelen) { int ret; struct file_lock_context *flctx = inode->i_flctx; @@ -1318,7 +1321,7 @@ static int nfs_can_extend_write(struct file *file, struct page *page, struct ino if (file->f_flags & O_DSYNC) return 0; - if (!nfs_write_pageuptodate(page, inode)) + if (!nfs_write_pageuptodate(page, inode, pagelen)) return 0; if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) return 1; @@ -1356,6 +1359,7 @@ int nfs_updatepage(struct file *file, struct page *page, struct nfs_open_context *ctx = nfs_file_open_context(file); struct address_space *mapping = page_file_mapping(page); struct inode *inode = mapping->host; + unsigned int pagelen = nfs_page_length(page); int status = 0; nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE); @@ -1366,8 +1370,8 @@ int nfs_updatepage(struct file *file, struct page *page, if (!count) goto out; - if (nfs_can_extend_write(file, page, inode)) { - count = max(count + offset, nfs_page_length(page)); + if (nfs_can_extend_write(file, page, inode, pagelen)) { + count = max(count + offset, pagelen); offset = 0; } -- cgit v1.2.3 From 28aa2f9e73e762dbaa28fdca20cccb59c74cc139 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 8 Feb 2021 08:55:47 -0500 Subject: NFS: Always clear an invalid mapping when attempting a buffered write If the page cache is invalid, then we can't do read-modify-write, so ensure that we do clear it when we know it is invalid. Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/file.c | 2 ++ fs/nfs/inode.c | 97 ++++++++++++++++++++++++++++---------------------- include/linux/nfs_fs.h | 1 + 3 files changed, 57 insertions(+), 43 deletions(-) (limited to 'fs') diff --git a/fs/nfs/file.c b/fs/nfs/file.c index 02795a01c7ef..03fd1dcc96bd 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -632,6 +632,8 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from) goto out; } + nfs_clear_invalid_mapping(file->f_mapping); + since = filemap_sample_wb_err(file->f_mapping); nfs_start_io_write(inode); result = generic_write_checks(iocb, from); diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 522aa10a1a3e..2533053d764a 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -1257,55 +1257,19 @@ static int nfs_invalidate_mapping(struct inode *inode, struct address_space *map return 0; } -bool nfs_mapping_need_revalidate_inode(struct inode *inode) -{ - return nfs_check_cache_invalid(inode, NFS_INO_REVAL_PAGECACHE) || - NFS_STALE(inode); -} - -int nfs_revalidate_mapping_rcu(struct inode *inode) -{ - struct nfs_inode *nfsi = NFS_I(inode); - unsigned long *bitlock = &nfsi->flags; - int ret = 0; - - if (IS_SWAPFILE(inode)) - goto out; - if (nfs_mapping_need_revalidate_inode(inode)) { - ret = -ECHILD; - goto out; - } - spin_lock(&inode->i_lock); - if (test_bit(NFS_INO_INVALIDATING, bitlock) || - (nfsi->cache_validity & NFS_INO_INVALID_DATA)) - ret = -ECHILD; - spin_unlock(&inode->i_lock); -out: - return ret; -} - /** - * nfs_revalidate_mapping - Revalidate the pagecache - * @inode: pointer to host inode + * nfs_clear_invalid_mapping - Conditionally clear a mapping * @mapping: pointer to mapping + * + * If the NFS_INO_INVALID_DATA inode flag is set, clear the mapping. */ -int nfs_revalidate_mapping(struct inode *inode, - struct address_space *mapping) +int nfs_clear_invalid_mapping(struct address_space *mapping) { + struct inode *inode = mapping->host; struct nfs_inode *nfsi = NFS_I(inode); unsigned long *bitlock = &nfsi->flags; int ret = 0; - /* swapfiles are not supposed to be shared. */ - if (IS_SWAPFILE(inode)) - goto out; - - if (nfs_mapping_need_revalidate_inode(inode)) { - ret = __nfs_revalidate_inode(NFS_SERVER(inode), inode); - if (ret < 0) - goto out; - } - /* * We must clear NFS_INO_INVALID_DATA first to ensure that * invalidations that come in while we're shooting down the mappings @@ -1336,8 +1300,8 @@ int nfs_revalidate_mapping(struct inode *inode, set_bit(NFS_INO_INVALIDATING, bitlock); smp_wmb(); - nfsi->cache_validity &= ~(NFS_INO_INVALID_DATA| - NFS_INO_DATA_INVAL_DEFER); + nfsi->cache_validity &= + ~(NFS_INO_INVALID_DATA | NFS_INO_DATA_INVAL_DEFER); spin_unlock(&inode->i_lock); trace_nfs_invalidate_mapping_enter(inode); ret = nfs_invalidate_mapping(inode, mapping); @@ -1350,6 +1314,53 @@ out: return ret; } +bool nfs_mapping_need_revalidate_inode(struct inode *inode) +{ + return nfs_check_cache_invalid(inode, NFS_INO_REVAL_PAGECACHE) || + NFS_STALE(inode); +} + +int nfs_revalidate_mapping_rcu(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + unsigned long *bitlock = &nfsi->flags; + int ret = 0; + + if (IS_SWAPFILE(inode)) + goto out; + if (nfs_mapping_need_revalidate_inode(inode)) { + ret = -ECHILD; + goto out; + } + spin_lock(&inode->i_lock); + if (test_bit(NFS_INO_INVALIDATING, bitlock) || + (nfsi->cache_validity & NFS_INO_INVALID_DATA)) + ret = -ECHILD; + spin_unlock(&inode->i_lock); +out: + return ret; +} + +/** + * nfs_revalidate_mapping - Revalidate the pagecache + * @inode: pointer to host inode + * @mapping: pointer to mapping + */ +int nfs_revalidate_mapping(struct inode *inode, struct address_space *mapping) +{ + /* swapfiles are not supposed to be shared. */ + if (IS_SWAPFILE(inode)) + return 0; + + if (nfs_mapping_need_revalidate_inode(inode)) { + int ret = __nfs_revalidate_inode(NFS_SERVER(inode), inode); + if (ret < 0) + return ret; + } + + return nfs_clear_invalid_mapping(mapping); +} + static bool nfs_file_has_writers(struct nfs_inode *nfsi) { struct inode *inode = &nfsi->vfs_inode; diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 3cfcf219e96b..2c662857247a 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -387,6 +387,7 @@ extern int nfs_open(struct inode *, struct file *); extern int nfs_attribute_cache_expired(struct inode *inode); extern int nfs_revalidate_inode(struct nfs_server *server, struct inode *inode); extern int __nfs_revalidate_inode(struct nfs_server *, struct inode *); +extern int nfs_clear_invalid_mapping(struct address_space *mapping); extern bool nfs_mapping_need_revalidate_inode(struct inode *inode); extern int nfs_revalidate_mapping(struct inode *inode, struct address_space *mapping); extern int nfs_revalidate_mapping_rcu(struct inode *inode); -- cgit v1.2.3 From d50dfc0c7df7bf037442045fbe63952ae0c4ce46 Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Mon, 8 Feb 2021 13:42:21 -0800 Subject: f2fs: don't grab superblock freeze for flush/ckpt thread There are controlled by f2fs_freeze(). This fixes xfstests/generic/068 which is stuck at task:f2fs_ckpt-252:3 state:D stack: 0 pid: 5761 ppid: 2 flags:0x00004000 Call Trace: __schedule+0x44c/0x8a0 schedule+0x4f/0xc0 percpu_rwsem_wait+0xd8/0x140 ? percpu_down_write+0xf0/0xf0 __percpu_down_read+0x56/0x70 issue_checkpoint_thread+0x12c/0x160 [f2fs] ? wait_woken+0x80/0x80 kthread+0x114/0x150 ? __checkpoint_and_complete_reqs+0x110/0x110 [f2fs] ? kthread_park+0x90/0x90 ret_from_fork+0x22/0x30 Signed-off-by: Jaegeuk Kim --- fs/f2fs/checkpoint.c | 4 ---- fs/f2fs/segment.c | 4 ---- fs/f2fs/super.c | 4 ++++ 3 files changed, 4 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 579b9c3603cc..174a0819ad96 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -1763,13 +1763,9 @@ repeat: if (kthread_should_stop()) return 0; - sb_start_intwrite(sbi->sb); - if (!llist_empty(&cprc->issue_list)) __checkpoint_and_complete_reqs(sbi); - sb_end_intwrite(sbi->sb); - wait_event_interruptible(*q, kthread_should_stop() || !llist_empty(&cprc->issue_list)); goto repeat; diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 7d34f1cacdee..440634dfaa56 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -607,8 +607,6 @@ repeat: if (kthread_should_stop()) return 0; - sb_start_intwrite(sbi->sb); - if (!llist_empty(&fcc->issue_list)) { struct flush_cmd *cmd, *next; int ret; @@ -629,8 +627,6 @@ repeat: fcc->dispatch_list = NULL; } - sb_end_intwrite(sbi->sb); - wait_event_interruptible(*q, kthread_should_stop() || !llist_empty(&fcc->issue_list)); goto repeat; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 1000d21120ca..4aa533cb4340 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -1472,6 +1472,10 @@ static int f2fs_freeze(struct super_block *sb) /* must be clean, since sync_filesystem() was already called */ if (is_sbi_flag_set(F2FS_SB(sb), SBI_IS_DIRTY)) return -EINVAL; + + /* ensure no checkpoint required */ + if (!llist_empty(&F2FS_SB(sb)->cprc_info.issue_list)) + return -EINVAL; return 0; } -- cgit v1.2.3 From 4f4317c13a40194940acf4a71670179c4faca2b5 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Tue, 1 Dec 2020 09:53:23 -0500 Subject: btrfs: fix error handling in commit_fs_roots While doing error injection I would sometimes get a corrupt file system. This is because I was injecting errors at btrfs_search_slot, but would only do it one time per stack. This uncovered a problem in commit_fs_roots, where if we get an error we would just break. However we're in a nested loop, the first loop being a loop to find all the dirty fs roots, and then subsequent root updates would succeed clearing the error value. This isn't likely to happen in real scenarios, however we could potentially get a random ENOMEM once and then not again, and we'd end up with a corrupted file system. Fix this by moving the error checking around a bit to the main loop, as this is the only place where something will fail, and return the error as soon as it occurs. With this patch my reproducer no longer corrupts the file system. Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/transaction.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 6af7f2bf92de..fbf93067642a 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -1319,7 +1319,6 @@ static noinline int commit_fs_roots(struct btrfs_trans_handle *trans) struct btrfs_root *gang[8]; int i; int ret; - int err = 0; spin_lock(&fs_info->fs_roots_radix_lock); while (1) { @@ -1331,6 +1330,8 @@ static noinline int commit_fs_roots(struct btrfs_trans_handle *trans) break; for (i = 0; i < ret; i++) { struct btrfs_root *root = gang[i]; + int ret2; + radix_tree_tag_clear(&fs_info->fs_roots_radix, (unsigned long)root->root_key.objectid, BTRFS_ROOT_TRANS_TAG); @@ -1350,17 +1351,17 @@ static noinline int commit_fs_roots(struct btrfs_trans_handle *trans) root->node); } - err = btrfs_update_root(trans, fs_info->tree_root, + ret2 = btrfs_update_root(trans, fs_info->tree_root, &root->root_key, &root->root_item); + if (ret2) + return ret2; spin_lock(&fs_info->fs_roots_radix_lock); - if (err) - break; btrfs_qgroup_free_meta_all_pertrans(root); } } spin_unlock(&fs_info->fs_roots_radix_lock); - return err; + return 0; } /* -- cgit v1.2.3 From 3cc64e7ebfb0d7faaba2438334c43466955a96e8 Mon Sep 17 00:00:00 2001 From: Zhihao Cheng Date: Fri, 20 Nov 2020 09:08:04 +0800 Subject: btrfs: clarify error returns values in __load_free_space_cache Return value in __load_free_space_cache is not properly set after (unlikely) memory allocation failures and 0 is returned instead. This is not a problem for the caller load_free_space_cache because only value 1 is considered as 'cache loaded' but for clarity it's better to set the errors accordingly. Fixes: a67509c30079 ("Btrfs: add a io_ctl struct and helpers for dealing with the space cache") Reported-by: Hulk Robot Signed-off-by: Zhihao Cheng Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/free-space-cache.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 4d8897879c9c..71d0d14bc18b 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -775,8 +775,10 @@ static int __load_free_space_cache(struct btrfs_root *root, struct inode *inode, while (num_entries) { e = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS); - if (!e) + if (!e) { + ret = -ENOMEM; goto free_cache; + } ret = io_ctl_read_entry(&io_ctl, e, &type); if (ret) { @@ -785,6 +787,7 @@ static int __load_free_space_cache(struct btrfs_root *root, struct inode *inode, } if (!e->bytes) { + ret = -1; kmem_cache_free(btrfs_free_space_cachep, e); goto free_cache; } @@ -805,6 +808,7 @@ static int __load_free_space_cache(struct btrfs_root *root, struct inode *inode, e->bitmap = kmem_cache_zalloc( btrfs_free_space_bitmap_cachep, GFP_NOFS); if (!e->bitmap) { + ret = -ENOMEM; kmem_cache_free( btrfs_free_space_cachep, e); goto free_cache; -- cgit v1.2.3 From 149716570be98185150860fe922bf89ed080bd3c Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Thu, 10 Dec 2020 10:38:32 +0200 Subject: btrfs: cleanup local variables in btrfs_file_write_iter First replace all inode instances with a pointer to btrfs_inode. This removes multiple invocations of the BTRFS_I macro, subsequently remove 2 local variables as they are called only once and simply refer to them directly. Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/file.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 0e41459b8de6..e65223e3510d 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1997,9 +1997,7 @@ static ssize_t btrfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; - struct inode *inode = file_inode(file); - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_inode *inode = BTRFS_I(file_inode(file)); ssize_t num_written = 0; const bool sync = iocb->ki_flags & IOCB_DSYNC; @@ -2008,7 +2006,7 @@ static ssize_t btrfs_file_write_iter(struct kiocb *iocb, * have opened a file as writable, we have to stop this write operation * to ensure consistency. */ - if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) + if (test_bit(BTRFS_FS_STATE_ERROR, &inode->root->fs_info->fs_state)) return -EROFS; if (!(iocb->ki_flags & IOCB_DIRECT) && @@ -2016,7 +2014,7 @@ static ssize_t btrfs_file_write_iter(struct kiocb *iocb, return -EOPNOTSUPP; if (sync) - atomic_inc(&BTRFS_I(inode)->sync_writers); + atomic_inc(&inode->sync_writers); if (iocb->ki_flags & IOCB_DIRECT) num_written = btrfs_direct_write(iocb, from); @@ -2028,14 +2026,14 @@ static ssize_t btrfs_file_write_iter(struct kiocb *iocb, * otherwise subsequent syncs to a file that's been synced in this * transaction will appear to have already occurred. */ - spin_lock(&BTRFS_I(inode)->lock); - BTRFS_I(inode)->last_sub_trans = root->log_transid; - spin_unlock(&BTRFS_I(inode)->lock); + spin_lock(&inode->lock); + inode->last_sub_trans = inode->root->log_transid; + spin_unlock(&inode->lock); if (num_written > 0) num_written = generic_write_sync(iocb, num_written); if (sync) - atomic_dec(&BTRFS_I(inode)->sync_writers); + atomic_dec(&inode->sync_writers); current->backing_dev_info = NULL; return num_written; -- cgit v1.2.3 From 453e4873869f5e967188d8b018efc34a57eed44f Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 7 Dec 2020 17:32:32 +0200 Subject: btrfs: rename btrfs_find_highest_objectid to btrfs_init_root_free_objectid This function is used to initialize the in-memory btrfs_root::highest_objectid member, which is used to get an available objectid. Rename it to better reflect its semantics. Reviewed-by: Josef Bacik Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 12 +++++------- fs/btrfs/disk-io.h | 2 +- fs/btrfs/tree-log.c | 3 +-- 3 files changed, 7 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 6b35b7e88136..47646a79d3fc 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1367,8 +1367,7 @@ static int btrfs_init_fs_root(struct btrfs_root *root, dev_t anon_dev) } mutex_lock(&root->objectid_mutex); - ret = btrfs_find_highest_objectid(root, - &root->highest_objectid); + ret = btrfs_init_root_free_objectid(root); if (ret) { mutex_unlock(&root->objectid_mutex); goto fail; @@ -2646,8 +2645,7 @@ static int __cold init_tree_roots(struct btrfs_fs_info *fs_info) * No need to hold btrfs_root::objectid_mutex since the fs * hasn't been fully initialised and we are the only user */ - ret = btrfs_find_highest_objectid(tree_root, - &tree_root->highest_objectid); + ret = btrfs_init_root_free_objectid(tree_root); if (ret < 0) { handle_error = true; continue; @@ -4745,7 +4743,7 @@ static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info) return 0; } -int btrfs_find_highest_objectid(struct btrfs_root *root, u64 *objectid) +int btrfs_init_root_free_objectid(struct btrfs_root *root) { struct btrfs_path *path; int ret; @@ -4769,10 +4767,10 @@ int btrfs_find_highest_objectid(struct btrfs_root *root, u64 *objectid) slot = path->slots[0] - 1; l = path->nodes[0]; btrfs_item_key_to_cpu(l, &found_key, slot); - *objectid = max_t(u64, found_key.objectid, + root->highest_objectid = max_t(u64, found_key.objectid, BTRFS_FIRST_FREE_OBJECTID - 1); } else { - *objectid = BTRFS_FIRST_FREE_OBJECTID - 1; + root->highest_objectid = BTRFS_FIRST_FREE_OBJECTID - 1; } ret = 0; error: diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index e45057c0c016..5e5bc603fbdf 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -134,7 +134,7 @@ int btree_lock_page_hook(struct page *page, void *data, void (*flush_fn)(void *)); int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags); int btrfs_find_free_objectid(struct btrfs_root *root, u64 *objectid); -int btrfs_find_highest_objectid(struct btrfs_root *root, u64 *objectid); +int btrfs_init_root_free_objectid(struct btrfs_root *root); int __init btrfs_end_io_wq_init(void); void __cold btrfs_end_io_wq_exit(void); diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 254c2ee43aae..8ee0700a980f 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -6307,8 +6307,7 @@ again: * root->objectid_mutex is not acquired as log replay * could only happen during mount. */ - ret = btrfs_find_highest_objectid(root, - &root->highest_objectid); + ret = btrfs_init_root_free_objectid(root); } wc.replay_dest->log_root = NULL; -- cgit v1.2.3 From 543068a217a877bb6fa831fc448c9cc131db4feb Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 7 Dec 2020 17:32:33 +0200 Subject: btrfs: rename btrfs_find_free_objectid to btrfs_get_free_objectid This better reflects the semantics of the function i.e no search is performed whatsoever. Reviewed-by: Josef Bacik Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 2 +- fs/btrfs/disk-io.h | 2 +- fs/btrfs/free-space-cache.c | 2 +- fs/btrfs/inode.c | 12 ++++++------ fs/btrfs/ioctl.c | 2 +- fs/btrfs/relocation.c | 2 +- fs/btrfs/transaction.c | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 47646a79d3fc..98384d3e41ac 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -4778,7 +4778,7 @@ error: return ret; } -int btrfs_find_free_objectid(struct btrfs_root *root, u64 *objectid) +int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid) { int ret; mutex_lock(&root->objectid_mutex); diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index 5e5bc603fbdf..9f4a2a1e3d36 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -133,7 +133,7 @@ struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans, int btree_lock_page_hook(struct page *page, void *data, void (*flush_fn)(void *)); int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags); -int btrfs_find_free_objectid(struct btrfs_root *root, u64 *objectid); +int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid); int btrfs_init_root_free_objectid(struct btrfs_root *root); int __init btrfs_end_io_wq_init(void); void __cold btrfs_end_io_wq_exit(void); diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 71d0d14bc18b..fd6ddd6b8165 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -198,7 +198,7 @@ int create_free_space_inode(struct btrfs_trans_handle *trans, int ret; u64 ino; - ret = btrfs_find_free_objectid(trans->fs_info->tree_root, &ino); + ret = btrfs_get_free_objectid(trans->fs_info->tree_root, &ino); if (ret < 0) return ret; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index a8e0a6b038d3..0fe4df05006d 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -6371,7 +6371,7 @@ static int btrfs_mknod(struct inode *dir, struct dentry *dentry, if (IS_ERR(trans)) return PTR_ERR(trans); - err = btrfs_find_free_objectid(root, &objectid); + err = btrfs_get_free_objectid(root, &objectid); if (err) goto out_unlock; @@ -6435,7 +6435,7 @@ static int btrfs_create(struct inode *dir, struct dentry *dentry, if (IS_ERR(trans)) return PTR_ERR(trans); - err = btrfs_find_free_objectid(root, &objectid); + err = btrfs_get_free_objectid(root, &objectid); if (err) goto out_unlock; @@ -6579,7 +6579,7 @@ static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) if (IS_ERR(trans)) return PTR_ERR(trans); - err = btrfs_find_free_objectid(root, &objectid); + err = btrfs_get_free_objectid(root, &objectid); if (err) goto out_fail; @@ -9079,7 +9079,7 @@ static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans, u64 objectid; u64 index; - ret = btrfs_find_free_objectid(root, &objectid); + ret = btrfs_get_free_objectid(root, &objectid); if (ret) return ret; @@ -9575,7 +9575,7 @@ static int btrfs_symlink(struct inode *dir, struct dentry *dentry, if (IS_ERR(trans)) return PTR_ERR(trans); - err = btrfs_find_free_objectid(root, &objectid); + err = btrfs_get_free_objectid(root, &objectid); if (err) goto out_unlock; @@ -9909,7 +9909,7 @@ static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) if (IS_ERR(trans)) return PTR_ERR(trans); - ret = btrfs_find_free_objectid(root, &objectid); + ret = btrfs_get_free_objectid(root, &objectid); if (ret) goto out; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index dde49a791f3e..ec83803800c5 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -613,7 +613,7 @@ static noinline int create_subvol(struct inode *dir, if (!root_item) return -ENOMEM; - ret = btrfs_find_free_objectid(fs_info->tree_root, &objectid); + ret = btrfs_get_free_objectid(fs_info->tree_root, &objectid); if (ret) goto fail_free; diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index df63ef64c5c0..2698805ebd26 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -3434,7 +3434,7 @@ struct inode *create_reloc_inode(struct btrfs_fs_info *fs_info, return ERR_CAST(trans); } - err = btrfs_find_free_objectid(root, &objectid); + err = btrfs_get_free_objectid(root, &objectid); if (err) goto out; diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index fbf93067642a..3bcb5444536e 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -1526,7 +1526,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans, ASSERT(pending->root_item); new_root_item = pending->root_item; - pending->error = btrfs_find_free_objectid(tree_root, &objectid); + pending->error = btrfs_get_free_objectid(tree_root, &objectid); if (pending->error) goto no_free_objectid; -- cgit v1.2.3 From 6b8fad576a3c8f822a888873c5acdfb31de53c4c Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 7 Dec 2020 17:32:35 +0200 Subject: btrfs: rename btrfs_root::highest_objectid to free_objectid This reflects the true purpose of the member as it's being used solely in context where a new objectid is being allocated. Future changes will also change the way it's being used to closely follow this semantics. Reviewed-by: Josef Bacik Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 2 +- fs/btrfs/disk-io.c | 14 +++++++------- fs/btrfs/ioctl.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 4debdbdde2ab..dc77aac2476c 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -1104,7 +1104,7 @@ struct btrfs_root { u32 type; - u64 highest_objectid; + u64 free_objectid; struct btrfs_key defrag_progress; struct btrfs_key defrag_max; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 98384d3e41ac..a5ae0bbd983e 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1016,7 +1016,7 @@ static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info, root->orphan_cleanup_state = 0; root->last_trans = 0; - root->highest_objectid = 0; + root->free_objectid = 0; root->nr_delalloc_inodes = 0; root->nr_ordered_extents = 0; root->inode_tree = RB_ROOT; @@ -1373,7 +1373,7 @@ static int btrfs_init_fs_root(struct btrfs_root *root, dev_t anon_dev) goto fail; } - ASSERT(root->highest_objectid <= BTRFS_LAST_FREE_OBJECTID); + ASSERT(root->free_objectid <= BTRFS_LAST_FREE_OBJECTID); mutex_unlock(&root->objectid_mutex); @@ -2651,7 +2651,7 @@ static int __cold init_tree_roots(struct btrfs_fs_info *fs_info) continue; } - ASSERT(tree_root->highest_objectid <= BTRFS_LAST_FREE_OBJECTID); + ASSERT(tree_root->free_objectid <= BTRFS_LAST_FREE_OBJECTID); ret = btrfs_read_roots(fs_info); if (ret < 0) { @@ -4767,10 +4767,10 @@ int btrfs_init_root_free_objectid(struct btrfs_root *root) slot = path->slots[0] - 1; l = path->nodes[0]; btrfs_item_key_to_cpu(l, &found_key, slot); - root->highest_objectid = max_t(u64, found_key.objectid, + root->free_objectid = max_t(u64, found_key.objectid, BTRFS_FIRST_FREE_OBJECTID - 1); } else { - root->highest_objectid = BTRFS_FIRST_FREE_OBJECTID - 1; + root->free_objectid = BTRFS_FIRST_FREE_OBJECTID - 1; } ret = 0; error: @@ -4783,7 +4783,7 @@ int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid) int ret; mutex_lock(&root->objectid_mutex); - if (unlikely(root->highest_objectid >= BTRFS_LAST_FREE_OBJECTID)) { + if (unlikely(root->free_objectid >= BTRFS_LAST_FREE_OBJECTID)) { btrfs_warn(root->fs_info, "the objectid of root %llu reaches its highest value", root->root_key.objectid); @@ -4791,7 +4791,7 @@ int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid) goto out; } - *objectid = ++root->highest_objectid; + *objectid = ++root->free_objectid; ret = 0; out: mutex_unlock(&root->objectid_mutex); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index ec83803800c5..2041c4b6fd0e 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -725,7 +725,7 @@ static noinline int create_subvol(struct inode *dir, } mutex_lock(&new_root->objectid_mutex); - new_root->highest_objectid = new_dirid; + new_root->free_objectid = new_dirid; mutex_unlock(&new_root->objectid_mutex); /* -- cgit v1.2.3 From 23125104d8485505cd19581025a3d6fc14e9945a Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 7 Dec 2020 17:32:36 +0200 Subject: btrfs: make btrfs_root::free_objectid hold the next available objectid Adjust the way free_objectid is being initialized, it now stores BTRFS_FIRST_FREE_OBJECTID rather than the, somewhat arbitrary, BTRFS_FIRST_FREE_OBJECTID - 1. This change also has the added benefit that now it becomes unnecessary to explicitly initialize free_objectid for a newly create fs root. Reviewed-by: Josef Bacik Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 8 ++++---- fs/btrfs/inode.c | 8 ++++++-- fs/btrfs/ioctl.c | 4 ---- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index a5ae0bbd983e..5473bed6a7e8 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -4767,10 +4767,10 @@ int btrfs_init_root_free_objectid(struct btrfs_root *root) slot = path->slots[0] - 1; l = path->nodes[0]; btrfs_item_key_to_cpu(l, &found_key, slot); - root->free_objectid = max_t(u64, found_key.objectid, - BTRFS_FIRST_FREE_OBJECTID - 1); + root->free_objectid = max_t(u64, found_key.objectid + 1, + BTRFS_FIRST_FREE_OBJECTID); } else { - root->free_objectid = BTRFS_FIRST_FREE_OBJECTID - 1; + root->free_objectid = BTRFS_FIRST_FREE_OBJECTID; } ret = 0; error: @@ -4791,7 +4791,7 @@ int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid) goto out; } - *objectid = ++root->free_objectid; + *objectid = root->free_objectid++; ret = 0; out: mutex_unlock(&root->objectid_mutex); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 0fe4df05006d..356905d97656 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8598,9 +8598,13 @@ int btrfs_create_subvol_root(struct btrfs_trans_handle *trans, struct inode *inode; int err; u64 index = 0; + u64 ino; + + err = btrfs_get_free_objectid(new_root, &ino); + if (err < 0) + return err; - inode = btrfs_new_inode(trans, new_root, NULL, "..", 2, - new_dirid, new_dirid, + inode = btrfs_new_inode(trans, new_root, NULL, "..", 2, ino, ino, S_IFDIR | (~current_umask() & S_IRWXUGO), &index); if (IS_ERR(inode)) diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 2041c4b6fd0e..d8422074da02 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -724,10 +724,6 @@ static noinline int create_subvol(struct inode *dir, goto fail; } - mutex_lock(&new_root->objectid_mutex); - new_root->free_objectid = new_dirid; - mutex_unlock(&new_root->objectid_mutex); - /* * insert the directory item */ -- cgit v1.2.3 From 69948022c9261a87c3c256bfa21c132f5099c690 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 7 Dec 2020 17:32:37 +0200 Subject: btrfs: remove new_dirid argument from btrfs_create_subvol_root It's no longer used. While at it also remove new_dirid in create_subvol as it's used in a single place and open code it. No functional changes. Reviewed-by: Josef Bacik Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 3 +-- fs/btrfs/inode.c | 3 +-- fs/btrfs/ioctl.c | 5 ++--- 3 files changed, 4 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index dc77aac2476c..f5c636b29451 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -3107,8 +3107,7 @@ int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, struct extent_state **cached_state); int btrfs_create_subvol_root(struct btrfs_trans_handle *trans, struct btrfs_root *new_root, - struct btrfs_root *parent_root, - u64 new_dirid); + struct btrfs_root *parent_root); void btrfs_set_delalloc_extent(struct inode *inode, struct extent_state *state, unsigned *bits); void btrfs_clear_delalloc_extent(struct inode *inode, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 356905d97656..af5558f87243 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8592,8 +8592,7 @@ out: */ int btrfs_create_subvol_root(struct btrfs_trans_handle *trans, struct btrfs_root *new_root, - struct btrfs_root *parent_root, - u64 new_dirid) + struct btrfs_root *parent_root) { struct inode *inode; int err; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index d8422074da02..5b9b0a390f0e 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -606,7 +606,6 @@ static noinline int create_subvol(struct inode *dir, int err; dev_t anon_dev = 0; u64 objectid; - u64 new_dirid = BTRFS_FIRST_FREE_OBJECTID; u64 index = 0; root_item = kzalloc(sizeof(*root_item), GFP_KERNEL); @@ -693,7 +692,7 @@ static noinline int create_subvol(struct inode *dir, free_extent_buffer(leaf); leaf = NULL; - btrfs_set_root_dirid(root_item, new_dirid); + btrfs_set_root_dirid(root_item, BTRFS_FIRST_FREE_OBJECTID); key.objectid = objectid; key.offset = 0; @@ -716,7 +715,7 @@ static noinline int create_subvol(struct inode *dir, btrfs_record_root_in_trans(trans, new_root); - ret = btrfs_create_subvol_root(trans, new_root, root, new_dirid); + ret = btrfs_create_subvol_root(trans, new_root, root); btrfs_put_root(new_root); if (ret) { /* We potentially lose an unused inode item here */ -- cgit v1.2.3 From f75e2b79b5ba9dd3e0899840a329c3da02dc8937 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Wed, 16 Dec 2020 11:18:43 -0500 Subject: btrfs: allow error injection for btrfs_search_slot and btrfs_cow_block The following patches are going to address error handling in relocation, in order to test those patches I need to be able to inject errors in btrfs_search_slot and btrfs_cow_block, as we call both of these pretty often in different cases during relocation. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index cc89b63d65a4..56e132d825a2 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -1494,6 +1494,7 @@ noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, return ret; } +ALLOW_ERROR_INJECTION(btrfs_cow_block, ERRNO); /* * helper function for defrag to decide if two blocks pointed to by a @@ -2821,6 +2822,7 @@ done: btrfs_release_path(p); return ret; } +ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO); /* * Like btrfs_search_slot, this looks for a key in the given tree. It uses the -- cgit v1.2.3 From 1fec12a560033ebe8fa6857dd3cbf9677371fbee Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Wed, 16 Dec 2020 11:18:45 -0500 Subject: btrfs: noinline btrfs_should_cancel_balance I was attempting to reproduce a problem that Zygo hit, but my error injection wasn't firing for a few of the common calls to btrfs_should_cancel_balance. This is because the compiler decided to inline it at these spots. Keep this from happening by explicitly marking the function as noinline so that error injection will always work. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 2698805ebd26..8e51b39cbfbb 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -2615,7 +2615,7 @@ int setup_extent_mapping(struct inode *inode, u64 start, u64 end, /* * Allow error injection to test balance cancellation */ -int btrfs_should_cancel_balance(struct btrfs_fs_info *fs_info) +noinline int btrfs_should_cancel_balance(struct btrfs_fs_info *fs_info) { return atomic_read(&fs_info->balance_cancel_req) || fatal_signal_pending(current); -- cgit v1.2.3 From 0d73a11c62642a25b688d09ae04b3b1f1b58ebb9 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Wed, 16 Dec 2020 11:18:46 -0500 Subject: btrfs: ref-verify: pass down tree block level when building refs I noticed that sometimes I would have the wrong level printed out with ref-verify while testing some error injection related problems. This is because we only get the level from the main extent item, but our references could go off the current leaf into another, and at that point we lose our level. Fix this by keeping track of the last tree block level that we found, the same way we keep track of our bytenr and num_bytes, in case we happen to wander into another leaf while still processing the references for a bytenr. Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ref-verify.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ref-verify.c b/fs/btrfs/ref-verify.c index 4b9b6c52a83b..409b02566b25 100644 --- a/fs/btrfs/ref-verify.c +++ b/fs/btrfs/ref-verify.c @@ -495,14 +495,15 @@ static int process_extent_item(struct btrfs_fs_info *fs_info, } static int process_leaf(struct btrfs_root *root, - struct btrfs_path *path, u64 *bytenr, u64 *num_bytes) + struct btrfs_path *path, u64 *bytenr, u64 *num_bytes, + int *tree_block_level) { struct btrfs_fs_info *fs_info = root->fs_info; struct extent_buffer *leaf = path->nodes[0]; struct btrfs_extent_data_ref *dref; struct btrfs_shared_data_ref *sref; u32 count; - int i = 0, tree_block_level = 0, ret = 0; + int i = 0, ret = 0; struct btrfs_key key; int nritems = btrfs_header_nritems(leaf); @@ -515,15 +516,15 @@ static int process_leaf(struct btrfs_root *root, case BTRFS_METADATA_ITEM_KEY: *bytenr = key.objectid; ret = process_extent_item(fs_info, path, &key, i, - &tree_block_level); + tree_block_level); break; case BTRFS_TREE_BLOCK_REF_KEY: ret = add_tree_block(fs_info, key.offset, 0, - key.objectid, tree_block_level); + key.objectid, *tree_block_level); break; case BTRFS_SHARED_BLOCK_REF_KEY: ret = add_tree_block(fs_info, 0, key.offset, - key.objectid, tree_block_level); + key.objectid, *tree_block_level); break; case BTRFS_EXTENT_DATA_REF_KEY: dref = btrfs_item_ptr(leaf, i, @@ -549,7 +550,8 @@ static int process_leaf(struct btrfs_root *root, /* Walk down to the leaf from the given level */ static int walk_down_tree(struct btrfs_root *root, struct btrfs_path *path, - int level, u64 *bytenr, u64 *num_bytes) + int level, u64 *bytenr, u64 *num_bytes, + int *tree_block_level) { struct extent_buffer *eb; int ret = 0; @@ -565,7 +567,8 @@ static int walk_down_tree(struct btrfs_root *root, struct btrfs_path *path, path->slots[level-1] = 0; path->locks[level-1] = BTRFS_READ_LOCK; } else { - ret = process_leaf(root, path, bytenr, num_bytes); + ret = process_leaf(root, path, bytenr, num_bytes, + tree_block_level); if (ret) break; } @@ -974,6 +977,7 @@ int btrfs_build_ref_tree(struct btrfs_fs_info *fs_info) { struct btrfs_path *path; struct extent_buffer *eb; + int tree_block_level = 0; u64 bytenr = 0, num_bytes = 0; int ret, level; @@ -998,7 +1002,7 @@ int btrfs_build_ref_tree(struct btrfs_fs_info *fs_info) * different leaf from the original extent item. */ ret = walk_down_tree(fs_info->extent_root, path, level, - &bytenr, &num_bytes); + &bytenr, &num_bytes, &tree_block_level); if (ret) break; ret = walk_up_tree(path, &level); -- cgit v1.2.3 From 1478143ac81acc4094f8501a88e9e6ef9ff0e4a5 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Wed, 16 Dec 2020 11:18:47 -0500 Subject: btrfs: ref-verify: make sure owner is set for all refs I noticed that shared ref entries in ref-verify didn't have the proper owner set, which caused me to think there was something seriously wrong. However the problem is if we have a parent we simply weren't filling out the owner part of the reference, even though we have it. Fix this by making sure we set all the proper fields when we modify a reference, this way we'll have the proper owner if a problem happens and we don't waste time thinking we're updating the wrong level. Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/ref-verify.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ref-verify.c b/fs/btrfs/ref-verify.c index 409b02566b25..2b490becbe67 100644 --- a/fs/btrfs/ref-verify.c +++ b/fs/btrfs/ref-verify.c @@ -669,18 +669,18 @@ int btrfs_ref_tree_mod(struct btrfs_fs_info *fs_info, u64 bytenr = generic_ref->bytenr; u64 num_bytes = generic_ref->len; u64 parent = generic_ref->parent; - u64 ref_root; - u64 owner; - u64 offset; + u64 ref_root = 0; + u64 owner = 0; + u64 offset = 0; if (!btrfs_test_opt(fs_info, REF_VERIFY)) return 0; if (generic_ref->type == BTRFS_REF_METADATA) { - ref_root = generic_ref->tree_ref.root; + if (!parent) + ref_root = generic_ref->tree_ref.root; owner = generic_ref->tree_ref.level; - offset = 0; - } else { + } else if (!parent) { ref_root = generic_ref->data_ref.ref_root; owner = generic_ref->data_ref.ino; offset = generic_ref->data_ref.offset; @@ -696,13 +696,10 @@ int btrfs_ref_tree_mod(struct btrfs_fs_info *fs_info, goto out; } - if (parent) { - ref->parent = parent; - } else { - ref->root_objectid = ref_root; - ref->owner = owner; - ref->offset = offset; - } + ref->parent = parent; + ref->owner = owner; + ref->root_objectid = ref_root; + ref->offset = offset; ref->num_refs = (action == BTRFS_DROP_DELAYED_REF) ? -1 : 1; memcpy(&ra->ref, ref, sizeof(struct ref_entry)); -- cgit v1.2.3 From 7056bf69e5a338811738a7932b8e707aaca9fdd0 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Thu, 17 Dec 2020 15:21:16 +0200 Subject: btrfs: consolidate btrfs_previous_item ret val handling in btrfs_shrink_device Instead of having three 'if' to handle non-NULL return value consolidate this in one 'if (ret)'. That way the code is more obvious: - Always drop delete_unused_bgs_mutex if ret is not NULL - If ret is negative -> goto done - If it's 1 -> reset ret to 0, release the path and finish the loop. Reviewed-by: Josef Bacik Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index d6c24c8ad749..a8ec8539cd8d 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -4668,11 +4668,10 @@ again: } ret = btrfs_previous_item(root, path, 0, key.type); - if (ret) - mutex_unlock(&fs_info->delete_unused_bgs_mutex); - if (ret < 0) - goto done; if (ret) { + mutex_unlock(&fs_info->delete_unused_bgs_mutex); + if (ret < 0) + goto done; ret = 0; btrfs_release_path(path); break; -- cgit v1.2.3 From 9c4a062a94752dabd3954ef39c4dfed581c664b9 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Mon, 11 Jan 2021 11:42:32 +0000 Subject: btrfs: send: remove stale code when checking for shared extents After commit 040ee6120cb670 ("Btrfs: send, improve clone range") we do not use anymore the data_offset field of struct backref_ctx, as after that we do all the necessary checks for the data offset of file extent items at clone_range(). Since there are no more users of data_offset from that structure, remove it. Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 78a35374d492..3bcbf2bcb869 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -1191,9 +1191,6 @@ struct backref_ctx { /* may be truncated in case it's the last extent in a file */ u64 extent_len; - /* data offset in the file extent item */ - u64 data_offset; - /* Just to check for bugs in backref resolving */ int found_itself; }; @@ -1401,19 +1398,6 @@ static int find_extent_clone(struct send_ctx *sctx, backref_ctx->cur_offset = data_offset; backref_ctx->found_itself = 0; backref_ctx->extent_len = num_bytes; - /* - * For non-compressed extents iterate_extent_inodes() gives us extent - * offsets that already take into account the data offset, but not for - * compressed extents, since the offset is logical and not relative to - * the physical extent locations. We must take this into account to - * avoid sending clone offsets that go beyond the source file's size, - * which would result in the clone ioctl failing with -EINVAL on the - * receiving end. - */ - if (compressed == BTRFS_COMPRESS_NONE) - backref_ctx->data_offset = 0; - else - backref_ctx->data_offset = btrfs_file_extent_offset(eb, fi); /* * The last extent of a file may be too large due to page alignment. -- cgit v1.2.3 From 9db4dc241e87fccd8301357d5ef908f40b50f2e3 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 11 Jan 2021 12:58:11 +0200 Subject: btrfs: make btrfs_start_delalloc_root's nr argument a long It's currently u64 which gets instantly translated either to LONG_MAX (if U64_MAX is passed) or cast to an unsigned long (which is in fact, wrong because writeback_control::nr_to_write is a signed, long type). Just convert the function's argument to be long time which obviates the need to manually convert u64 value to a long. Adjust all call sites which pass U64_MAX to pass LONG_MAX. Finally ensure that in shrink_delalloc the u64 is converted to a long without overflowing, resulting in a negative number. Reviewed-by: Josef Bacik Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 2 +- fs/btrfs/dev-replace.c | 2 +- fs/btrfs/inode.c | 6 +++--- fs/btrfs/ioctl.c | 2 +- fs/btrfs/space-info.c | 3 ++- 5 files changed, 8 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index f5c636b29451..ed6bb46a2572 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -3100,7 +3100,7 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, u32 min_type); int btrfs_start_delalloc_snapshot(struct btrfs_root *root); -int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, u64 nr, +int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, bool in_reclaim_context); int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, unsigned int extra_bits, diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index 324f646d6e5e..bc73f798ce3a 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -715,7 +715,7 @@ static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info, * flush all outstanding I/O and inode extent mappings before the * copy operation is declared as being finished */ - ret = btrfs_start_delalloc_roots(fs_info, U64_MAX, false); + ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false); if (ret) { mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); return ret; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index af5558f87243..17418a75e3c8 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -9489,11 +9489,11 @@ int btrfs_start_delalloc_snapshot(struct btrfs_root *root) return start_delalloc_inodes(root, &wbc, true, false); } -int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, u64 nr, +int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, bool in_reclaim_context) { struct writeback_control wbc = { - .nr_to_write = (nr == U64_MAX) ? LONG_MAX : (unsigned long)nr, + .nr_to_write = nr, .sync_mode = WB_SYNC_NONE, .range_start = 0, .range_end = LLONG_MAX, @@ -9515,7 +9515,7 @@ int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, u64 nr, * Reset nr_to_write here so we know that we're doing a full * flush. */ - if (nr == U64_MAX) + if (nr == LONG_MAX) wbc.nr_to_write = LONG_MAX; root = list_first_entry(&splice, struct btrfs_root, diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 5b9b0a390f0e..7f2935ea8d3a 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -4946,7 +4946,7 @@ long btrfs_ioctl(struct file *file, unsigned int case BTRFS_IOC_SYNC: { int ret; - ret = btrfs_start_delalloc_roots(fs_info, U64_MAX, false); + ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false); if (ret) return ret; ret = btrfs_sync_fs(inode->i_sb, 1); diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index e8347461c8dd..84fb94e78a8f 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -532,7 +532,8 @@ static void shrink_delalloc(struct btrfs_fs_info *fs_info, loops = 0; while ((delalloc_bytes || dio_bytes) && loops < 3) { - u64 nr_pages = min(delalloc_bytes, to_reclaim) >> PAGE_SHIFT; + u64 temp = min(delalloc_bytes, to_reclaim) >> PAGE_SHIFT; + long nr_pages = min_t(u64, temp, LONG_MAX); btrfs_start_delalloc_roots(fs_info, nr_pages, true); -- cgit v1.2.3 From d7830b7155ab43952ec8f2b95f326f63936ecd03 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 11 Jan 2021 12:58:12 +0200 Subject: btrfs: remove always true condition in btrfs_start_delalloc_roots Following the rework in e076ab2a2ca7 ("btrfs: shrink delalloc pages instead of full inodes") the nr variable is no longer passed by reference to start_delalloc_inodes hence it cannot change. Additionally we are always guaranteed for it to be positive number hence it's redundant to have it as a condition in the loop. Simply remove that usage. Reviewed-by: Josef Bacik Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 17418a75e3c8..4056fc3b39cf 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -9510,7 +9510,7 @@ int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, mutex_lock(&fs_info->delalloc_root_mutex); spin_lock(&fs_info->delalloc_root_lock); list_splice_init(&fs_info->delalloc_roots, &splice); - while (!list_empty(&splice) && nr) { + while (!list_empty(&splice)) { /* * Reset nr_to_write here so we know that we're doing a full * flush. -- cgit v1.2.3 From 523929f1cac3e869492ea376c9d86af11ec0e5c5 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 22 Dec 2020 13:59:23 +0800 Subject: btrfs: make btrfs_dio_private::bytes u32 btrfs_dio_private::bytes is only assigned from bio::bi_iter::bi_size, which is never larger than U32. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index d9bf53d9ff90..28e202e89660 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -325,7 +325,8 @@ struct btrfs_dio_private { struct inode *inode; u64 logical_offset; u64 disk_bytenr; - u64 bytes; + /* Used for bio::bi_size */ + u32 bytes; /* * References to this structure. There is one reference per in-flight -- cgit v1.2.3 From 58f74b2203d786da37128cbf786873996145bfdc Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 22 Dec 2020 13:59:24 +0800 Subject: btrfs: refactor btrfs_dec_test_* functions for ordered extents The refactoring involves the following modifications: - Return bool instead of int - Parameter update for @cached of btrfs_dec_test_first_ordered_pending() For btrfs_dec_test_first_ordered_pending(), @cached is only used to return the finished ordered extent. Rename it to @finished_ret. - Comment updates * Change one stale comment Which still refers to btrfs_dec_test_ordered_pending(), but the context is calling btrfs_dec_test_first_ordered_pending(). * Follow the common comment style for both functions Add more detailed descriptions for parameters and the return value * Move the reason why test_and_set_bit() is used into the call sites - Change how the return value is calculated The most anti-human part of the return value is: if (...) ret = 1; ... return ret == 0; This means, when we set ret to 1, the function returns 0. Change the local variable name to @finished, and directly return the value of it. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 6 +-- fs/btrfs/ordered-data.c | 104 +++++++++++++++++++++++++++--------------------- fs/btrfs/ordered-data.h | 10 ++--- 3 files changed, 65 insertions(+), 55 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 4056fc3b39cf..ef6cb7b620d0 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7797,10 +7797,8 @@ static void __endio_write_update_ordered(struct btrfs_inode *inode, NULL); btrfs_queue_work(wq, &ordered->work); } - /* - * If btrfs_dec_test_ordered_pending does not find any ordered - * extent in the range, we can exit. - */ + + /* No ordered extent found in the range, exit */ if (ordered_offset == last_offset) return; /* diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 79d366a36223..d5d326c674b1 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -297,26 +297,33 @@ void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry, } /* - * this is used to account for finished IO across a given range - * of the file. The IO may span ordered extents. If - * a given ordered_extent is completely done, 1 is returned, otherwise - * 0. + * Finish IO for one ordered extent across a given range. The range can + * contain several ordered extents. * - * test_and_set_bit on a flag in the struct btrfs_ordered_extent is used - * to make sure this function only returns 1 once for a given ordered extent. + * @found_ret: Return the finished ordered extent + * @file_offset: File offset for the finished IO + * Will also be updated to one byte past the range that is + * recordered as finished. This allows caller to walk forward. + * @io_size: Length of the finish IO range + * @uptodate: If the IO finished without problem * - * file_offset is updated to one byte past the range that is recorded as - * complete. This allows you to walk forward in the file. + * Return true if any ordered extent is finished in the range, and update + * @found_ret and @file_offset. + * Return false otherwise. + * + * NOTE: Although The range can cross multiple ordered extents, only one + * ordered extent will be updated during one call. The caller is responsible to + * iterate all ordered extents in the range. */ -int btrfs_dec_test_first_ordered_pending(struct btrfs_inode *inode, - struct btrfs_ordered_extent **cached, +bool btrfs_dec_test_first_ordered_pending(struct btrfs_inode *inode, + struct btrfs_ordered_extent **finished_ret, u64 *file_offset, u64 io_size, int uptodate) { struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree; struct rb_node *node; struct btrfs_ordered_extent *entry = NULL; - int ret; + bool finished = false; unsigned long flags; u64 dec_end; u64 dec_start; @@ -324,16 +331,12 @@ int btrfs_dec_test_first_ordered_pending(struct btrfs_inode *inode, spin_lock_irqsave(&tree->lock, flags); node = tree_search(tree, *file_offset); - if (!node) { - ret = 1; + if (!node) goto out; - } entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); - if (!offset_in_entry(entry, *file_offset)) { - ret = 1; + if (!offset_in_entry(entry, *file_offset)) goto out; - } dec_start = max(*file_offset, entry->file_offset); dec_end = min(*file_offset + io_size, @@ -354,39 +357,50 @@ int btrfs_dec_test_first_ordered_pending(struct btrfs_inode *inode, set_bit(BTRFS_ORDERED_IOERR, &entry->flags); if (entry->bytes_left == 0) { - ret = test_and_set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags); + /* + * Ensure only one caller can set the flag and finished_ret + * accordingly + */ + finished = !test_and_set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags); /* test_and_set_bit implies a barrier */ cond_wake_up_nomb(&entry->wait); - } else { - ret = 1; } out: - if (!ret && cached && entry) { - *cached = entry; + if (finished && finished_ret && entry) { + *finished_ret = entry; refcount_inc(&entry->refs); } spin_unlock_irqrestore(&tree->lock, flags); - return ret == 0; + return finished; } /* - * this is used to account for finished IO across a given range - * of the file. The IO should not span ordered extents. If - * a given ordered_extent is completely done, 1 is returned, otherwise - * 0. + * Finish IO for one ordered extent across a given range. The range can only + * contain one ordered extent. + * + * @cached: The cached ordered extent. If not NULL, we can skip the tree + * search and use the ordered extent directly. + * Will be also used to store the finished ordered extent. + * @file_offset: File offset for the finished IO + * @io_size: Length of the finish IO range + * @uptodate: If the IO finishes without problem + * + * Return true if the ordered extent is finished in the range, and update + * @cached. + * Return false otherwise. * - * test_and_set_bit on a flag in the struct btrfs_ordered_extent is used - * to make sure this function only returns 1 once for a given ordered extent. + * NOTE: The range can NOT cross multiple ordered extents. + * Thus caller should ensure the range doesn't cross ordered extents. */ -int btrfs_dec_test_ordered_pending(struct btrfs_inode *inode, - struct btrfs_ordered_extent **cached, - u64 file_offset, u64 io_size, int uptodate) +bool btrfs_dec_test_ordered_pending(struct btrfs_inode *inode, + struct btrfs_ordered_extent **cached, + u64 file_offset, u64 io_size, int uptodate) { struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree; struct rb_node *node; struct btrfs_ordered_extent *entry = NULL; unsigned long flags; - int ret; + bool finished = false; spin_lock_irqsave(&tree->lock, flags); if (cached && *cached) { @@ -395,41 +409,39 @@ int btrfs_dec_test_ordered_pending(struct btrfs_inode *inode, } node = tree_search(tree, file_offset); - if (!node) { - ret = 1; + if (!node) goto out; - } entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); have_entry: - if (!offset_in_entry(entry, file_offset)) { - ret = 1; + if (!offset_in_entry(entry, file_offset)) goto out; - } - if (io_size > entry->bytes_left) { + if (io_size > entry->bytes_left) btrfs_crit(inode->root->fs_info, "bad ordered accounting left %llu size %llu", entry->bytes_left, io_size); - } + entry->bytes_left -= io_size; if (!uptodate) set_bit(BTRFS_ORDERED_IOERR, &entry->flags); if (entry->bytes_left == 0) { - ret = test_and_set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags); + /* + * Ensure only one caller can set the flag and finished_ret + * accordingly + */ + finished = !test_and_set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags); /* test_and_set_bit implies a barrier */ cond_wake_up_nomb(&entry->wait); - } else { - ret = 1; } out: - if (!ret && cached && entry) { + if (finished && cached && entry) { *cached = entry; refcount_inc(&entry->refs); } spin_unlock_irqrestore(&tree->lock, flags); - return ret == 0; + return finished; } /* diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index 0bfa82b58e23..46194c2c05d4 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -152,11 +152,11 @@ btrfs_ordered_inode_tree_init(struct btrfs_ordered_inode_tree *t) void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry); void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode, struct btrfs_ordered_extent *entry); -int btrfs_dec_test_ordered_pending(struct btrfs_inode *inode, - struct btrfs_ordered_extent **cached, - u64 file_offset, u64 io_size, int uptodate); -int btrfs_dec_test_first_ordered_pending(struct btrfs_inode *inode, - struct btrfs_ordered_extent **cached, +bool btrfs_dec_test_ordered_pending(struct btrfs_inode *inode, + struct btrfs_ordered_extent **cached, + u64 file_offset, u64 io_size, int uptodate); +bool btrfs_dec_test_first_ordered_pending(struct btrfs_inode *inode, + struct btrfs_ordered_extent **finished_ret, u64 *file_offset, u64 io_size, int uptodate); int btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset, -- cgit v1.2.3 From 0c64c33c603f692ceb91d9fe17cc10028cff7da8 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Wed, 6 Jan 2021 09:01:40 +0800 Subject: btrfs: rename parameter offset to disk_bytenr in submit_extent_page The parameter offset is confusing, it's supposed to be the disk bytenr of metadata/data. Rename it to disk_bytenr and update the comment. Also rename each offset passed to submit_extent_page() as @disk_bytenr so they're consistent. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index c9cee458e001..60990616b895 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -3062,10 +3062,10 @@ struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size) * @opf: bio REQ_OP_* and REQ_* flags as one value * @wbc: optional writeback control for io accounting * @page: page to add to the bio + * @disk_bytenr: logical bytenr where the write will be + * @size: portion of page that we want to write to * @pg_offset: offset of the new bio or to check whether we are adding * a contiguous page to the previous one - * @size: portion of page that we want to write - * @offset: starting offset in the page * @bio_ret: must be valid pointer, newly allocated bio will be stored there * @end_io_func: end_io callback for new bio * @mirror_num: desired mirror to read/write @@ -3074,7 +3074,7 @@ struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size) */ static int submit_extent_page(unsigned int opf, struct writeback_control *wbc, - struct page *page, u64 offset, + struct page *page, u64 disk_bytenr, size_t size, unsigned long pg_offset, struct bio **bio_ret, bio_end_io_t end_io_func, @@ -3086,7 +3086,7 @@ static int submit_extent_page(unsigned int opf, int ret = 0; struct bio *bio; size_t io_size = min_t(size_t, size, PAGE_SIZE); - sector_t sector = offset >> 9; + sector_t sector = disk_bytenr >> 9; struct extent_io_tree *tree = &BTRFS_I(page->mapping->host)->io_tree; ASSERT(bio_ret); @@ -3120,7 +3120,7 @@ static int submit_extent_page(unsigned int opf, } } - bio = btrfs_bio_alloc(offset); + bio = btrfs_bio_alloc(disk_bytenr); bio_add_page(bio, page, io_size, pg_offset); bio->bi_end_io = end_io_func; bio->bi_private = tree; @@ -3242,7 +3242,7 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, } while (cur <= end) { bool force_bio_submit = false; - u64 offset; + u64 disk_bytenr; if (cur >= last_byte) { char *userpage; @@ -3280,9 +3280,9 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, cur_end = min(extent_map_end(em) - 1, end); iosize = ALIGN(iosize, blocksize); if (this_bio_flag & EXTENT_BIO_COMPRESSED) - offset = em->block_start; + disk_bytenr = em->block_start; else - offset = em->block_start + extent_offset; + disk_bytenr = em->block_start + extent_offset; block_start = em->block_start; if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) block_start = EXTENT_MAP_HOLE; @@ -3371,7 +3371,7 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, } ret = submit_extent_page(REQ_OP_READ | read_flags, NULL, - page, offset, iosize, + page, disk_bytenr, iosize, pg_offset, bio, end_bio_extent_readpage, 0, *bio_flags, @@ -3548,8 +3548,8 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, blocksize = inode->vfs_inode.i_sb->s_blocksize; while (cur <= end) { + u64 disk_bytenr; u64 em_end; - u64 offset; if (cur >= i_size) { btrfs_writepage_endio_finish_ordered(page, cur, @@ -3569,7 +3569,7 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, BUG_ON(end < cur); iosize = min(em_end - cur, end - cur + 1); iosize = ALIGN(iosize, blocksize); - offset = em->block_start + extent_offset; + disk_bytenr = em->block_start + extent_offset; block_start = em->block_start; compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags); free_extent_map(em); @@ -3599,7 +3599,7 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, } ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, - page, offset, iosize, pg_offset, + page, disk_bytenr, iosize, pg_offset, &epd->bio, end_bio_extent_writepage, 0, 0, 0, false); @@ -3923,7 +3923,7 @@ static noinline_for_stack int write_one_eb(struct extent_buffer *eb, struct writeback_control *wbc, struct extent_page_data *epd) { - u64 offset = eb->start; + u64 disk_bytenr = eb->start; u32 nritems; int i, num_pages; unsigned long start, end; @@ -3956,7 +3956,7 @@ static noinline_for_stack int write_one_eb(struct extent_buffer *eb, clear_page_dirty_for_io(p); set_page_writeback(p); ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, - p, offset, PAGE_SIZE, 0, + p, disk_bytenr, PAGE_SIZE, 0, &epd->bio, end_bio_extent_buffer_writepage, 0, 0, 0, false); @@ -3969,7 +3969,7 @@ static noinline_for_stack int write_one_eb(struct extent_buffer *eb, ret = -EIO; break; } - offset += PAGE_SIZE; + disk_bytenr += PAGE_SIZE; update_nr_written(wbc, 1); unlock_page(p); } -- cgit v1.2.3 From 6bc5636a67bf489d95ebc06c0449396fd487d309 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Wed, 6 Jan 2021 09:01:41 +0800 Subject: btrfs: refactor __extent_writepage_io() to improve readability The refactoring involves the following modifications: - iosize alignment In fact we don't really need to manually do alignment at all. All extent maps should already be aligned, thus basic ASSERT() check would be enough. - redundant variables We have extra variable like blocksize/pg_offset/end. They are all unnecessary. @blocksize can be replaced by sectorsize size directly, and it's only used to verify the em start/size is aligned. @pg_offset can be easily calculated using @cur and page_offset(page). @end is just assigned from @page_end and never modified, use "start + PAGE_SIZE - 1" directly and remove @page_end. - remove some BUG_ON()s The BUG_ON()s are for extent map, which we have tree-checker to check on-disk extent data item and runtime check. ASSERT() should be enough. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 60990616b895..74c0a32d04fc 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -3513,23 +3513,20 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, unsigned long nr_written, int *nr_ret) { + struct btrfs_fs_info *fs_info = inode->root->fs_info; struct extent_io_tree *tree = &inode->io_tree; u64 start = page_offset(page); - u64 page_end = start + PAGE_SIZE - 1; - u64 end; + u64 end = start + PAGE_SIZE - 1; u64 cur = start; u64 extent_offset; u64 block_start; - u64 iosize; struct extent_map *em; - size_t pg_offset = 0; - size_t blocksize; int ret = 0; int nr = 0; const unsigned int write_flags = wbc_to_write_flags(wbc); bool compressed; - ret = btrfs_writepage_cow_fixup(page, start, page_end); + ret = btrfs_writepage_cow_fixup(page, start, end); if (ret) { /* Fixup worker will requeue */ redirty_page_for_writepage(wbc, page); @@ -3544,16 +3541,13 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, */ update_nr_written(wbc, nr_written + 1); - end = page_end; - blocksize = inode->vfs_inode.i_sb->s_blocksize; - while (cur <= end) { u64 disk_bytenr; u64 em_end; + u32 iosize; if (cur >= i_size) { - btrfs_writepage_endio_finish_ordered(page, cur, - page_end, 1); + btrfs_writepage_endio_finish_ordered(page, cur, end, 1); break; } em = btrfs_get_extent(inode, NULL, 0, cur, end - cur + 1); @@ -3565,13 +3559,16 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, extent_offset = cur - em->start; em_end = extent_map_end(em); - BUG_ON(em_end <= cur); - BUG_ON(end < cur); - iosize = min(em_end - cur, end - cur + 1); - iosize = ALIGN(iosize, blocksize); - disk_bytenr = em->block_start + extent_offset; + ASSERT(cur <= em_end); + ASSERT(cur < end); + ASSERT(IS_ALIGNED(em->start, fs_info->sectorsize)); + ASSERT(IS_ALIGNED(em->len, fs_info->sectorsize)); block_start = em->block_start; compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags); + disk_bytenr = em->block_start + extent_offset; + + /* Note that em_end from extent_map_end() is exclusive */ + iosize = min(em_end, end + 1) - cur; free_extent_map(em); em = NULL; @@ -3587,7 +3584,6 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, btrfs_writepage_endio_finish_ordered(page, cur, cur + iosize - 1, 1); cur += iosize; - pg_offset += iosize; continue; } @@ -3599,8 +3595,8 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, } ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, - page, disk_bytenr, iosize, pg_offset, - &epd->bio, + page, disk_bytenr, iosize, + cur - page_offset(page), &epd->bio, end_bio_extent_writepage, 0, 0, 0, false); if (ret) { @@ -3609,8 +3605,7 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, end_page_writeback(page); } - cur = cur + iosize; - pg_offset += iosize; + cur += iosize; nr++; } *nr_ret = nr; -- cgit v1.2.3 From c0fab480955c4a943cc77be58269d97128ac3ef9 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Wed, 6 Jan 2021 09:01:42 +0800 Subject: btrfs: update comment for btrfs_dirty_pages The original comment is from the initial merge, which has several problems: - No holes check any more - No inline decision is made Update the out-of-date comment with more correct one. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/file.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index e65223e3510d..d81ae1f518f2 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -453,12 +453,11 @@ static void btrfs_drop_pages(struct page **pages, size_t num_pages) } /* - * after copy_from_user, pages need to be dirtied and we need to make - * sure holes are created between the current EOF and the start of - * any next extents (if required). - * - * this also makes the decision about creating an inline extent vs - * doing real data extents, marking pages dirty and delalloc as required. + * After btrfs_copy_from_user(), update the following things for delalloc: + * - Mark newly dirtied pages as DELALLOC in the io tree. + * Used to advise which range is to be written back. + * - Mark modified pages as Uptodate/Dirty and not needing COW fixup + * - Update inode size for past EOF write */ int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages, size_t num_pages, loff_t pos, size_t write_bytes, -- cgit v1.2.3 From c0f0a9e71653b33c003433f2248cec88f6942f35 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Wed, 6 Jan 2021 09:01:45 +0800 Subject: btrfs: introduce helper to grab an existing extent buffer from a page This patch will extract the code to grab an extent buffer from a page into a helper, grab_extent_buffer_from_page(). This reduces one indent level, and provides the work place for later expansion for subapge support. Reviewed-by: Johannes Thumshirn Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 50 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 20 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 74c0a32d04fc..7f689ad7709c 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5247,6 +5247,29 @@ free_eb: } #endif +static struct extent_buffer *grab_extent_buffer(struct page *page) +{ + struct extent_buffer *exists; + + /* Page not yet attached to an extent buffer */ + if (!PagePrivate(page)) + return NULL; + + /* + * We could have already allocated an eb for this page and attached one + * so lets see if we can get a ref on the existing eb, and if we can we + * know it's good and we can just return that one, else we know we can + * just overwrite page->private. + */ + exists = (struct extent_buffer *)page->private; + if (atomic_inc_not_zero(&exists->refs)) + return exists; + + WARN_ON(PageDirty(page)); + detach_page_private(page); + return NULL; +} + struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, u64 owner_root, int level) { @@ -5292,26 +5315,13 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, } spin_lock(&mapping->private_lock); - if (PagePrivate(p)) { - /* - * We could have already allocated an eb for this page - * and attached one so lets see if we can get a ref on - * the existing eb, and if we can we know it's good and - * we can just return that one, else we know we can just - * overwrite page->private. - */ - exists = (struct extent_buffer *)p->private; - if (atomic_inc_not_zero(&exists->refs)) { - spin_unlock(&mapping->private_lock); - unlock_page(p); - put_page(p); - mark_extent_buffer_accessed(exists, p); - goto free_eb; - } - exists = NULL; - - WARN_ON(PageDirty(p)); - detach_page_private(p); + exists = grab_extent_buffer(p); + if (exists) { + spin_unlock(&mapping->private_lock); + unlock_page(p); + put_page(p); + mark_extent_buffer_accessed(exists, p); + goto free_eb; } attach_extent_buffer_page(eb, p); spin_unlock(&mapping->private_lock); -- cgit v1.2.3 From f7ba2d37519dd6e15af9f00e9b4bbc7d1aba267a Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Wed, 16 Dec 2020 11:22:15 -0500 Subject: btrfs: keep track of the root owner for relocation reads While testing the error paths in relocation, I hit the following lockdep splat: ====================================================== WARNING: possible circular locking dependency detected 5.10.0-rc3+ #206 Not tainted ------------------------------------------------------ btrfs-balance/1571 is trying to acquire lock: ffff8cdbcc8f77d0 (&head_ref->mutex){+.+.}-{3:3}, at: btrfs_lookup_extent_info+0x156/0x3b0 but task is already holding lock: ffff8cdbc54adbf8 (btrfs-tree-00){++++}-{3:3}, at: __btrfs_tree_lock+0x27/0x100 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #2 (btrfs-tree-00){++++}-{3:3}: down_write_nested+0x43/0x80 __btrfs_tree_lock+0x27/0x100 btrfs_search_slot+0x248/0x890 relocate_tree_blocks+0x490/0x650 relocate_block_group+0x1ba/0x5d0 kretprobe_trampoline+0x0/0x50 -> #1 (btrfs-csum-01){++++}-{3:3}: down_read_nested+0x43/0x130 __btrfs_tree_read_lock+0x27/0x100 btrfs_read_lock_root_node+0x31/0x40 btrfs_search_slot+0x5ab/0x890 btrfs_del_csums+0x10b/0x3c0 __btrfs_free_extent+0x49d/0x8e0 __btrfs_run_delayed_refs+0x283/0x11f0 btrfs_run_delayed_refs+0x86/0x220 btrfs_start_dirty_block_groups+0x2ba/0x520 kretprobe_trampoline+0x0/0x50 -> #0 (&head_ref->mutex){+.+.}-{3:3}: __lock_acquire+0x1167/0x2150 lock_acquire+0x116/0x3e0 __mutex_lock+0x7e/0x7b0 btrfs_lookup_extent_info+0x156/0x3b0 walk_down_proc+0x1c3/0x280 walk_down_tree+0x64/0xe0 btrfs_drop_subtree+0x182/0x260 do_relocation+0x52e/0x660 relocate_tree_blocks+0x2ae/0x650 relocate_block_group+0x1ba/0x5d0 kretprobe_trampoline+0x0/0x50 other info that might help us debug this: Chain exists of: &head_ref->mutex --> btrfs-csum-01 --> btrfs-tree-00 Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(btrfs-tree-00); lock(btrfs-csum-01); lock(btrfs-tree-00); lock(&head_ref->mutex); *** DEADLOCK *** 5 locks held by btrfs-balance/1571: #0: ffff8cdb89749ff8 (&fs_info->delete_unused_bgs_mutex){+.+.}-{3:3}, at: btrfs_balance+0x563/0xf40 #1: ffff8cdb89748838 (&fs_info->cleaner_mutex){+.+.}-{3:3}, at: btrfs_relocate_block_group+0x156/0x300 #2: ffff8cdbc2c16650 (sb_internal#2){.+.+}-{0:0}, at: start_transaction+0x413/0x5c0 #3: ffff8cdbc135f538 (btrfs-treloc-01){+.+.}-{3:3}, at: __btrfs_tree_lock+0x27/0x100 #4: ffff8cdbc54adbf8 (btrfs-tree-00){++++}-{3:3}, at: __btrfs_tree_lock+0x27/0x100 stack backtrace: CPU: 1 PID: 1571 Comm: btrfs-balance Not tainted 5.10.0-rc3+ #206 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.13.0-2.fc32 04/01/2014 Call Trace: dump_stack+0x8b/0xb0 check_noncircular+0xcf/0xf0 ? trace_call_bpf+0x139/0x260 __lock_acquire+0x1167/0x2150 lock_acquire+0x116/0x3e0 ? btrfs_lookup_extent_info+0x156/0x3b0 __mutex_lock+0x7e/0x7b0 ? btrfs_lookup_extent_info+0x156/0x3b0 ? btrfs_lookup_extent_info+0x156/0x3b0 ? release_extent_buffer+0x124/0x170 ? _raw_spin_unlock+0x1f/0x30 ? release_extent_buffer+0x124/0x170 btrfs_lookup_extent_info+0x156/0x3b0 walk_down_proc+0x1c3/0x280 walk_down_tree+0x64/0xe0 btrfs_drop_subtree+0x182/0x260 do_relocation+0x52e/0x660 relocate_tree_blocks+0x2ae/0x650 ? add_tree_block+0x149/0x1b0 relocate_block_group+0x1ba/0x5d0 elfcorehdr_read+0x40/0x40 ? elfcorehdr_read+0x40/0x40 ? btrfs_balance+0x796/0xf40 ? __kthread_parkme+0x66/0x90 ? btrfs_balance+0xf40/0xf40 ? balance_kthread+0x37/0x50 ? kthread+0x137/0x150 ? __kthread_bind_mask+0x60/0x60 ? ret_from_fork+0x1f/0x30 As you can see this is bogus, we never take another tree's lock under the csum lock. This happens because sometimes we have to read tree blocks from disk without knowing which root they belong to during relocation. We defaulted to an owner of 0, which translates to an fs tree. This is fine as all fs trees have the same class, but obviously isn't fine if the block belongs to a COW only tree. Thankfully COW only trees only have their owners root as a reference to them, and since we already look up the extent information during relocation, go ahead and check and see if this block might belong to a COW only tree, and if so save the owner in the tree_block struct. This allows us to read_tree_block with the proper owner, which gets rid of this lockdep splat. Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 8e51b39cbfbb..9f2289bcdde6 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -97,6 +97,7 @@ struct tree_block { struct rb_node rb_node; u64 bytenr; }; /* Use rb_simple_node for search/insert */ + u64 owner; struct btrfs_key key; unsigned int level:8; unsigned int key_ready:1; @@ -2393,8 +2394,8 @@ static int get_tree_block_key(struct btrfs_fs_info *fs_info, { struct extent_buffer *eb; - eb = read_tree_block(fs_info, block->bytenr, 0, block->key.offset, - block->level, NULL); + eb = read_tree_block(fs_info, block->bytenr, block->owner, + block->key.offset, block->level, NULL); if (IS_ERR(eb)) { return PTR_ERR(eb); } else if (!extent_buffer_uptodate(eb)) { @@ -2493,7 +2494,8 @@ int relocate_tree_blocks(struct btrfs_trans_handle *trans, /* Kick in readahead for tree blocks with missing keys */ rbtree_postorder_for_each_entry_safe(block, next, blocks, rb_node) { if (!block->key_ready) - btrfs_readahead_tree_block(fs_info, block->bytenr, 0, 0, + btrfs_readahead_tree_block(fs_info, block->bytenr, + block->owner, 0, block->level); } @@ -2801,21 +2803,58 @@ static int add_tree_block(struct reloc_control *rc, u32 item_size; int level = -1; u64 generation; + u64 owner = 0; eb = path->nodes[0]; item_size = btrfs_item_size_nr(eb, path->slots[0]); if (extent_key->type == BTRFS_METADATA_ITEM_KEY || item_size >= sizeof(*ei) + sizeof(*bi)) { + unsigned long ptr = 0, end; + ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item); + end = (unsigned long)ei + item_size; if (extent_key->type == BTRFS_EXTENT_ITEM_KEY) { bi = (struct btrfs_tree_block_info *)(ei + 1); level = btrfs_tree_block_level(eb, bi); + ptr = (unsigned long)(bi + 1); } else { level = (int)extent_key->offset; + ptr = (unsigned long)(ei + 1); } generation = btrfs_extent_generation(eb, ei); + + /* + * We're reading random blocks without knowing their owner ahead + * of time. This is ok most of the time, as all reloc roots and + * fs roots have the same lock type. However normal trees do + * not, and the only way to know ahead of time is to read the + * inline ref offset. We know it's an fs root if + * + * 1. There's more than one ref. + * 2. There's a SHARED_DATA_REF_KEY set. + * 3. FULL_BACKREF is set on the flags. + * + * Otherwise it's safe to assume that the ref offset == the + * owner of this block, so we can use that when calling + * read_tree_block. + */ + if (btrfs_extent_refs(eb, ei) == 1 && + !(btrfs_extent_flags(eb, ei) & + BTRFS_BLOCK_FLAG_FULL_BACKREF) && + ptr < end) { + struct btrfs_extent_inline_ref *iref; + int type; + + iref = (struct btrfs_extent_inline_ref *)ptr; + type = btrfs_get_extent_inline_ref_type(eb, iref, + BTRFS_REF_TYPE_BLOCK); + if (type == BTRFS_REF_TYPE_INVALID) + return -EINVAL; + if (type == BTRFS_TREE_BLOCK_REF_KEY) + owner = btrfs_extent_inline_ref_offset(eb, iref); + } } else if (unlikely(item_size == sizeof(struct btrfs_extent_item_v0))) { btrfs_print_v0_err(eb->fs_info); btrfs_handle_fs_error(eb->fs_info, -EINVAL, NULL); @@ -2837,6 +2876,7 @@ static int add_tree_block(struct reloc_control *rc, block->key.offset = generation; block->level = level; block->key_ready = 0; + block->owner = owner; rb_node = rb_simple_insert(blocks, block->bytenr, &block->rb_node); if (rb_node) -- cgit v1.2.3 From 7e2a870a599d4699a626ec26430c7a1ab14a2a49 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Wed, 16 Dec 2020 11:22:16 -0500 Subject: btrfs: do not cleanup upper nodes in btrfs_backref_cleanup_node Zygo reported the following panic when testing my error handling patches for relocation: kernel BUG at fs/btrfs/backref.c:2545! invalid opcode: 0000 [#1] SMP KASAN PTI CPU: 3 PID: 8472 Comm: btrfs Tainted: G W 14 Hardware name: QEMU Standard PC (i440FX + PIIX, Call Trace: btrfs_backref_error_cleanup+0x4df/0x530 build_backref_tree+0x1a5/0x700 ? _raw_spin_unlock+0x22/0x30 ? release_extent_buffer+0x225/0x280 ? free_extent_buffer.part.52+0xd7/0x140 relocate_tree_blocks+0x2a6/0xb60 ? kasan_unpoison_shadow+0x35/0x50 ? do_relocation+0xc10/0xc10 ? kasan_kmalloc+0x9/0x10 ? kmem_cache_alloc_trace+0x6a3/0xcb0 ? free_extent_buffer.part.52+0xd7/0x140 ? rb_insert_color+0x342/0x360 ? add_tree_block.isra.36+0x236/0x2b0 relocate_block_group+0x2eb/0x780 ? merge_reloc_roots+0x470/0x470 btrfs_relocate_block_group+0x26e/0x4c0 btrfs_relocate_chunk+0x52/0x120 btrfs_balance+0xe2e/0x18f0 ? pvclock_clocksource_read+0xeb/0x190 ? btrfs_relocate_chunk+0x120/0x120 ? lock_contended+0x620/0x6e0 ? do_raw_spin_lock+0x1e0/0x1e0 ? do_raw_spin_unlock+0xa8/0x140 btrfs_ioctl_balance+0x1f9/0x460 btrfs_ioctl+0x24c8/0x4380 ? __kasan_check_read+0x11/0x20 ? check_chain_key+0x1f4/0x2f0 ? __asan_loadN+0xf/0x20 ? btrfs_ioctl_get_supported_features+0x30/0x30 ? kvm_sched_clock_read+0x18/0x30 ? check_chain_key+0x1f4/0x2f0 ? lock_downgrade+0x3f0/0x3f0 ? handle_mm_fault+0xad6/0x2150 ? do_vfs_ioctl+0xfc/0x9d0 ? ioctl_file_clone+0xe0/0xe0 ? check_flags.part.50+0x6c/0x1e0 ? check_flags.part.50+0x6c/0x1e0 ? check_flags+0x26/0x30 ? lock_is_held_type+0xc3/0xf0 ? syscall_enter_from_user_mode+0x1b/0x60 ? do_syscall_64+0x13/0x80 ? rcu_read_lock_sched_held+0xa1/0xd0 ? __kasan_check_read+0x11/0x20 ? __fget_light+0xae/0x110 __x64_sys_ioctl+0xc3/0x100 do_syscall_64+0x37/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xa9 This occurs because of this check if (RB_EMPTY_NODE(&upper->rb_node)) BUG_ON(!list_empty(&node->upper)); As we are dropping the backref node, if we discover that our upper node in the edge we just cleaned up isn't linked into the cache that we are now done with this node, thus the BUG_ON(). However this is an erroneous assumption, as we will look up all the references for a node first, and then process the pending edges. All of the 'upper' nodes in our pending edges won't be in the cache's rb_tree yet, because they haven't been processed. We could very well have many edges still left to cleanup on this node. The fact is we simply do not need this check, we can just process all of the edges only for this node, because below this check we do the following if (list_empty(&upper->lower)) { list_add_tail(&upper->lower, &cache->leaves); upper->lowest = 1; } If the upper node truly isn't used yet, then we add it to the cache->leaves list to be cleaned up later. If it is still used then the last child node that has it linked into its node will add it to the leaves list and then it will be cleaned up. Fix this problem by dropping this logic altogether. With this fix I no longer see the panic when testing with error injection in the backref code. CC: stable@vger.kernel.org # 4.4+ Reviewed-by: Qu Wenruo Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/backref.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 9cadacf3ec27..ef71aba5bc15 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2541,13 +2541,6 @@ void btrfs_backref_cleanup_node(struct btrfs_backref_cache *cache, list_del(&edge->list[UPPER]); btrfs_backref_free_edge(cache, edge); - if (RB_EMPTY_NODE(&upper->rb_node)) { - BUG_ON(!list_empty(&node->upper)); - btrfs_backref_drop_node(cache, node); - node = upper; - node->lowest = 1; - continue; - } /* * Add the node to leaf node list if no other child block * cached. -- cgit v1.2.3 From fe3b7bb085a0b1fb26d622a5eccc7dbb5c4f82fb Mon Sep 17 00:00:00 2001 From: Yang Li Date: Thu, 21 Jan 2021 16:19:47 +0800 Subject: btrfs: remove redundant NULL check before kvfree Fix below warnings reported by coccicheck: ./fs/btrfs/raid56.c:237:2-8: WARNING: NULL check before some freeing functions is not needed. Reported-by: Abaci Robot Reviewed-by: Anand Jain Signed-off-by: Yang Li Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 93fbf87bdc8d..5394641541f7 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -233,8 +233,7 @@ int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info) } x = cmpxchg(&info->stripe_hash_table, NULL, table); - if (x) - kvfree(x); + kvfree(x); return 0; } -- cgit v1.2.3 From 3c198fe064491dcceaed9e15c6c997e92e71293e Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 21 Jan 2021 14:13:54 +0800 Subject: btrfs: rework the order of btrfs_ordered_extent::flags [BUG] There is a long existing bug in the last parameter of btrfs_add_ordered_extent(), in commit 771ed689d2cd ("Btrfs: Optimize compressed writeback and reads") back to 2008. In that ancient commit btrfs_add_ordered_extent() expects the @type parameter to be one of the following: - BTRFS_ORDERED_REGULAR - BTRFS_ORDERED_NOCOW - BTRFS_ORDERED_PREALLOC - BTRFS_ORDERED_COMPRESSED But we pass 0 in cow_file_range(), which means BTRFS_ORDERED_IO_DONE. Ironically extra check in __btrfs_add_ordered_extent() won't set the bit if we see (type == IO_DONE || type == IO_COMPLETE), and avoid any obvious bug. But this still leads to regular COW ordered extent having no bit to indicate its type in various trace events, rendering REGULAR bit useless. [FIX] Change the following aspects to avoid such problem: - Reorder btrfs_ordered_extent::flags Now the type bits go first (REGULAR/NOCOW/PREALLCO/COMPRESSED), then DIRECT bit, finally extra status bits like IO_DONE/COMPLETE/IOERR. - Add extra ASSERT() for btrfs_add_ordered_extent_*() - Remove @type parameter for btrfs_add_ordered_extent_compress() As the only valid @type here is BTRFS_ORDERED_COMPRESSED. - Remove the unnecessary special check for IO_DONE/COMPLETE in __btrfs_add_ordered_extent() This is just to make the code work, with extra ASSERT(), there are limited values can be passed in. Reviewed-by: Nikolay Borisov Reviewed-by: Filipe Manana Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 4 ++-- fs/btrfs/ordered-data.c | 21 ++++++++++++++++----- fs/btrfs/ordered-data.h | 37 ++++++++++++++++++++++++------------- include/trace/events/btrfs.h | 7 ++++--- 4 files changed, 46 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index ef6cb7b620d0..ea9056cc5559 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -917,7 +917,6 @@ retry: ins.objectid, async_extent->ram_size, ins.offset, - BTRFS_ORDERED_COMPRESSED, async_extent->compress_type); if (ret) { btrfs_drop_extent_cache(inode, async_extent->start, @@ -1127,7 +1126,8 @@ static noinline int cow_file_range(struct btrfs_inode *inode, free_extent_map(em); ret = btrfs_add_ordered_extent(inode, start, ins.objectid, - ram_size, cur_alloc_size, 0); + ram_size, cur_alloc_size, + BTRFS_ORDERED_REGULAR); if (ret) goto out_drop_extent_cache; diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index d5d326c674b1..b4e6500548a2 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -199,8 +199,12 @@ static int __btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset entry->compress_type = compress_type; entry->truncated_len = (u64)-1; entry->qgroup_rsv = ret; - if (type != BTRFS_ORDERED_IO_DONE && type != BTRFS_ORDERED_COMPLETE) - set_bit(type, &entry->flags); + + ASSERT(type == BTRFS_ORDERED_REGULAR || + type == BTRFS_ORDERED_NOCOW || + type == BTRFS_ORDERED_PREALLOC || + type == BTRFS_ORDERED_COMPRESSED); + set_bit(type, &entry->flags); if (dio) { percpu_counter_add_batch(&fs_info->dio_bytes, num_bytes, @@ -256,6 +260,9 @@ int btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset, u64 disk_bytenr, u64 num_bytes, u64 disk_num_bytes, int type) { + ASSERT(type == BTRFS_ORDERED_REGULAR || + type == BTRFS_ORDERED_NOCOW || + type == BTRFS_ORDERED_PREALLOC); return __btrfs_add_ordered_extent(inode, file_offset, disk_bytenr, num_bytes, disk_num_bytes, type, 0, BTRFS_COMPRESS_NONE); @@ -265,6 +272,9 @@ int btrfs_add_ordered_extent_dio(struct btrfs_inode *inode, u64 file_offset, u64 disk_bytenr, u64 num_bytes, u64 disk_num_bytes, int type) { + ASSERT(type == BTRFS_ORDERED_REGULAR || + type == BTRFS_ORDERED_NOCOW || + type == BTRFS_ORDERED_PREALLOC); return __btrfs_add_ordered_extent(inode, file_offset, disk_bytenr, num_bytes, disk_num_bytes, type, 1, BTRFS_COMPRESS_NONE); @@ -272,11 +282,12 @@ int btrfs_add_ordered_extent_dio(struct btrfs_inode *inode, u64 file_offset, int btrfs_add_ordered_extent_compress(struct btrfs_inode *inode, u64 file_offset, u64 disk_bytenr, u64 num_bytes, - u64 disk_num_bytes, int type, - int compress_type) + u64 disk_num_bytes, int compress_type) { + ASSERT(compress_type != BTRFS_COMPRESS_NONE); return __btrfs_add_ordered_extent(inode, file_offset, disk_bytenr, - num_bytes, disk_num_bytes, type, 0, + num_bytes, disk_num_bytes, + BTRFS_ORDERED_COMPRESSED, 0, compress_type); } diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index 46194c2c05d4..cca3307807e8 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -27,7 +27,7 @@ struct btrfs_ordered_sum { }; /* - * bits for the flags field: + * Bits for btrfs_ordered_extent::flags. * * BTRFS_ORDERED_IO_DONE is set when all of the blocks are written. * It is used to make sure metadata is inserted into the tree only once @@ -38,24 +38,36 @@ struct btrfs_ordered_sum { * IO is done and any metadata is inserted into the tree. */ enum { + /* + * Different types for direct io, one and only one of the 4 type can + * be set when creating ordered extent. + * + * REGULAR: For regular non-compressed COW write + * NOCOW: For NOCOW write into existing non-hole extent + * PREALLOC: For NOCOW write into preallocated extent + * COMPRESSED: For compressed COW write + */ + BTRFS_ORDERED_REGULAR, + BTRFS_ORDERED_NOCOW, + BTRFS_ORDERED_PREALLOC, + BTRFS_ORDERED_COMPRESSED, + + /* + * Extra bit for direct io, can only be set for + * REGULAR/NOCOW/PREALLOC. No direct io for compressed extent. + */ + BTRFS_ORDERED_DIRECT, + + /* Extra status bits for ordered extents */ + /* set when all the pages are written */ BTRFS_ORDERED_IO_DONE, /* set when removed from the tree */ BTRFS_ORDERED_COMPLETE, - /* set when we want to write in place */ - BTRFS_ORDERED_NOCOW, - /* writing a zlib compressed extent */ - BTRFS_ORDERED_COMPRESSED, - /* set when writing to preallocated extent */ - BTRFS_ORDERED_PREALLOC, - /* set when we're doing DIO with this extent */ - BTRFS_ORDERED_DIRECT, /* We had an io error when writing this out */ BTRFS_ORDERED_IOERR, /* Set when we have to truncate an extent */ BTRFS_ORDERED_TRUNCATED, - /* Regular IO for COW */ - BTRFS_ORDERED_REGULAR, /* Used during fsync to track already logged extents */ BTRFS_ORDERED_LOGGED, /* We have already logged all the csums of the ordered extent */ @@ -167,8 +179,7 @@ int btrfs_add_ordered_extent_dio(struct btrfs_inode *inode, u64 file_offset, u64 disk_num_bytes, int type); int btrfs_add_ordered_extent_compress(struct btrfs_inode *inode, u64 file_offset, u64 disk_bytenr, u64 num_bytes, - u64 disk_num_bytes, int type, - int compress_type); + u64 disk_num_bytes, int compress_type); void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry, struct btrfs_ordered_sum *sum); struct btrfs_ordered_extent *btrfs_lookup_ordered_extent(struct btrfs_inode *inode, diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index ecd24c719de4..b9896fc06160 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -499,12 +499,13 @@ DEFINE_EVENT( #define show_ordered_flags(flags) \ __print_flags(flags, "|", \ - { (1 << BTRFS_ORDERED_IO_DONE), "IO_DONE" }, \ - { (1 << BTRFS_ORDERED_COMPLETE), "COMPLETE" }, \ + { (1 << BTRFS_ORDERED_REGULAR), "REGULAR" }, \ { (1 << BTRFS_ORDERED_NOCOW), "NOCOW" }, \ - { (1 << BTRFS_ORDERED_COMPRESSED), "COMPRESSED" }, \ { (1 << BTRFS_ORDERED_PREALLOC), "PREALLOC" }, \ + { (1 << BTRFS_ORDERED_COMPRESSED), "COMPRESSED" }, \ { (1 << BTRFS_ORDERED_DIRECT), "DIRECT" }, \ + { (1 << BTRFS_ORDERED_IO_DONE), "IO_DONE" }, \ + { (1 << BTRFS_ORDERED_COMPLETE), "COMPLETE" }, \ { (1 << BTRFS_ORDERED_IOERR), "IOERR" }, \ { (1 << BTRFS_ORDERED_TRUNCATED), "TRUNCATED" }) -- cgit v1.2.3 From 401bd2dd1299dd384849707c6577b2089ab9f615 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:57:52 +0200 Subject: btrfs: document modified parameter of add_extent_mapping Fixes fs/btrfs/extent_map.c:399: warning: Function parameter or member 'modified' not described in 'add_extent_mapping' Reviewed-by: Johannes Thumshirn Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_map.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c index bd6229fb2b6f..8bda6c89e23e 100644 --- a/fs/btrfs/extent_map.c +++ b/fs/btrfs/extent_map.c @@ -385,9 +385,12 @@ static void extent_map_device_clear_bits(struct extent_map *em, unsigned bits) } /** - * add_extent_mapping - add new extent map to the extent tree + * Add new extent map to the extent tree + * * @tree: tree to insert new map in * @em: map to insert + * @modified: indicate whether the given @em should be added to the + * modified list, which indicates the extent needs to be logged * * Insert @em into @tree or perform a simple forward/backward merge with * existing mappings. The extent_map struct passed in will be inserted -- cgit v1.2.3 From 9ad37bb3ffc51fbd9c48ba4d85414b4aa3e21c6d Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:57:53 +0200 Subject: btrfs: fix parameter description of btrfs_add_extent_mapping This fixes the following compiler warnings: fs/btrfs/extent_map.c:601: warning: Function parameter or member 'fs_info' not described in 'btrfs_add_extent_mapping' fs/btrfs/extent_map.c:601: warning: Function parameter or member 'em_tree' not described in 'btrfs_add_extent_mapping' fs/btrfs/extent_map.c:601: warning: Function parameter or member 'em_in' not described in 'btrfs_add_extent_mapping' fs/btrfs/extent_map.c:601: warning: Function parameter or member 'start' not described in 'btrfs_add_extent_mapping' fs/btrfs/extent_map.c:601: warning: Function parameter or member 'len' not described in 'btrfs_add_extent_mapping' Reviewed-by: Johannes Thumshirn Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_map.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c index 8bda6c89e23e..4a8e02f7b6c7 100644 --- a/fs/btrfs/extent_map.c +++ b/fs/btrfs/extent_map.c @@ -577,12 +577,13 @@ static noinline int merge_extent_mapping(struct extent_map_tree *em_tree, } /** - * btrfs_add_extent_mapping - add extent mapping into em_tree - * @fs_info - used for tracepoint - * @em_tree - the extent tree into which we want to insert the extent mapping - * @em_in - extent we are inserting - * @start - start of the logical range btrfs_get_extent() is requesting - * @len - length of the logical range btrfs_get_extent() is requesting + * Add extent mapping into em_tree + * + * @fs_info: the filesystem + * @em_tree: extent tree into which we want to insert the extent mapping + * @em_in: extent we are inserting + * @start: start of the logical range btrfs_get_extent() is requesting + * @len: length of the logical range btrfs_get_extent() is requesting * * Note that @em_in's range may be different from [start, start+len), * but they must be overlapped. -- cgit v1.2.3 From ca4207ae1385190f7d62926f107ede1edced4c1f Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:57:54 +0200 Subject: btrfs: fix function description formats in file-item.c This fixes following W=1 warnings: fs/btrfs/file-item.c:27: warning: Cannot understand * @inode: the inode we want to update the disk_i_size for on line 27 - I thought it was a doc line fs/btrfs/file-item.c:65: warning: Cannot understand * @inode - the inode we're modifying on line 65 - I thought it was a doc line fs/btrfs/file-item.c:91: warning: Cannot understand * @inode - the inode we're modifying on line 91 - I thought it was a doc line Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/file-item.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 6ccfc019ad90..47cd3a6dc635 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -24,8 +24,10 @@ PAGE_SIZE)) /** - * @inode - the inode we want to update the disk_i_size for - * @new_i_size - the i_size we want to set to, 0 if we use i_size + * Set inode's size according to filesystem options + * + * @inode: inode we want to update the disk_i_size for + * @new_i_size: i_size we want to set to, 0 if we use i_size * * With NO_HOLES set this simply sets the disk_is_size to whatever i_size_read() * returns as it is perfectly fine with a file that has holes without hole file @@ -62,9 +64,11 @@ void btrfs_inode_safe_disk_i_size_write(struct btrfs_inode *inode, u64 new_i_siz } /** - * @inode - the inode we're modifying - * @start - the start file offset of the file extent we've inserted - * @len - the logical length of the file extent item + * Mark range within a file as having a new extent inserted + * + * @inode: inode being modified + * @start: start file offset of the file extent we've inserted + * @len: logical length of the file extent item * * Call when we are inserting a new file extent where there was none before. * Does not need to call this in the case where we're replacing an existing file @@ -88,9 +92,11 @@ int btrfs_inode_set_file_extent_range(struct btrfs_inode *inode, u64 start, } /** - * @inode - the inode we're modifying - * @start - the start file offset of the file extent we've inserted - * @len - the logical length of the file extent item + * Marks an inode range as not having a backing extent + * + * @inode: inode being modified + * @start: start file offset of the file extent we've inserted + * @len: logical length of the file extent item * * Called when we drop a file extent, for example when we truncate. Doesn't * need to be called for cases where we're replacing a file extent, like when -- cgit v1.2.3 From 696eb22b67add04e13f26cebe9f63eeb9477becd Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:57:55 +0200 Subject: btrfs: fix parameter description in delayed-ref.c functions This fixes the following warnings: fs/btrfs/delayed-ref.c:80: warning: Function parameter or member 'fs_info' not described in 'btrfs_delayed_refs_rsv_release' fs/btrfs/delayed-ref.c:80: warning: Function parameter or member 'nr' not described in 'btrfs_delayed_refs_rsv_release' fs/btrfs/delayed-ref.c:128: warning: Function parameter or member 'fs_info' not described in 'btrfs_migrate_to_delayed_refs_rsv' fs/btrfs/delayed-ref.c:128: warning: Function parameter or member 'src' not described in 'btrfs_migrate_to_delayed_refs_rsv' fs/btrfs/delayed-ref.c:128: warning: Function parameter or member 'num_bytes' not described in 'btrfs_migrate_to_delayed_refs_rsv' fs/btrfs/delayed-ref.c:174: warning: Function parameter or member 'fs_info' not described in 'btrfs_delayed_refs_rsv_refill' fs/btrfs/delayed-ref.c:174: warning: Function parameter or member 'flush' not described in 'btrfs_delayed_refs_rsv_refill' Reviewed-by: Johannes Thumshirn Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/delayed-ref.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index 353cc2994d10..88a1e27d2fc2 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -69,9 +69,10 @@ int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans) } /** - * btrfs_delayed_refs_rsv_release - release a ref head's reservation. - * @fs_info - the fs_info for our fs. - * @nr - the number of items to drop. + * Release a ref head's reservation + * + * @fs_info: the filesystem + * @nr: number of items to drop * * This drops the delayed ref head's count from the delayed refs rsv and frees * any excess reservation we had. @@ -114,10 +115,11 @@ void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans) } /** - * btrfs_migrate_to_delayed_refs_rsv - transfer bytes to our delayed refs rsv. - * @fs_info - the fs info for our fs. - * @src - the source block rsv to transfer from. - * @num_bytes - the number of bytes to transfer. + * Transfer bytes to our delayed refs rsv + * + * @fs_info: the filesystem + * @src: source block rsv to transfer from + * @num_bytes: number of bytes to transfer * * This transfers up to the num_bytes amount from the src rsv to the * delayed_refs_rsv. Any extra bytes are returned to the space info. @@ -162,9 +164,10 @@ void btrfs_migrate_to_delayed_refs_rsv(struct btrfs_fs_info *fs_info, } /** - * btrfs_delayed_refs_rsv_refill - refill based on our delayed refs usage. - * @fs_info - the fs_info for our fs. - * @flush - control how we can flush for this reservation. + * Refill based on our delayed refs usage + * + * @fs_info: the filesystem + * @flush: control how we can flush for this reservation. * * This will refill the delayed block_rsv up to 1 items size worth of space and * will return -ENOSPC if we can't make the reservation. -- cgit v1.2.3 From f092cf3cfd0144bdaf6110176ea9d2cef1f3b4a8 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:57:56 +0200 Subject: btrfs: improve parameter description for __btrfs_write_out_cache Fixes following W=1 warnings: fs/btrfs/free-space-cache.c:1317: warning: Function parameter or member 'root' not described in '__btrfs_write_out_cache' fs/btrfs/free-space-cache.c:1317: warning: Function parameter or member 'inode' not described in '__btrfs_write_out_cache' fs/btrfs/free-space-cache.c:1317: warning: Function parameter or member 'ctl' not described in '__btrfs_write_out_cache' fs/btrfs/free-space-cache.c:1317: warning: Function parameter or member 'block_group' not described in '__btrfs_write_out_cache' fs/btrfs/free-space-cache.c:1317: warning: Function parameter or member 'io_ctl' not described in '__btrfs_write_out_cache' fs/btrfs/free-space-cache.c:1317: warning: Function parameter or member 'trans' not described in '__btrfs_write_out_cache' Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/free-space-cache.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index fd6ddd6b8165..0d6dcb5ff963 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -1299,11 +1299,14 @@ int btrfs_wait_cache_io(struct btrfs_trans_handle *trans, } /** - * __btrfs_write_out_cache - write out cached info to an inode - * @root - the root the inode belongs to - * @ctl - the free space cache we are going to write out - * @block_group - the block_group for this cache if it belongs to a block_group - * @trans - the trans handle + * Write out cached info to an inode + * + * @root: root the inode belongs to + * @inode: freespace inode we are writing out + * @ctl: free space cache we are going to write out + * @block_group: block_group for this cache if it belongs to a block_group + * @io_ctl: holds context for the io + * @trans: the trans handle * * This function writes out a free space cache struct to disk for quick recovery * on mount. This will return 0 if it was successful in writing the cache out, -- cgit v1.2.3 From 92419695478b6a75ca85e9f8e06b08a4a35bfb20 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:57:57 +0200 Subject: btrfs: document now parameter of peek_discard_list Fixes fs/btrfs/discard.c:203: warning: Function parameter or member 'now' not described in 'peek_discard_list' Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/discard.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/discard.c b/fs/btrfs/discard.c index 2b8383d41144..306ff20af70f 100644 --- a/fs/btrfs/discard.c +++ b/fs/btrfs/discard.c @@ -185,10 +185,12 @@ static struct btrfs_block_group *find_next_block_group( } /** - * peek_discard_list - wrap find_next_block_group() - * @discard_ctl: discard control + * Wrap find_next_block_group() + * + * @discard_ctl: discard control * @discard_state: the discard_state of the block_group after state management * @discard_index: the discard_index of the block_group after state management + * @now: time when discard was invoked, in ns * * This wraps find_next_block_group() and sets the block_group to be in use. * discard_state's control flow is managed here. Variables related to -- cgit v1.2.3 From 9ee9b97990d6eff9cea64303c640dfb4b3a40253 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:57:58 +0200 Subject: btrfs: document fs_info in btrfs_rmap_block Fixes fs/btrfs/block-group.c:1570: warning: Function parameter or member 'fs_info' not described in 'btrfs_rmap_block' Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 48ebc106a606..2d7294d81616 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1561,7 +1561,9 @@ static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) } /** - * btrfs_rmap_block - Map a physical disk address to a list of logical addresses + * Map a physical disk address to a list of logical addresses + * + * @fs_info: the filesystem * @chunk_start: logical address of block group * @physical: physical address to map to logical addresses * @logical: return array of logical addresses which map to @physical -- cgit v1.2.3 From 2639631d34941db1ebbc74fb879855e0cd286cec Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:57:59 +0200 Subject: btrfs: fix description format of fs_info of btrfs_wait_on_delayed_iputs Fixes fs/btrfs/inode.c:3101: warning: Function parameter or member 'fs_info' not described in 'btrfs_wait_on_delayed_iputs' Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index ea9056cc5559..0dbe1aaa0b71 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3103,14 +3103,16 @@ void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) } /** - * btrfs_wait_on_delayed_iputs - wait on the delayed iputs to be done running - * @fs_info - the fs_info for this fs - * @return - EINTR if we were killed, 0 if nothing's pending + * Wait for flushing all delayed iputs + * + * @fs_info: the filesystem * * This will wait on any delayed iputs that are currently running with KILLABLE * set. Once they are all done running we will return, unless we are killed in * which case we return EINTR. This helps in user operations like fallocate etc * that might get blocked on the iputs. + * + * Return EINTR if we were killed, 0 if nothing's pending */ int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info) { -- cgit v1.2.3 From 6e353e3b3c5545524d718d528548f7c8c95536c5 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:58:00 +0200 Subject: btrfs: document btrfs_check_shared parameters Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index ef71aba5bc15..701124c3e0b1 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -1501,7 +1501,13 @@ int btrfs_find_all_roots(struct btrfs_trans_handle *trans, } /** - * btrfs_check_shared - tell us whether an extent is shared + * Check if an extent is shared or not + * + * @root: root inode belongs to + * @inum: inode number of the inode whose extent we are checking + * @bytenr: logical bytenr of the extent we are checking + * @roots: list of roots this extent is shared among + * @tmp: temporary list used for iteration * * btrfs_check_shared uses the backref walking code but will short * circuit as soon as it finds a root or inode that doesn't match the -- cgit v1.2.3 From b762d1d08dacdc444ffd6417fc17805408da7af4 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:58:01 +0200 Subject: btrfs: fix parameter description of btrfs_inode_rsv_release/btrfs_delalloc_release_space Fixes following warnings: fs/btrfs/delalloc-space.c:205: warning: Function parameter or member 'inode' not described in 'btrfs_inode_rsv_release' fs/btrfs/delalloc-space.c:205: warning: Function parameter or member 'qgroup_free' not described in 'btrfs_inode_rsv_release' fs/btrfs/delalloc-space.c:472: warning: Function parameter or member 'reserved' not described in 'btrfs_delalloc_release_space' fs/btrfs/delalloc-space.c:472: warning: Function parameter or member 'qgroup_free' not described in 'btrfs_delalloc_release_space' fs/btrfs/delalloc-space.c:472: warning: Excess function parameter 'release_bytes' description in 'btrfs_delalloc_release_space' Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/delalloc-space.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c index bacee09b7bfd..56642ca7af10 100644 --- a/fs/btrfs/delalloc-space.c +++ b/fs/btrfs/delalloc-space.c @@ -191,12 +191,14 @@ void btrfs_free_reserved_data_space(struct btrfs_inode *inode, } /** - * btrfs_inode_rsv_release - release any excessive reservation. - * @inode - the inode we need to release from. - * @qgroup_free - free or convert qgroup meta. - * Unlike normal operation, qgroup meta reservation needs to know if we are - * freeing qgroup reservation or just converting it into per-trans. Normally - * @qgroup_free is true for error handling, and false for normal release. + * Release any excessive reservation + * + * @inode: the inode we need to release from + * @qgroup_free: free or convert qgroup meta. Unlike normal operation, qgroup + * meta reservation needs to know if we are freeing qgroup + * reservation or just converting it into per-trans. Normally + * @qgroup_free is true for error handling, and false for normal + * release. * * This is the same as btrfs_block_rsv_release, except that it handles the * tracepoint for the reservation. @@ -361,7 +363,8 @@ int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes) } /** - * btrfs_delalloc_release_metadata - release a metadata reservation for an inode + * Release a metadata reservation for an inode + * * @inode: the inode to release the reservation for. * @num_bytes: the number of bytes we are releasing. * @qgroup_free: free qgroup reservation or convert it to per-trans reservation @@ -455,11 +458,13 @@ int btrfs_delalloc_reserve_space(struct btrfs_inode *inode, } /** - * btrfs_delalloc_release_space - release data and metadata space for delalloc - * @inode: inode we're releasing space for - * @start: start position of the space already reserved - * @len: the len of the space already reserved - * @release_bytes: the len of the space we consumed or didn't use + * Release data and metadata space for delalloc + * + * @inode: inode we're releasing space for + * @reserved: list of changed/reserved ranges + * @start: start position of the space already reserved + * @len: length of the space already reserved + * @qgroup_free: should qgroup reserved-space also be freed * * This function will release the metadata space that was not used and will * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes -- cgit v1.2.3 From d98b188ea463281ee89663c36d8ac0a030e93b0c Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:58:02 +0200 Subject: btrfs: fix parameter description in space-info.c With these fixes space-info.c is clear for W=1 warnings, namely the following ones are fixed: fs/btrfs/space-info.c:575: warning: Function parameter or member 'fs_info' not described in 'may_commit_transaction' fs/btrfs/space-info.c:575: warning: Function parameter or member 'space_info' not described in 'may_commit_transaction' fs/btrfs/space-info.c:1231: warning: Function parameter or member 'fs_info' not described in 'handle_reserve_ticket' fs/btrfs/space-info.c:1231: warning: Function parameter or member 'space_info' not described in 'handle_reserve_ticket' fs/btrfs/space-info.c:1231: warning: Function parameter or member 'ticket' not described in 'handle_reserve_ticket' fs/btrfs/space-info.c:1231: warning: Function parameter or member 'flush' not described in 'handle_reserve_ticket' fs/btrfs/space-info.c:1315: warning: Function parameter or member 'fs_info' not described in '__reserve_bytes' fs/btrfs/space-info.c:1315: warning: Function parameter or member 'space_info' not described in '__reserve_bytes' fs/btrfs/space-info.c:1315: warning: Function parameter or member 'orig_bytes' not described in '__reserve_bytes' fs/btrfs/space-info.c:1315: warning: Function parameter or member 'flush' not described in '__reserve_bytes' fs/btrfs/space-info.c:1427: warning: Function parameter or member 'root' not described in 'btrfs_reserve_metadata_bytes' fs/btrfs/space-info.c:1427: warning: Function parameter or member 'block_rsv' not described in 'btrfs_reserve_metadata_bytes' fs/btrfs/space-info.c:1427: warning: Function parameter or member 'orig_bytes' not described in 'btrfs_reserve_metadata_bytes' fs/btrfs/space-info.c:1427: warning: Function parameter or member 'flush' not described in 'btrfs_reserve_metadata_bytes' fs/btrfs/space-info.c:1462: warning: Function parameter or member 'fs_info' not described in 'btrfs_reserve_data_bytes' fs/btrfs/space-info.c:1462: warning: Function parameter or member 'bytes' not described in 'btrfs_reserve_data_bytes' fs/btrfs/space-info.c:1462: warning: Function parameter or member 'flush' not described in 'btrfs_reserve_data_bytes' Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 84fb94e78a8f..fd8e79e3c10e 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -561,10 +561,10 @@ static void shrink_delalloc(struct btrfs_fs_info *fs_info, } /** - * maybe_commit_transaction - possibly commit the transaction if its ok to - * @root - the root we're allocating for - * @bytes - the number of bytes we want to reserve - * @force - force the commit + * Possibly commit the transaction if its ok to + * + * @fs_info: the filesystem + * @space_info: space_info we are checking for commit, either data or metadata * * This will check to make sure that committing the transaction will actually * get us somewhere and then commit the transaction if it does. Otherwise it @@ -1215,11 +1215,12 @@ static void wait_reserve_ticket(struct btrfs_fs_info *fs_info, } /** - * handle_reserve_ticket - do the appropriate flushing and waiting for a ticket - * @fs_info - the fs - * @space_info - the space_info for the reservation - * @ticket - the ticket for the reservation - * @flush - how much we can flush + * Do the appropriate flushing and waiting for a ticket + * + * @fs_info: the filesystem + * @space_info: space info for the reservation + * @ticket: ticket for the reservation + * @flush: how much we can flush * * This does the work of figuring out how to flush for the ticket, waiting for * the reservation, and returning the appropriate error if there is one. @@ -1296,11 +1297,12 @@ static inline bool is_normal_flushing(enum btrfs_reserve_flush_enum flush) } /** - * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space - * @root - the root we're allocating for - * @space_info - the space info we want to allocate from - * @orig_bytes - the number of bytes we want - * @flush - whether or not we can flush to make our reservation + * Try to reserve bytes from the block_rsv's space + * + * @fs_info: the filesystem + * @space_info: space info we want to allocate from + * @orig_bytes: number of bytes we want + * @flush: whether or not we can flush to make our reservation * * This will reserve orig_bytes number of bytes from the space info associated * with the block_rsv. If there is not enough space it will make an attempt to @@ -1407,11 +1409,12 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, } /** - * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space - * @root - the root we're allocating for - * @block_rsv - the block_rsv we're allocating for - * @orig_bytes - the number of bytes we want - * @flush - whether or not we can flush to make our reservation + * Trye to reserve metadata bytes from the block_rsv's space + * + * @root: the root we're allocating for + * @block_rsv: block_rsv we're allocating for + * @orig_bytes: number of bytes we want + * @flush: whether or not we can flush to make our reservation * * This will reserve orig_bytes number of bytes from the space info associated * with the block_rsv. If there is not enough space it will make an attempt to @@ -1449,10 +1452,11 @@ int btrfs_reserve_metadata_bytes(struct btrfs_root *root, } /** - * btrfs_reserve_data_bytes - try to reserve data bytes for an allocation - * @fs_info - the filesystem - * @bytes - the number of bytes we need - * @flush - how we are allowed to flush + * Try to reserve data bytes for an allocation + * + * @fs_info: the filesystem + * @bytes: number of bytes we need + * @flush: how we are allowed to flush * * This will reserve bytes from the data space info. If there is not enough * space then we will attempt to flush space as specified by flush. -- cgit v1.2.3 From 3bed2da1b00f554e70d16f44db9357a7670d776c Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:58:03 +0200 Subject: btrfs: fix parameter description for functions in extent_io.c This makes the file W=1 clean and fixes the following warnings: fs/btrfs/extent_io.c:414: warning: Function parameter or member 'tree' not described in '__etree_search' fs/btrfs/extent_io.c:414: warning: Function parameter or member 'offset' not described in '__etree_search' fs/btrfs/extent_io.c:414: warning: Function parameter or member 'next_ret' not described in '__etree_search' fs/btrfs/extent_io.c:414: warning: Function parameter or member 'prev_ret' not described in '__etree_search' fs/btrfs/extent_io.c:414: warning: Function parameter or member 'p_ret' not described in '__etree_search' fs/btrfs/extent_io.c:414: warning: Function parameter or member 'parent_ret' not described in '__etree_search' fs/btrfs/extent_io.c:1607: warning: Function parameter or member 'tree' not described in 'find_contiguous_extent_bit' fs/btrfs/extent_io.c:1607: warning: Function parameter or member 'start' not described in 'find_contiguous_extent_bit' fs/btrfs/extent_io.c:1607: warning: Function parameter or member 'start_ret' not described in 'find_contiguous_extent_bit' fs/btrfs/extent_io.c:1607: warning: Function parameter or member 'end_ret' not described in 'find_contiguous_extent_bit' fs/btrfs/extent_io.c:1607: warning: Function parameter or member 'bits' not described in 'find_contiguous_extent_bit' fs/btrfs/extent_io.c:1644: warning: Function parameter or member 'tree' not described in 'find_first_clear_extent_bit' fs/btrfs/extent_io.c:1644: warning: Function parameter or member 'start' not described in 'find_first_clear_extent_bit' fs/btrfs/extent_io.c:1644: warning: Function parameter or member 'start_ret' not described in 'find_first_clear_extent_bit' fs/btrfs/extent_io.c:1644: warning: Function parameter or member 'end_ret' not described in 'find_first_clear_extent_bit' fs/btrfs/extent_io.c:1644: warning: Function parameter or member 'bits' not described in 'find_first_clear_extent_bit' fs/btrfs/extent_io.c:4187: warning: Function parameter or member 'epd' not described in 'extent_write_cache_pages' fs/btrfs/extent_io.c:4187: warning: Excess function parameter 'data' description in 'extent_write_cache_pages' Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 7f689ad7709c..2fa563da65bd 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -389,16 +389,16 @@ do_insert: } /** - * __etree_search - searche @tree for an entry that contains @offset. Such - * entry would have entry->start <= offset && entry->end >= offset. + * Search @tree for an entry that contains @offset. Such entry would have + * entry->start <= offset && entry->end >= offset. * - * @tree - the tree to search - * @offset - offset that should fall within an entry in @tree - * @next_ret - pointer to the first entry whose range ends after @offset - * @prev - pointer to the first entry whose range begins before @offset - * @p_ret - pointer where new node should be anchored (used when inserting an - * entry in the tree) - * @parent_ret - points to entry which would have been the parent of the entry, + * @tree: the tree to search + * @offset: offset that should fall within an entry in @tree + * @next_ret: pointer to the first entry whose range ends after @offset + * @prev_ret: pointer to the first entry whose range begins before @offset + * @p_ret: pointer where new node should be anchored (used when inserting an + * entry in the tree) + * @parent_ret: points to entry which would have been the parent of the entry, * containing @offset * * This function returns a pointer to the entry that contains @offset byte @@ -1588,12 +1588,13 @@ out: } /** - * find_contiguous_extent_bit: find a contiguous area of bits - * @tree - io tree to check - * @start - offset to start the search from - * @start_ret - the first offset we found with the bits set - * @end_ret - the final contiguous range of the bits that were set - * @bits - bits to look for + * Find a contiguous area of bits + * + * @tree: io tree to check + * @start: offset to start the search from + * @start_ret: the first offset we found with the bits set + * @end_ret: the final contiguous range of the bits that were set + * @bits: bits to look for * * set_extent_bit and clear_extent_bit can temporarily split contiguous ranges * to set bits appropriately, and then merge them again. During this time it @@ -1625,14 +1626,14 @@ int find_contiguous_extent_bit(struct extent_io_tree *tree, u64 start, } /** - * find_first_clear_extent_bit - find the first range that has @bits not set. - * This range could start before @start. + * Find the first range that has @bits not set. This range could start before + * @start. * - * @tree - the tree to search - * @start - the offset at/after which the found extent should start - * @start_ret - records the beginning of the range - * @end_ret - records the end of the range (inclusive) - * @bits - the set of bits which must be unset + * @tree: the tree to search + * @start: offset at/after which the found extent should start + * @start_ret: records the beginning of the range + * @end_ret: records the end of the range (inclusive) + * @bits: the set of bits which must be unset * * Since unallocated range is also considered one which doesn't have the bits * set it's possible that @end_ret contains -1, this happens in case the range @@ -4168,10 +4169,11 @@ retry: } /** - * write_cache_pages - walk the list of dirty pages of the given address space and write all of them. + * Walk the list of dirty pages of the given address space and write all of them. + * * @mapping: address space structure to write - * @wbc: subtract the number of written pages from *@wbc->nr_to_write - * @data: data passed to __extent_writepage function + * @wbc: subtract the number of written pages from *@wbc->nr_to_write + * @epd: holds context for the write, namely the bio * * If a page is already under I/O, write_cache_pages() skips it, even * if it's dirty. This is desirable behaviour for memory-cleaning writeback, -- cgit v1.2.3 From 8c31a3dbaa356b1fce97bf55026410649e4dd0f1 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Sun, 24 Jan 2021 18:03:21 +0200 Subject: btrfs: zoned: remove unused variable in btrfs_sb_log_location_bdev MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes warning: fs/btrfs/zoned.c:491:6: warning: variable ‘zone_size’ set but not used [-Wunused-but-set-variable] 491 | u64 zone_size; which got introduced in 12659251ca5d ("btrfs: implement log-structured superblock for ZONED mode"). We'll enable the warning by default and want clean build until the relevant zoned patches land. Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/zoned.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index c38846659019..41d27fefd306 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -488,7 +488,6 @@ int btrfs_sb_log_location_bdev(struct block_device *bdev, int mirror, int rw, unsigned int zone_sectors; u32 sb_zone; int ret; - u64 zone_size; u8 zone_sectors_shift; sector_t nr_sectors; u32 nr_zones; @@ -503,7 +502,6 @@ int btrfs_sb_log_location_bdev(struct block_device *bdev, int mirror, int rw, zone_sectors = bdev_zone_sectors(bdev); if (!is_power_of_2(zone_sectors)) return -EINVAL; - zone_size = zone_sectors << SECTOR_SHIFT; zone_sectors_shift = ilog2(zone_sectors); nr_sectors = bdev_nr_sectors(bdev); nr_zones = nr_sectors >> zone_sectors_shift; -- cgit v1.2.3 From e9aa7c285d20a69ce1fb940ec846686780af9e56 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:58:05 +0200 Subject: btrfs: enable W=1 checks for btrfs Now that the btrfs' codebase is clean of almost all W=1 warnings let's enable those checks unconditionally for the entire fs/btrfs/ and its subdirectories to catch potential errors during development. Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba [ add some comments ] Signed-off-by: David Sterba --- fs/btrfs/Makefile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile index 9f1b1a88e317..e45957319424 100644 --- a/fs/btrfs/Makefile +++ b/fs/btrfs/Makefile @@ -1,5 +1,21 @@ # SPDX-License-Identifier: GPL-2.0 +# Subset of W=1 warnings +subdir-ccflags-y += -Wextra -Wunused -Wno-unused-parameter +subdir-ccflags-y += -Wmissing-declarations +subdir-ccflags-y += -Wmissing-format-attribute +subdir-ccflags-y += -Wmissing-prototypes +subdir-ccflags-y += -Wold-style-definition +subdir-ccflags-y += -Wmissing-include-dirs +subdir-ccflags-y += $(call cc-option, -Wunused-but-set-variable) +subdir-ccflags-y += $(call cc-option, -Wunused-const-variable) +subdir-ccflags-y += $(call cc-option, -Wpacked-not-aligned) +subdir-ccflags-y += $(call cc-option, -Wstringop-truncation) +# The following turn off the warnings enabled by -Wextra +subdir-ccflags-y += -Wno-missing-field-initializers +subdir-ccflags-y += -Wno-sign-compare +subdir-ccflags-y += -Wno-type-limits + obj-$(CONFIG_BTRFS_FS) := btrfs.o btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \ -- cgit v1.2.3 From 2187374f35fe9cadbddaa9fcf0c4121365d914e8 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 15 Jan 2021 16:48:55 -0500 Subject: btrfs: handle space_info::total_bytes_pinned inside the delayed ref itself Currently we pass things around to figure out if we maybe freeing data based on the state of the delayed refs head. This makes the accounting sort of confusing and hard to follow, as it's distinctly separate from the delayed ref heads stuff, but also depends on it entirely. Fix this by explicitly adjusting the space_info->total_bytes_pinned in the delayed refs code. We now have two places where we modify this counter, once where we create the delayed and destroy the delayed refs, and once when we pin and unpin the extents. This means there is a slight overlap between delayed refs and the pin/unpin mechanisms, but this is simply used by the ENOSPC infrastructure to determine if we need to commit the transaction, so there's no adverse affect from this, we might simply commit thinking it will give us enough space when it might not. CC: stable@vger.kernel.org # 5.10 Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 10 ++---- fs/btrfs/delayed-ref.c | 51 +++++++++++++++----------- fs/btrfs/delayed-ref.h | 16 ++++++--- fs/btrfs/extent-tree.c | 97 +++++++------------------------------------------- fs/btrfs/space-info.h | 17 +++++++++ 5 files changed, 74 insertions(+), 117 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 2d7294d81616..763a3671b7af 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1371,9 +1371,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) btrfs_space_info_update_bytes_pinned(fs_info, space_info, -block_group->pinned); space_info->bytes_readonly += block_group->pinned; - percpu_counter_add_batch(&space_info->total_bytes_pinned, - -block_group->pinned, - BTRFS_TOTAL_BYTES_PINNED_BATCH); + __btrfs_mod_total_bytes_pinned(space_info, -block_group->pinned); block_group->pinned = 0; spin_unlock(&block_group->lock); @@ -2898,10 +2896,8 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, spin_unlock(&cache->lock); spin_unlock(&cache->space_info->lock); - percpu_counter_add_batch( - &cache->space_info->total_bytes_pinned, - num_bytes, - BTRFS_TOTAL_BYTES_PINNED_BATCH); + __btrfs_mod_total_bytes_pinned(cache->space_info, + num_bytes); set_extent_dirty(&trans->transaction->pinned_extents, bytenr, bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL); diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index 88a1e27d2fc2..a540ace3e03a 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -651,12 +651,12 @@ inserted: */ static noinline void update_existing_head_ref(struct btrfs_trans_handle *trans, struct btrfs_delayed_ref_head *existing, - struct btrfs_delayed_ref_head *update, - int *old_ref_mod_ret) + struct btrfs_delayed_ref_head *update) { struct btrfs_delayed_ref_root *delayed_refs = &trans->transaction->delayed_refs; struct btrfs_fs_info *fs_info = trans->fs_info; + u64 flags = btrfs_ref_head_to_space_flags(existing); int old_ref_mod; BUG_ON(existing->is_data != update->is_data); @@ -704,8 +704,6 @@ static noinline void update_existing_head_ref(struct btrfs_trans_handle *trans, * currently, for refs we just added we know we're a-ok. */ old_ref_mod = existing->total_ref_mod; - if (old_ref_mod_ret) - *old_ref_mod_ret = old_ref_mod; existing->ref_mod += update->ref_mod; existing->total_ref_mod += update->ref_mod; @@ -727,6 +725,22 @@ static noinline void update_existing_head_ref(struct btrfs_trans_handle *trans, trans->delayed_ref_updates += csum_leaves; } } + + /* + * This handles the following conditions: + * + * 1. We had a ref mod of 0 or more and went negative, indicating that + * we may be freeing space, so add our space to the + * total_bytes_pinned counter. + * 2. We were negative and went to 0 or positive, so no longer can say + * that the space would be pinned, decrement our counter from the + * total_bytes_pinned counter. + */ + if (existing->total_ref_mod < 0 && old_ref_mod >= 0) + btrfs_mod_total_bytes_pinned(fs_info, flags, existing->num_bytes); + else if (existing->total_ref_mod >= 0 && old_ref_mod < 0) + btrfs_mod_total_bytes_pinned(fs_info, flags, -existing->num_bytes); + spin_unlock(&existing->lock); } @@ -801,8 +815,7 @@ static noinline struct btrfs_delayed_ref_head * add_delayed_ref_head(struct btrfs_trans_handle *trans, struct btrfs_delayed_ref_head *head_ref, struct btrfs_qgroup_extent_record *qrecord, - int action, int *qrecord_inserted_ret, - int *old_ref_mod, int *new_ref_mod) + int action, int *qrecord_inserted_ret) { struct btrfs_delayed_ref_head *existing; struct btrfs_delayed_ref_root *delayed_refs; @@ -824,8 +837,7 @@ add_delayed_ref_head(struct btrfs_trans_handle *trans, existing = htree_insert(&delayed_refs->href_root, &head_ref->href_node); if (existing) { - update_existing_head_ref(trans, existing, head_ref, - old_ref_mod); + update_existing_head_ref(trans, existing, head_ref); /* * we've updated the existing ref, free the newly * allocated ref @@ -833,14 +845,17 @@ add_delayed_ref_head(struct btrfs_trans_handle *trans, kmem_cache_free(btrfs_delayed_ref_head_cachep, head_ref); head_ref = existing; } else { - if (old_ref_mod) - *old_ref_mod = 0; + u64 flags = btrfs_ref_head_to_space_flags(head_ref); + if (head_ref->is_data && head_ref->ref_mod < 0) { delayed_refs->pending_csums += head_ref->num_bytes; trans->delayed_ref_updates += btrfs_csum_bytes_to_leaves(trans->fs_info, head_ref->num_bytes); } + if (head_ref->ref_mod < 0) + btrfs_mod_total_bytes_pinned(trans->fs_info, flags, + head_ref->num_bytes); delayed_refs->num_heads++; delayed_refs->num_heads_ready++; atomic_inc(&delayed_refs->num_entries); @@ -848,8 +863,6 @@ add_delayed_ref_head(struct btrfs_trans_handle *trans, } if (qrecord_inserted_ret) *qrecord_inserted_ret = qrecord_inserted; - if (new_ref_mod) - *new_ref_mod = head_ref->total_ref_mod; return head_ref; } @@ -912,8 +925,7 @@ static void init_delayed_ref_common(struct btrfs_fs_info *fs_info, */ int btrfs_add_delayed_tree_ref(struct btrfs_trans_handle *trans, struct btrfs_ref *generic_ref, - struct btrfs_delayed_extent_op *extent_op, - int *old_ref_mod, int *new_ref_mod) + struct btrfs_delayed_extent_op *extent_op) { struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_delayed_tree_ref *ref; @@ -980,8 +992,7 @@ int btrfs_add_delayed_tree_ref(struct btrfs_trans_handle *trans, * the spin lock */ head_ref = add_delayed_ref_head(trans, head_ref, record, - action, &qrecord_inserted, - old_ref_mod, new_ref_mod); + action, &qrecord_inserted); ret = insert_delayed_ref(trans, delayed_refs, head_ref, &ref->node); spin_unlock(&delayed_refs->lock); @@ -1009,8 +1020,7 @@ int btrfs_add_delayed_tree_ref(struct btrfs_trans_handle *trans, */ int btrfs_add_delayed_data_ref(struct btrfs_trans_handle *trans, struct btrfs_ref *generic_ref, - u64 reserved, int *old_ref_mod, - int *new_ref_mod) + u64 reserved) { struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_delayed_data_ref *ref; @@ -1076,8 +1086,7 @@ int btrfs_add_delayed_data_ref(struct btrfs_trans_handle *trans, * the spin lock */ head_ref = add_delayed_ref_head(trans, head_ref, record, - action, &qrecord_inserted, - old_ref_mod, new_ref_mod); + action, &qrecord_inserted); ret = insert_delayed_ref(trans, delayed_refs, head_ref, &ref->node); spin_unlock(&delayed_refs->lock); @@ -1120,7 +1129,7 @@ int btrfs_add_delayed_extent_op(struct btrfs_trans_handle *trans, spin_lock(&delayed_refs->lock); add_delayed_ref_head(trans, head_ref, NULL, BTRFS_UPDATE_DELAYED_HEAD, - NULL, NULL, NULL); + NULL); spin_unlock(&delayed_refs->lock); diff --git a/fs/btrfs/delayed-ref.h b/fs/btrfs/delayed-ref.h index 1c977e6d45dc..3ba140468f12 100644 --- a/fs/btrfs/delayed-ref.h +++ b/fs/btrfs/delayed-ref.h @@ -326,6 +326,16 @@ static inline void btrfs_put_delayed_ref(struct btrfs_delayed_ref_node *ref) } } +static inline u64 btrfs_ref_head_to_space_flags( + struct btrfs_delayed_ref_head *head_ref) +{ + if (head_ref->is_data) + return BTRFS_BLOCK_GROUP_DATA; + else if (head_ref->is_system) + return BTRFS_BLOCK_GROUP_SYSTEM; + return BTRFS_BLOCK_GROUP_METADATA; +} + static inline void btrfs_put_delayed_ref_head(struct btrfs_delayed_ref_head *head) { if (refcount_dec_and_test(&head->refs)) @@ -334,12 +344,10 @@ static inline void btrfs_put_delayed_ref_head(struct btrfs_delayed_ref_head *hea int btrfs_add_delayed_tree_ref(struct btrfs_trans_handle *trans, struct btrfs_ref *generic_ref, - struct btrfs_delayed_extent_op *extent_op, - int *old_ref_mod, int *new_ref_mod); + struct btrfs_delayed_extent_op *extent_op); int btrfs_add_delayed_data_ref(struct btrfs_trans_handle *trans, struct btrfs_ref *generic_ref, - u64 reserved, int *old_ref_mod, - int *new_ref_mod); + u64 reserved); int btrfs_add_delayed_extent_op(struct btrfs_trans_handle *trans, u64 bytenr, u64 num_bytes, struct btrfs_delayed_extent_op *extent_op); diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 0c335dae5af7..2f591036ffc1 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -82,41 +82,6 @@ void btrfs_free_excluded_extents(struct btrfs_block_group *cache) EXTENT_UPTODATE); } -static u64 generic_ref_to_space_flags(struct btrfs_ref *ref) -{ - if (ref->type == BTRFS_REF_METADATA) { - if (ref->tree_ref.root == BTRFS_CHUNK_TREE_OBJECTID) - return BTRFS_BLOCK_GROUP_SYSTEM; - else - return BTRFS_BLOCK_GROUP_METADATA; - } - return BTRFS_BLOCK_GROUP_DATA; -} - -static void add_pinned_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_ref *ref) -{ - struct btrfs_space_info *space_info; - u64 flags = generic_ref_to_space_flags(ref); - - space_info = btrfs_find_space_info(fs_info, flags); - ASSERT(space_info); - percpu_counter_add_batch(&space_info->total_bytes_pinned, ref->len, - BTRFS_TOTAL_BYTES_PINNED_BATCH); -} - -static void sub_pinned_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_ref *ref) -{ - struct btrfs_space_info *space_info; - u64 flags = generic_ref_to_space_flags(ref); - - space_info = btrfs_find_space_info(fs_info, flags); - ASSERT(space_info); - percpu_counter_add_batch(&space_info->total_bytes_pinned, -ref->len, - BTRFS_TOTAL_BYTES_PINNED_BATCH); -} - /* simple helper to search for an existing data extent at a given offset */ int btrfs_lookup_data_extent(struct btrfs_fs_info *fs_info, u64 start, u64 len) { @@ -1388,7 +1353,6 @@ int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, struct btrfs_ref *generic_ref) { struct btrfs_fs_info *fs_info = trans->fs_info; - int old_ref_mod, new_ref_mod; int ret; ASSERT(generic_ref->type != BTRFS_REF_NOT_SET && @@ -1397,17 +1361,12 @@ int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, generic_ref->tree_ref.root == BTRFS_TREE_LOG_OBJECTID); if (generic_ref->type == BTRFS_REF_METADATA) - ret = btrfs_add_delayed_tree_ref(trans, generic_ref, - NULL, &old_ref_mod, &new_ref_mod); + ret = btrfs_add_delayed_tree_ref(trans, generic_ref, NULL); else - ret = btrfs_add_delayed_data_ref(trans, generic_ref, 0, - &old_ref_mod, &new_ref_mod); + ret = btrfs_add_delayed_data_ref(trans, generic_ref, 0); btrfs_ref_tree_mod(fs_info, generic_ref); - if (ret == 0 && old_ref_mod < 0 && new_ref_mod >= 0) - sub_pinned_bytes(fs_info, generic_ref); - return ret; } @@ -1796,20 +1755,9 @@ void btrfs_cleanup_ref_head_accounting(struct btrfs_fs_info *fs_info, int nr_items = 1; /* Dropping this ref head update. */ if (head->total_ref_mod < 0) { - struct btrfs_space_info *space_info; - u64 flags; + u64 flags = btrfs_ref_head_to_space_flags(head); - if (head->is_data) - flags = BTRFS_BLOCK_GROUP_DATA; - else if (head->is_system) - flags = BTRFS_BLOCK_GROUP_SYSTEM; - else - flags = BTRFS_BLOCK_GROUP_METADATA; - space_info = btrfs_find_space_info(fs_info, flags); - ASSERT(space_info); - percpu_counter_add_batch(&space_info->total_bytes_pinned, - -head->num_bytes, - BTRFS_TOTAL_BYTES_PINNED_BATCH); + btrfs_mod_total_bytes_pinned(fs_info, flags, -head->num_bytes); /* * We had csum deletions accounted for in our delayed refs rsv, @@ -2572,8 +2520,7 @@ static int pin_down_extent(struct btrfs_trans_handle *trans, spin_unlock(&cache->lock); spin_unlock(&cache->space_info->lock); - percpu_counter_add_batch(&cache->space_info->total_bytes_pinned, - num_bytes, BTRFS_TOTAL_BYTES_PINNED_BATCH); + __btrfs_mod_total_bytes_pinned(cache->space_info, num_bytes); set_extent_dirty(&trans->transaction->pinned_extents, bytenr, bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL); return 0; @@ -2784,8 +2731,7 @@ static int unpin_extent_range(struct btrfs_fs_info *fs_info, cache->pinned -= len; btrfs_space_info_update_bytes_pinned(fs_info, space_info, -len); space_info->max_extent_size = 0; - percpu_counter_add_batch(&space_info->total_bytes_pinned, - -len, BTRFS_TOTAL_BYTES_PINNED_BATCH); + __btrfs_mod_total_bytes_pinned(space_info, -len); if (cache->ro) { space_info->bytes_readonly += len; readonly = true; @@ -3318,7 +3264,6 @@ void btrfs_free_tree_block(struct btrfs_trans_handle *trans, { struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_ref generic_ref = { 0 }; - int pin = 1; int ret; btrfs_init_generic_ref(&generic_ref, BTRFS_DROP_DELAYED_REF, @@ -3327,13 +3272,9 @@ void btrfs_free_tree_block(struct btrfs_trans_handle *trans, root->root_key.objectid); if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { - int old_ref_mod, new_ref_mod; - btrfs_ref_tree_mod(fs_info, &generic_ref); - ret = btrfs_add_delayed_tree_ref(trans, &generic_ref, NULL, - &old_ref_mod, &new_ref_mod); + ret = btrfs_add_delayed_tree_ref(trans, &generic_ref, NULL); BUG_ON(ret); /* -ENOMEM */ - pin = old_ref_mod >= 0 && new_ref_mod < 0; } if (last_ref && btrfs_header_generation(buf) == trans->transid) { @@ -3345,7 +3286,6 @@ void btrfs_free_tree_block(struct btrfs_trans_handle *trans, goto out; } - pin = 0; cache = btrfs_lookup_block_group(fs_info, buf->start); if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) { @@ -3362,9 +3302,6 @@ void btrfs_free_tree_block(struct btrfs_trans_handle *trans, trace_btrfs_reserved_extent_free(fs_info, buf->start, buf->len); } out: - if (pin) - add_pinned_bytes(fs_info, &generic_ref); - if (last_ref) { /* * Deleting the buffer, clear the corrupt flag since it doesn't @@ -3378,7 +3315,6 @@ out: int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_ref *ref) { struct btrfs_fs_info *fs_info = trans->fs_info; - int old_ref_mod, new_ref_mod; int ret; if (btrfs_is_testing(fs_info)) @@ -3394,14 +3330,11 @@ int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_ref *ref) ref->data_ref.ref_root == BTRFS_TREE_LOG_OBJECTID)) { /* unlocks the pinned mutex */ btrfs_pin_extent(trans, ref->bytenr, ref->len, 1); - old_ref_mod = new_ref_mod = 0; ret = 0; } else if (ref->type == BTRFS_REF_METADATA) { - ret = btrfs_add_delayed_tree_ref(trans, ref, NULL, - &old_ref_mod, &new_ref_mod); + ret = btrfs_add_delayed_tree_ref(trans, ref, NULL); } else { - ret = btrfs_add_delayed_data_ref(trans, ref, 0, - &old_ref_mod, &new_ref_mod); + ret = btrfs_add_delayed_data_ref(trans, ref, 0); } if (!((ref->type == BTRFS_REF_METADATA && @@ -3410,9 +3343,6 @@ int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_ref *ref) ref->data_ref.ref_root == BTRFS_TREE_LOG_OBJECTID))) btrfs_ref_tree_mod(fs_info, ref); - if (ret == 0 && old_ref_mod >= 0 && new_ref_mod < 0) - add_pinned_bytes(fs_info, ref); - return ret; } @@ -4528,7 +4458,6 @@ int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans, struct btrfs_key *ins) { struct btrfs_ref generic_ref = { 0 }; - int ret; BUG_ON(root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID); @@ -4536,9 +4465,8 @@ int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans, ins->objectid, ins->offset, 0); btrfs_init_data_ref(&generic_ref, root->root_key.objectid, owner, offset); btrfs_ref_tree_mod(root->fs_info, &generic_ref); - ret = btrfs_add_delayed_data_ref(trans, &generic_ref, - ram_bytes, NULL, NULL); - return ret; + + return btrfs_add_delayed_data_ref(trans, &generic_ref, ram_bytes); } /* @@ -4730,8 +4658,7 @@ struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans, generic_ref.real_root = root->root_key.objectid; btrfs_init_tree_ref(&generic_ref, level, root_objectid); btrfs_ref_tree_mod(fs_info, &generic_ref); - ret = btrfs_add_delayed_tree_ref(trans, &generic_ref, - extent_op, NULL, NULL); + ret = btrfs_add_delayed_tree_ref(trans, &generic_ref, extent_op); if (ret) goto out_free_delayed; } diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index 5646393b928c..74706f604bce 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -152,4 +152,21 @@ static inline void btrfs_space_info_free_bytes_may_use( int btrfs_reserve_data_bytes(struct btrfs_fs_info *fs_info, u64 bytes, enum btrfs_reserve_flush_enum flush); +static inline void __btrfs_mod_total_bytes_pinned( + struct btrfs_space_info *space_info, + s64 mod) +{ + percpu_counter_add_batch(&space_info->total_bytes_pinned, mod, + BTRFS_TOTAL_BYTES_PINNED_BATCH); +} + +static inline void btrfs_mod_total_bytes_pinned(struct btrfs_fs_info *fs_info, + u64 flags, s64 mod) +{ + struct btrfs_space_info *space_info = btrfs_find_space_info(fs_info, flags); + + ASSERT(space_info); + __btrfs_mod_total_bytes_pinned(space_info, mod); +} + #endif /* BTRFS_SPACE_INFO_H */ -- cgit v1.2.3 From 81e75ac74ecba929d1e922bf93f9fc467232e39f Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 15 Jan 2021 16:48:56 -0500 Subject: btrfs: account for new extents being deleted in total_bytes_pinned My recent patch set "A variety of lock contention fixes", found here https://lore.kernel.org/linux-btrfs/cover.1608319304.git.josef@toxicpanda.com/ (Tracked in https://github.com/btrfs/linux/issues/86) that reduce lock contention on the extent root by running delayed refs less often resulted in a regression in generic/371. This test fallocate()'s the fs until it's full, deletes all the files, and then tries to fallocate() until full again. Before these patches we would run all of the delayed refs during flushing, and then would commit the transaction because we had plenty of pinned space to recover in order to allocate. However my patches made it so we weren't running the delayed refs as aggressively, which meant that we appeared to have less pinned space when we were deciding to commit the transaction. We use the space_info->total_bytes_pinned to approximate how much space we have pinned. It's approximate because if we remove a reference to an extent we may free it, but there may be more references to it than we know of at that point, but we account it as pinned at the creation time, and then it's properly accounted when the delayed ref runs. The way we account for pinned space is if the delayed_ref_head->total_ref_mod is < 0, because that is clearly a freeing option. However there is another case, and that is where ->total_ref_mod == 0 && ->must_insert_reserved == 1. When we allocate a new extent, we have ->total_ref_mod == 1 and we have ->must_insert_reserved == 1. This is used to indicate that it is a brand new extent and will need to have its extent entry added before we modify any references on the delayed ref head. But if we subsequently remove that extent reference, our ->total_ref_mod will be 0, and that space will be pinned and freed. Accounting for this case properly allows for generic/371 to pass with my delayed refs patches applied. It's important to note that this problem exists without the referenced patches, it just was uncovered by them. CC: stable@vger.kernel.org # 5.10 Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/delayed-ref.c | 5 +++++ fs/btrfs/extent-tree.c | 33 +++++++++++++++++++-------------- 2 files changed, 24 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index a540ace3e03a..63be7d01a9a3 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -735,11 +735,16 @@ static noinline void update_existing_head_ref(struct btrfs_trans_handle *trans, * 2. We were negative and went to 0 or positive, so no longer can say * that the space would be pinned, decrement our counter from the * total_bytes_pinned counter. + * 3. We are now at 0 and have ->must_insert_reserved set, which means + * this was a new allocation and then we dropped it, and thus must + * add our space to the total_bytes_pinned counter. */ if (existing->total_ref_mod < 0 && old_ref_mod >= 0) btrfs_mod_total_bytes_pinned(fs_info, flags, existing->num_bytes); else if (existing->total_ref_mod >= 0 && old_ref_mod < 0) btrfs_mod_total_bytes_pinned(fs_info, flags, -existing->num_bytes); + else if (existing->total_ref_mod == 0 && existing->must_insert_reserved) + btrfs_mod_total_bytes_pinned(fs_info, flags, existing->num_bytes); spin_unlock(&existing->lock); } diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 2f591036ffc1..6f0c59debc2b 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -1754,23 +1754,28 @@ void btrfs_cleanup_ref_head_accounting(struct btrfs_fs_info *fs_info, { int nr_items = 1; /* Dropping this ref head update. */ - if (head->total_ref_mod < 0) { + /* + * We had csum deletions accounted for in our delayed refs rsv, we need + * to drop the csum leaves for this update from our delayed_refs_rsv. + */ + if (head->total_ref_mod < 0 && head->is_data) { + spin_lock(&delayed_refs->lock); + delayed_refs->pending_csums -= head->num_bytes; + spin_unlock(&delayed_refs->lock); + nr_items += btrfs_csum_bytes_to_leaves(fs_info, head->num_bytes); + } + + /* + * We were dropping refs, or had a new ref and dropped it, and thus must + * adjust down our total_bytes_pinned, the space may or may not have + * been pinned and so is accounted for properly in the pinned space by + * now. + */ + if (head->total_ref_mod < 0 || + (head->total_ref_mod == 0 && head->must_insert_reserved)) { u64 flags = btrfs_ref_head_to_space_flags(head); btrfs_mod_total_bytes_pinned(fs_info, flags, -head->num_bytes); - - /* - * We had csum deletions accounted for in our delayed refs rsv, - * we need to drop the csum leaves for this update from our - * delayed_refs_rsv. - */ - if (head->is_data) { - spin_lock(&delayed_refs->lock); - delayed_refs->pending_csums -= head->num_bytes; - spin_unlock(&delayed_refs->lock); - nr_items += btrfs_csum_bytes_to_leaves(fs_info, - head->num_bytes); - } } btrfs_delayed_refs_rsv_release(fs_info, nr_items); -- cgit v1.2.3 From 2e626e5673c2a3b4ce8200b961e28edd613ab6a9 Mon Sep 17 00:00:00 2001 From: Nigel Christian Date: Sun, 24 Jan 2021 20:41:41 -0500 Subject: btrfs: remove repeated word in struct member comment Comment for processed extent end of range has an unnecessary "in", remove it. Signed-off-by: Nigel Christian Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 2fa563da65bd..edcdbd739a1e 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2776,7 +2776,7 @@ struct processed_extent { struct btrfs_inode *inode; /* Start of the range in @inode */ u64 start; - /* End of the range in in @inode */ + /* End of the range in @inode */ u64 end; bool uptodate; }; -- cgit v1.2.3 From c78a10aebb275c38d0cfccae129a803fe622e305 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 14 Jan 2021 14:02:42 -0500 Subject: btrfs: fix reloc root leak with 0 ref reloc roots on recovery When recovering a relocation, if we run into a reloc root that has 0 refs we simply add it to the reloc_control->reloc_roots list, and then clean it up later. The problem with this is __del_reloc_root() doesn't do anything if the root isn't in the radix tree, which in this case it won't be because we never call __add_reloc_root() on the reloc_root. This exit condition simply isn't correct really. During normal operation we can remove ourselves from the rb tree and then we're meant to clean up later at merge_reloc_roots() time, and this happens correctly. During recovery we're depending on free_reloc_roots() to drop our references, but we're short-circuiting. Fix this by continuing to check if we're on the list and dropping ourselves from the reloc_control root list and dropping our reference appropriately. Change the corresponding BUG_ON() to an ASSERT() that does the correct thing if we aren't in the rb tree. CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 9f2289bcdde6..d29baf3822a7 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -669,9 +669,7 @@ static void __del_reloc_root(struct btrfs_root *root) RB_CLEAR_NODE(&node->rb_node); } spin_unlock(&rc->reloc_root_tree.lock); - if (!node) - return; - BUG_ON((struct btrfs_root *)node->data != root); + ASSERT(!node || (struct btrfs_root *)node->data == root); } /* -- cgit v1.2.3 From 938fcbfb0cbcf532a1869efab58e6009446b1ced Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 14 Jan 2021 14:02:43 -0500 Subject: btrfs: splice remaining dirty_bg's onto the transaction dirty bg list While doing error injection testing with my relocation patches I hit the following assert: assertion failed: list_empty(&block_group->dirty_list), in fs/btrfs/block-group.c:3356 ------------[ cut here ]------------ kernel BUG at fs/btrfs/ctree.h:3357! invalid opcode: 0000 [#1] SMP NOPTI CPU: 0 PID: 24351 Comm: umount Tainted: G W 5.10.0-rc3+ #193 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.13.0-2.fc32 04/01/2014 RIP: 0010:assertfail.constprop.0+0x18/0x1a RSP: 0018:ffffa09b019c7e00 EFLAGS: 00010282 RAX: 0000000000000056 RBX: ffff8f6492c18000 RCX: 0000000000000000 RDX: ffff8f64fbc27c60 RSI: ffff8f64fbc19050 RDI: ffff8f64fbc19050 RBP: ffff8f6483bbdc00 R08: 0000000000000000 R09: 0000000000000000 R10: ffffa09b019c7c38 R11: ffffffff85d70928 R12: ffff8f6492c18100 R13: ffff8f6492c18148 R14: ffff8f6483bbdd70 R15: dead000000000100 FS: 00007fbfda4cdc40(0000) GS:ffff8f64fbc00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007fbfda666fd0 CR3: 000000013cf66002 CR4: 0000000000370ef0 Call Trace: btrfs_free_block_groups.cold+0x55/0x55 close_ctree+0x2c5/0x306 ? fsnotify_destroy_marks+0x14/0x100 generic_shutdown_super+0x6c/0x100 kill_anon_super+0x14/0x30 btrfs_kill_super+0x12/0x20 deactivate_locked_super+0x36/0xa0 cleanup_mnt+0x12d/0x190 task_work_run+0x5c/0xa0 exit_to_user_mode_prepare+0x1b1/0x1d0 syscall_exit_to_user_mode+0x54/0x280 entry_SYSCALL_64_after_hwframe+0x44/0xa9 This happened because I injected an error in btrfs_cow_block() while running the dirty block groups. When we run the dirty block groups, we splice the list onto a local list to process. However if an error occurs, we only cleanup the transactions dirty block group list, not any pending block groups we have on our locally spliced list. In fact if we fail to allocate a path in this function we'll also fail to clean up the splice list. Fix this by splicing the list back onto the transaction dirty block group list so that the block groups are cleaned up. Then add a 'out' label and have the error conditions jump to out so that the errors are handled properly. This also has the side-effect of fixing a problem where we would clear 'ret' on error because we unconditionally ran btrfs_run_delayed_refs(). CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 763a3671b7af..dda495b2a862 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -2564,8 +2564,10 @@ again: if (!path) { path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; + if (!path) { + ret = -ENOMEM; + goto out; + } } /* @@ -2659,16 +2661,14 @@ again: btrfs_put_block_group(cache); if (drop_reserve) btrfs_delayed_refs_rsv_release(fs_info, 1); - - if (ret) - break; - /* * Avoid blocking other tasks for too long. It might even save * us from writing caches for block groups that are going to be * removed. */ mutex_unlock(&trans->transaction->cache_write_mutex); + if (ret) + goto out; mutex_lock(&trans->transaction->cache_write_mutex); } mutex_unlock(&trans->transaction->cache_write_mutex); @@ -2692,7 +2692,12 @@ again: goto again; } spin_unlock(&cur_trans->dirty_bgs_lock); - } else if (ret < 0) { + } +out: + if (ret < 0) { + spin_lock(&cur_trans->dirty_bgs_lock); + list_splice_init(&dirty, &cur_trans->dirty_bgs); + spin_unlock(&cur_trans->dirty_bgs_lock); btrfs_cleanup_dirty_bgs(cur_trans, fs_info); } -- cgit v1.2.3 From f78743fbdae1bb31bc9c9233c3590a5048782381 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 14 Jan 2021 14:02:44 -0500 Subject: btrfs: do not warn if we can't find the reloc root when looking up backref The backref code is looking for a reloc_root that corresponds to the given fs root. However any number of things could have gone wrong while initializing that reloc_root, like ENOMEM while trying to allocate the root itself, or EIO while trying to write the root item. This would result in no corresponding reloc_root being in the reloc root cache, and thus would return NULL when we do the find_reloc_root() call. Because of this we do not want to WARN_ON(). This presumably was meant to catch developer errors, cases where we messed up adding the reloc root. However we can easily hit this case with error injection, and thus should not do a WARN_ON(). CC: stable@vger.kernel.org # 5.10+ Reported-by: Zygo Blaxell Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 701124c3e0b1..f47c1528eb9a 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2623,7 +2623,7 @@ static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, /* Only reloc backref cache cares about a specific root */ if (cache->is_reloc) { root = find_reloc_root(cache->fs_info, cur->bytenr); - if (WARN_ON(!root)) + if (!root) return -ENOENT; cur->root = root; } else { -- cgit v1.2.3 From eddda68d97732ce05ca145f8e85e8a447f65cdad Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 14 Jan 2021 14:02:45 -0500 Subject: btrfs: add asserts for deleting backref cache nodes A weird KASAN problem that Zygo reported could have been easily caught if we checked for basic things in our backref freeing code. We have two methods of freeing a backref node - btrfs_backref_free_node: this just is kfree() essentially. - btrfs_backref_drop_node: this actually unlinks the node and cleans up everything and then calls btrfs_backref_free_node(). We should mostly be using btrfs_backref_drop_node(), to make sure the node is properly unlinked from the backref cache, and only use btrfs_backref_free_node() when we know the node isn't actually linked to the backref cache. We made a mistake here and thus got the KASAN splat. Make this style of issue easier to find by adding some ASSERT()'s to btrfs_backref_free_node() and adjusting our deletion stuff to properly init the list so we can rely on list_empty() checks working properly. BUG: KASAN: use-after-free in btrfs_backref_cleanup_node+0x18a/0x420 Read of size 8 at addr ffff888112402950 by task btrfs/28836 CPU: 0 PID: 28836 Comm: btrfs Tainted: G W 5.10.0-e35f27394290-for-next+ #23 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 Call Trace: dump_stack+0xbc/0xf9 ? btrfs_backref_cleanup_node+0x18a/0x420 print_address_description.constprop.8+0x21/0x210 ? record_print_text.cold.34+0x11/0x11 ? btrfs_backref_cleanup_node+0x18a/0x420 ? btrfs_backref_cleanup_node+0x18a/0x420 kasan_report.cold.10+0x20/0x37 ? btrfs_backref_cleanup_node+0x18a/0x420 __asan_load8+0x69/0x90 btrfs_backref_cleanup_node+0x18a/0x420 btrfs_backref_release_cache+0x83/0x1b0 relocate_block_group+0x394/0x780 ? merge_reloc_roots+0x4a0/0x4a0 btrfs_relocate_block_group+0x26e/0x4c0 btrfs_relocate_chunk+0x52/0x120 btrfs_balance+0xe2e/0x1900 ? check_flags.part.50+0x6c/0x1e0 ? btrfs_relocate_chunk+0x120/0x120 ? kmem_cache_alloc_trace+0xa06/0xcb0 ? _copy_from_user+0x83/0xc0 btrfs_ioctl_balance+0x3a7/0x460 btrfs_ioctl+0x24c8/0x4360 ? __kasan_check_read+0x11/0x20 ? check_chain_key+0x1f4/0x2f0 ? __asan_loadN+0xf/0x20 ? btrfs_ioctl_get_supported_features+0x30/0x30 ? kvm_sched_clock_read+0x18/0x30 ? check_chain_key+0x1f4/0x2f0 ? lock_downgrade+0x3f0/0x3f0 ? handle_mm_fault+0xad6/0x2150 ? do_vfs_ioctl+0xfc/0x9d0 ? ioctl_file_clone+0xe0/0xe0 ? check_flags.part.50+0x6c/0x1e0 ? check_flags.part.50+0x6c/0x1e0 ? check_flags+0x26/0x30 ? lock_is_held_type+0xc3/0xf0 ? syscall_enter_from_user_mode+0x1b/0x60 ? do_syscall_64+0x13/0x80 ? rcu_read_lock_sched_held+0xa1/0xd0 ? __kasan_check_read+0x11/0x20 ? __fget_light+0xae/0x110 __x64_sys_ioctl+0xc3/0x100 do_syscall_64+0x37/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xa9 RIP: 0033:0x7f4c4bdfe427 RSP: 002b:00007fff33ee6df8 EFLAGS: 00000202 ORIG_RAX: 0000000000000010 RAX: ffffffffffffffda RBX: 00007fff33ee6e98 RCX: 00007f4c4bdfe427 RDX: 00007fff33ee6e98 RSI: 00000000c4009420 RDI: 0000000000000003 RBP: 0000000000000003 R08: 0000000000000003 R09: 0000000000000078 R10: fffffffffffff59d R11: 0000000000000202 R12: 0000000000000001 R13: 0000000000000000 R14: 00007fff33ee8a34 R15: 0000000000000001 Allocated by task 28836: kasan_save_stack+0x21/0x50 __kasan_kmalloc.constprop.18+0xbe/0xd0 kasan_kmalloc+0x9/0x10 kmem_cache_alloc_trace+0x410/0xcb0 btrfs_backref_alloc_node+0x46/0xf0 btrfs_backref_add_tree_node+0x60d/0x11d0 build_backref_tree+0xc5/0x700 relocate_tree_blocks+0x2be/0xb90 relocate_block_group+0x2eb/0x780 btrfs_relocate_block_group+0x26e/0x4c0 btrfs_relocate_chunk+0x52/0x120 btrfs_balance+0xe2e/0x1900 btrfs_ioctl_balance+0x3a7/0x460 btrfs_ioctl+0x24c8/0x4360 __x64_sys_ioctl+0xc3/0x100 do_syscall_64+0x37/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xa9 Freed by task 28836: kasan_save_stack+0x21/0x50 kasan_set_track+0x20/0x30 kasan_set_free_info+0x1f/0x30 __kasan_slab_free+0xf3/0x140 kasan_slab_free+0xe/0x10 kfree+0xde/0x200 btrfs_backref_error_cleanup+0x452/0x530 build_backref_tree+0x1a5/0x700 relocate_tree_blocks+0x2be/0xb90 relocate_block_group+0x2eb/0x780 btrfs_relocate_block_group+0x26e/0x4c0 btrfs_relocate_chunk+0x52/0x120 btrfs_balance+0xe2e/0x1900 btrfs_ioctl_balance+0x3a7/0x460 btrfs_ioctl+0x24c8/0x4360 __x64_sys_ioctl+0xc3/0x100 do_syscall_64+0x37/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xa9 The buggy address belongs to the object at ffff888112402900 which belongs to the cache kmalloc-128 of size 128 The buggy address is located 80 bytes inside of 128-byte region [ffff888112402900, ffff888112402980) The buggy address belongs to the page: page:0000000028b1cd08 refcount:1 mapcount:0 mapping:0000000000000000 index:0xffff888131c810c0 pfn:0x112402 flags: 0x17ffe0000000200(slab) raw: 017ffe0000000200 ffffea000424f308 ffffea0007d572c8 ffff888100040440 raw: ffff888131c810c0 ffff888112402000 0000000100000009 0000000000000000 page dumped because: kasan: bad access detected Memory state around the buggy address: ffff888112402800: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb ffff888112402880: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc >ffff888112402900: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb ^ ffff888112402980: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ffff888112402a00: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb Link: https://lore.kernel.org/linux-btrfs/20201208194607.GI31381@hungrycats.org/ CC: stable@vger.kernel.org # 5.10+ Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index ff705cc564a9..17abde7f794c 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -296,6 +296,9 @@ static inline void btrfs_backref_free_node(struct btrfs_backref_cache *cache, struct btrfs_backref_node *node) { if (node) { + ASSERT(list_empty(&node->list)); + ASSERT(list_empty(&node->lower)); + ASSERT(node->eb == NULL); cache->nr_nodes--; btrfs_put_root(node->root); kfree(node); @@ -340,11 +343,11 @@ static inline void btrfs_backref_drop_node_buffer( static inline void btrfs_backref_drop_node(struct btrfs_backref_cache *tree, struct btrfs_backref_node *node) { - BUG_ON(!list_empty(&node->upper)); + ASSERT(list_empty(&node->upper)); btrfs_backref_drop_node_buffer(node); - list_del(&node->list); - list_del(&node->lower); + list_del_init(&node->list); + list_del_init(&node->lower); if (!RB_EMPTY_NODE(&node->rb_node)) rb_erase(&node->rb_node, &tree->rb_root); btrfs_backref_free_node(tree, node); -- cgit v1.2.3 From 867ed321f90d06aaba84e2c91de51cd3038825ef Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 14 Jan 2021 14:02:46 -0500 Subject: btrfs: abort the transaction if we fail to inc ref in btrfs_copy_root While testing my error handling patches, I added a error injection site at btrfs_inc_extent_ref, to validate the error handling I added was doing the correct thing. However I hit a pretty ugly corruption while doing this check, with the following error injection stack trace: btrfs_inc_extent_ref btrfs_copy_root create_reloc_root btrfs_init_reloc_root btrfs_record_root_in_trans btrfs_start_transaction btrfs_update_inode btrfs_update_time touch_atime file_accessed btrfs_file_mmap This is because we do not catch the error from btrfs_inc_extent_ref, which in practice would be ENOMEM, which means we lose the extent references for a root that has already been allocated and inserted, which is the problem. Fix this by aborting the transaction if we fail to do the reference modification. CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 56e132d825a2..95d9bae764ab 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -221,9 +221,10 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans, ret = btrfs_inc_ref(trans, root, cow, 1); else ret = btrfs_inc_ref(trans, root, cow, 0); - - if (ret) + if (ret) { + btrfs_abort_transaction(trans, ret); return ret; + } btrfs_mark_buffer_dirty(cow); *cow_ret = cow; -- cgit v1.2.3 From ddfd08cb0484e491cae47a76ead051a168a0e644 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 18 Dec 2020 14:24:19 -0500 Subject: btrfs: do not block on deleted bgs mutex in the cleaner While running some stress tests I started getting hung task messages. This is because the delete unused block groups code has to take the delete_unused_bgs_mutex to do it's work, which is taken by balance to make sure we don't delete block groups while we're balancing. The problem is that balance can take a while, and so we were getting hung task warnings. We don't need to block and run these things, and the cleaner is needed to do other work, so trylock on this mutex and just bail if we can't acquire it right away. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index dda495b2a862..5fa6b3d540f4 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1262,6 +1262,13 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags)) return; + /* + * Long running balances can keep us blocked here for eternity, so + * simply skip deletion if we're unable to get the mutex. + */ + if (!mutex_trylock(&fs_info->delete_unused_bgs_mutex)) + return; + spin_lock(&fs_info->unused_bgs_lock); while (!list_empty(&fs_info->unused_bgs)) { int trimming; @@ -1281,8 +1288,6 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group); - mutex_lock(&fs_info->delete_unused_bgs_mutex); - /* Don't want to race with allocators so take the groups_sem */ down_write(&space_info->groups_sem); @@ -1426,11 +1431,11 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) end_trans: btrfs_end_transaction(trans); next: - mutex_unlock(&fs_info->delete_unused_bgs_mutex); btrfs_put_block_group(block_group); spin_lock(&fs_info->unused_bgs_lock); } spin_unlock(&fs_info->unused_bgs_lock); + mutex_unlock(&fs_info->delete_unused_bgs_mutex); return; flip_async: -- cgit v1.2.3 From e19eb11f4f3d3b0463cd897016064a79cb6d8c6d Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 18 Dec 2020 14:24:20 -0500 Subject: btrfs: only let one thread pre-flush delayed refs in commit I've been running a stress test that runs 20 workers in their own subvolume, which are running an fsstress instance with 4 threads per worker, which is 80 total fsstress threads. In addition to this I'm running balance in the background as well as creating and deleting snapshots. This test takes around 12 hours to run normally, going slower and slower as the test goes on. The reason for this is because fsstress is running fsync sometimes, and because we're messing with block groups we often fall through to btrfs_commit_transaction, so will often have 20-30 threads all calling btrfs_commit_transaction at the same time. These all get stuck contending on the extent tree while they try to run delayed refs during the initial part of the commit. This is suboptimal, really because the extent tree is a single point of failure we only want one thread acting on that tree at once to reduce lock contention. Fix this by making the flushing mechanism a bit operation, to make it easy to use test_and_set_bit() in order to make sure only one task does this initial flush. Once we're into the transaction commit we only have one thread doing delayed ref running, it's just this initial pre-flush that is problematic. With this patch my stress test takes around 90 minutes to run, instead of 12 hours. The memory barrier is not necessary for the flushing bit as it's ordered, unlike plain int. The transaction state accessed in btrfs_should_end_transaction could be affected by that too as it's not always used under transaction lock. Upon Nikolay's analysis in [1] it's not necessary: In should_end_transaction it's read without holding any locks. (U) It's modified in btrfs_cleanup_transaction without holding the fs_info->trans_lock (U), but the STATE_ERROR flag is going to be set. set in cleanup_transaction under fs_info->trans_lock (L) set in btrfs_commit_trans to COMMIT_START under fs_info->trans_lock.(L) set in btrfs_commit_trans to COMMIT_DOING under fs_info->trans_lock.(L) set in btrfs_commit_trans to COMMIT_UNBLOCK under fs_info->trans_lock.(L) set in btrfs_commit_trans to COMMIT_COMPLETED without locks but at this point the transaction is finished and fs_info->running_trans is NULL (U but irrelevant). So by the looks of it we can have a concurrent READ race with a WRITE, due to reads not taking a lock. In this case what we want to ensure is we either see new or old state. I consulted with Will Deacon and he said that in such a case we'd want to annotate the accesses to ->state with (READ|WRITE)_ONCE so as to avoid a theoretical tear, in this case I don't think this could happen but I imagine at some point KCSAN would flag such an access as racy (which it is). [1] https://lore.kernel.org/linux-btrfs/e1fd5cc1-0f28-f670-69f4-e9958b4964e6@suse.com Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik [ add comments regarding memory barrier ] Signed-off-by: David Sterba --- fs/btrfs/delayed-ref.h | 12 ++++++------ fs/btrfs/transaction.c | 32 +++++++++++++++----------------- 2 files changed, 21 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/delayed-ref.h b/fs/btrfs/delayed-ref.h index 3ba140468f12..e22fba272e4f 100644 --- a/fs/btrfs/delayed-ref.h +++ b/fs/btrfs/delayed-ref.h @@ -135,6 +135,11 @@ struct btrfs_delayed_data_ref { u64 offset; }; +enum btrfs_delayed_ref_flags { + /* Indicate that we are flushing delayed refs for the commit */ + BTRFS_DELAYED_REFS_FLUSHING, +}; + struct btrfs_delayed_ref_root { /* head ref rbtree */ struct rb_root_cached href_root; @@ -158,12 +163,7 @@ struct btrfs_delayed_ref_root { u64 pending_csums; - /* - * set when the tree is flushing before a transaction commit, - * used by the throttling code to decide if new updates need - * to be run right away - */ - int flushing; + unsigned long flags; u64 run_delayed_start; diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 3bcb5444536e..1485f7722f47 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -909,9 +909,8 @@ bool btrfs_should_end_transaction(struct btrfs_trans_handle *trans) { struct btrfs_transaction *cur_trans = trans->transaction; - smp_mb(); if (cur_trans->state >= TRANS_STATE_COMMIT_START || - cur_trans->delayed_refs.flushing) + test_bit(BTRFS_DELAYED_REFS_FLUSHING, &cur_trans->delayed_refs.flags)) return true; return should_end_transaction(trans); @@ -2043,23 +2042,22 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) btrfs_trans_release_metadata(trans); trans->block_rsv = NULL; - /* make a pass through all the delayed refs we have so far - * any runnings procs may add more while we are here - */ - ret = btrfs_run_delayed_refs(trans, 0); - if (ret) { - btrfs_end_transaction(trans); - return ret; - } - - cur_trans = trans->transaction; - /* - * set the flushing flag so procs in this transaction have to - * start sending their work down. + * We only want one transaction commit doing the flushing so we do not + * waste a bunch of time on lock contention on the extent root node. */ - cur_trans->delayed_refs.flushing = 1; - smp_wmb(); + if (!test_and_set_bit(BTRFS_DELAYED_REFS_FLUSHING, + &cur_trans->delayed_refs.flags)) { + /* + * Make a pass through all the delayed refs we have so far. + * Any running threads may add more while we are here. + */ + ret = btrfs_run_delayed_refs(trans, 0); + if (ret) { + btrfs_end_transaction(trans); + return ret; + } + } btrfs_create_pending_block_groups(trans); -- cgit v1.2.3 From 61a56a992fcfc694a54de77d896350b9d0588e86 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 18 Dec 2020 14:24:21 -0500 Subject: btrfs: delayed refs pre-flushing should only run the heads we have Previously our delayed ref running used the total number of items as the items to run. However we changed that to number of heads to run with the delayed_refs_rsv, as generally we want to run all of the operations for one bytenr. But with btrfs_run_delayed_refs(trans, 0) we set our count to 2x the number of items that we have. This is generally fine, but if we have some operation generation loads of delayed refs while we're doing this pre-flushing in the transaction commit, we'll just spin forever doing delayed refs. Fix this to simply pick the number of delayed refs we currently have, that way we do not end up doing a lot of extra work that's being generated in other threads. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 6f0c59debc2b..0943731f7edd 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -2113,7 +2113,7 @@ int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, delayed_refs = &trans->transaction->delayed_refs; if (count == 0) - count = atomic_read(&delayed_refs->num_entries) * 2; + count = delayed_refs->num_heads_ready; again: #ifdef SCRAMBLE_DELAYED_REFS -- cgit v1.2.3 From ad368f3394b796fd7faa46da8d326c98718f21d7 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 18 Dec 2020 14:24:22 -0500 Subject: btrfs: only run delayed refs once before committing We try to pre-flush the delayed refs when committing, because we want to do as little work as possible in the critical section of the transaction commit. However doing this twice can lead to very long transaction commit delays as other threads are allowed to continue to generate more delayed refs, which potentially delays the commit by multiple minutes in very extreme cases. So simply stick to one pre-flush, and then continue the rest of the transaction commit. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/transaction.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 1485f7722f47..7bb58c3ddcd1 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -2061,12 +2061,6 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) btrfs_create_pending_block_groups(trans); - ret = btrfs_run_delayed_refs(trans, 0); - if (ret) { - btrfs_end_transaction(trans); - return ret; - } - if (!test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &cur_trans->flags)) { int run_it = 0; -- cgit v1.2.3 From 2a4d84c11a872551a335cfe3ee8b60af67ded109 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 18 Dec 2020 14:24:23 -0500 Subject: btrfs: move delayed ref flushing for qgroup into qgroup helper The commit d67263354541 ("btrfs: qgroup: Make snapshot accounting work with new extent-oriented qgroup.") added a flush of the delayed refs during snapshot creation in order to get the qgroup accounting properly. However this code has changed and been moved to it's own helper that is skipped if qgroups are turned off. Move the flushing to the helper, as we do not need it when qgroups are turned off. Also add a comment explaining why it exists, and why it doesn't actually save us. This will be helpful later when we try to fix qgroup accounting properly. Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/transaction.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 7bb58c3ddcd1..dbbd46417534 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -1432,6 +1432,23 @@ static int qgroup_account_snapshot(struct btrfs_trans_handle *trans, */ record_root_in_trans(trans, src, 1); + /* + * btrfs_qgroup_inherit relies on a consistent view of the usage for the + * src root, so we must run the delayed refs here. + * + * However this isn't particularly fool proof, because there's no + * synchronization keeping us from changing the tree after this point + * before we do the qgroup_inherit, or even from making changes while + * we're doing the qgroup_inherit. But that's a problem for the future, + * for now flush the delayed refs to narrow the race window where the + * qgroup counters could end up wrong. + */ + ret = btrfs_run_delayed_refs(trans, (unsigned long)-1); + if (ret) { + btrfs_abort_transaction(trans, ret); + goto out; + } + /* * We are going to commit transaction, see btrfs_commit_transaction() * comment for reason locking tree_log_mutex @@ -1685,12 +1702,6 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans, goto fail; } - ret = btrfs_run_delayed_refs(trans, (unsigned long)-1); - if (ret) { - btrfs_abort_transaction(trans, ret); - goto fail; - } - /* * Do special qgroup accounting for snapshot, as we do some qgroup * snapshot hack to do fast snapshot. -- cgit v1.2.3 From b7774425e0c08d8558be3a072b0c3e0b806b95f6 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 18 Dec 2020 14:24:24 -0500 Subject: btrfs: remove bogus BUG_ON in alloc_reserved_tree_block The fix 361048f586f5 ("Btrfs: fix full backref problem when inserting shared block reference") added a delayed ref flushing at subvolume creation time in order to avoid hitting this particular BUG_ON(). Before this fix, we were tripping the BUG_ON() by 1. Modify snapshot A, which creates blocks with a normal reference for snapshot A, as A is the owner of these blocks. We now have delayed refs for these blocks. 2. Create a snapshot of A named B, which pushes references for the children blocks of the root node for the new root B, thus creating more delayed refs for newly allocated blocks. 3. A is modified, and because the metadata blocks can now be shared, it must push FULL_BACKREF references to the children of any block that A COWs down it's path to its target key. 4. Delayed refs are run. Because these are newly allocated blocks, we have ->must_insert_reserved reserved set on the delayed ref head, we call into alloc_reserved_tree_block() to add the extent item, and then add our ref. At the time of this fix, we were ordering FULL_BACKREF delayed ref operations first, so we'd go to add this reference and then BUG_ON() because we didn't have the FULL_BACKREF flag set. The patch fixed this problem by making sure we ran the delayed refs before we had the chance to modify A. This meant that any *new* blocks would have had their extent items created _before_ we would ever actually COW down and generate FULL_BACKREF entries. Thus the problem went away. However this BUG_ON() is actually completely bogus. The existence of a full backref doesn't necessarily mean that FULL_BACKREF must be set on that block, it must only be set on the actual parent itself. Consider the example provided above. If we COW down one path from A, any nodes are going to have a FULL_BACKREF ref pushed down to _all_ of their children, but not all of the children are going to have FULL_BACKREF set. It is completely valid to have an extent item with normal and full backrefs without FULL_BACKREF actually set on the block itself. As a final note, I have been testing with the patch (applied after this one) btrfs: stop running all delayed refs during snapshot which removed this flushing. My test was a torture test which did a lot of operations while snapshotting and deleting snapshots as well as relocation, and I never tripped this BUG_ON(). This is actually because at the time of 361048f586f5, we ordered SHARED keys _before_ normal references, and thus they would get run first. However currently they are ordered _after_ normal references, so we'd do the initial creation without having a shared reference, and thus not hit this BUG_ON(), which explains why I didn't start hitting this problem during my testing with my other patch applied. Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 1 - 1 file changed, 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 0943731f7edd..5476ab84e544 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -4426,7 +4426,6 @@ static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans, } if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) { - BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); btrfs_set_extent_inline_ref_type(leaf, iref, BTRFS_SHARED_BLOCK_REF_KEY); btrfs_set_extent_inline_ref_offset(leaf, iref, ref->parent); -- cgit v1.2.3 From dac348e9257051e7a39224747695b53e3fc737d7 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 18 Dec 2020 14:24:25 -0500 Subject: btrfs: stop running all delayed refs during snapshot This was added in commit 361048f586f5 ("Btrfs: fix full backref problem when inserting shared block reference") to address a problem where we hit the following BUG_ON() in alloc_reserved_tree_block if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) { BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); However this BUG_ON() is bogus, and was removed by previous commit: btrfs: remove bogus BUG_ON in alloc_reserved_tree_block We no longer need to run delayed refs because of this, and can remove this flushing here. Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/transaction.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index dbbd46417534..02592c6ce755 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -1749,12 +1749,6 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans, } } - ret = btrfs_run_delayed_refs(trans, (unsigned long)-1); - if (ret) { - btrfs_abort_transaction(trans, ret); - goto fail; - } - fail: pending->error = ret; dir_item_existed: -- cgit v1.2.3 From 488bc2a2d21e5faf14f9f695bb592ae9dd0e7465 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 18 Dec 2020 14:24:26 -0500 Subject: btrfs: run delayed refs less often in commit_cowonly_roots We love running delayed refs in commit_cowonly_roots, but it is a bit excessive. I was seeing cases of running 3 or 4 refs a few times in a row during this time. Instead simply: - update all of the roots first - then run delayed refs - then handle the empty block groups case - and then if we have any more dirty roots do the whole thing again This allows us to be much more efficient with our delayed ref running, as we can batch a few more operations at once. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/transaction.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 02592c6ce755..b83e8ae38cfc 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -1226,10 +1226,6 @@ static noinline int commit_cowonly_roots(struct btrfs_trans_handle *trans) btrfs_tree_unlock(eb); free_extent_buffer(eb); - if (ret) - return ret; - - ret = btrfs_run_delayed_refs(trans, (unsigned long)-1); if (ret) return ret; @@ -1247,10 +1243,6 @@ static noinline int commit_cowonly_roots(struct btrfs_trans_handle *trans) if (ret) return ret; - /* run_qgroups might have added some more refs */ - ret = btrfs_run_delayed_refs(trans, (unsigned long)-1); - if (ret) - return ret; again: while (!list_empty(&fs_info->dirty_cowonly_roots)) { struct btrfs_root *root; @@ -1265,15 +1257,24 @@ again: ret = update_cowonly_root(trans, root); if (ret) return ret; - ret = btrfs_run_delayed_refs(trans, (unsigned long)-1); - if (ret) - return ret; } + /* Now flush any delayed refs generated by updating all of the roots */ + ret = btrfs_run_delayed_refs(trans, (unsigned long)-1); + if (ret) + return ret; + while (!list_empty(dirty_bgs) || !list_empty(io_bgs)) { ret = btrfs_write_dirty_block_groups(trans); if (ret) return ret; + + /* + * We're writing the dirty block groups, which could generate + * delayed refs, which could generate more dirty block groups, + * so we want to keep this flushing in this loop to make sure + * everything gets run. + */ ret = btrfs_run_delayed_refs(trans, (unsigned long)-1); if (ret) return ret; -- cgit v1.2.3 From 8898038309876e5b8e535eac9d4b9fe4e3d6f5b3 Mon Sep 17 00:00:00 2001 From: Roman Anasal Date: Mon, 25 Jan 2021 20:43:25 +0100 Subject: btrfs: send: use struct send_ctx *sctx for btrfs_compare_trees and changed_cb btrfs_compare_trees and changed_cb use a void *ctx parameter instead of struct send_ctx *sctx but when used in changed_cb it is immediately cast to `struct send_ctx *sctx = ctx;`. changed_cb is only ever called from btrfs_compare_trees and full_send_tree: - full_send_tree already passes a struct send_ctx *sctx - btrfs_compare_trees is only called by send_subvol with a struct send_ctx *sctx - void *ctx in btrfs_compare_trees is only used to be passed to changed_cb So casting to/from void *ctx seems unnecessary and directly using struct send_ctx *sctx instead provides better type-safety. The original reason for using void *ctx in the first place seems to have been dropped with 1b51d6fce45e ("btrfs: send: remove indirect callback parameter for changed_cb"). Signed-off-by: Roman Anasal Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 3bcbf2bcb869..f87878274e9f 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -6591,10 +6591,9 @@ static int changed_cb(struct btrfs_path *left_path, struct btrfs_path *right_path, struct btrfs_key *key, enum btrfs_compare_tree_result result, - void *ctx) + struct send_ctx *sctx) { int ret = 0; - struct send_ctx *sctx = ctx; if (result == BTRFS_COMPARE_TREE_SAME) { if (key->type == BTRFS_INODE_REF_KEY || @@ -6799,7 +6798,7 @@ static int tree_compare_item(struct btrfs_path *left_path, * If it detects a change, it aborts immediately. */ static int btrfs_compare_trees(struct btrfs_root *left_root, - struct btrfs_root *right_root, void *ctx) + struct btrfs_root *right_root, struct send_ctx *sctx) { struct btrfs_fs_info *fs_info = left_root->fs_info; int ret; @@ -6951,7 +6950,7 @@ static int btrfs_compare_trees(struct btrfs_root *left_root, ret = changed_cb(left_path, right_path, &right_key, BTRFS_COMPARE_TREE_DELETED, - ctx); + sctx); if (ret < 0) goto out; } @@ -6962,7 +6961,7 @@ static int btrfs_compare_trees(struct btrfs_root *left_root, ret = changed_cb(left_path, right_path, &left_key, BTRFS_COMPARE_TREE_NEW, - ctx); + sctx); if (ret < 0) goto out; } @@ -6976,7 +6975,7 @@ static int btrfs_compare_trees(struct btrfs_root *left_root, ret = changed_cb(left_path, right_path, &left_key, BTRFS_COMPARE_TREE_NEW, - ctx); + sctx); if (ret < 0) goto out; advance_left = ADVANCE; @@ -6984,7 +6983,7 @@ static int btrfs_compare_trees(struct btrfs_root *left_root, ret = changed_cb(left_path, right_path, &right_key, BTRFS_COMPARE_TREE_DELETED, - ctx); + sctx); if (ret < 0) goto out; advance_right = ADVANCE; @@ -6999,7 +6998,7 @@ static int btrfs_compare_trees(struct btrfs_root *left_root, else result = BTRFS_COMPARE_TREE_SAME; ret = changed_cb(left_path, right_path, - &left_key, result, ctx); + &left_key, result, sctx); if (ret < 0) goto out; advance_left = ADVANCE; -- cgit v1.2.3 From 91e79a83fff663283341c8c29293faec8255099a Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:18 -0400 Subject: btrfs: make flush_space take a enum btrfs_flush_state instead of int I got a automated message from somebody who runs clang against our kernels and it's because I used the wrong enum type for what I passed into flush_space, caught by -Wenum-conversion. Change the argument to be explicitly the enum we're expecting to make everything consistent. Maybe eventually gcc will catch errors like this. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index fd8e79e3c10e..4eab581b1b9c 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -670,7 +670,7 @@ enospc: */ static void flush_space(struct btrfs_fs_info *fs_info, struct btrfs_space_info *space_info, u64 num_bytes, - int state) + enum btrfs_flush_state state) { struct btrfs_root *root = fs_info->extent_root; struct btrfs_trans_handle *trans; @@ -923,7 +923,7 @@ static void btrfs_async_reclaim_metadata_space(struct work_struct *work) struct btrfs_fs_info *fs_info; struct btrfs_space_info *space_info; u64 to_reclaim; - int flush_state; + enum btrfs_flush_state flush_state; int commit_cycles = 0; u64 last_tickets_id; @@ -1055,7 +1055,7 @@ static void btrfs_async_reclaim_data_space(struct work_struct *work) struct btrfs_fs_info *fs_info; struct btrfs_space_info *space_info; u64 last_tickets_id; - int flush_state = 0; + enum btrfs_flush_state flush_state = 0; fs_info = container_of(work, struct btrfs_fs_info, async_data_reclaim_work); space_info = fs_info->data_sinfo; -- cgit v1.2.3 From ac1ea10e757a57fb61512ae9beb2ef67e5340e31 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:19 -0400 Subject: btrfs: add a trace point for reserve tickets While debugging a ENOSPC related performance problem I needed to see the time difference between start and end of a reserve ticket, so add a trace point to report when we handle a reserve ticket. I opted to spit out start_ns itself without calculating the difference because there could be a gap between enabling the tracepoint and setting start_ns. Doing it this way allows us to filter on 0 start_ns so we don't get bogus entries, and we can easily calculate the time difference with bpftrace or something else. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 12 +++++++++++- include/trace/events/btrfs.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 4eab581b1b9c..d879e3fea0b6 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -1220,6 +1220,8 @@ static void wait_reserve_ticket(struct btrfs_fs_info *fs_info, * @fs_info: the filesystem * @space_info: space info for the reservation * @ticket: ticket for the reservation + * @start_ns: timestamp when the reservation started + * @orig_bytes: amount of bytes originally reserved * @flush: how much we can flush * * This does the work of figuring out how to flush for the ticket, waiting for @@ -1228,6 +1230,7 @@ static void wait_reserve_ticket(struct btrfs_fs_info *fs_info, static int handle_reserve_ticket(struct btrfs_fs_info *fs_info, struct btrfs_space_info *space_info, struct reserve_ticket *ticket, + u64 start_ns, u64 orig_bytes, enum btrfs_reserve_flush_enum flush) { int ret; @@ -1283,6 +1286,8 @@ static int handle_reserve_ticket(struct btrfs_fs_info *fs_info, * space wasn't reserved at all). */ ASSERT(!(ticket->bytes == 0 && ticket->error)); + trace_btrfs_reserve_ticket(fs_info, space_info->flags, orig_bytes, + start_ns, flush, ticket->error); return ret; } @@ -1317,6 +1322,7 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, { struct work_struct *async_work; struct reserve_ticket ticket; + u64 start_ns = 0; u64 used; int ret = 0; bool pending_tickets; @@ -1369,6 +1375,9 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, space_info->reclaim_size += ticket.bytes; init_waitqueue_head(&ticket.wait); ticket.steal = (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL); + if (trace_btrfs_reserve_ticket_enabled()) + start_ns = ktime_get_ns(); + if (flush == BTRFS_RESERVE_FLUSH_ALL || flush == BTRFS_RESERVE_FLUSH_ALL_STEAL || flush == BTRFS_RESERVE_FLUSH_DATA) { @@ -1405,7 +1414,8 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, if (!ret || flush == BTRFS_RESERVE_NO_FLUSH) return ret; - return handle_reserve_ticket(fs_info, space_info, &ticket, flush); + return handle_reserve_ticket(fs_info, space_info, &ticket, start_ns, + orig_bytes, flush); } /** diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index b9896fc06160..b0ea2a108be3 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -2026,6 +2026,35 @@ TRACE_EVENT(btrfs_convert_extent_bit, __print_flags(__entry->clear_bits, "|", EXTENT_FLAGS)) ); +TRACE_EVENT(btrfs_reserve_ticket, + TP_PROTO(const struct btrfs_fs_info *fs_info, u64 flags, u64 bytes, + u64 start_ns, int flush, int error), + + TP_ARGS(fs_info, flags, bytes, start_ns, flush, error), + + TP_STRUCT__entry_btrfs( + __field( u64, flags ) + __field( u64, bytes ) + __field( u64, start_ns ) + __field( int, flush ) + __field( int, error ) + ), + + TP_fast_assign_btrfs(fs_info, + __entry->flags = flags; + __entry->bytes = bytes; + __entry->start_ns = start_ns; + __entry->flush = flush; + __entry->error = error; + ), + + TP_printk_btrfs("flags=%s bytes=%llu start_ns=%llu flush=%s error=%d", + __print_flags(__entry->flags, "|", BTRFS_GROUP_FLAGS), + __entry->bytes, __entry->start_ns, + __print_symbolic(__entry->flush, FLUSH_ACTIONS), + __entry->error) +); + DECLARE_EVENT_CLASS(btrfs_sleep_tree_lock, TP_PROTO(const struct extent_buffer *eb, u64 start_ns), -- cgit v1.2.3 From 5deb17e18e27a3502f21581ba4d086e762b86b31 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:20 -0400 Subject: btrfs: track ordered bytes instead of just dio ordered bytes We track dio_bytes because the shrink delalloc code needs to know if we have more DIO in flight than we have normal buffered IO. The reason for this is because we can't "flush" DIO, we have to just wait on the ordered extents to finish. However this is true of all ordered extents. If we have more ordered space outstanding than dirty pages we should be waiting on ordered extents. We already are ok on this front technically, because we always do a FLUSH_DELALLOC_WAIT loop, but I want to use the ordered counter in the preemptive flushing code as well, so change this to count all ordered bytes instead of just DIO ordered bytes. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 2 +- fs/btrfs/disk-io.c | 8 ++++---- fs/btrfs/ordered-data.c | 13 ++++++------- fs/btrfs/space-info.c | 18 +++++++----------- 4 files changed, 18 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index ed6bb46a2572..7d8660227520 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -797,7 +797,7 @@ struct btrfs_fs_info { /* used to keep from writing metadata until there is a nice batch */ struct percpu_counter dirty_metadata_bytes; struct percpu_counter delalloc_bytes; - struct percpu_counter dio_bytes; + struct percpu_counter ordered_bytes; s32 dirty_metadata_batch; s32 delalloc_batch; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 5473bed6a7e8..e0d56b3d1223 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1469,7 +1469,7 @@ void btrfs_free_fs_info(struct btrfs_fs_info *fs_info) { percpu_counter_destroy(&fs_info->dirty_metadata_bytes); percpu_counter_destroy(&fs_info->delalloc_bytes); - percpu_counter_destroy(&fs_info->dio_bytes); + percpu_counter_destroy(&fs_info->ordered_bytes); percpu_counter_destroy(&fs_info->dev_replace.bio_counter); btrfs_free_csum_hash(fs_info); btrfs_free_stripe_hash_table(fs_info); @@ -2802,7 +2802,7 @@ static int init_mount_fs_info(struct btrfs_fs_info *fs_info, struct super_block sb->s_blocksize = BTRFS_BDEV_BLOCKSIZE; sb->s_blocksize_bits = blksize_bits(BTRFS_BDEV_BLOCKSIZE); - ret = percpu_counter_init(&fs_info->dio_bytes, 0, GFP_KERNEL); + ret = percpu_counter_init(&fs_info->ordered_bytes, 0, GFP_KERNEL); if (ret) return ret; @@ -4163,9 +4163,9 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) percpu_counter_sum(&fs_info->delalloc_bytes)); } - if (percpu_counter_sum(&fs_info->dio_bytes)) + if (percpu_counter_sum(&fs_info->ordered_bytes)) btrfs_info(fs_info, "at unmount dio bytes count %lld", - percpu_counter_sum(&fs_info->dio_bytes)); + percpu_counter_sum(&fs_info->ordered_bytes)); btrfs_sysfs_remove_mounted(fs_info); btrfs_sysfs_remove_fsid(fs_info->fs_devices); diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index b4e6500548a2..e8dee1578d4a 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -206,11 +206,11 @@ static int __btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset type == BTRFS_ORDERED_COMPRESSED); set_bit(type, &entry->flags); - if (dio) { - percpu_counter_add_batch(&fs_info->dio_bytes, num_bytes, - fs_info->delalloc_batch); + percpu_counter_add_batch(&fs_info->ordered_bytes, num_bytes, + fs_info->delalloc_batch); + + if (dio) set_bit(BTRFS_ORDERED_DIRECT, &entry->flags); - } /* one ref for the tree */ refcount_set(&entry->refs, 1); @@ -503,9 +503,8 @@ void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode, btrfs_delalloc_release_metadata(btrfs_inode, entry->num_bytes, false); - if (test_bit(BTRFS_ORDERED_DIRECT, &entry->flags)) - percpu_counter_add_batch(&fs_info->dio_bytes, -entry->num_bytes, - fs_info->delalloc_batch); + percpu_counter_add_batch(&fs_info->ordered_bytes, -entry->num_bytes, + fs_info->delalloc_batch); tree = &btrfs_inode->ordered_tree; spin_lock_irq(&tree->lock); diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index d879e3fea0b6..711beacd75d6 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -489,7 +489,7 @@ static void shrink_delalloc(struct btrfs_fs_info *fs_info, { struct btrfs_trans_handle *trans; u64 delalloc_bytes; - u64 dio_bytes; + u64 ordered_bytes; u64 items; long time_left; int loops; @@ -513,25 +513,20 @@ static void shrink_delalloc(struct btrfs_fs_info *fs_info, delalloc_bytes = percpu_counter_sum_positive( &fs_info->delalloc_bytes); - dio_bytes = percpu_counter_sum_positive(&fs_info->dio_bytes); - if (delalloc_bytes == 0 && dio_bytes == 0) { - if (trans) - return; - if (wait_ordered) - btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1); + ordered_bytes = percpu_counter_sum_positive(&fs_info->ordered_bytes); + if (delalloc_bytes == 0 && ordered_bytes == 0) return; - } /* * If we are doing more ordered than delalloc we need to just wait on * ordered extents, otherwise we'll waste time trying to flush delalloc * that likely won't give us the space back we need. */ - if (dio_bytes > delalloc_bytes) + if (ordered_bytes > delalloc_bytes) wait_ordered = true; loops = 0; - while ((delalloc_bytes || dio_bytes) && loops < 3) { + while ((delalloc_bytes || ordered_bytes) && loops < 3) { u64 temp = min(delalloc_bytes, to_reclaim) >> PAGE_SHIFT; long nr_pages = min_t(u64, temp, LONG_MAX); @@ -556,7 +551,8 @@ static void shrink_delalloc(struct btrfs_fs_info *fs_info, delalloc_bytes = percpu_counter_sum_positive( &fs_info->delalloc_bytes); - dio_bytes = percpu_counter_sum_positive(&fs_info->dio_bytes); + ordered_bytes = percpu_counter_sum_positive( + &fs_info->ordered_bytes); } } -- cgit v1.2.3 From f00c42dd4cc8b856e68638e6a88b51f88b8e849e Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:21 -0400 Subject: btrfs: introduce a FORCE_COMMIT_TRANS flush operation Solely for preemptive flushing, we want to be able to force the transaction commit without any of the ambiguity of may_commit_transaction(). This is because may_commit_transaction() checks tickets and such, and in preemptive flushing we already know it'll be helpful, so use this to keep the code nice and clean and straightforward. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik [ add comment ] Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 1 + fs/btrfs/space-info.c | 14 ++++++++++++++ include/trace/events/btrfs.h | 3 ++- 3 files changed, 17 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 7d8660227520..90726954b883 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -2740,6 +2740,7 @@ enum btrfs_flush_state { ALLOC_CHUNK_FORCE = 8, RUN_DELAYED_IPUTS = 9, COMMIT_TRANS = 10, + FORCE_COMMIT_TRANS = 11, }; int btrfs_subvolume_reserve_metadata(struct btrfs_root *root, diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 711beacd75d6..e677b5451f82 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -140,6 +140,12 @@ * be freed, plus any delayed work we may not have gotten rid of in the case * of metadata. * + * FORCE_COMMIT_TRANS + * For use by the preemptive flusher. We use this to bypass the ticketing + * checks in may_commit_transaction, as we have more information about the + * overall state of the system and may want to commit the transaction ahead + * of actual ENOSPC conditions. + * * OVERCOMMIT * * Because we hold so many reservations for metadata we will allow you to @@ -735,6 +741,14 @@ static void flush_space(struct btrfs_fs_info *fs_info, case COMMIT_TRANS: ret = may_commit_transaction(fs_info, space_info); break; + case FORCE_COMMIT_TRANS: + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + break; + } + ret = btrfs_commit_transaction(trans); + break; default: ret = -ENOSPC; break; diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index b0ea2a108be3..8a7c163907a2 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -99,7 +99,8 @@ struct btrfs_space_info; EM( ALLOC_CHUNK, "ALLOC_CHUNK") \ EM( ALLOC_CHUNK_FORCE, "ALLOC_CHUNK_FORCE") \ EM( RUN_DELAYED_IPUTS, "RUN_DELAYED_IPUTS") \ - EMe(COMMIT_TRANS, "COMMIT_TRANS") + EM( COMMIT_TRANS, "COMMIT_TRANS") \ + EMe(FORCE_COMMIT_TRANS, "FORCE_COMMIT_TRANS") /* * First define the enums in the above macros to be exported to userspace via -- cgit v1.2.3 From 576fa34830afac6a40cd19c777f1ab49c914e87c Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:22 -0400 Subject: btrfs: improve preemptive background space flushing Currently if we ever have to flush space because we do not have enough we allocate a ticket and attach it to the space_info, and then systematically flush things in the filesystem that hold space reservations until our space is reclaimed. However this has a latency cost, we must go to sleep and wait for the flushing to make progress before we are woken up and allowed to continue doing our work. In order to address that we used to kick off the async worker to flush space preemptively, so that we could be reclaiming space hopefully before any tasks needed to stop and wait for space to reclaim. When I introduced the ticketed ENOSPC stuff this broke slightly in the fact that we were using tickets to indicate if we were done flushing. No tickets, no more flushing. However this meant that we essentially never preemptively flushed. This caused a write performance regression that Nikolay noticed in an unrelated patch that removed the committing of the transaction during btrfs_end_transaction. The behavior that happened pre that patch was btrfs_end_transaction() would see that we were low on space, and it would commit the transaction. This was bad because in this particular case you could end up with thousands and thousands of transactions being committed during the 5 minute reproducer. With the patch to remove this behavior we got much more sane transaction commits, but we ended up slower because we would write for a while, flush, write for a while, flush again. To address this we need to reinstate a preemptive flushing mechanism. However it is distinctly different from our ticketing flushing in that it doesn't have tickets to base it's decisions on. Instead of bolting this logic into our existing flushing work, add another worker to handle this preemptive flushing. Here we will attempt to be slightly intelligent about the things that we flushing, attempting to balance between whichever pool is taking up the most space. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 1 + fs/btrfs/disk-io.c | 1 + fs/btrfs/space-info.c | 100 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 100 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 90726954b883..a9b0521d9e89 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -933,6 +933,7 @@ struct btrfs_fs_info { /* Used to reclaim the metadata space in the background. */ struct work_struct async_reclaim_work; struct work_struct async_data_reclaim_work; + struct work_struct preempt_reclaim_work; spinlock_t unused_bgs_lock; struct list_head unused_bgs; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index e0d56b3d1223..e0d1b328397e 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -4111,6 +4111,7 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) cancel_work_sync(&fs_info->async_reclaim_work); cancel_work_sync(&fs_info->async_data_reclaim_work); + cancel_work_sync(&fs_info->preempt_reclaim_work); /* Cancel or finish ongoing discard work */ btrfs_discard_cleanup(fs_info); diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index e677b5451f82..8a27c193f8a8 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -1000,6 +1000,100 @@ static void btrfs_async_reclaim_metadata_space(struct work_struct *work) } while (flush_state <= COMMIT_TRANS); } +/* + * This handles pre-flushing of metadata space before we get to the point that + * we need to start blocking threads on tickets. The logic here is different + * from the other flush paths because it doesn't rely on tickets to tell us how + * much we need to flush, instead it attempts to keep us below the 80% full + * watermark of space by flushing whichever reservation pool is currently the + * largest. + */ +static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_space_info *space_info; + struct btrfs_block_rsv *delayed_block_rsv; + struct btrfs_block_rsv *delayed_refs_rsv; + struct btrfs_block_rsv *global_rsv; + struct btrfs_block_rsv *trans_rsv; + u64 used; + + fs_info = container_of(work, struct btrfs_fs_info, + preempt_reclaim_work); + space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); + delayed_block_rsv = &fs_info->delayed_block_rsv; + delayed_refs_rsv = &fs_info->delayed_refs_rsv; + global_rsv = &fs_info->global_block_rsv; + trans_rsv = &fs_info->trans_block_rsv; + + spin_lock(&space_info->lock); + used = btrfs_space_info_used(space_info, true); + while (need_do_async_reclaim(fs_info, space_info, used)) { + enum btrfs_flush_state flush; + u64 delalloc_size = 0; + u64 to_reclaim, block_rsv_size; + u64 global_rsv_size = global_rsv->reserved; + + /* + * We don't have a precise counter for the metadata being + * reserved for delalloc, so we'll approximate it by subtracting + * out the block rsv's space from the bytes_may_use. If that + * amount is higher than the individual reserves, then we can + * assume it's tied up in delalloc reservations. + */ + block_rsv_size = global_rsv_size + + delayed_block_rsv->reserved + + delayed_refs_rsv->reserved + + trans_rsv->reserved; + if (block_rsv_size < space_info->bytes_may_use) + delalloc_size = space_info->bytes_may_use - block_rsv_size; + spin_unlock(&space_info->lock); + + /* + * We don't want to include the global_rsv in our calculation, + * because that's space we can't touch. Subtract it from the + * block_rsv_size for the next checks. + */ + block_rsv_size -= global_rsv_size; + + /* + * We really want to avoid flushing delalloc too much, as it + * could result in poor allocation patterns, so only flush it if + * it's larger than the rest of the pools combined. + */ + if (delalloc_size > block_rsv_size) { + to_reclaim = delalloc_size; + flush = FLUSH_DELALLOC; + } else if (space_info->bytes_pinned > + (delayed_block_rsv->reserved + + delayed_refs_rsv->reserved)) { + to_reclaim = space_info->bytes_pinned; + flush = FORCE_COMMIT_TRANS; + } else if (delayed_block_rsv->reserved > + delayed_refs_rsv->reserved) { + to_reclaim = delayed_block_rsv->reserved; + flush = FLUSH_DELAYED_ITEMS_NR; + } else { + to_reclaim = delayed_refs_rsv->reserved; + flush = FLUSH_DELAYED_REFS_NR; + } + + /* + * We don't want to reclaim everything, just a portion, so scale + * down the to_reclaim by 1/4. If it takes us down to 0, + * reclaim 1 items worth. + */ + to_reclaim >>= 2; + if (!to_reclaim) + to_reclaim = btrfs_calc_insert_metadata_size(fs_info, 1); + flush_space(fs_info, space_info, to_reclaim, flush); + cond_resched(); + spin_lock(&space_info->lock); + used = btrfs_space_info_used(space_info, true); + } + spin_unlock(&space_info->lock); +} + /* * FLUSH_DELALLOC_WAIT: * Space is freed from flushing delalloc in one of two ways. @@ -1126,6 +1220,8 @@ void btrfs_init_async_reclaim_work(struct btrfs_fs_info *fs_info) { INIT_WORK(&fs_info->async_reclaim_work, btrfs_async_reclaim_metadata_space); INIT_WORK(&fs_info->async_data_reclaim_work, btrfs_async_reclaim_data_space); + INIT_WORK(&fs_info->preempt_reclaim_work, + btrfs_preempt_reclaim_metadata_space); } static const enum btrfs_flush_state priority_flush_states[] = { @@ -1413,11 +1509,11 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, */ if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) && need_do_async_reclaim(fs_info, space_info, used) && - !work_busy(&fs_info->async_reclaim_work)) { + !work_busy(&fs_info->preempt_reclaim_work)) { trace_btrfs_trigger_flush(fs_info, space_info->flags, orig_bytes, flush, "preempt"); queue_work(system_unbound_wq, - &fs_info->async_reclaim_work); + &fs_info->preempt_reclaim_work); } } spin_unlock(&space_info->lock); -- cgit v1.2.3 From ae7913ba52ec4a2883eb073c6bc99f1a8d9d636b Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:23 -0400 Subject: btrfs: rename need_do_async_reclaim All of our normal flushing is asynchronous reclaim, so this helper is poorly named. This is more checking if we need to preemptively flush space, so rename it to need_preemptive_reclaim. Also switch it to bool and make it plain static as followup patches will move more code here. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 8a27c193f8a8..effb9b73a418 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -808,18 +808,18 @@ btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info, return to_reclaim; } -static inline int need_do_async_reclaim(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - u64 used) +static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + u64 used) { u64 thresh = div_factor_fine(space_info->total_bytes, 98); /* If we're just plain full then async reclaim just slows us down. */ if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh) - return 0; + return false; if (!btrfs_calc_reclaim_metadata_size(fs_info, space_info)) - return 0; + return false; return (used >= thresh && !btrfs_fs_closing(fs_info) && !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)); @@ -1028,7 +1028,7 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) spin_lock(&space_info->lock); used = btrfs_space_info_used(space_info, true); - while (need_do_async_reclaim(fs_info, space_info, used)) { + while (need_preemptive_reclaim(fs_info, space_info, used)) { enum btrfs_flush_state flush; u64 delalloc_size = 0; u64 to_reclaim, block_rsv_size; @@ -1508,7 +1508,7 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, * the async reclaim as we will panic. */ if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) && - need_do_async_reclaim(fs_info, space_info, used) && + need_preemptive_reclaim(fs_info, space_info, used) && !work_busy(&fs_info->preempt_reclaim_work)) { trace_btrfs_trigger_flush(fs_info, space_info->flags, orig_bytes, flush, "preempt"); -- cgit v1.2.3 From f205edf77315a33eee82a7615fb57e9297957fe9 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:24 -0400 Subject: btrfs: check reclaim_size in need_preemptive_reclaim If we're flushing space for tickets then we have space_info->reclaim_size set and we do not need to do background reclaim. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index effb9b73a418..9f30d6837eb5 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -818,6 +818,13 @@ static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh) return false; + /* + * We have tickets queued, bail so we don't compete with the async + * flushers. + */ + if (space_info->reclaim_size) + return false; + if (!btrfs_calc_reclaim_metadata_size(fs_info, space_info)) return false; -- cgit v1.2.3 From 9f42d37748264d65ca611b60c22b9c003030b0b3 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:25 -0400 Subject: btrfs: rework btrfs_calc_reclaim_metadata_size Currently btrfs_calc_reclaim_metadata_size does two things, it returns the space currently required for flushing by the tickets, and if there are no tickets it calculates a value for the preemptive flushing. However for the normal ticketed flushing we really only care about the space required for tickets. We will accidentally come in and flush one time, but as soon as we see there are no tickets we bail out of our flushing. Fix this by making btrfs_calc_reclaim_metadata_size really only tell us what is required for flushing if we have people waiting on space. Then move the preemptive flushing logic into need_preemptive_reclaim(). We ignore btrfs_calc_reclaim_metadata_size() in need_preemptive_reclaim() because if we are in this path then we made our reservation and there are not pending tickets currently, so we do not need to check it, simply do the fuzzy logic to check if we're getting low on space. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 9f30d6837eb5..636a42620b11 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -765,7 +765,6 @@ btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info, { u64 used; u64 avail; - u64 expected; u64 to_reclaim = space_info->reclaim_size; lockdep_assert_held(&space_info->lock); @@ -783,28 +782,6 @@ btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info, if (space_info->total_bytes + avail < used) to_reclaim += used - (space_info->total_bytes + avail); - if (to_reclaim) - return to_reclaim; - - to_reclaim = min_t(u64, num_online_cpus() * SZ_1M, SZ_16M); - if (btrfs_can_overcommit(fs_info, space_info, to_reclaim, - BTRFS_RESERVE_FLUSH_ALL)) - return 0; - - used = btrfs_space_info_used(space_info, true); - - if (btrfs_can_overcommit(fs_info, space_info, SZ_1M, - BTRFS_RESERVE_FLUSH_ALL)) - expected = div_factor_fine(space_info->total_bytes, 95); - else - expected = div_factor_fine(space_info->total_bytes, 90); - - if (used > expected) - to_reclaim = used - expected; - else - to_reclaim = 0; - to_reclaim = min(to_reclaim, space_info->bytes_may_use + - space_info->bytes_reserved); return to_reclaim; } @@ -813,6 +790,7 @@ static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, u64 used) { u64 thresh = div_factor_fine(space_info->total_bytes, 98); + u64 to_reclaim, expected; /* If we're just plain full then async reclaim just slows us down. */ if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh) @@ -825,7 +803,25 @@ static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, if (space_info->reclaim_size) return false; - if (!btrfs_calc_reclaim_metadata_size(fs_info, space_info)) + to_reclaim = min_t(u64, num_online_cpus() * SZ_1M, SZ_16M); + if (btrfs_can_overcommit(fs_info, space_info, to_reclaim, + BTRFS_RESERVE_FLUSH_ALL)) + return false; + + used = btrfs_space_info_used(space_info, true); + if (btrfs_can_overcommit(fs_info, space_info, SZ_1M, + BTRFS_RESERVE_FLUSH_ALL)) + expected = div_factor_fine(space_info->total_bytes, 95); + else + expected = div_factor_fine(space_info->total_bytes, 90); + + if (used > expected) + to_reclaim = used - expected; + else + to_reclaim = 0; + to_reclaim = min(to_reclaim, space_info->bytes_may_use + + space_info->bytes_reserved); + if (!to_reclaim) return false; return (used >= thresh && !btrfs_fs_closing(fs_info) && -- cgit v1.2.3 From 2e294c60497f29ab8791f4b99f348b22d70dd3c3 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:26 -0400 Subject: btrfs: simplify the logic in need_preemptive_flushing A lot of this was added all in one go with no explanation, and is a bit unwieldy and confusing. Simplify the logic to start preemptive flushing if we've reserved more than half of our available free space. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 73 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 25 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 636a42620b11..9befd22a2316 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -786,11 +786,11 @@ btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info, } static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - u64 used) + struct btrfs_space_info *space_info) { + u64 ordered, delalloc; u64 thresh = div_factor_fine(space_info->total_bytes, 98); - u64 to_reclaim, expected; + u64 used; /* If we're just plain full then async reclaim just slows us down. */ if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh) @@ -803,26 +803,52 @@ static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, if (space_info->reclaim_size) return false; - to_reclaim = min_t(u64, num_online_cpus() * SZ_1M, SZ_16M); - if (btrfs_can_overcommit(fs_info, space_info, to_reclaim, - BTRFS_RESERVE_FLUSH_ALL)) - return false; + /* + * If we have over half of the free space occupied by reservations or + * pinned then we want to start flushing. + * + * We do not do the traditional thing here, which is to say + * + * if (used >= ((total_bytes + avail) / 2)) + * return 1; + * + * because this doesn't quite work how we want. If we had more than 50% + * of the space_info used by bytes_used and we had 0 available we'd just + * constantly run the background flusher. Instead we want it to kick in + * if our reclaimable space exceeds 50% of our available free space. + */ + thresh = calc_available_free_space(fs_info, space_info, + BTRFS_RESERVE_FLUSH_ALL); + thresh += (space_info->total_bytes - space_info->bytes_used - + space_info->bytes_reserved - space_info->bytes_readonly); + thresh >>= 1; - used = btrfs_space_info_used(space_info, true); - if (btrfs_can_overcommit(fs_info, space_info, SZ_1M, - BTRFS_RESERVE_FLUSH_ALL)) - expected = div_factor_fine(space_info->total_bytes, 95); - else - expected = div_factor_fine(space_info->total_bytes, 90); + used = space_info->bytes_pinned; - if (used > expected) - to_reclaim = used - expected; + /* + * If we have more ordered bytes than delalloc bytes then we're either + * doing a lot of DIO, or we simply don't have a lot of delalloc waiting + * around. Preemptive flushing is only useful in that it can free up + * space before tickets need to wait for things to finish. In the case + * of ordered extents, preemptively waiting on ordered extents gets us + * nothing, if our reservations are tied up in ordered extents we'll + * simply have to slow down writers by forcing them to wait on ordered + * extents. + * + * In the case that ordered is larger than delalloc, only include the + * block reserves that we would actually be able to directly reclaim + * from. In this case if we're heavy on metadata operations this will + * clearly be heavy enough to warrant preemptive flushing. In the case + * of heavy DIO or ordered reservations, preemptive flushing will just + * waste time and cause us to slow down. + */ + ordered = percpu_counter_sum_positive(&fs_info->ordered_bytes); + delalloc = percpu_counter_sum_positive(&fs_info->delalloc_bytes); + if (ordered >= delalloc) + used += fs_info->delayed_refs_rsv.reserved + + fs_info->delayed_block_rsv.reserved; else - to_reclaim = 0; - to_reclaim = min(to_reclaim, space_info->bytes_may_use + - space_info->bytes_reserved); - if (!to_reclaim) - return false; + used += space_info->bytes_may_use; return (used >= thresh && !btrfs_fs_closing(fs_info) && !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)); @@ -1019,7 +1045,6 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) struct btrfs_block_rsv *delayed_refs_rsv; struct btrfs_block_rsv *global_rsv; struct btrfs_block_rsv *trans_rsv; - u64 used; fs_info = container_of(work, struct btrfs_fs_info, preempt_reclaim_work); @@ -1030,8 +1055,7 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) trans_rsv = &fs_info->trans_block_rsv; spin_lock(&space_info->lock); - used = btrfs_space_info_used(space_info, true); - while (need_preemptive_reclaim(fs_info, space_info, used)) { + while (need_preemptive_reclaim(fs_info, space_info)) { enum btrfs_flush_state flush; u64 delalloc_size = 0; u64 to_reclaim, block_rsv_size; @@ -1092,7 +1116,6 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) flush_space(fs_info, space_info, to_reclaim, flush); cond_resched(); spin_lock(&space_info->lock); - used = btrfs_space_info_used(space_info, true); } spin_unlock(&space_info->lock); } @@ -1511,7 +1534,7 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, * the async reclaim as we will panic. */ if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) && - need_preemptive_reclaim(fs_info, space_info, used) && + need_preemptive_reclaim(fs_info, space_info) && !work_busy(&fs_info->preempt_reclaim_work)) { trace_btrfs_trigger_flush(fs_info, space_info->flags, orig_bytes, flush, "preempt"); -- cgit v1.2.3 From 88a777a6e5272106bdc96b1032d89b0ddc0e526f Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:27 -0400 Subject: btrfs: implement space clamping for preemptive flushing Starting preemptive flushing at 50% of available free space is a good start, but some workloads are particularly abusive and can quickly overwhelm the preemptive flushing code and drive us into using tickets. Handle this by clamping down on our threshold for starting and continuing to run preemptive flushing. This is particularly important for our overcommit case, as we can really drive the file system into overages and then it's more difficult to pull it back as we start to actually fill up the file system. The clamping is essentially 2^CLAMP, but we start at 1 so whatever we calculate for overcommit is the baseline. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++-- fs/btrfs/space-info.h | 4 ++++ 2 files changed, 55 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 9befd22a2316..e8acf087dcee 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -212,6 +212,7 @@ static int create_space_info(struct btrfs_fs_info *info, u64 flags) INIT_LIST_HEAD(&space_info->ro_bgs); INIT_LIST_HEAD(&space_info->tickets); INIT_LIST_HEAD(&space_info->priority_tickets); + space_info->clamp = 1; ret = btrfs_sysfs_add_space_info_type(info, space_info); if (ret) @@ -815,13 +816,28 @@ static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, * because this doesn't quite work how we want. If we had more than 50% * of the space_info used by bytes_used and we had 0 available we'd just * constantly run the background flusher. Instead we want it to kick in - * if our reclaimable space exceeds 50% of our available free space. + * if our reclaimable space exceeds our clamped free space. + * + * Our clamping range is 2^1 -> 2^8. Practically speaking that means + * the following: + * + * Amount of RAM Minimum threshold Maximum threshold + * + * 256GiB 1GiB 128GiB + * 128GiB 512MiB 64GiB + * 64GiB 256MiB 32GiB + * 32GiB 128MiB 16GiB + * 16GiB 64MiB 8GiB + * + * These are the range our thresholds will fall in, corresponding to how + * much delalloc we need for the background flusher to kick in. */ + thresh = calc_available_free_space(fs_info, space_info, BTRFS_RESERVE_FLUSH_ALL); thresh += (space_info->total_bytes - space_info->bytes_used - space_info->bytes_reserved - space_info->bytes_readonly); - thresh >>= 1; + thresh >>= space_info->clamp; used = space_info->bytes_pinned; @@ -1045,6 +1061,7 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) struct btrfs_block_rsv *delayed_refs_rsv; struct btrfs_block_rsv *global_rsv; struct btrfs_block_rsv *trans_rsv; + int loops = 0; fs_info = container_of(work, struct btrfs_fs_info, preempt_reclaim_work); @@ -1061,6 +1078,8 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) u64 to_reclaim, block_rsv_size; u64 global_rsv_size = global_rsv->reserved; + loops++; + /* * We don't have a precise counter for the metadata being * reserved for delalloc, so we'll approximate it by subtracting @@ -1117,6 +1136,10 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) cond_resched(); spin_lock(&space_info->lock); } + + /* We only went through once, back off our clamping. */ + if (loops == 1 && !space_info->reclaim_size) + space_info->clamp = max(1, space_info->clamp - 1); spin_unlock(&space_info->lock); } @@ -1433,6 +1456,24 @@ static inline bool is_normal_flushing(enum btrfs_reserve_flush_enum flush) (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL); } +static inline void maybe_clamp_preempt(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info) +{ + u64 ordered = percpu_counter_sum_positive(&fs_info->ordered_bytes); + u64 delalloc = percpu_counter_sum_positive(&fs_info->delalloc_bytes); + + /* + * If we're heavy on ordered operations then clamping won't help us. We + * need to clamp specifically to keep up with dirty'ing buffered + * writers, because there's not a 1:1 correlation of writing delalloc + * and freeing space, like there is with flushing delayed refs or + * delayed nodes. If we're already more ordered than delalloc then + * we're keeping up, otherwise we aren't and should probably clamp. + */ + if (ordered < delalloc) + space_info->clamp = min(space_info->clamp + 1, 8); +} + /** * Try to reserve bytes from the block_rsv's space * @@ -1526,6 +1567,14 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, list_add_tail(&ticket.list, &space_info->priority_tickets); } + + /* + * We were forced to add a reserve ticket, so our preemptive + * flushing is unable to keep up. Clamp down on the threshold + * for the preemptive flushing in order to keep up with the + * workload. + */ + maybe_clamp_preempt(fs_info, space_info); } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { used += orig_bytes; /* diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index 74706f604bce..e237156ce888 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -22,6 +22,10 @@ struct btrfs_space_info { the space info if we had an ENOSPC in the allocator. */ + int clamp; /* Used to scale our threshold for preemptive + flushing. The value is >> clamp, so turns + out to be a 2^clamp divisor. */ + unsigned int full:1; /* indicates that we cannot allocate any more chunks for this space */ unsigned int chunk_alloc:1; /* set if we are allocating a chunk */ -- cgit v1.2.3 From 4b02b00fe5f1377f3dbfb168dfcfebf3d7a9632f Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:28 -0400 Subject: btrfs: adjust the flush trace point to include the source Since we have normal ticketed flushing and preemptive flushing, adjust the tracepoint so that we know the source of the flushing action to make it easier to debug problems. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 17 +++++++++-------- include/trace/events/btrfs.h | 10 ++++++---- 2 files changed, 15 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index e8acf087dcee..bb4f85c07738 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -673,7 +673,7 @@ enospc: */ static void flush_space(struct btrfs_fs_info *fs_info, struct btrfs_space_info *space_info, u64 num_bytes, - enum btrfs_flush_state state) + enum btrfs_flush_state state, bool for_preempt) { struct btrfs_root *root = fs_info->extent_root; struct btrfs_trans_handle *trans; @@ -756,7 +756,7 @@ static void flush_space(struct btrfs_fs_info *fs_info, } trace_btrfs_flush_space(fs_info, space_info->flags, num_bytes, state, - ret); + ret, for_preempt); return; } @@ -997,7 +997,7 @@ static void btrfs_async_reclaim_metadata_space(struct work_struct *work) flush_state = FLUSH_DELAYED_ITEMS_NR; do { - flush_space(fs_info, space_info, to_reclaim, flush_state); + flush_space(fs_info, space_info, to_reclaim, flush_state, false); spin_lock(&space_info->lock); if (list_empty(&space_info->tickets)) { space_info->flush = 0; @@ -1132,7 +1132,7 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) to_reclaim >>= 2; if (!to_reclaim) to_reclaim = btrfs_calc_insert_metadata_size(fs_info, 1); - flush_space(fs_info, space_info, to_reclaim, flush); + flush_space(fs_info, space_info, to_reclaim, flush, true); cond_resched(); spin_lock(&space_info->lock); } @@ -1223,7 +1223,7 @@ static void btrfs_async_reclaim_data_space(struct work_struct *work) spin_unlock(&space_info->lock); while (!space_info->full) { - flush_space(fs_info, space_info, U64_MAX, ALLOC_CHUNK_FORCE); + flush_space(fs_info, space_info, U64_MAX, ALLOC_CHUNK_FORCE, false); spin_lock(&space_info->lock); if (list_empty(&space_info->tickets)) { space_info->flush = 0; @@ -1236,7 +1236,7 @@ static void btrfs_async_reclaim_data_space(struct work_struct *work) while (flush_state < ARRAY_SIZE(data_flush_states)) { flush_space(fs_info, space_info, U64_MAX, - data_flush_states[flush_state]); + data_flush_states[flush_state], false); spin_lock(&space_info->lock); if (list_empty(&space_info->tickets)) { space_info->flush = 0; @@ -1309,7 +1309,8 @@ static void priority_reclaim_metadata_space(struct btrfs_fs_info *fs_info, flush_state = 0; do { - flush_space(fs_info, space_info, to_reclaim, states[flush_state]); + flush_space(fs_info, space_info, to_reclaim, states[flush_state], + false); flush_state++; spin_lock(&space_info->lock); if (ticket->bytes == 0) { @@ -1325,7 +1326,7 @@ static void priority_reclaim_data_space(struct btrfs_fs_info *fs_info, struct reserve_ticket *ticket) { while (!space_info->full) { - flush_space(fs_info, space_info, U64_MAX, ALLOC_CHUNK_FORCE); + flush_space(fs_info, space_info, U64_MAX, ALLOC_CHUNK_FORCE, false); spin_lock(&space_info->lock); if (ticket->bytes == 0) { spin_unlock(&space_info->lock); diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index 8a7c163907a2..807921de6b32 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -1113,15 +1113,16 @@ TRACE_EVENT(btrfs_trigger_flush, TRACE_EVENT(btrfs_flush_space, TP_PROTO(const struct btrfs_fs_info *fs_info, u64 flags, u64 num_bytes, - int state, int ret), + int state, int ret, bool for_preempt), - TP_ARGS(fs_info, flags, num_bytes, state, ret), + TP_ARGS(fs_info, flags, num_bytes, state, ret, for_preempt), TP_STRUCT__entry_btrfs( __field( u64, flags ) __field( u64, num_bytes ) __field( int, state ) __field( int, ret ) + __field( bool, for_preempt ) ), TP_fast_assign_btrfs(fs_info, @@ -1129,15 +1130,16 @@ TRACE_EVENT(btrfs_flush_space, __entry->num_bytes = num_bytes; __entry->state = state; __entry->ret = ret; + __entry->for_preempt = for_preempt; ), - TP_printk_btrfs("state=%d(%s) flags=%llu(%s) num_bytes=%llu ret=%d", + TP_printk_btrfs("state=%d(%s) flags=%llu(%s) num_bytes=%llu ret=%d for_preempt=%d", __entry->state, __print_symbolic(__entry->state, FLUSH_STATES), __entry->flags, __print_flags((unsigned long)__entry->flags, "|", BTRFS_GROUP_FLAGS), - __entry->num_bytes, __entry->ret) + __entry->num_bytes, __entry->ret, __entry->for_preempt) ); DECLARE_EVENT_CLASS(btrfs__reserved_extent, -- cgit v1.2.3 From e5ad49e215a07562f0a765c68161d13d7c23d8d1 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 9 Oct 2020 09:28:29 -0400 Subject: btrfs: add a trace class for dumping the current ENOSPC state Often when I'm debugging ENOSPC related issues I have to resort to printing the entire ENOSPC state with trace_printk() in different spots. This gets pretty annoying, so add a trace state that does this for us. Then add a trace point at the end of preemptive flushing so you can see the state of the space_info when we decide to exit preemptive flushing. This helped me figure out we weren't kicking in the preemptive flushing soon enough. Reviewed-by: Nikolay Borisov Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 1 + include/trace/events/btrfs.h | 62 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index bb4f85c07738..bccd98141a6e 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -1140,6 +1140,7 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) /* We only went through once, back off our clamping. */ if (loops == 1 && !space_info->reclaim_size) space_info->clamp = max(1, space_info->clamp - 1); + trace_btrfs_done_preemptive_reclaim(fs_info, space_info); spin_unlock(&space_info->lock); } diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index 807921de6b32..0551ea65374f 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -2029,6 +2029,68 @@ TRACE_EVENT(btrfs_convert_extent_bit, __print_flags(__entry->clear_bits, "|", EXTENT_FLAGS)) ); +DECLARE_EVENT_CLASS(btrfs_dump_space_info, + TP_PROTO(const struct btrfs_fs_info *fs_info, + const struct btrfs_space_info *sinfo), + + TP_ARGS(fs_info, sinfo), + + TP_STRUCT__entry_btrfs( + __field( u64, flags ) + __field( u64, total_bytes ) + __field( u64, bytes_used ) + __field( u64, bytes_pinned ) + __field( u64, bytes_reserved ) + __field( u64, bytes_may_use ) + __field( u64, bytes_readonly ) + __field( u64, reclaim_size ) + __field( int, clamp ) + __field( u64, global_reserved ) + __field( u64, trans_reserved ) + __field( u64, delayed_refs_reserved ) + __field( u64, delayed_reserved ) + __field( u64, free_chunk_space ) + ), + + TP_fast_assign_btrfs(fs_info, + __entry->flags = sinfo->flags; + __entry->total_bytes = sinfo->total_bytes; + __entry->bytes_used = sinfo->bytes_used; + __entry->bytes_pinned = sinfo->bytes_pinned; + __entry->bytes_reserved = sinfo->bytes_reserved; + __entry->bytes_may_use = sinfo->bytes_may_use; + __entry->bytes_readonly = sinfo->bytes_readonly; + __entry->reclaim_size = sinfo->reclaim_size; + __entry->clamp = sinfo->clamp; + __entry->global_reserved = fs_info->global_block_rsv.reserved; + __entry->trans_reserved = fs_info->trans_block_rsv.reserved; + __entry->delayed_refs_reserved = fs_info->delayed_refs_rsv.reserved; + __entry->delayed_reserved = fs_info->delayed_block_rsv.reserved; + __entry->free_chunk_space = atomic64_read(&fs_info->free_chunk_space); + ), + + TP_printk_btrfs("flags=%s total_bytes=%llu bytes_used=%llu " + "bytes_pinned=%llu bytes_reserved=%llu " + "bytes_may_use=%llu bytes_readonly=%llu " + "reclaim_size=%llu clamp=%d global_reserved=%llu " + "trans_reserved=%llu delayed_refs_reserved=%llu " + "delayed_reserved=%llu chunk_free_space=%llu", + __print_flags(__entry->flags, "|", BTRFS_GROUP_FLAGS), + __entry->total_bytes, __entry->bytes_used, + __entry->bytes_pinned, __entry->bytes_reserved, + __entry->bytes_may_use, __entry->bytes_readonly, + __entry->reclaim_size, __entry->clamp, + __entry->global_reserved, __entry->trans_reserved, + __entry->delayed_refs_reserved, + __entry->delayed_reserved, __entry->free_chunk_space) +); + +DEFINE_EVENT(btrfs_dump_space_info, btrfs_done_preemptive_reclaim, + TP_PROTO(const struct btrfs_fs_info *fs_info, + const struct btrfs_space_info *sinfo), + TP_ARGS(fs_info, sinfo) +); + TRACE_EVENT(btrfs_reserve_ticket, TP_PROTO(const struct btrfs_fs_info *fs_info, u64 flags, u64 bytes, u64 start_ns, int flush, int error), -- cgit v1.2.3 From 2965194b7700f9405860557826520fd6e8e8b9ad Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 Jan 2021 15:05:41 +0000 Subject: btrfs: remove wrong comment for can_nocow_extent() The comment for can_nocow_extent() says that the function will flush ordered extents, however that never happens and was never true before the comment was added in commit e4ecaf90bc13 ("btrfs: add comments for btrfs_check_can_nocow() and can_nocow_extent()"). This is true only for the function btrfs_check_can_nocow(), which after that commit was renamed to check_can_nocow(). So just remove that part of the comment. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 0dbe1aaa0b71..589030cefd90 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7105,9 +7105,6 @@ static struct extent_map *btrfs_new_extent_direct(struct btrfs_inode *inode, * @strict: if true, omit optimizations that might force us into unnecessary * cow. e.g., don't trust generation number. * - * This function will flush ordered extents in the range to ensure proper - * nocow checks for (nowait == false) case. - * * Return: * >0 and update @len if we can do nocow write * 0 if we can't do nocow write -- cgit v1.2.3 From a4559e6f6f3a4e84cb788ac158fb419ece473527 Mon Sep 17 00:00:00 2001 From: Abaci Team Date: Wed, 27 Jan 2021 16:11:37 +0800 Subject: btrfs: simplify condition in __btrfs_run_delayed_items Fix the following coccicheck warnings: ./fs/btrfs/delayed-inode.c:1157:39-41: WARNING !A || A && B is equivalent to !A || B. Reported-by: Abaci Robot Suggested-by: Jiapeng Zhong Reviewed-by: Josef Bacik Signed-off-by: Abaci Team Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/delayed-inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index 70c0340d839c..ec0b50b8c5d6 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -1154,7 +1154,7 @@ static int __btrfs_run_delayed_items(struct btrfs_trans_handle *trans, int nr) delayed_root = fs_info->delayed_root; curr_node = btrfs_first_delayed_node(delayed_root); - while (curr_node && (!count || (count && nr--))) { + while (curr_node && (!count || nr--)) { ret = __btrfs_commit_inode_delayed_items(trans, path, curr_node); if (ret) { -- cgit v1.2.3 From 951c80f83d61bd4b21794c8aba829c3c1a45c2d0 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Wed, 27 Jan 2021 14:38:48 +0800 Subject: btrfs: fix double accounting of ordered extent for subpage case in btrfs_invalidapge Commit dbfdb6d1b369 ("Btrfs: Search for all ordered extents that could span across a page") make btrfs_invalidapage() to search all ordered extents. The offending code looks like this: again: start = page_start; ordered = btrfs_lookup_ordered_range(inode, start, page_end - start + 1); if (ordred) { end = min(page_end, ordered->file_offset + ordered->num_bytes - 1); /* Do the cleanup */ start = end + 1; if (start < page_end) goto again; } The behavior is indeed necessary for the incoming subpage support, but when it iterates through all the ordered extents, it also resets the search range @start. This means, for the following cases, we can double account the ordered extents, causing its bytes_left underflow: Page offset 0 16K 32K |<--- OE 1 --->|<--- OE 2 ---->| As the first iteration will find ordered extent (OE) 1, which doesn't cover the full page, thus after cleanup code, we need to retry again. But again label will reset start to page_start, and we got OE 1 again, which causes double accounting on OE 1, and cause OE 1's byte_left to underflow. This problem can only happen for subpage case, as for regular sectorsize == PAGE_SIZE case, we will always find a OE ends at or after page end, thus no way to trigger the problem. Move the again label after start = page_start. There will be more comprehensive rework to convert the open coded loop to a proper while loop for subpage support. Fixes: dbfdb6d1b369 ("Btrfs: Search for all ordered extents that could span across a page") Reviewed-by: Filipe Manana Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/inode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 589030cefd90..680cd0eea6d1 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8183,8 +8183,9 @@ static void btrfs_invalidatepage(struct page *page, unsigned int offset, if (!inode_evicting) lock_extent_bits(tree, page_start, page_end, &cached_state); -again: + start = page_start; +again: ordered = btrfs_lookup_ordered_range(inode, start, page_end - start + 1); if (ordered) { found_ordered = true; -- cgit v1.2.3 From 420343131970fd29db129b308612f9364b06df0b Mon Sep 17 00:00:00 2001 From: Michal Rostecki Date: Wed, 27 Jan 2021 14:57:27 +0100 Subject: btrfs: let callers of btrfs_get_io_geometry pass the em Before this change, the btrfs_get_io_geometry() function was calling btrfs_get_chunk_map() to get the extent mapping, necessary for calculating the I/O geometry. It was using that extent mapping only internally and freeing the pointer after its execution. That resulted in calling btrfs_get_chunk_map() de facto twice by the __btrfs_map_block() function. It was calling btrfs_get_io_geometry() first and then calling btrfs_get_chunk_map() directly to get the extent mapping, used by the rest of the function. Change that to passing the extent mapping to the btrfs_get_io_geometry() function as an argument. This could improve performance in some cases. For very large filesystems, i.e. several thousands of allocated chunks, not only this avoids searching two times the rbtree, saving time, it may also help reducing contention on the lock that protects the tree - thinking of writeback starting for multiple inodes, other tasks allocating or removing chunks, and anything else that requires access to the rbtree. Reviewed-by: Filipe Manana Signed-off-by: Michal Rostecki Reviewed-by: David Sterba [ add Filipe's analysis ] Signed-off-by: David Sterba --- fs/btrfs/inode.c | 41 ++++++++++++++++++++++++++++++----------- fs/btrfs/volumes.c | 43 ++++++++++++++++++------------------------- fs/btrfs/volumes.h | 5 +++-- 3 files changed, 51 insertions(+), 38 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 680cd0eea6d1..04cd95899ac8 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2183,9 +2183,10 @@ int btrfs_bio_fits_in_stripe(struct page *page, size_t size, struct bio *bio, struct inode *inode = page->mapping->host; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); u64 logical = bio->bi_iter.bi_sector << 9; + struct extent_map *em; u64 length = 0; u64 map_length; - int ret; + int ret = 0; struct btrfs_io_geometry geom; if (bio_flags & EXTENT_BIO_COMPRESSED) @@ -2193,14 +2194,19 @@ int btrfs_bio_fits_in_stripe(struct page *page, size_t size, struct bio *bio, length = bio->bi_iter.bi_size; map_length = length; - ret = btrfs_get_io_geometry(fs_info, btrfs_op(bio), logical, map_length, - &geom); + em = btrfs_get_chunk_map(fs_info, logical, map_length); + if (IS_ERR(em)) + return PTR_ERR(em); + ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(bio), logical, + map_length, &geom); if (ret < 0) - return ret; + goto out; if (geom.len < length + size) - return 1; - return 0; + ret = 1; +out: + free_extent_map(em); + return ret; } /* @@ -7938,10 +7944,12 @@ static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap, u64 submit_len; int clone_offset = 0; int clone_len; + u64 logical; int ret; blk_status_t status; struct btrfs_io_geometry geom; struct btrfs_dio_data *dio_data = iomap->private; + struct extent_map *em = NULL; dip = btrfs_create_dio_private(dio_bio, inode, file_offset); if (!dip) { @@ -7970,12 +7978,18 @@ static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap, submit_len = dio_bio->bi_iter.bi_size; do { - ret = btrfs_get_io_geometry(fs_info, btrfs_op(dio_bio), - start_sector << 9, submit_len, - &geom); + logical = start_sector << 9; + em = btrfs_get_chunk_map(fs_info, logical, submit_len); + if (IS_ERR(em)) { + status = errno_to_blk_status(PTR_ERR(em)); + em = NULL; + goto out_err_em; + } + ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(dio_bio), + logical, submit_len, &geom); if (ret) { status = errno_to_blk_status(ret); - goto out_err; + goto out_err_em; } ASSERT(geom.len <= INT_MAX); @@ -8020,19 +8034,24 @@ static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap, bio_put(bio); if (submit_len > 0) refcount_dec(&dip->refs); - goto out_err; + goto out_err_em; } dio_data->submitted += clone_len; clone_offset += clone_len; start_sector += clone_len >> 9; file_offset += clone_len; + + free_extent_map(em); } while (submit_len > 0); return BLK_QC_T_NONE; +out_err_em: + free_extent_map(em); out_err: dip->dio_bio->bi_status = status; btrfs_dio_private_put(dip); + return BLK_QC_T_NONE; } diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index a8ec8539cd8d..3948f5b50d11 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -5940,23 +5940,24 @@ static bool need_full_stripe(enum btrfs_map_op op) } /* - * btrfs_get_io_geometry - calculates the geomery of a particular (address, len) - * tuple. This information is used to calculate how big a - * particular bio can get before it straddles a stripe. + * Calculate the geometry of a particular (address, len) tuple. This + * information is used to calculate how big a particular bio can get before it + * straddles a stripe. * - * @fs_info - the filesystem - * @logical - address that we want to figure out the geometry of - * @len - the length of IO we are going to perform, starting at @logical - * @op - type of operation - write or read - * @io_geom - pointer used to return values + * @fs_info: the filesystem + * @em: mapping containing the logical extent + * @op: type of operation - write or read + * @logical: address that we want to figure out the geometry of + * @len: the length of IO we are going to perform, starting at @logical + * @io_geom: pointer used to return values * * Returns < 0 in case a chunk for the given logical address cannot be found, * usually shouldn't happen unless @logical is corrupted, 0 otherwise. */ -int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, - u64 logical, u64 len, struct btrfs_io_geometry *io_geom) +int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *em, + enum btrfs_map_op op, u64 logical, u64 len, + struct btrfs_io_geometry *io_geom) { - struct extent_map *em; struct map_lookup *map; u64 offset; u64 stripe_offset; @@ -5964,14 +5965,9 @@ int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, u64 stripe_len; u64 raid56_full_stripe_start = (u64)-1; int data_stripes; - int ret = 0; ASSERT(op != BTRFS_MAP_DISCARD); - em = btrfs_get_chunk_map(fs_info, logical, len); - if (IS_ERR(em)) - return PTR_ERR(em); - map = em->map_lookup; /* Offset of this logical address in the chunk */ offset = logical - em->start; @@ -5985,8 +5981,7 @@ int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, btrfs_crit(fs_info, "stripe math has gone wrong, stripe_offset=%llu offset=%llu start=%llu logical=%llu stripe_len=%llu", stripe_offset, offset, em->start, logical, stripe_len); - ret = -EINVAL; - goto out; + return -EINVAL; } /* stripe_offset is the offset of this block in its stripe */ @@ -6033,10 +6028,7 @@ int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, io_geom->stripe_offset = stripe_offset; io_geom->raid56_stripe_offset = raid56_full_stripe_start; -out: - /* once for us */ - free_extent_map(em); - return ret; + return 0; } static int __btrfs_map_block(struct btrfs_fs_info *fs_info, @@ -6069,12 +6061,13 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, ASSERT(bbio_ret); ASSERT(op != BTRFS_MAP_DISCARD); - ret = btrfs_get_io_geometry(fs_info, op, logical, *length, &geom); + em = btrfs_get_chunk_map(fs_info, logical, *length); + ASSERT(!IS_ERR(em)); + + ret = btrfs_get_io_geometry(fs_info, em, op, logical, *length, &geom); if (ret < 0) return ret; - em = btrfs_get_chunk_map(fs_info, logical, *length); - ASSERT(!IS_ERR(em)); map = em->map_lookup; *length = geom.len; diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index c43663d9c22e..04e2b26823c2 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -440,8 +440,9 @@ int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, int btrfs_map_sblock(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, u64 logical, u64 *length, struct btrfs_bio **bbio_ret); -int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, - u64 logical, u64 len, struct btrfs_io_geometry *io_geom); +int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *map, + enum btrfs_map_op op, u64 logical, u64 len, + struct btrfs_io_geometry *io_geom); int btrfs_read_sys_array(struct btrfs_fs_info *fs_info); int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info); int btrfs_alloc_chunk(struct btrfs_trans_handle *trans, u64 type); -- cgit v1.2.3 From ddffcf6fb5ac54ffcd7e90b10554d89dbd10b47b Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 Jan 2021 10:34:54 +0000 Subject: btrfs: remove unnecessary directory inode item update when deleting dir entry When we remove a directory entry, as part of an unlink operation, if the directory was logged before we must remove the directory index items from the log. We are also updating the inode item of the directory to update its i_size, but that is not necessary because during log replay we do not need it and we correctly adjust the i_size in the inode item of the subvolume as we process directory index items and replay deletes. This is not needed since commit d555438b6e1dad ("Btrfs: drop dir i_size when adding new names on replay"), where we explicitly ignore the i_size of directory inode items on log replay. Before that we used it but it was buggy as mentioned in that commit's change log (i_size got a larger value then it should have). So stop updating the i_size of the directory inode item in the log, as that is a waste of time, adds more log contention to the log tree and often results in COWing more extent buffers for the log tree. This code path is triggered often during dbench workloads for example. This patch is part of a patchset comprised of the following patches: btrfs: remove unnecessary directory inode item update when deleting dir entry btrfs: stop setting nbytes when filling inode item for logging btrfs: avoid logging new ancestor inodes when logging new inode btrfs: skip logging directories already logged when logging all parents btrfs: skip logging inodes already logged when logging new entries btrfs: remove unnecessary check_parent_dirs_for_sync() btrfs: make concurrent fsyncs wait less when waiting for a transaction commit Performance results, after applying all patches, are mentioned in the change log of the last patch. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 39 ++++----------------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 8ee0700a980f..5d87afc6058a 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3379,7 +3379,6 @@ int btrfs_del_dir_entries_in_log(struct btrfs_trans_handle *trans, struct btrfs_path *path; int ret; int err = 0; - int bytes_del = 0; u64 dir_ino = btrfs_ino(dir); if (!inode_logged(trans, dir)) @@ -3406,7 +3405,6 @@ int btrfs_del_dir_entries_in_log(struct btrfs_trans_handle *trans, } if (di) { ret = btrfs_delete_one_dir_name(trans, log, path, di); - bytes_del += name_len; if (ret) { err = ret; goto fail; @@ -3421,46 +3419,17 @@ int btrfs_del_dir_entries_in_log(struct btrfs_trans_handle *trans, } if (di) { ret = btrfs_delete_one_dir_name(trans, log, path, di); - bytes_del += name_len; if (ret) { err = ret; goto fail; } } - /* update the directory size in the log to reflect the names - * we have removed + /* + * We do not need to update the size field of the directory's inode item + * because on log replay we update the field to reflect all existing + * entries in the directory (see overwrite_item()). */ - if (bytes_del) { - struct btrfs_key key; - - key.objectid = dir_ino; - key.offset = 0; - key.type = BTRFS_INODE_ITEM_KEY; - btrfs_release_path(path); - - ret = btrfs_search_slot(trans, log, &key, path, 0, 1); - if (ret < 0) { - err = ret; - goto fail; - } - if (ret == 0) { - struct btrfs_inode_item *item; - u64 i_size; - - item = btrfs_item_ptr(path->nodes[0], path->slots[0], - struct btrfs_inode_item); - i_size = btrfs_inode_size(path->nodes[0], item); - if (i_size > bytes_del) - i_size -= bytes_del; - else - i_size = 0; - btrfs_set_inode_size(path->nodes[0], item, i_size); - btrfs_mark_buffer_dirty(path->nodes[0]); - } else - ret = 0; - btrfs_release_path(path); - } fail: btrfs_free_path(path); out_unlock: -- cgit v1.2.3 From e593e54ed1f643f5007ab4656188b7c3c9a9cb11 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 Jan 2021 10:34:55 +0000 Subject: btrfs: stop setting nbytes when filling inode item for logging When we fill an inode item for logging we are setting its nbytes field with the value returned by inode_get_bytes() (a VFS API), however we do not need it because it is not used during log replay. In fact, for fast fsyncs, when we call inode_get_bytes() we may even get an outdated value for nbytes because the nbytes field of the inode is only updated when ordered extents complete, and a fast fsync only waits for writeback to complete, it does not wait for ordered extent completion. So just remove the setup of nbytes and add an explicit comment mentioning why we do not set it. This also avoids adding contention on the inode's i_lock (VFS) with concurrent stat() calls, since that spinlock is used by inode_get_bytes() which is also called by our stat callback (btrfs_getattr()). This patch is part of a patchset comprised of the following patches: btrfs: remove unnecessary directory inode item update when deleting dir entry btrfs: stop setting nbytes when filling inode item for logging btrfs: avoid logging new ancestor inodes when logging new inode btrfs: skip logging directories already logged when logging all parents btrfs: skip logging inodes already logged when logging new entries btrfs: remove unnecessary check_parent_dirs_for_sync() btrfs: make concurrent fsyncs wait less when waiting for a transaction commit Performance results, after applying all patches, are mentioned in the change log of the last patch. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 5d87afc6058a..be62759f0aac 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3858,7 +3858,14 @@ static void fill_inode_item(struct btrfs_trans_handle *trans, btrfs_set_token_timespec_nsec(&token, &item->ctime, inode->i_ctime.tv_nsec); - btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode)); + /* + * We do not need to set the nbytes field, in fact during a fast fsync + * its value may not even be correct, since a fast fsync does not wait + * for ordered extent completion, which is where we update nbytes, it + * only waits for writeback to complete. During log replay as we find + * file extent items and replay them, we adjust the nbytes field of the + * inode item in subvolume tree as needed (see overwrite_item()). + */ btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode)); btrfs_set_token_inode_transid(&token, item, trans->transid); -- cgit v1.2.3 From ab12313a9f56b939529abc80ac26bedefb3d5b62 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 Jan 2021 10:34:56 +0000 Subject: btrfs: avoid logging new ancestor inodes when logging new inode When we fsync a new file, created in the current transaction, we check all its ancestor inodes and always log them if they were created in the current transaction - even if we have already logged them before, which is a waste of time. So avoid logging new ancestor inodes if they were already logged before and have no xattrs added/updated/removed since they were last logged. This patch is part of a patchset comprised of the following patches: btrfs: remove unnecessary directory inode item update when deleting dir entry btrfs: stop setting nbytes when filling inode item for logging btrfs: avoid logging new ancestor inodes when logging new inode btrfs: skip logging directories already logged when logging all parents btrfs: skip logging inodes already logged when logging new entries btrfs: remove unnecessary check_parent_dirs_for_sync() btrfs: make concurrent fsyncs wait less when waiting for a transaction commit Performance results, after applying all patches, are mentioned in the change log of the last patch. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index be62759f0aac..105cf316ee27 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -5272,6 +5272,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, if (S_ISDIR(inode->vfs_inode.i_mode)) { int max_key_type = BTRFS_DIR_LOG_INDEX_KEY; + clear_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags); if (inode_only == LOG_INODE_EXISTS) max_key_type = BTRFS_XATTR_ITEM_KEY; ret = drop_objectid_items(trans, log, path, ino, max_key_type); @@ -5520,6 +5521,34 @@ out: return ret; } +/* + * Check if we need to log an inode. This is used in contexts where while + * logging an inode we need to log another inode (either that it exists or in + * full mode). This is used instead of btrfs_inode_in_log() because the later + * requires the inode to be in the log and have the log transaction committed, + * while here we do not care if the log transaction was already committed - our + * caller will commit the log later - and we want to avoid logging an inode + * multiple times when multiple tasks have joined the same log transaction. + */ +static bool need_log_inode(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + /* + * If this inode does not have new/updated/deleted xattrs since the last + * time it was logged and is flagged as logged in the current transaction, + * we can skip logging it. As for new/deleted names, those are updated in + * the log by link/unlink/rename operations. + * In case the inode was logged and then evicted and reloaded, its + * logged_trans will be 0, in which case we have to fully log it since + * logged_trans is a transient field, not persisted. + */ + if (inode->logged_trans == trans->transid && + !test_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags)) + return false; + + return true; +} + struct btrfs_dir_list { u64 ino; struct list_head list; @@ -5848,7 +5877,8 @@ static int log_new_ancestors(struct btrfs_trans_handle *trans, if (IS_ERR(inode)) return PTR_ERR(inode); - if (BTRFS_I(inode)->generation >= trans->transid) + if (BTRFS_I(inode)->generation >= trans->transid && + need_log_inode(trans, BTRFS_I(inode))) ret = btrfs_log_inode(trans, root, BTRFS_I(inode), LOG_INODE_EXISTS, ctx); btrfs_add_delayed_iput(inode); @@ -5902,7 +5932,8 @@ static int log_new_ancestors_fast(struct btrfs_trans_handle *trans, if (root != inode->root) break; - if (inode->generation >= trans->transid) { + if (inode->generation >= trans->transid && + need_log_inode(trans, inode)) { ret = btrfs_log_inode(trans, root, inode, LOG_INODE_EXISTS, ctx); if (ret) -- cgit v1.2.3 From 3e6a86a193b08039a382807c56421622c3ff4368 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 Jan 2021 10:34:57 +0000 Subject: btrfs: skip logging directories already logged when logging all parents Some times when we fsync an inode we need to do a full log of all its ancestors (due to unlink, link or rename operations), which can be an expensive operation, specially if the directories are large. However if we find an ancestor directory inode that is already logged in the current transaction, and has no inserted/updated/deleted xattrs since it was last logged, we can skip logging the directory again. We are safe to skip that since we know that for logged directories, any link, unlink or rename operations that implicate the directory will update the log as necessary. So use the helper need_log_dir(), introduced in a previous commit, to detect already logged directories that can be skipped. This patch is part of a patchset comprised of the following patches: btrfs: remove unnecessary directory inode item update when deleting dir entry btrfs: stop setting nbytes when filling inode item for logging btrfs: avoid logging new ancestor inodes when logging new inode btrfs: skip logging directories already logged when logging all parents btrfs: skip logging inodes already logged when logging new entries btrfs: remove unnecessary check_parent_dirs_for_sync() btrfs: make concurrent fsyncs wait less when waiting for a transaction commit Performance results, after applying all patches, are mentioned in the change log of the last patch. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 105cf316ee27..c0dce99c2c14 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -5826,6 +5826,11 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, goto out; } + if (!need_log_inode(trans, BTRFS_I(dir_inode))) { + btrfs_add_delayed_iput(dir_inode); + continue; + } + if (ctx) ctx->log_new_dentries = false; ret = btrfs_log_inode(trans, root, BTRFS_I(dir_inode), -- cgit v1.2.3 From 0e44cb3f94284d33067fc74e30990a0ed5b3540d Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 Jan 2021 10:34:58 +0000 Subject: btrfs: skip logging inodes already logged when logging new entries When logging new directory entries of a directory, we log the inodes of new dentries and the inodes of dentries pointing to directories that may have been created in past transactions. For the case of directories we log in full mode, which can be particularly expensive for large directories. We do use btrfs_inode_in_log() to skip already logged inodes, however for that helper to return true, it requires that the log transaction used to log the inode to be already committed. This means that when we have more than one task using the same log transaction we can end up logging an inode multiple times, which is a waste of time and not necessary since the log will be committed by one of the tasks and the others will wait for the log transaction to be committed before returning to user space. So simply replace the use of btrfs_inode_in_log() with the new helper function need_log_inode(), introduced in a previous commit. This patch is part of a patchset comprised of the following patches: btrfs: remove unnecessary directory inode item update when deleting dir entry btrfs: stop setting nbytes when filling inode item for logging btrfs: avoid logging new ancestor inodes when logging new inode btrfs: skip logging directories already logged when logging all parents btrfs: skip logging inodes already logged when logging new entries btrfs: remove unnecessary check_parent_dirs_for_sync() btrfs: make concurrent fsyncs wait less when waiting for a transaction commit Performance results, after applying all patches, are mentioned in the change log of the last patch. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index c0dce99c2c14..6dc376a16cf2 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -5676,7 +5676,7 @@ process_leaf: goto next_dir_inode; } - if (btrfs_inode_in_log(BTRFS_I(di_inode), trans->transid)) { + if (!need_log_inode(trans, BTRFS_I(di_inode))) { btrfs_add_delayed_iput(di_inode); break; } -- cgit v1.2.3 From 64d6b281ba4db044c946158387c74e1149b9487e Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 Jan 2021 10:34:59 +0000 Subject: btrfs: remove unnecessary check_parent_dirs_for_sync() Whenever we fsync an inode, if it is a directory, a regular file that was created in the current transaction or has last_unlink_trans set to the generation of the current transaction, we check if any of its ancestor inodes (and the inode itself if it is a directory) can not be logged and need a fallback to a full transaction commit - if so, we return with a value of 1 in order to fallback to a transaction commit. However we often do not need to fallback to a transaction commit because: 1) The ancestor inode is not an immediate parent, and therefore there is not an explicit request to log it and it is not needed neither to guarantee the consistency of the inode originally asked to be logged (fsynced) nor its immediate parent; 2) The ancestor inode was already logged before, in which case any link, unlink or rename operation updates the log as needed. So for these two cases we can avoid an unnecessary transaction commit. Therefore remove check_parent_dirs_for_sync() and add a check at the top of btrfs_log_inode() to make us fallback immediately to a transaction commit when we are logging a directory inode that can not be logged and needs a full transaction commit. All we need to protect is the case where after renaming a file someone fsyncs only the old directory, which would result is losing the renamed file after a log replay. This patch is part of a patchset comprised of the following patches: btrfs: remove unnecessary directory inode item update when deleting dir entry btrfs: stop setting nbytes when filling inode item for logging btrfs: avoid logging new ancestor inodes when logging new inode btrfs: skip logging directories already logged when logging all parents btrfs: skip logging inodes already logged when logging new entries btrfs: remove unnecessary check_parent_dirs_for_sync() btrfs: make concurrent fsyncs wait less when waiting for a transaction commit Performance results, after applying all patches, are mentioned in the change log of the last patch. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 121 +++++++--------------------------------------------- 1 file changed, 15 insertions(+), 106 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 6dc376a16cf2..4c7b283ed2b2 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -5265,6 +5265,21 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, mutex_lock(&inode->log_mutex); } + /* + * This is for cases where logging a directory could result in losing a + * a file after replaying the log. For example, if we move a file from a + * directory A to a directory B, then fsync directory A, we have no way + * to known the file was moved from A to B, so logging just A would + * result in losing the file after a log replay. + */ + if (S_ISDIR(inode->vfs_inode.i_mode) && + inode_only == LOG_INODE_ALL && + inode->last_unlink_trans >= trans->transid) { + btrfs_set_log_full_commit(trans); + err = 1; + goto out_unlock; + } + /* * a brute force approach to making sure we get the most uptodate * copies of everything. @@ -5428,99 +5443,6 @@ out_unlock: return err; } -/* - * Check if we must fallback to a transaction commit when logging an inode. - * This must be called after logging the inode and is used only in the context - * when fsyncing an inode requires the need to log some other inode - in which - * case we can't lock the i_mutex of each other inode we need to log as that - * can lead to deadlocks with concurrent fsync against other inodes (as we can - * log inodes up or down in the hierarchy) or rename operations for example. So - * we take the log_mutex of the inode after we have logged it and then check for - * its last_unlink_trans value - this is safe because any task setting - * last_unlink_trans must take the log_mutex and it must do this before it does - * the actual unlink operation, so if we do this check before a concurrent task - * sets last_unlink_trans it means we've logged a consistent version/state of - * all the inode items, otherwise we are not sure and must do a transaction - * commit (the concurrent task might have only updated last_unlink_trans before - * we logged the inode or it might have also done the unlink). - */ -static bool btrfs_must_commit_transaction(struct btrfs_trans_handle *trans, - struct btrfs_inode *inode) -{ - bool ret = false; - - mutex_lock(&inode->log_mutex); - if (inode->last_unlink_trans >= trans->transid) { - /* - * Make sure any commits to the log are forced to be full - * commits. - */ - btrfs_set_log_full_commit(trans); - ret = true; - } - mutex_unlock(&inode->log_mutex); - - return ret; -} - -/* - * follow the dentry parent pointers up the chain and see if any - * of the directories in it require a full commit before they can - * be logged. Returns zero if nothing special needs to be done or 1 if - * a full commit is required. - */ -static noinline int check_parent_dirs_for_sync(struct btrfs_trans_handle *trans, - struct btrfs_inode *inode, - struct dentry *parent, - struct super_block *sb) -{ - int ret = 0; - struct dentry *old_parent = NULL; - - /* - * for regular files, if its inode is already on disk, we don't - * have to worry about the parents at all. This is because - * we can use the last_unlink_trans field to record renames - * and other fun in this file. - */ - if (S_ISREG(inode->vfs_inode.i_mode) && - inode->generation < trans->transid && - inode->last_unlink_trans < trans->transid) - goto out; - - if (!S_ISDIR(inode->vfs_inode.i_mode)) { - if (!parent || d_really_is_negative(parent) || sb != parent->d_sb) - goto out; - inode = BTRFS_I(d_inode(parent)); - } - - while (1) { - if (btrfs_must_commit_transaction(trans, inode)) { - ret = 1; - break; - } - - if (!parent || d_really_is_negative(parent) || sb != parent->d_sb) - break; - - if (IS_ROOT(parent)) { - inode = BTRFS_I(d_inode(parent)); - if (btrfs_must_commit_transaction(trans, inode)) - ret = 1; - break; - } - - parent = dget_parent(parent); - dput(old_parent); - old_parent = parent; - inode = BTRFS_I(d_inode(parent)); - - } - dput(old_parent); -out: - return ret; -} - /* * Check if we need to log an inode. This is used in contexts where while * logging an inode we need to log another inode (either that it exists or in @@ -5686,9 +5608,6 @@ process_leaf: log_mode = LOG_INODE_ALL; ret = btrfs_log_inode(trans, root, BTRFS_I(di_inode), log_mode, ctx); - if (!ret && - btrfs_must_commit_transaction(trans, BTRFS_I(di_inode))) - ret = 1; btrfs_add_delayed_iput(di_inode); if (ret) goto next_dir_inode; @@ -5835,9 +5754,6 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, ctx->log_new_dentries = false; ret = btrfs_log_inode(trans, root, BTRFS_I(dir_inode), LOG_INODE_ALL, ctx); - if (!ret && - btrfs_must_commit_transaction(trans, BTRFS_I(dir_inode))) - ret = 1; if (!ret && ctx && ctx->log_new_dentries) ret = log_new_dir_dentries(trans, root, BTRFS_I(dir_inode), ctx); @@ -6053,12 +5969,9 @@ static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, { struct btrfs_root *root = inode->root; struct btrfs_fs_info *fs_info = root->fs_info; - struct super_block *sb; int ret = 0; bool log_dentries = false; - sb = inode->vfs_inode.i_sb; - if (btrfs_test_opt(fs_info, NOTREELOG)) { ret = 1; goto end_no_trans; @@ -6069,10 +5982,6 @@ static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, goto end_no_trans; } - ret = check_parent_dirs_for_sync(trans, inode, parent, sb); - if (ret) - goto end_no_trans; - /* * Skip already logged inodes or inodes corresponding to tmpfiles * (since logging them is pointless, a link count of 0 means they -- cgit v1.2.3 From d0c2f4fa555e70324ec2a129b822ab58f172cc62 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 Jan 2021 10:35:00 +0000 Subject: btrfs: make concurrent fsyncs wait less when waiting for a transaction commit Often an fsync needs to fallback to a transaction commit for several reasons (to ensure consistency after a power failure, a new block group was allocated or a temporary error such as ENOMEM or ENOSPC happened). In that case the log is marked as needing a full commit and any concurrent tasks attempting to log inodes or commit the log will also fallback to the transaction commit. When this happens they all wait for the task that first started the transaction commit to finish the transaction commit - however they wait until the full transaction commit happens, which is not needed, as they only need to wait for the superblocks to be persisted and not for unpinning all the extents pinned during the transaction's lifetime, which even for short lived transactions can be a few thousand and take some significant amount of time to complete - for dbench workloads I have observed up to 4~5 milliseconds of time spent unpinning extents in the worst cases, and the number of pinned extents was between 2 to 3 thousand. So allow fsync tasks to skip waiting for the unpinning of extents when they call btrfs_commit_transaction() and they were not the task that started the transaction commit (that one has to do it, the alternative would be to offload the transaction commit to another task so that it could avoid waiting for the extent unpinning or offload the extent unpinning to another task). This patch is part of a patchset comprised of the following patches: btrfs: remove unnecessary directory inode item update when deleting dir entry btrfs: stop setting nbytes when filling inode item for logging btrfs: avoid logging new ancestor inodes when logging new inode btrfs: skip logging directories already logged when logging all parents btrfs: skip logging inodes already logged when logging new entries btrfs: remove unnecessary check_parent_dirs_for_sync() btrfs: make concurrent fsyncs wait less when waiting for a transaction commit After applying the entire patchset, dbench shows improvements in respect to throughput and latency. The script used to measure it is the following: $ cat dbench-test.sh #!/bin/bash DEV=/dev/sdk MNT=/mnt/sdk MOUNT_OPTIONS="-o ssd" MKFS_OPTIONS="-m single -d single" echo "performance" | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor umount $DEV &> /dev/null mkfs.btrfs -f $MKFS_OPTIONS $DEV mount $MOUNT_OPTIONS $DEV $MNT dbench -D $MNT -t 300 64 umount $MNT The test was run on a physical machine with 12 cores (Intel corei7), 64G of ram, using a NVMe device and a non-debug kernel configuration (Debian's default configuration). Before applying patchset, 32 clients: Operation Count AvgLat MaxLat ---------------------------------------- NTCreateX 9627107 0.153 61.938 Close 7072076 0.001 3.175 Rename 407633 1.222 44.439 Unlink 1943895 0.658 44.440 Deltree 256 17.339 110.891 Mkdir 128 0.003 0.009 Qpathinfo 8725406 0.064 17.850 Qfileinfo 1529516 0.001 2.188 Qfsinfo 1599884 0.002 1.457 Sfileinfo 784200 0.005 3.562 Find 3373513 0.411 30.312 WriteX 4802132 0.053 29.054 ReadX 15089959 0.002 5.801 LockX 31344 0.002 0.425 UnlockX 31344 0.001 0.173 Flush 674724 5.952 341.830 Throughput 1008.02 MB/sec 32 clients 32 procs max_latency=341.833 ms After applying patchset, 32 clients: After patchset, with 32 clients: Operation Count AvgLat MaxLat ---------------------------------------- NTCreateX 9931568 0.111 25.597 Close 7295730 0.001 2.171 Rename 420549 0.982 49.714 Unlink 2005366 0.497 39.015 Deltree 256 11.149 89.242 Mkdir 128 0.002 0.014 Qpathinfo 9001863 0.049 20.761 Qfileinfo 1577730 0.001 2.546 Qfsinfo 1650508 0.002 3.531 Sfileinfo 809031 0.005 5.846 Find 3480259 0.309 23.977 WriteX 4952505 0.043 41.283 ReadX 15568127 0.002 5.476 LockX 32338 0.002 0.978 UnlockX 32338 0.001 2.032 Flush 696017 7.485 228.835 Throughput 1049.91 MB/sec 32 clients 32 procs max_latency=228.847 ms --> +4.1% throughput, -39.6% max latency Before applying patchset, 64 clients: Operation Count AvgLat MaxLat ---------------------------------------- NTCreateX 8956748 0.342 108.312 Close 6579660 0.001 3.823 Rename 379209 2.396 81.897 Unlink 1808625 1.108 131.148 Deltree 256 25.632 172.176 Mkdir 128 0.003 0.018 Qpathinfo 8117615 0.131 55.916 Qfileinfo 1423495 0.001 2.635 Qfsinfo 1488496 0.002 5.412 Sfileinfo 729472 0.007 8.643 Find 3138598 0.855 78.321 WriteX 4470783 0.102 79.442 ReadX 14038139 0.002 7.578 LockX 29158 0.002 0.844 UnlockX 29158 0.001 0.567 Flush 627746 14.168 506.151 Throughput 924.738 MB/sec 64 clients 64 procs max_latency=506.154 ms After applying patchset, 64 clients: Operation Count AvgLat MaxLat ---------------------------------------- NTCreateX 9069003 0.303 43.193 Close 6662328 0.001 3.888 Rename 383976 2.194 46.418 Unlink 1831080 1.022 43.873 Deltree 256 24.037 155.763 Mkdir 128 0.002 0.005 Qpathinfo 8219173 0.137 30.233 Qfileinfo 1441203 0.001 3.204 Qfsinfo 1507092 0.002 4.055 Sfileinfo 738775 0.006 5.431 Find 3177874 0.936 38.170 WriteX 4526152 0.084 39.518 ReadX 14213562 0.002 24.760 LockX 29522 0.002 1.221 UnlockX 29522 0.001 0.694 Flush 635652 14.358 422.039 Throughput 990.13 MB/sec 64 clients 64 procs max_latency=422.043 ms --> +6.8% throughput, -18.1% max latency Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/file.c | 1 + fs/btrfs/transaction.c | 39 +++++++++++++++++++++++++++++++-------- fs/btrfs/transaction.h | 2 ++ 3 files changed, 34 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index d81ae1f518f2..be5350f5bedf 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -2238,6 +2238,7 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) ret = PTR_ERR(trans); goto out_release_extents; } + trans->in_fsync = true; ret = btrfs_log_dentry_safe(trans, dentry, &ctx); btrfs_release_log_ctx_extents(&ctx); diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index b83e8ae38cfc..00c0680dac3a 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -107,6 +107,11 @@ static const unsigned int btrfs_blocked_trans_types[TRANS_STATE_MAX] = { __TRANS_JOIN | __TRANS_JOIN_NOLOCK | __TRANS_JOIN_NOSTART), + [TRANS_STATE_SUPER_COMMITTED] = (__TRANS_START | + __TRANS_ATTACH | + __TRANS_JOIN | + __TRANS_JOIN_NOLOCK | + __TRANS_JOIN_NOSTART), [TRANS_STATE_COMPLETED] = (__TRANS_START | __TRANS_ATTACH | __TRANS_JOIN | @@ -826,10 +831,11 @@ btrfs_attach_transaction_barrier(struct btrfs_root *root) return trans; } -/* wait for a transaction commit to be fully complete */ -static noinline void wait_for_commit(struct btrfs_transaction *commit) +/* Wait for a transaction commit to reach at least the given state. */ +static noinline void wait_for_commit(struct btrfs_transaction *commit, + const enum btrfs_trans_state min_state) { - wait_event(commit->commit_wait, commit->state == TRANS_STATE_COMPLETED); + wait_event(commit->commit_wait, commit->state >= min_state); } int btrfs_wait_for_commit(struct btrfs_fs_info *fs_info, u64 transid) @@ -884,7 +890,7 @@ int btrfs_wait_for_commit(struct btrfs_fs_info *fs_info, u64 transid) goto out; /* nothing committing|committed */ } - wait_for_commit(cur_trans); + wait_for_commit(cur_trans, TRANS_STATE_COMPLETED); btrfs_put_transaction(cur_trans); out: return ret; @@ -2100,11 +2106,15 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) spin_lock(&fs_info->trans_lock); if (cur_trans->state >= TRANS_STATE_COMMIT_START) { + enum btrfs_trans_state want_state = TRANS_STATE_COMPLETED; + spin_unlock(&fs_info->trans_lock); refcount_inc(&cur_trans->use_count); - ret = btrfs_end_transaction(trans); - wait_for_commit(cur_trans); + if (trans->in_fsync) + want_state = TRANS_STATE_SUPER_COMMITTED; + ret = btrfs_end_transaction(trans); + wait_for_commit(cur_trans, want_state); if (TRANS_ABORTED(cur_trans)) ret = cur_trans->aborted; @@ -2118,13 +2128,19 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) wake_up(&fs_info->transaction_blocked_wait); if (cur_trans->list.prev != &fs_info->trans_list) { + enum btrfs_trans_state want_state = TRANS_STATE_COMPLETED; + + if (trans->in_fsync) + want_state = TRANS_STATE_SUPER_COMMITTED; + prev_trans = list_entry(cur_trans->list.prev, struct btrfs_transaction, list); - if (prev_trans->state != TRANS_STATE_COMPLETED) { + if (prev_trans->state < want_state) { refcount_inc(&prev_trans->use_count); spin_unlock(&fs_info->trans_lock); - wait_for_commit(prev_trans); + wait_for_commit(prev_trans, want_state); + ret = READ_ONCE(prev_trans->aborted); btrfs_put_transaction(prev_trans); @@ -2343,6 +2359,13 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) if (ret) goto scrub_continue; + /* + * We needn't acquire the lock here because there is no other task + * which can change it. + */ + cur_trans->state = TRANS_STATE_SUPER_COMMITTED; + wake_up(&cur_trans->commit_wait); + btrfs_finish_extent_commit(trans); if (test_bit(BTRFS_TRANS_HAVE_FREE_BGS, &cur_trans->flags)) diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 31ca81bad822..935bd6958a8a 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -16,6 +16,7 @@ enum btrfs_trans_state { TRANS_STATE_COMMIT_START, TRANS_STATE_COMMIT_DOING, TRANS_STATE_UNBLOCKED, + TRANS_STATE_SUPER_COMMITTED, TRANS_STATE_COMPLETED, TRANS_STATE_MAX, }; @@ -133,6 +134,7 @@ struct btrfs_trans_handle { bool can_flush_pending_bgs; bool reloc_reserved; bool dirty; + bool in_fsync; struct btrfs_root *root; struct btrfs_fs_info *fs_info; struct list_head new_bgs; -- cgit v1.2.3 From 6869b0a8be775e920be54ee9b69a743ca20d8332 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:45 +0800 Subject: btrfs: merge PAGE_CLEAR_DIRTY and PAGE_SET_WRITEBACK to PAGE_START_WRITEBACK PAGE_CLEAR_DIRTY and PAGE_SET_WRITEBACK are two defines used in __process_pages_contig(), to let the function know to clear page dirty bit and then set page writeback. However page writeback and dirty bits are conflicting (at least for sector size == PAGE_SIZE case), this means these two have to be always updated together. This means we can merge PAGE_CLEAR_DIRTY and PAGE_SET_WRITEBACK to PAGE_START_WRITEBACK. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 4 ++-- fs/btrfs/extent_io.h | 12 ++++++------ fs/btrfs/inode.c | 28 ++++++++++------------------ 3 files changed, 18 insertions(+), 26 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index edcdbd739a1e..7c14ccf76838 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -1976,10 +1976,10 @@ static int __process_pages_contig(struct address_space *mapping, pages_processed++; continue; } - if (page_ops & PAGE_CLEAR_DIRTY) + if (page_ops & PAGE_START_WRITEBACK) { clear_page_dirty_for_io(pages[i]); - if (page_ops & PAGE_SET_WRITEBACK) set_page_writeback(pages[i]); + } if (page_ops & PAGE_SET_ERROR) SetPageError(pages[i]); if (page_ops & PAGE_END_WRITEBACK) diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 19221095c635..2d8187c84812 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -35,12 +35,12 @@ enum { /* these are flags for __process_pages_contig */ #define PAGE_UNLOCK (1 << 0) -#define PAGE_CLEAR_DIRTY (1 << 1) -#define PAGE_SET_WRITEBACK (1 << 2) -#define PAGE_END_WRITEBACK (1 << 3) -#define PAGE_SET_PRIVATE2 (1 << 4) -#define PAGE_SET_ERROR (1 << 5) -#define PAGE_LOCK (1 << 6) +/* Page starts writeback, clear dirty bit and set writeback bit */ +#define PAGE_START_WRITEBACK (1 << 1) +#define PAGE_END_WRITEBACK (1 << 2) +#define PAGE_SET_PRIVATE2 (1 << 3) +#define PAGE_SET_ERROR (1 << 4) +#define PAGE_LOCK (1 << 5) /* * page->private values. Every page that is controlled by the extent diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 04cd95899ac8..3337c8ee7928 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -692,8 +692,7 @@ cont: NULL, clear_flags, PAGE_UNLOCK | - PAGE_CLEAR_DIRTY | - PAGE_SET_WRITEBACK | + PAGE_START_WRITEBACK | page_error_op | PAGE_END_WRITEBACK); @@ -933,8 +932,7 @@ retry: async_extent->start + async_extent->ram_size - 1, NULL, EXTENT_LOCKED | EXTENT_DELALLOC, - PAGE_UNLOCK | PAGE_CLEAR_DIRTY | - PAGE_SET_WRITEBACK); + PAGE_UNLOCK | PAGE_START_WRITEBACK); if (btrfs_submit_compressed_write(inode, async_extent->start, async_extent->ram_size, ins.objectid, @@ -970,9 +968,8 @@ out_free: NULL, EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, - PAGE_UNLOCK | PAGE_CLEAR_DIRTY | - PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK | - PAGE_SET_ERROR); + PAGE_UNLOCK | PAGE_START_WRITEBACK | + PAGE_END_WRITEBACK | PAGE_SET_ERROR); free_async_extent_pages(async_extent); kfree(async_extent); goto again; @@ -1070,8 +1067,7 @@ static noinline int cow_file_range(struct btrfs_inode *inode, EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | - PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | - PAGE_END_WRITEBACK); + PAGE_START_WRITEBACK | PAGE_END_WRITEBACK); *nr_written = *nr_written + (end - start + PAGE_SIZE) / PAGE_SIZE; *page_started = 1; @@ -1194,8 +1190,7 @@ out_reserve: out_unlock: clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV; - page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK | - PAGE_END_WRITEBACK; + page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK; /* * If we reserved an extent for our delalloc range (or a subrange) and * failed to create the respective ordered extent, then it means that @@ -1320,9 +1315,8 @@ static int cow_file_range_async(struct btrfs_inode *inode, unsigned clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING; - unsigned long page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | - PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK | - PAGE_SET_ERROR; + unsigned long page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | + PAGE_END_WRITEBACK | PAGE_SET_ERROR; extent_clear_unlock_delalloc(inode, start, end, locked_page, clear_bits, page_ops); @@ -1519,8 +1513,7 @@ static noinline int run_delalloc_nocow(struct btrfs_inode *inode, EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, PAGE_UNLOCK | - PAGE_CLEAR_DIRTY | - PAGE_SET_WRITEBACK | + PAGE_START_WRITEBACK | PAGE_END_WRITEBACK); return -ENOMEM; } @@ -1842,8 +1835,7 @@ error: locked_page, EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | - PAGE_CLEAR_DIRTY | - PAGE_SET_WRITEBACK | + PAGE_START_WRITEBACK | PAGE_END_WRITEBACK); btrfs_free_path(path); return ret; -- cgit v1.2.3 From 62c053fbb2d1816def1d353d9abed4c2f1f0abe9 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:46 +0800 Subject: btrfs: set UNMAPPED bit early in btrfs_clone_extent_buffer() for subpage support For the incoming subpage support, UNMAPPED extent buffer will have different behavior in btrfs_release_extent_buffer(). This means we need to set UNMAPPED bit early before calling btrfs_release_extent_buffer(). Currently there is only one caller which relies on btrfs_release_extent_buffer() in its error path while set UNMAPPED bit late: - btrfs_clone_extent_buffer() Make it subpage compatible by setting the UNMAPPED bit early, since we're here, also move the UPTODATE bit early. There is another caller, __alloc_dummy_extent_buffer(), setting UNMAPPED bit late, but that function clean up the allocated page manually, thus no need for any modification. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 7c14ccf76838..d3819dde8952 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5064,6 +5064,13 @@ struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src) if (new == NULL) return NULL; + /* + * Set UNMAPPED before calling btrfs_release_extent_buffer(), as + * btrfs_release_extent_buffer() have different behavior for + * UNMAPPED subpage extent buffer. + */ + set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags); + for (i = 0; i < num_pages; i++) { p = alloc_page(GFP_NOFS); if (!p) { @@ -5076,9 +5083,7 @@ struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src) new->pages[i] = p; copy_page(page_address(p), page_address(src->pages[i])); } - set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags); - set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags); return new; } -- cgit v1.2.3 From cac06d843f259ebc4d03e4bc8af7304c17f76ee5 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:47 +0800 Subject: btrfs: introduce the skeleton of btrfs_subpage structure For sectorsize < page size support, we need a structure to record extra status info for each sector of a page. Introduce the skeleton structure, all subpage related code would go to subpage.[ch]. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/Makefile | 3 ++- fs/btrfs/subpage.c | 43 +++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/subpage.h | 33 +++++++++++++++++++++++++++++++++ fs/btrfs/super.c | 1 - 4 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 fs/btrfs/subpage.c create mode 100644 fs/btrfs/subpage.h (limited to 'fs') diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile index e45957319424..b634c42115ea 100644 --- a/fs/btrfs/Makefile +++ b/fs/btrfs/Makefile @@ -27,7 +27,8 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \ compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \ reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \ uuid-tree.o props.o free-space-tree.o tree-checker.o space-info.o \ - block-rsv.o delalloc-space.o block-group.o discard.o reflink.o + block-rsv.o delalloc-space.o block-group.o discard.o reflink.o \ + subpage.o btrfs-$(CONFIG_BTRFS_FS_POSIX_ACL) += acl.o btrfs-$(CONFIG_BTRFS_FS_CHECK_INTEGRITY) += check-integrity.o diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c new file mode 100644 index 000000000000..a3e5b6a13d54 --- /dev/null +++ b/fs/btrfs/subpage.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "ctree.h" +#include "subpage.h" + +int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, + struct page *page, enum btrfs_subpage_type type) +{ + struct btrfs_subpage *subpage; + + /* + * We have cases like a dummy extent buffer page, which is not mappped + * and doesn't need to be locked. + */ + if (page->mapping) + ASSERT(PageLocked(page)); + /* Either not subpage, or the page already has private attached */ + if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page)) + return 0; + + subpage = kzalloc(sizeof(struct btrfs_subpage), GFP_NOFS); + if (!subpage) + return -ENOMEM; + + spin_lock_init(&subpage->lock); + attach_page_private(page, subpage); + return 0; +} + +void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, + struct page *page) +{ + struct btrfs_subpage *subpage; + + /* Either not subpage, or already detached */ + if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page)) + return; + + subpage = (struct btrfs_subpage *)detach_page_private(page); + ASSERT(subpage); + kfree(subpage); +} diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h new file mode 100644 index 000000000000..676280bc7562 --- /dev/null +++ b/fs/btrfs/subpage.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_SUBPAGE_H +#define BTRFS_SUBPAGE_H + +#include + +/* + * Maximum page size we support is 64K, minimum sector size is 4K, u16 bitmap + * is sufficient. Regular bitmap_* is not used due to size reasons. + */ +#define BTRFS_SUBPAGE_BITMAP_SIZE 16 + +/* + * Structure to trace status of each sector inside a page, attached to + * page::private for both data and metadata inodes. + */ +struct btrfs_subpage { + /* Common members for both data and metadata pages */ + spinlock_t lock; +}; + +enum btrfs_subpage_type { + BTRFS_SUBPAGE_METADATA, + BTRFS_SUBPAGE_DATA, +}; + +int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, + struct page *page, enum btrfs_subpage_type type); +void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, + struct page *page); + +#endif diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 12d7d3be7cd4..919ed5c357e9 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -48,7 +48,6 @@ #include "tests/btrfs-tests.h" #include "block-group.h" #include "discard.h" - #include "qgroup.h" #define CREATE_TRACE_POINTS #include -- cgit v1.2.3 From 760f991f1428f25fd18b8638004c95f0a2a43b2f Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:48 +0800 Subject: btrfs: make attach_extent_buffer_page() handle subpage case For subpage case, we need to allocate additional memory for each metadata page. So we need to: - Allow attach_extent_buffer_page() to return int to indicate allocation failure - Allow manually pre-allocate subpage memory for alloc_extent_buffer() As we don't want to use GFP_ATOMIC under spinlock, we introduce btrfs_alloc_subpage() and btrfs_free_subpage() functions for this purpose. (The simple wrap for btrfs_free_subpage() is for later convert to kmem_cache. Already internally tested without problem) - Preallocate btrfs_subpage structure for alloc_extent_buffer() We don't want to call memory allocation with spinlock held, so do preallocation before we acquire mapping->private_lock. - Handle subpage and regular case differently in attach_extent_buffer_page() For regular case, no change, just do the usual thing. For subpage case, allocate new memory or use the preallocated memory. For future subpage metadata, we will make use of radix tree to grab extent buffer. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++------ fs/btrfs/subpage.c | 30 ++++++++++++++++++----- fs/btrfs/subpage.h | 10 ++++++++ 3 files changed, 96 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index d3819dde8952..e498d496560b 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -24,6 +24,7 @@ #include "rcu-string.h" #include "backref.h" #include "disk-io.h" +#include "subpage.h" static struct kmem_cache *extent_state_cache; static struct kmem_cache *extent_buffer_cache; @@ -3141,9 +3142,13 @@ static int submit_extent_page(unsigned int opf, return ret; } -static void attach_extent_buffer_page(struct extent_buffer *eb, - struct page *page) +static int attach_extent_buffer_page(struct extent_buffer *eb, + struct page *page, + struct btrfs_subpage *prealloc) { + struct btrfs_fs_info *fs_info = eb->fs_info; + int ret = 0; + /* * If the page is mapped to btree inode, we should hold the private * lock to prevent race. @@ -3153,10 +3158,28 @@ static void attach_extent_buffer_page(struct extent_buffer *eb, if (page->mapping) lockdep_assert_held(&page->mapping->private_lock); - if (!PagePrivate(page)) - attach_page_private(page, eb); + if (fs_info->sectorsize == PAGE_SIZE) { + if (!PagePrivate(page)) + attach_page_private(page, eb); + else + WARN_ON(page->private != (unsigned long)eb); + return 0; + } + + /* Already mapped, just free prealloc */ + if (PagePrivate(page)) { + btrfs_free_subpage(prealloc); + return 0; + } + + if (prealloc) + /* Has preallocated memory for subpage */ + attach_page_private(page, prealloc); else - WARN_ON(page->private != (unsigned long)eb); + /* Do new allocation to attach subpage */ + ret = btrfs_attach_subpage(fs_info, page, + BTRFS_SUBPAGE_METADATA); + return ret; } void set_page_extent_mapped(struct page *page) @@ -5072,12 +5095,19 @@ struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src) set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags); for (i = 0; i < num_pages; i++) { + int ret; + p = alloc_page(GFP_NOFS); if (!p) { btrfs_release_extent_buffer(new); return NULL; } - attach_extent_buffer_page(new, p); + ret = attach_extent_buffer_page(new, p, NULL); + if (ret < 0) { + put_page(p); + btrfs_release_extent_buffer(new); + return NULL; + } WARN_ON(PageDirty(p)); SetPageUptodate(p); new->pages[i] = p; @@ -5315,12 +5345,33 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, num_pages = num_extent_pages(eb); for (i = 0; i < num_pages; i++, index++) { + struct btrfs_subpage *prealloc = NULL; + p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL); if (!p) { exists = ERR_PTR(-ENOMEM); goto free_eb; } + /* + * Preallocate page->private for subpage case, so that we won't + * allocate memory with private_lock hold. The memory will be + * freed by attach_extent_buffer_page() or freed manually if + * we exit earlier. + * + * Although we have ensured one subpage eb can only have one + * page, but it may change in the future for 16K page size + * support, so we still preallocate the memory in the loop. + */ + ret = btrfs_alloc_subpage(fs_info, &prealloc, + BTRFS_SUBPAGE_METADATA); + if (ret < 0) { + unlock_page(p); + put_page(p); + exists = ERR_PTR(ret); + goto free_eb; + } + spin_lock(&mapping->private_lock); exists = grab_extent_buffer(p); if (exists) { @@ -5328,10 +5379,14 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, unlock_page(p); put_page(p); mark_extent_buffer_accessed(exists, p); + btrfs_free_subpage(prealloc); goto free_eb; } - attach_extent_buffer_page(eb, p); + /* Should not fail, as we have preallocated the memory */ + ret = attach_extent_buffer_page(eb, p, prealloc); + ASSERT(!ret); spin_unlock(&mapping->private_lock); + WARN_ON(PageDirty(p)); eb->pages[i] = p; if (!PageUptodate(p)) diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c index a3e5b6a13d54..61b28dfca20c 100644 --- a/fs/btrfs/subpage.c +++ b/fs/btrfs/subpage.c @@ -7,7 +7,8 @@ int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, struct page *page, enum btrfs_subpage_type type) { - struct btrfs_subpage *subpage; + struct btrfs_subpage *subpage = NULL; + int ret; /* * We have cases like a dummy extent buffer page, which is not mappped @@ -19,11 +20,9 @@ int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page)) return 0; - subpage = kzalloc(sizeof(struct btrfs_subpage), GFP_NOFS); - if (!subpage) - return -ENOMEM; - - spin_lock_init(&subpage->lock); + ret = btrfs_alloc_subpage(fs_info, &subpage, type); + if (ret < 0) + return ret; attach_page_private(page, subpage); return 0; } @@ -39,5 +38,24 @@ void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, subpage = (struct btrfs_subpage *)detach_page_private(page); ASSERT(subpage); + btrfs_free_subpage(subpage); +} + +int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, + struct btrfs_subpage **ret, + enum btrfs_subpage_type type) +{ + if (fs_info->sectorsize == PAGE_SIZE) + return 0; + + *ret = kzalloc(sizeof(struct btrfs_subpage), GFP_NOFS); + if (!*ret) + return -ENOMEM; + spin_lock_init(&(*ret)->lock); + return 0; +} + +void btrfs_free_subpage(struct btrfs_subpage *subpage) +{ kfree(subpage); } diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index 676280bc7562..7ba544bcc9c6 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -18,6 +18,10 @@ struct btrfs_subpage { /* Common members for both data and metadata pages */ spinlock_t lock; + union { + /* Structures only used by metadata */ + /* Structures only used by data */ + }; }; enum btrfs_subpage_type { @@ -30,4 +34,10 @@ int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info, void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, struct page *page); +/* Allocate additional data where page represents more than one sector */ +int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, + struct btrfs_subpage **ret, + enum btrfs_subpage_type type); +void btrfs_free_subpage(struct btrfs_subpage *subpage); + #endif -- cgit v1.2.3 From 819822107d8837fc3363ceaeb172b981c8600a2b Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:49 +0800 Subject: btrfs: make grab_extent_buffer_from_page() handle subpage case For subpage case, grab_extent_buffer() can't really get an extent buffer just from btrfs_subpage. We have radix tree lock protecting us from inserting the same eb into the tree. Thus we don't really need to do the extra hassle, just let alloc_extent_buffer() handle the existing eb in radix tree. Now if two ebs are being allocated as the same time, one will fail with -EEIXST when inserting into the radix tree. So for grab_extent_buffer(), just always return NULL for subpage case. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index e498d496560b..133ff4531472 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5284,10 +5284,19 @@ free_eb: } #endif -static struct extent_buffer *grab_extent_buffer(struct page *page) +static struct extent_buffer *grab_extent_buffer( + struct btrfs_fs_info *fs_info, struct page *page) { struct extent_buffer *exists; + /* + * For subpage case, we completely rely on radix tree to ensure we + * don't try to insert two ebs for the same bytenr. So here we always + * return NULL and just continue. + */ + if (fs_info->sectorsize < PAGE_SIZE) + return NULL; + /* Page not yet attached to an extent buffer */ if (!PagePrivate(page)) return NULL; @@ -5373,7 +5382,7 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, } spin_lock(&mapping->private_lock); - exists = grab_extent_buffer(p); + exists = grab_extent_buffer(fs_info, p); if (exists) { spin_unlock(&mapping->private_lock); unlock_page(p); -- cgit v1.2.3 From 8ff8466d29efc226648c3c5e57590428d798a6ea Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:50 +0800 Subject: btrfs: support subpage for extent buffer page release In btrfs_release_extent_buffer_pages(), we need to add extra handling for subpage. Introduce a helper, detach_extent_buffer_page(), to do different handling for regular and subpage cases. For subpage case, handle detaching page private. For unmapped (dummy or cloned) ebs, we can detach the page private immediately as the page can only be attached to one unmapped eb. For mapped ebs, we have to ensure there are no eb in the page range before we delete it, as page->private is shared between all ebs in the same page. But there is a subpage specific race, where we can race with extent buffer allocation, and clear the page private while new eb is still being utilized, like this: Extent buffer A is the new extent buffer which will be allocated, while extent buffer B is the last existing extent buffer of the page. T1 (eb A) | T2 (eb B) -------------------------------+------------------------------ alloc_extent_buffer() | btrfs_release_extent_buffer_pages() |- p = find_or_create_page() | | |- attach_extent_buffer_page() | | | | |- detach_extent_buffer_page() | | |- if (!page_range_has_eb()) | | | No new eb in the page range yet | | | As new eb A hasn't yet been | | | inserted into radix tree. | | |- btrfs_detach_subpage() | | |- detach_page_private(); |- radix_tree_insert() | Then we have a metadata eb whose page has no private bit. To avoid such race, we introduce a subpage metadata-specific member, btrfs_subpage::eb_refs. In alloc_extent_buffer() we increase eb_refs in the critical section of private_lock. Then page_range_has_eb() will return true for detach_extent_buffer_page(), and will not detach page private. The section is marked by: - btrfs_page_inc_eb_refs() - btrfs_page_dec_eb_refs() Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 94 +++++++++++++++++++++++++++++++++++++++++++--------- fs/btrfs/subpage.c | 42 +++++++++++++++++++++++ fs/btrfs/subpage.h | 13 +++++++- 3 files changed, 133 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 133ff4531472..1812813bdf63 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -4995,25 +4995,39 @@ int extent_buffer_under_io(const struct extent_buffer *eb) test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); } -/* - * Release all pages attached to the extent buffer. - */ -static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb) +static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page) { - int i; - int num_pages; - int mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags); + struct btrfs_subpage *subpage; - BUG_ON(extent_buffer_under_io(eb)); + lockdep_assert_held(&page->mapping->private_lock); - num_pages = num_extent_pages(eb); - for (i = 0; i < num_pages; i++) { - struct page *page = eb->pages[i]; + if (PagePrivate(page)) { + subpage = (struct btrfs_subpage *)page->private; + if (atomic_read(&subpage->eb_refs)) + return true; + } + return false; +} - if (!page) - continue; +static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *page) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags); + + /* + * For mapped eb, we're going to change the page private, which should + * be done under the private_lock. + */ + if (mapped) + spin_lock(&page->mapping->private_lock); + + if (!PagePrivate(page)) { if (mapped) - spin_lock(&page->mapping->private_lock); + spin_unlock(&page->mapping->private_lock); + return; + } + + if (fs_info->sectorsize == PAGE_SIZE) { /* * We do this since we'll remove the pages after we've * removed the eb from the radix tree, so we could race @@ -5032,9 +5046,49 @@ static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb) */ detach_page_private(page); } - if (mapped) spin_unlock(&page->mapping->private_lock); + return; + } + + /* + * For subpage, we can have dummy eb with page private. In this case, + * we can directly detach the private as such page is only attached to + * one dummy eb, no sharing. + */ + if (!mapped) { + btrfs_detach_subpage(fs_info, page); + return; + } + + btrfs_page_dec_eb_refs(fs_info, page); + + /* + * We can only detach the page private if there are no other ebs in the + * page range. + */ + if (!page_range_has_eb(fs_info, page)) + btrfs_detach_subpage(fs_info, page); + + spin_unlock(&page->mapping->private_lock); +} + +/* Release all pages attached to the extent buffer */ +static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb) +{ + int i; + int num_pages; + + ASSERT(!extent_buffer_under_io(eb)); + + num_pages = num_extent_pages(eb); + for (i = 0; i < num_pages; i++) { + struct page *page = eb->pages[i]; + + if (!page) + continue; + + detach_extent_buffer_page(eb, page); /* One for when we allocated the page */ put_page(page); @@ -5394,6 +5448,16 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, /* Should not fail, as we have preallocated the memory */ ret = attach_extent_buffer_page(eb, p, prealloc); ASSERT(!ret); + /* + * To inform we have extra eb under allocation, so that + * detach_extent_buffer_page() won't release the page private + * when the eb hasn't yet been inserted into radix tree. + * + * The ref will be decreased when the eb released the page, in + * detach_extent_buffer_page(). + * Thus needs no special handling in error path. + */ + btrfs_page_inc_eb_refs(fs_info, p); spin_unlock(&mapping->private_lock); WARN_ON(PageDirty(p)); diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c index 61b28dfca20c..a2a21fa0ea35 100644 --- a/fs/btrfs/subpage.c +++ b/fs/btrfs/subpage.c @@ -52,6 +52,8 @@ int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, if (!*ret) return -ENOMEM; spin_lock_init(&(*ret)->lock); + if (type == BTRFS_SUBPAGE_METADATA) + atomic_set(&(*ret)->eb_refs, 0); return 0; } @@ -59,3 +61,43 @@ void btrfs_free_subpage(struct btrfs_subpage *subpage) { kfree(subpage); } + +/* + * Increase the eb_refs of current subpage. + * + * This is important for eb allocation, to prevent race with last eb freeing + * of the same page. + * With the eb_refs increased before the eb inserted into radix tree, + * detach_extent_buffer_page() won't detach the page private while we're still + * allocating the extent buffer. + */ +void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, + struct page *page) +{ + struct btrfs_subpage *subpage; + + if (fs_info->sectorsize == PAGE_SIZE) + return; + + ASSERT(PagePrivate(page) && page->mapping); + lockdep_assert_held(&page->mapping->private_lock); + + subpage = (struct btrfs_subpage *)page->private; + atomic_inc(&subpage->eb_refs); +} + +void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, + struct page *page) +{ + struct btrfs_subpage *subpage; + + if (fs_info->sectorsize == PAGE_SIZE) + return; + + ASSERT(PagePrivate(page) && page->mapping); + lockdep_assert_held(&page->mapping->private_lock); + + subpage = (struct btrfs_subpage *)page->private; + ASSERT(atomic_read(&subpage->eb_refs)); + atomic_dec(&subpage->eb_refs); +} diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index 7ba544bcc9c6..fe51cc237a66 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -19,7 +19,13 @@ struct btrfs_subpage { /* Common members for both data and metadata pages */ spinlock_t lock; union { - /* Structures only used by metadata */ + /* + * Structures only used by metadata + * + * @eb_refs should only be operated under private_lock, as it + * manages whether the subpage can be detached. + */ + atomic_t eb_refs; /* Structures only used by data */ }; }; @@ -40,4 +46,9 @@ int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, enum btrfs_subpage_type type); void btrfs_free_subpage(struct btrfs_subpage *subpage); +void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, + struct page *page); +void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, + struct page *page); + #endif -- cgit v1.2.3 From 09bc1f0fb845a6435e2c6c5d3c937f7a674e816a Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:51 +0800 Subject: btrfs: attach private to dummy extent buffer pages There are locations where we allocate dummy extent buffers for temporary usage, like in tree_mod_log_rewind() or get_old_root(). These dummy extent buffers will be handled by the same eb accessors, and if they don't have page::private subpage eb accessors could fail. To address such problems, make __alloc_dummy_extent_buffer() attach page private for dummy extent buffers too. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 1812813bdf63..b8ff05916b8f 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5185,9 +5185,14 @@ struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, num_pages = num_extent_pages(eb); for (i = 0; i < num_pages; i++) { + int ret; + eb->pages[i] = alloc_page(GFP_NOFS); if (!eb->pages[i]) goto err; + ret = attach_extent_buffer_page(eb, eb->pages[i], NULL); + if (ret < 0) + goto err; } set_extent_buffer_uptodate(eb); btrfs_set_header_nritems(eb, 0); @@ -5195,8 +5200,10 @@ struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, return eb; err: - for (; i > 0; i--) + for (; i > 0; i--) { + detach_extent_buffer_page(eb, eb->pages[i - 1]); __free_page(eb->pages[i - 1]); + } __free_extent_buffer(eb); return NULL; } -- cgit v1.2.3 From a1d767c11cca0f9b6ddc56ea9561d441340d91a9 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:52 +0800 Subject: btrfs: introduce helpers for subpage uptodate status Introduce the following functions to handle subpage uptodate status: - btrfs_subpage_set_uptodate() - btrfs_subpage_clear_uptodate() - btrfs_subpage_test_uptodate() These helpers can only be called when the page has subpage attached and the range is ensured to be inside the page. - btrfs_page_set_uptodate() - btrfs_page_clear_uptodate() - btrfs_page_test_uptodate() These helpers can handle both regular sector size and subpage. Although caller should still ensure that the range is inside the page. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/subpage.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/subpage.h | 27 +++++++++++++ 2 files changed, 140 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c index a2a21fa0ea35..4e1b187b9607 100644 --- a/fs/btrfs/subpage.c +++ b/fs/btrfs/subpage.c @@ -101,3 +101,116 @@ void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, ASSERT(atomic_read(&subpage->eb_refs)); atomic_dec(&subpage->eb_refs); } + +/* + * Convert the [start, start + len) range into a u16 bitmap + * + * For example: if start == page_offset() + 16K, len = 16K, we get 0x00f0. + */ +static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + const int bit_start = offset_in_page(start) >> fs_info->sectorsize_bits; + const int nbits = len >> fs_info->sectorsize_bits; + + /* Basic checks */ + ASSERT(PagePrivate(page) && page->private); + ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && + IS_ALIGNED(len, fs_info->sectorsize)); + + /* + * The range check only works for mapped page, we can still have + * unmapped page like dummy extent buffer pages. + */ + if (page->mapping) + ASSERT(page_offset(page) <= start && + start + len <= page_offset(page) + PAGE_SIZE); + /* + * Here nbits can be 16, thus can go beyond u16 range. We make the + * first left shift to be calculate in unsigned long (at least u32), + * then truncate the result to u16. + */ + return (u16)(((1UL << nbits) - 1) << bit_start); +} + +void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); + unsigned long flags; + + spin_lock_irqsave(&subpage->lock, flags); + subpage->uptodate_bitmap |= tmp; + if (subpage->uptodate_bitmap == U16_MAX) + SetPageUptodate(page); + spin_unlock_irqrestore(&subpage->lock, flags); +} + +void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); + unsigned long flags; + + spin_lock_irqsave(&subpage->lock, flags); + subpage->uptodate_bitmap &= ~tmp; + ClearPageUptodate(page); + spin_unlock_irqrestore(&subpage->lock, flags); +} + +/* + * Unlike set/clear which is dependent on each page status, for test all bits + * are tested in the same way. + */ +#define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ +bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len) \ +{ \ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \ + const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); \ + unsigned long flags; \ + bool ret; \ + \ + spin_lock_irqsave(&subpage->lock, flags); \ + ret = ((subpage->name##_bitmap & tmp) == tmp); \ + spin_unlock_irqrestore(&subpage->lock, flags); \ + return ret; \ +} +IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); + +/* + * Note that, in selftests (extent-io-tests), we can have empty fs_info passed + * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall + * back to regular sectorsize branch. + */ +#define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func, \ + test_page_func) \ +void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ + set_page_func(page); \ + return; \ + } \ + btrfs_subpage_set_##name(fs_info, page, start, len); \ +} \ +void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ + clear_page_func(page); \ + return; \ + } \ + btrfs_subpage_clear_##name(fs_info, page, start, len); \ +} \ +bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ + return test_page_func(page); \ + return btrfs_subpage_test_##name(fs_info, page, start, len); \ +} +IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate, + PageUptodate); diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index fe51cc237a66..98d511cd75bf 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -18,6 +18,7 @@ struct btrfs_subpage { /* Common members for both data and metadata pages */ spinlock_t lock; + u16 uptodate_bitmap; union { /* * Structures only used by metadata @@ -51,4 +52,30 @@ void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct page *page); +/* + * Template for subpage related operations. + * + * btrfs_subpage_*() are for call sites where the page has subpage attached and + * the range is ensured to be inside the page. + * + * btrfs_page_*() are for call sites where the page can either be subpage + * specific or regular page. The function will handle both cases. + * But the range still needs to be inside the page. + */ +#define DECLARE_BTRFS_SUBPAGE_OPS(name) \ +void btrfs_subpage_set_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len); \ +void btrfs_subpage_clear_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len); \ +bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len); \ +void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len); \ +void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len); \ +bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len); + +DECLARE_BTRFS_SUBPAGE_OPS(uptodate); + #endif -- cgit v1.2.3 From 03a816b32be577fdeed2e17d95c2636b68f6860c Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:53 +0800 Subject: btrfs: introduce helpers for subpage error status Introduce the following functions to handle subpage error status: - btrfs_subpage_set_error() - btrfs_subpage_clear_error() - btrfs_subpage_test_error() These helpers can only be called when the page has subpage attached and the range is ensured to be inside the page. - btrfs_page_set_error() - btrfs_page_clear_error() - btrfs_page_test_error() These helpers can handle both regular sector size and subpage without problem. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/subpage.c | 29 +++++++++++++++++++++++++++++ fs/btrfs/subpage.h | 2 ++ 2 files changed, 31 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c index 4e1b187b9607..2c51ab71e000 100644 --- a/fs/btrfs/subpage.c +++ b/fs/btrfs/subpage.c @@ -160,6 +160,33 @@ void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, spin_unlock_irqrestore(&subpage->lock, flags); } +void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); + unsigned long flags; + + spin_lock_irqsave(&subpage->lock, flags); + subpage->error_bitmap |= tmp; + SetPageError(page); + spin_unlock_irqrestore(&subpage->lock, flags); +} + +void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); + unsigned long flags; + + spin_lock_irqsave(&subpage->lock, flags); + subpage->error_bitmap &= ~tmp; + if (subpage->error_bitmap == 0) + ClearPageError(page); + spin_unlock_irqrestore(&subpage->lock, flags); +} + /* * Unlike set/clear which is dependent on each page status, for test all bits * are tested in the same way. @@ -179,6 +206,7 @@ bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ return ret; \ } IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); +IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error); /* * Note that, in selftests (extent-io-tests), we can have empty fs_info passed @@ -214,3 +242,4 @@ bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ } IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate, PageUptodate); +IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError); diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index 98d511cd75bf..f3c5def313a1 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -19,6 +19,7 @@ struct btrfs_subpage { /* Common members for both data and metadata pages */ spinlock_t lock; u16 uptodate_bitmap; + u16 error_bitmap; union { /* * Structures only used by metadata @@ -77,5 +78,6 @@ bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ struct page *page, u64 start, u32 len); DECLARE_BTRFS_SUBPAGE_OPS(uptodate); +DECLARE_BTRFS_SUBPAGE_OPS(error); #endif -- cgit v1.2.3 From 251f2acc719e99f00827814ea77cfd38080e1d62 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:54 +0800 Subject: btrfs: support subpage in set/clear_extent_buffer_uptodate() To support subpage in set_extent_buffer_uptodate and clear_extent_buffer_uptodate we only need to use the subpage-aware helpers to update the page bits. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index b8ff05916b8f..969de300a95b 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5672,30 +5672,33 @@ bool set_extent_buffer_dirty(struct extent_buffer *eb) void clear_extent_buffer_uptodate(struct extent_buffer *eb) { - int i; + struct btrfs_fs_info *fs_info = eb->fs_info; struct page *page; int num_pages; + int i; clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); num_pages = num_extent_pages(eb); for (i = 0; i < num_pages; i++) { page = eb->pages[i]; if (page) - ClearPageUptodate(page); + btrfs_page_clear_uptodate(fs_info, page, + eb->start, eb->len); } } void set_extent_buffer_uptodate(struct extent_buffer *eb) { - int i; + struct btrfs_fs_info *fs_info = eb->fs_info; struct page *page; int num_pages; + int i; set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); num_pages = num_extent_pages(eb); for (i = 0; i < num_pages; i++) { page = eb->pages[i]; - SetPageUptodate(page); + btrfs_page_set_uptodate(fs_info, page, eb->start, eb->len); } } -- cgit v1.2.3 From 92d83e94365706fa3250b0e43bdab5995ac03046 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:55 +0800 Subject: btrfs: support subpage in btrfs_clone_extent_buffer For btrfs_clone_extent_buffer(), it's mostly the same code of __alloc_dummy_extent_buffer(), except it has extra page copy. So to make it subpage compatible, we only need to: - Call set_extent_buffer_uptodate() instead of SetPageUptodate() This will set correct uptodate bit for subpage and regular sector size cases. Since we're calling set_extent_buffer_uptodate() which will also set EXTENT_BUFFER_UPTODATE bit, we don't need to manually set that bit either. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 969de300a95b..6b27daf62d94 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5163,11 +5163,10 @@ struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src) return NULL; } WARN_ON(PageDirty(p)); - SetPageUptodate(p); new->pages[i] = p; copy_page(page_address(p), page_address(src->pages[i])); } - set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags); + set_extent_buffer_uptodate(new); return new; } -- cgit v1.2.3 From d1e86e3fc34f24b090d86949ad7f3db7a4c1861f Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:56 +0800 Subject: btrfs: support subpage in try_release_extent_buffer() Unlike the original try_release_extent_buffer(), try_release_subpage_extent_buffer() will iterate through all the ebs in the page, and try to release each. We can release the full page only after there's no private attached, which means all ebs of that page have been released as well. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 6b27daf62d94..a6102e795af5 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -6318,13 +6318,115 @@ void memmove_extent_buffer(const struct extent_buffer *dst, } } +static struct extent_buffer *get_next_extent_buffer( + struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr) +{ + struct extent_buffer *gang[BTRFS_SUBPAGE_BITMAP_SIZE]; + struct extent_buffer *found = NULL; + u64 page_start = page_offset(page); + int ret; + int i; + + ASSERT(in_range(bytenr, page_start, PAGE_SIZE)); + ASSERT(PAGE_SIZE / fs_info->nodesize <= BTRFS_SUBPAGE_BITMAP_SIZE); + lockdep_assert_held(&fs_info->buffer_lock); + + ret = radix_tree_gang_lookup(&fs_info->buffer_radix, (void **)gang, + bytenr >> fs_info->sectorsize_bits, + PAGE_SIZE / fs_info->nodesize); + for (i = 0; i < ret; i++) { + /* Already beyond page end */ + if (gang[i]->start >= page_start + PAGE_SIZE) + break; + /* Found one */ + if (gang[i]->start >= bytenr) { + found = gang[i]; + break; + } + } + return found; +} + +static int try_release_subpage_extent_buffer(struct page *page) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); + u64 cur = page_offset(page); + const u64 end = page_offset(page) + PAGE_SIZE; + int ret; + + while (cur < end) { + struct extent_buffer *eb = NULL; + + /* + * Unlike try_release_extent_buffer() which uses page->private + * to grab buffer, for subpage case we rely on radix tree, thus + * we need to ensure radix tree consistency. + * + * We also want an atomic snapshot of the radix tree, thus go + * with spinlock rather than RCU. + */ + spin_lock(&fs_info->buffer_lock); + eb = get_next_extent_buffer(fs_info, page, cur); + if (!eb) { + /* No more eb in the page range after or at cur */ + spin_unlock(&fs_info->buffer_lock); + break; + } + cur = eb->start + eb->len; + + /* + * The same as try_release_extent_buffer(), to ensure the eb + * won't disappear out from under us. + */ + spin_lock(&eb->refs_lock); + if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) { + spin_unlock(&eb->refs_lock); + spin_unlock(&fs_info->buffer_lock); + break; + } + spin_unlock(&fs_info->buffer_lock); + + /* + * If tree ref isn't set then we know the ref on this eb is a + * real ref, so just return, this eb will likely be freed soon + * anyway. + */ + if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) { + spin_unlock(&eb->refs_lock); + break; + } + + /* + * Here we don't care about the return value, we will always + * check the page private at the end. And + * release_extent_buffer() will release the refs_lock. + */ + release_extent_buffer(eb); + } + /* + * Finally to check if we have cleared page private, as if we have + * released all ebs in the page, the page private should be cleared now. + */ + spin_lock(&page->mapping->private_lock); + if (!PagePrivate(page)) + ret = 1; + else + ret = 0; + spin_unlock(&page->mapping->private_lock); + return ret; + +} + int try_release_extent_buffer(struct page *page) { struct extent_buffer *eb; + if (btrfs_sb(page->mapping->host->i_sb)->sectorsize < PAGE_SIZE) + return try_release_subpage_extent_buffer(page); + /* - * We need to make sure nobody is attaching this page to an eb right - * now. + * We need to make sure nobody is changing page->private, as we rely on + * page->private as the pointer to extent buffer. */ spin_lock(&page->mapping->private_lock); if (!PagePrivate(page)) { -- cgit v1.2.3 From 4012daf769cb77dbf3bc36c3adecf480ad097682 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:57 +0800 Subject: btrfs: introduce read_extent_buffer_subpage() Introduce a helper, read_extent_buffer_subpage(), to do the subpage extent buffer read. The difference between regular and subpage routines are: - No page locking Here we completely rely on extent locking. Page locking can reduce the concurrency greatly, as if we lock one page to read one extent buffer, all the other extent buffers in the same page will have to wait. - Extent uptodate condition Despite the existing PageUptodate() and EXTENT_BUFFER_UPTODATE check, We also need to check btrfs_subpage::uptodate_bitmap. - No page iteration Just one page, no need to loop, this greatly simplified the subpage routine. This patch only implements the bio submit part, no endio support yet. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index a6102e795af5..2ffaa983c9dd 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5701,6 +5701,73 @@ void set_extent_buffer_uptodate(struct extent_buffer *eb) } } +static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, + int mirror_num) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + struct extent_io_tree *io_tree; + struct page *page = eb->pages[0]; + struct bio *bio = NULL; + int ret = 0; + + ASSERT(!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags)); + ASSERT(PagePrivate(page)); + io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree; + + if (wait == WAIT_NONE) { + ret = try_lock_extent(io_tree, eb->start, + eb->start + eb->len - 1); + if (ret <= 0) + return ret; + } else { + ret = lock_extent(io_tree, eb->start, eb->start + eb->len - 1); + if (ret < 0) + return ret; + } + + ret = 0; + if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags) || + PageUptodate(page) || + btrfs_subpage_test_uptodate(fs_info, page, eb->start, eb->len)) { + set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); + unlock_extent(io_tree, eb->start, eb->start + eb->len - 1); + return ret; + } + + clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags); + eb->read_mirror = 0; + atomic_set(&eb->io_pages, 1); + check_buffer_tree_ref(eb); + btrfs_subpage_clear_error(fs_info, page, eb->start, eb->len); + + ret = submit_extent_page(REQ_OP_READ | REQ_META, NULL, page, eb->start, + eb->len, eb->start - page_offset(page), &bio, + end_bio_extent_readpage, mirror_num, 0, 0, + true); + if (ret) { + /* + * In the endio function, if we hit something wrong we will + * increase the io_pages, so here we need to decrease it for + * error path. + */ + atomic_dec(&eb->io_pages); + } + if (bio) { + int tmp; + + tmp = submit_one_bio(bio, mirror_num, 0); + if (tmp < 0) + return tmp; + } + if (ret || wait != WAIT_COMPLETE) + return ret; + + wait_extent_bit(io_tree, eb->start, eb->start + eb->len - 1, EXTENT_LOCKED); + if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) + ret = -EIO; + return ret; +} + int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) { int i; @@ -5717,6 +5784,9 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) return 0; + if (eb->fs_info->sectorsize < PAGE_SIZE) + return read_extent_buffer_subpage(eb, wait, mirror_num); + num_pages = num_extent_pages(eb); for (i = 0; i < num_pages; i++) { page = eb->pages[i]; -- cgit v1.2.3 From 4325cb2293817cef3611c43d7a27d0937d1e6962 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:58 +0800 Subject: btrfs: support subpage in endio_readpage_update_page_status() To handle subpage status update, add the following: - Use btrfs_page_*() subpage-aware helpers to update page status Now we can handle both cases well. - No page unlock for subpage metadata Since subpage metadata doesn't utilize page locking at all, skip it. For subpage data locking, it's handled in later commits. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 2ffaa983c9dd..491fc0114672 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2840,15 +2840,24 @@ update: processed->uptodate = uptodate; } -static void endio_readpage_update_page_status(struct page *page, bool uptodate) +static void endio_readpage_update_page_status(struct page *page, bool uptodate, + u64 start, u32 len) { + struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); + + ASSERT(page_offset(page) <= start && + start + len <= page_offset(page) + PAGE_SIZE); + if (uptodate) { - SetPageUptodate(page); + btrfs_page_set_uptodate(fs_info, page, start, len); } else { - ClearPageUptodate(page); - SetPageError(page); + btrfs_page_clear_uptodate(fs_info, page, start, len); + btrfs_page_set_error(fs_info, page, start, len); } - unlock_page(page); + + if (fs_info->sectorsize == PAGE_SIZE) + unlock_page(page); + /* Subpage locking will be handled in later patches */ } /* @@ -2985,7 +2994,7 @@ readpage_ok: bio_offset += len; /* Update page status and unlock */ - endio_readpage_update_page_status(page, uptodate); + endio_readpage_update_page_status(page, uptodate, start, len); endio_readpage_release_extent(&processed, BTRFS_I(inode), start, end, uptodate); } -- cgit v1.2.3 From 371cdc0700c778b94ae8fa2c7d99401f13070d8f Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:33:59 +0800 Subject: btrfs: introduce subpage metadata validation check For subpage metadata validation check, there are some differences: - Read must finish in one bvec Since we're just reading one subpage range in one page, it should never be split into two bios nor two bvecs. - How to grab the existing eb Instead of grabbing eb using page->private, we have to go search radix tree as we don't have any direct pointer at hand. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index e0d1b328397e..d34c6d61928f 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -591,6 +591,59 @@ out: return ret; } +static int validate_subpage_buffer(struct page *page, u64 start, u64 end, + int mirror) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); + struct extent_buffer *eb; + bool reads_done; + int ret = 0; + + /* + * We don't allow bio merge for subpage metadata read, so we should + * only get one eb for each endio hook. + */ + ASSERT(end == start + fs_info->nodesize - 1); + ASSERT(PagePrivate(page)); + + eb = find_extent_buffer(fs_info, start); + /* + * When we are reading one tree block, eb must have been inserted into + * the radix tree. If not, something is wrong. + */ + ASSERT(eb); + + reads_done = atomic_dec_and_test(&eb->io_pages); + /* Subpage read must finish in page read */ + ASSERT(reads_done); + + eb->read_mirror = mirror; + if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) { + ret = -EIO; + goto err; + } + ret = validate_extent_buffer(eb); + if (ret < 0) + goto err; + + if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD, &eb->bflags)) + btree_readahead_hook(eb, ret); + + set_extent_buffer_uptodate(eb); + + free_extent_buffer(eb); + return ret; +err: + /* + * end_bio_extent_readpage decrements io_pages in case of error, + * make sure it has something to decrement. + */ + atomic_inc(&eb->io_pages); + clear_extent_buffer_uptodate(eb); + free_extent_buffer(eb); + return ret; +} + int btrfs_validate_metadata_buffer(struct btrfs_io_bio *io_bio, struct page *page, u64 start, u64 end, int mirror) @@ -600,6 +653,10 @@ int btrfs_validate_metadata_buffer(struct btrfs_io_bio *io_bio, int reads_done; ASSERT(page->private); + + if (btrfs_sb(page->mapping->host->i_sb)->sectorsize < PAGE_SIZE) + return validate_subpage_buffer(page, start, end, mirror); + eb = (struct extent_buffer *)page->private; /* -- cgit v1.2.3 From 32443de3382be98c0a8b8f6f50d23da2e10c4117 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:34:00 +0800 Subject: btrfs: introduce btrfs_subpage for data inodes To support subpage sector size, data also need extra info to make sure which sectors in a page are uptodate/dirty/... This patch will make pages for data inodes get btrfs_subpage structure attached, and detached when the page is freed. This patch also slightly changes the timing when set_page_extent_mapped() is called to make sure: - We have page->mapping set page->mapping->host is used to grab btrfs_fs_info, thus we can only call this function after page is mapped to an inode. One call site attaches pages to inode manually, thus we have to modify the timing of set_page_extent_mapped() a bit. - As soon as possible, before other operations Since memory allocation can fail, we have to do extra error handling. Calling set_page_extent_mapped() as soon as possible can simply the error handling for several call sites. The idea is pretty much the same as iomap_page, but with more bitmaps for btrfs specific cases. Currently the plan is to switch iomap if iomap can provide sector aligned write back (only write back dirty sectors, but not the full page, data balance require this feature). So we will stick to btrfs specific bitmap for now. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/compression.c | 10 ++++++++-- fs/btrfs/extent_io.c | 45 +++++++++++++++++++++++++++++++++++++++++---- fs/btrfs/extent_io.h | 3 ++- fs/btrfs/file.c | 24 +++++++++--------------- fs/btrfs/free-space-cache.c | 15 ++++++++++++--- fs/btrfs/inode.c | 15 +++++++++++---- fs/btrfs/ioctl.c | 8 +++++++- fs/btrfs/reflink.c | 5 ++++- fs/btrfs/relocation.c | 11 +++++++++-- 9 files changed, 103 insertions(+), 33 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 5ae3fa0386b7..6d203acfdeb3 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -542,13 +542,19 @@ static noinline int add_ra_bio_pages(struct inode *inode, goto next; } - end = last_offset + PAGE_SIZE - 1; /* * at this point, we have a locked page in the page cache * for these bytes in the file. But, we have to make * sure they map to this compressed extent on disk. */ - set_page_extent_mapped(page); + ret = set_page_extent_mapped(page); + if (ret < 0) { + unlock_page(page); + put_page(page); + break; + } + + end = last_offset + PAGE_SIZE - 1; lock_extent(tree, last_offset, end); read_lock(&em_tree->lock); em = lookup_extent_mapping(em_tree, last_offset, diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 491fc0114672..5ff85ecc6c3f 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -3191,10 +3191,38 @@ static int attach_extent_buffer_page(struct extent_buffer *eb, return ret; } -void set_page_extent_mapped(struct page *page) +int set_page_extent_mapped(struct page *page) { + struct btrfs_fs_info *fs_info; + + ASSERT(page->mapping); + + if (PagePrivate(page)) + return 0; + + fs_info = btrfs_sb(page->mapping->host->i_sb); + + if (fs_info->sectorsize < PAGE_SIZE) + return btrfs_attach_subpage(fs_info, page, BTRFS_SUBPAGE_DATA); + + attach_page_private(page, (void *)EXTENT_PAGE_PRIVATE); + return 0; +} + +void clear_page_extent_mapped(struct page *page) +{ + struct btrfs_fs_info *fs_info; + + ASSERT(page->mapping); + if (!PagePrivate(page)) - attach_page_private(page, (void *)EXTENT_PAGE_PRIVATE); + return; + + fs_info = btrfs_sb(page->mapping->host->i_sb); + if (fs_info->sectorsize < PAGE_SIZE) + return btrfs_detach_subpage(fs_info, page); + + detach_page_private(page); } static struct extent_map * @@ -3251,7 +3279,12 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, unsigned long this_bio_flag = 0; struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; - set_page_extent_mapped(page); + ret = set_page_extent_mapped(page); + if (ret < 0) { + unlock_extent(tree, start, end); + SetPageError(page); + goto out; + } if (!PageUptodate(page)) { if (cleancache_get_page(page) == 0) { @@ -3691,7 +3724,11 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc, flush_dcache_page(page); } - set_page_extent_mapped(page); + ret = set_page_extent_mapped(page); + if (ret < 0) { + SetPageError(page); + goto done; + } if (!epd->extent_locked) { ret = writepage_delalloc(BTRFS_I(inode), page, wbc, start, diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 2d8187c84812..047b3e66897f 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -178,7 +178,8 @@ int btree_write_cache_pages(struct address_space *mapping, void extent_readahead(struct readahead_control *rac); int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len); -void set_page_extent_mapped(struct page *page); +int set_page_extent_mapped(struct page *page); +void clear_page_extent_mapped(struct page *page); struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, u64 owner_root, int level); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index be5350f5bedf..bf52d7e85914 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1369,6 +1369,12 @@ again: goto fail; } + err = set_page_extent_mapped(pages[i]); + if (err < 0) { + faili = i; + goto fail; + } + if (i == 0) err = prepare_uptodate_page(inode, pages[i], pos, force_uptodate); @@ -1453,23 +1459,11 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages, } /* - * It's possible the pages are dirty right now, but we don't want - * to clean them yet because copy_from_user may catch a page fault - * and we might have to fall back to one page at a time. If that - * happens, we'll unlock these pages and we'd have a window where - * reclaim could sneak in and drop the once-dirty page on the floor - * without writing it. - * - * We have the pages locked and the extent range locked, so there's - * no way someone can start IO on any dirty pages in this range. - * - * We'll call btrfs_dirty_pages() later on, and that will flip around - * delalloc bits and dirty the pages as required. + * We should be called after prepare_pages() which should have locked + * all pages in the range. */ - for (i = 0; i < num_pages; i++) { - set_page_extent_mapped(pages[i]); + for (i = 0; i < num_pages; i++) WARN_ON(!PageLocked(pages[i])); - } return ret; } diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 0d6dcb5ff963..6134e10a6e7f 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -431,11 +431,22 @@ static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate) int i; for (i = 0; i < io_ctl->num_pages; i++) { + int ret; + page = find_or_create_page(inode->i_mapping, i, mask); if (!page) { io_ctl_drop_pages(io_ctl); return -ENOMEM; } + + ret = set_page_extent_mapped(page); + if (ret < 0) { + unlock_page(page); + put_page(page); + io_ctl_drop_pages(io_ctl); + return ret; + } + io_ctl->pages[i] = page; if (uptodate && !PageUptodate(page)) { btrfs_readpage(NULL, page); @@ -455,10 +466,8 @@ static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate) } } - for (i = 0; i < io_ctl->num_pages; i++) { + for (i = 0; i < io_ctl->num_pages; i++) clear_page_dirty_for_io(io_ctl->pages[i]); - set_page_extent_mapped(io_ctl->pages[i]); - } return 0; } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 3337c8ee7928..5522e9d09c8a 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -4720,6 +4720,9 @@ again: ret = -ENOMEM; goto out; } + ret = set_page_extent_mapped(page); + if (ret < 0) + goto out_unlock; if (!PageUptodate(page)) { ret = btrfs_readpage(NULL, page); @@ -4737,7 +4740,6 @@ again: wait_on_page_writeback(page); lock_extent_bits(io_tree, block_start, block_end, &cached_state); - set_page_extent_mapped(page); ordered = btrfs_lookup_ordered_extent(inode, block_start); if (ordered) { @@ -8125,7 +8127,7 @@ static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags) { int ret = try_release_extent_mapping(page, gfp_flags); if (ret == 1) - detach_page_private(page); + clear_page_extent_mapped(page); return ret; } @@ -8285,7 +8287,7 @@ again: } ClearPageChecked(page); - detach_page_private(page); + clear_page_extent_mapped(page); } /* @@ -8364,7 +8366,12 @@ again: wait_on_page_writeback(page); lock_extent_bits(io_tree, page_start, page_end, &cached_state); - set_page_extent_mapped(page); + ret2 = set_page_extent_mapped(page); + if (ret2 < 0) { + ret = vmf_error(ret2); + unlock_extent_cached(io_tree, page_start, page_end, &cached_state); + goto out_unlock; + } /* * we can't set the delalloc bits if there are pending ordered diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 7f2935ea8d3a..e6a63f652235 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -1314,6 +1314,13 @@ again: if (!page) break; + ret = set_page_extent_mapped(page); + if (ret < 0) { + unlock_page(page); + put_page(page); + break; + } + page_start = page_offset(page); page_end = page_start + PAGE_SIZE - 1; while (1) { @@ -1435,7 +1442,6 @@ again: for (i = 0; i < i_done; i++) { clear_page_dirty_for_io(pages[i]); ClearPageChecked(pages[i]); - set_page_extent_mapped(pages[i]); set_page_dirty(pages[i]); unlock_page(pages[i]); put_page(pages[i]); diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index b03e7891394e..b24396cf2f99 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -81,7 +81,10 @@ static int copy_inline_to_page(struct btrfs_inode *inode, goto out_unlock; } - set_page_extent_mapped(page); + ret = set_page_extent_mapped(page); + if (ret < 0) + goto out_unlock; + clear_extent_bit(&inode->io_tree, file_offset, range_end, EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 0, 0, NULL); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index d29baf3822a7..473b78874844 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -2679,6 +2679,15 @@ static int relocate_file_extent_cluster(struct inode *inode, goto out; } } + ret = set_page_extent_mapped(page); + if (ret < 0) { + btrfs_delalloc_release_metadata(BTRFS_I(inode), + PAGE_SIZE, true); + btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE); + unlock_page(page); + put_page(page); + goto out; + } if (PageReadahead(page)) { page_cache_async_readahead(inode->i_mapping, @@ -2706,8 +2715,6 @@ static int relocate_file_extent_cluster(struct inode *inode, lock_extent(&BTRFS_I(inode)->io_tree, page_start, page_end); - set_page_extent_mapped(page); - if (nr < cluster->nr && page_start + offset == cluster->boundary[nr]) { set_extent_bits(&BTRFS_I(inode)->io_tree, -- cgit v1.2.3 From 92082d40976ed0a421305e2264bde53944805627 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 2 Feb 2021 10:28:36 +0800 Subject: btrfs: integrate page status update for data read path into begin/end_page_read In btrfs data page read path, the page status update are handled in two different locations: btrfs_do_read_page() { while (cur <= end) { /* No need to read from disk */ if (HOLE/PREALLOC/INLINE){ memset(); set_extent_uptodate(); continue; } /* Read from disk */ ret = submit_extent_page(end_bio_extent_readpage); } end_bio_extent_readpage() { endio_readpage_uptodate_page_status(); } This is fine for sectorsize == PAGE_SIZE case, as for above loop we should only hit one branch and then exit. But for subpage, there is more work to be done in page status update: - Page Unlock condition Unlike regular page size == sectorsize case, we can no longer just unlock a page. Only the last reader of the page can unlock the page. This means, we can unlock the page either in the while() loop, or in the endio function. - Page uptodate condition Since we have multiple sectors to read for a page, we can only mark the full page uptodate if all sectors are uptodate. To handle both subpage and regular cases, introduce a pair of functions to help handling page status update: - begin_page_read() For regular case, it does nothing. For subpage case, it updates the reader counters so that later end_page_read() can know who is the last one to unlock the page. - end_page_read() This is just endio_readpage_uptodate_page_status() renamed. The original name is a little too long and too specific for endio. The new thing added is the condition for page unlock. Now for subpage data, we unlock the page if we're the last reader. This does not only provide the basis for subpage data read, but also hide the special handling of page read from the main read loop. Also, since we're changing how the page lock is handled, there are two existing error paths where we need to manually unlock the page before calling begin_page_read(). Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 42 ++++++++++++++++++++++++++++------------- fs/btrfs/subpage.c | 53 ++++++++++++++++++++++++++++++++++++++++++---------- fs/btrfs/subpage.h | 8 ++++++++ 3 files changed, 80 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 5ff85ecc6c3f..40d3bca6aaa4 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2840,8 +2840,17 @@ update: processed->uptodate = uptodate; } -static void endio_readpage_update_page_status(struct page *page, bool uptodate, - u64 start, u32 len) +static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page) +{ + ASSERT(PageLocked(page)); + if (fs_info->sectorsize == PAGE_SIZE) + return; + + ASSERT(PagePrivate(page)); + btrfs_subpage_start_reader(fs_info, page, page_offset(page), PAGE_SIZE); +} + +static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len) { struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); @@ -2857,7 +2866,12 @@ static void endio_readpage_update_page_status(struct page *page, bool uptodate, if (fs_info->sectorsize == PAGE_SIZE) unlock_page(page); - /* Subpage locking will be handled in later patches */ + else if (is_data_inode(page->mapping->host)) + /* + * For subpage data, unlock the page if we're the last reader. + * For subpage metadata, page lock is not utilized for read. + */ + btrfs_subpage_end_reader(fs_info, page, start, len); } /* @@ -2994,7 +3008,7 @@ readpage_ok: bio_offset += len; /* Update page status and unlock */ - endio_readpage_update_page_status(page, uptodate, start, len); + end_page_read(page, uptodate, start, len); endio_readpage_release_extent(&processed, BTRFS_I(inode), start, end, uptodate); } @@ -3263,6 +3277,7 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, unsigned int read_flags, u64 *prev_em_start) { struct inode *inode = page->mapping->host; + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); u64 start = page_offset(page); const u64 end = start + PAGE_SIZE - 1; u64 cur = start; @@ -3282,7 +3297,8 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, ret = set_page_extent_mapped(page); if (ret < 0) { unlock_extent(tree, start, end); - SetPageError(page); + btrfs_page_set_error(fs_info, page, start, PAGE_SIZE); + unlock_page(page); goto out; } @@ -3290,6 +3306,7 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, if (cleancache_get_page(page) == 0) { BUG_ON(blocksize != PAGE_SIZE); unlock_extent(tree, start, end); + unlock_page(page); goto out; } } @@ -3306,6 +3323,7 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, kunmap_atomic(userpage); } } + begin_page_read(fs_info, page); while (cur <= end) { bool force_bio_submit = false; u64 disk_bytenr; @@ -3323,13 +3341,14 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, &cached, GFP_NOFS); unlock_extent_cached(tree, cur, cur + iosize - 1, &cached); + end_page_read(page, true, cur, iosize); break; } em = __get_extent_map(inode, page, pg_offset, cur, end - cur + 1, em_cached); if (IS_ERR_OR_NULL(em)) { - SetPageError(page); unlock_extent(tree, cur, end); + end_page_read(page, false, cur, end + 1 - cur); break; } extent_offset = cur - em->start; @@ -3412,6 +3431,7 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, &cached, GFP_NOFS); unlock_extent_cached(tree, cur, cur + iosize - 1, &cached); + end_page_read(page, true, cur, iosize); cur = cur + iosize; pg_offset += iosize; continue; @@ -3421,6 +3441,7 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, EXTENT_UPTODATE, 1, NULL)) { check_page_uptodate(tree, page); unlock_extent(tree, cur, cur + iosize - 1); + end_page_read(page, true, cur, iosize); cur = cur + iosize; pg_offset += iosize; continue; @@ -3429,8 +3450,8 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, * to date. Error out */ if (block_start == EXTENT_MAP_INLINE) { - SetPageError(page); unlock_extent(tree, cur, cur + iosize - 1); + end_page_read(page, false, cur, iosize); cur = cur + iosize; pg_offset += iosize; continue; @@ -3447,19 +3468,14 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, nr++; *bio_flags = this_bio_flag; } else { - SetPageError(page); unlock_extent(tree, cur, cur + iosize - 1); + end_page_read(page, false, cur, iosize); goto out; } cur = cur + iosize; pg_offset += iosize; } out: - if (!nr) { - if (!PageError(page)) - SetPageUptodate(page); - unlock_page(page); - } return ret; } diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c index 2c51ab71e000..c69049e7daa9 100644 --- a/fs/btrfs/subpage.c +++ b/fs/btrfs/subpage.c @@ -54,6 +54,8 @@ int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, spin_lock_init(&(*ret)->lock); if (type == BTRFS_SUBPAGE_METADATA) atomic_set(&(*ret)->eb_refs, 0); + else + atomic_set(&(*ret)->readers, 0); return 0; } @@ -102,22 +104,13 @@ void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, atomic_dec(&subpage->eb_refs); } -/* - * Convert the [start, start + len) range into a u16 bitmap - * - * For example: if start == page_offset() + 16K, len = 16K, we get 0x00f0. - */ -static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info, +static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info, struct page *page, u64 start, u32 len) { - const int bit_start = offset_in_page(start) >> fs_info->sectorsize_bits; - const int nbits = len >> fs_info->sectorsize_bits; - /* Basic checks */ ASSERT(PagePrivate(page) && page->private); ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && IS_ALIGNED(len, fs_info->sectorsize)); - /* * The range check only works for mapped page, we can still have * unmapped page like dummy extent buffer pages. @@ -125,6 +118,46 @@ static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info, if (page->mapping) ASSERT(page_offset(page) <= start && start + len <= page_offset(page) + PAGE_SIZE); +} + +void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + const int nbits = len >> fs_info->sectorsize_bits; + int ret; + + btrfs_subpage_assert(fs_info, page, start, len); + + ret = atomic_add_return(nbits, &subpage->readers); + ASSERT(ret == nbits); +} + +void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + const int nbits = len >> fs_info->sectorsize_bits; + + btrfs_subpage_assert(fs_info, page, start, len); + ASSERT(atomic_read(&subpage->readers) >= nbits); + if (atomic_sub_and_test(nbits, &subpage->readers)) + unlock_page(page); +} + +/* + * Convert the [start, start + len) range into a u16 bitmap + * + * For example: if start == page_offset() + 16K, len = 16K, we get 0x00f0. + */ +static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + const int bit_start = offset_in_page(start) >> fs_info->sectorsize_bits; + const int nbits = len >> fs_info->sectorsize_bits; + + btrfs_subpage_assert(fs_info, page, start, len); + /* * Here nbits can be 16, thus can go beyond u16 range. We make the * first left shift to be calculate in unsigned long (at least u32), diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index f3c5def313a1..b86a4881475d 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -29,6 +29,9 @@ struct btrfs_subpage { */ atomic_t eb_refs; /* Structures only used by data */ + struct { + atomic_t readers; + }; }; }; @@ -53,6 +56,11 @@ void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info, void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct page *page); +void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len); +void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len); + /* * Template for subpage related operations. * -- cgit v1.2.3 From 0bb3eb3ee8674d5d20ad3c0c0767e18787bbd761 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 26 Jan 2021 16:34:02 +0800 Subject: btrfs: allow read-only mount of 4K sector size fs on 64K page system This adds the basic RO mount ability for 4K sector size on 64K page system. Currently we only plan to support 4K and 64K page system. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 25 ++++++++++++++++++++++--- fs/btrfs/super.c | 7 +++++++ 2 files changed, 29 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index d34c6d61928f..71fab77873a5 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -2483,13 +2483,21 @@ static int validate_super(struct btrfs_fs_info *fs_info, btrfs_err(fs_info, "invalid sectorsize %llu", sectorsize); ret = -EINVAL; } - /* Only PAGE SIZE is supported yet */ - if (sectorsize != PAGE_SIZE) { + + /* + * For 4K page size, we only support 4K sector size. + * For 64K page size, we support read-write for 64K sector size, and + * read-only for 4K sector size. + */ + if ((PAGE_SIZE == SZ_4K && sectorsize != PAGE_SIZE) || + (PAGE_SIZE == SZ_64K && (sectorsize != SZ_4K && + sectorsize != SZ_64K))) { btrfs_err(fs_info, - "sectorsize %llu not supported yet, only support %lu", + "sectorsize %llu not yet supported for page size %lu", sectorsize, PAGE_SIZE); ret = -EINVAL; } + if (!is_power_of_2(nodesize) || nodesize < sectorsize || nodesize > BTRFS_MAX_METADATA_BLOCKSIZE) { btrfs_err(fs_info, "invalid nodesize %llu", nodesize); @@ -3248,6 +3256,17 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device goto fail_alloc; } + /* For 4K sector size support, it's only read-only */ + if (PAGE_SIZE == SZ_64K && sectorsize == SZ_4K) { + if (!sb_rdonly(sb) || btrfs_super_log_root(disk_super)) { + btrfs_err(fs_info, + "subpage sectorsize %u only supported read-only for page size %lu", + sectorsize, PAGE_SIZE); + err = -EINVAL; + goto fail_alloc; + } + } + ret = btrfs_init_workqueues(fs_info, fs_devices); if (ret) { err = ret; diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 919ed5c357e9..f8435641b912 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -2027,6 +2027,13 @@ static int btrfs_remount(struct super_block *sb, int *flags, char *data) ret = -EINVAL; goto restore; } + if (fs_info->sectorsize < PAGE_SIZE) { + btrfs_warn(fs_info, + "read-write mount is not yet allowed for sectorsize %u page size %lu", + fs_info->sectorsize, PAGE_SIZE); + ret = -EINVAL; + goto restore; + } /* * NOTE: when remounting with a change that does writes, don't -- cgit v1.2.3 From 2c4d8cb737b805ca8d890e50c23f2b5eca270733 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 28 Jan 2021 19:25:08 +0800 Subject: btrfs: explain page locking and readahead in read_extent_buffer_pages() In read_extent_buffer_pages(), if we failed to lock the page atomically, we just exit with return value 0. This is counter-intuitive, as normally if we can't lock what we need, we would return something like EAGAIN. But that return hides under (wait == WAIT_NONE) branch, which only gets triggered for readahead. And for readahead, if we failed to lock the page, it means the extent buffer is either being read by other thread, or has been read and is under modification. Either way the eb will or has been cached, thus readahead has no need to wait for it. Add comment on this counter-intuitive behavior. Reported-by: Dan Carpenter Reviewed-by: Filipe Manana Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 40d3bca6aaa4..4be117adda33 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5853,6 +5853,13 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) for (i = 0; i < num_pages; i++) { page = eb->pages[i]; if (wait == WAIT_NONE) { + /* + * WAIT_NONE is only utilized by readahead. If we can't + * acquire the lock atomically it means either the eb + * is being read out or under modification. + * Either way the eb will be or has been cached, + * readahead can exit safely. + */ if (!trylock_page(page)) goto unlock_exit; } else { -- cgit v1.2.3 From 72c9925f87c8b74f36f8e75a4cd93d964538d3ca Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Thu, 4 Feb 2021 14:35:44 +0000 Subject: btrfs: fix extent buffer leak on failure to copy root At btrfs_copy_root(), if the call to btrfs_inc_ref() fails we end up returning without unlocking and releasing our reference on the extent buffer named "cow" we previously allocated with btrfs_alloc_tree_block(). So fix that by unlocking the extent buffer and dropping our reference on it before returning. Fixes: be20aa9dbadc8c ("Btrfs: Add mount option to turn off data cow") CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 95d9bae764ab..d56730a67885 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -222,6 +222,8 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans, else ret = btrfs_inc_ref(trans, root, cow, 0); if (ret) { + btrfs_tree_unlock(cow); + free_extent_buffer(cow); btrfs_abort_transaction(trans, ret); return ret; } -- cgit v1.2.3 From 3258386aba670e3406a499d2d0b7395e14c8d097 Mon Sep 17 00:00:00 2001 From: Eric Whitney Date: Wed, 13 Jan 2021 17:14:03 -0500 Subject: ext4: reset retry counter when ext4_alloc_file_blocks() makes progress Change the retry policy in ext4_alloc_file_blocks() to allow for a full retry cycle whenever a portion of an allocation request has been fulfilled. A large allocation request often results in multiple calls to ext4_map_blocks(), each of which is potentially subject to a temporary ENOSPC condition and retry cycle. The current code only allows for a single retry cycle. This patch does not address a known bug or reported complaint. However, it should make block allocation for fallocate and zero range more robust. In addition, simplify the conditional controlling the allocation while loop, where testing len alone is sufficient. Remove the assignment to ret2 in the error path after the call to ext4_map_blocks() since its value isn't subsequently used. Signed-off-by: Eric Whitney Link: https://lore.kernel.org/r/20210113221403.18258-1-enwlinux@gmail.com Signed-off-by: Theodore Ts'o --- fs/ext4/extents.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 3960b7ec3ab7..77c7c8a54da7 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -4382,8 +4382,7 @@ static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset, { struct inode *inode = file_inode(file); handle_t *handle; - int ret = 0; - int ret2 = 0, ret3 = 0; + int ret, ret2 = 0, ret3 = 0; int retries = 0; int depth = 0; struct ext4_map_blocks map; @@ -4408,7 +4407,7 @@ static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset, depth = ext_depth(inode); retry: - while (ret >= 0 && len) { + while (len) { /* * Recalculate credits when extent tree depth changes. */ @@ -4430,9 +4429,13 @@ retry: inode->i_ino, map.m_lblk, map.m_len, ret); ext4_mark_inode_dirty(handle, inode); - ret2 = ext4_journal_stop(handle); + ext4_journal_stop(handle); break; } + /* + * allow a full retry cycle for any remaining allocations + */ + retries = 0; map.m_lblk += ret; map.m_len = len = len - ret; epos = (loff_t)map.m_lblk << inode->i_blkbits; @@ -4450,11 +4453,8 @@ retry: if (unlikely(ret2)) break; } - if (ret == -ENOSPC && - ext4_should_retry_alloc(inode->i_sb, &retries)) { - ret = 0; + if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) goto retry; - } return ret > 0 ? ret2 : ret; } -- cgit v1.2.3 From c3b0e880bbfafab6beed92b1ee6db2cdaf4bc54c Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:41 +0900 Subject: iomap: support REQ_OP_ZONE_APPEND A ZONE_APPEND bio must follow hardware restrictions (e.g. not exceeding max_zone_append_sectors) not to be split. bio_iov_iter_get_pages builds such restricted bio using __bio_iov_append_get_pages if bio_op(bio) == REQ_OP_ZONE_APPEND. To utilize it, we need to set the bio_op before calling bio_iov_iter_get_pages(). This commit introduces IOMAP_F_ZONE_APPEND, so that iomap user can set the flag to indicate they want REQ_OP_ZONE_APPEND and restricted bio. Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/iomap/direct-io.c | 43 +++++++++++++++++++++++++++++++++++++------ include/linux/iomap.h | 1 + 2 files changed, 38 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 933f234d5bec..2273120d8ed7 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -201,6 +201,34 @@ iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos, iomap_dio_submit_bio(dio, iomap, bio, pos); } +/* + * Figure out the bio's operation flags from the dio request, the + * mapping, and whether or not we want FUA. Note that we can end up + * clearing the WRITE_FUA flag in the dio request. + */ +static inline unsigned int +iomap_dio_bio_opflags(struct iomap_dio *dio, struct iomap *iomap, bool use_fua) +{ + unsigned int opflags = REQ_SYNC | REQ_IDLE; + + if (!(dio->flags & IOMAP_DIO_WRITE)) { + WARN_ON_ONCE(iomap->flags & IOMAP_F_ZONE_APPEND); + return REQ_OP_READ; + } + + if (iomap->flags & IOMAP_F_ZONE_APPEND) + opflags |= REQ_OP_ZONE_APPEND; + else + opflags |= REQ_OP_WRITE; + + if (use_fua) + opflags |= REQ_FUA; + else + dio->flags &= ~IOMAP_DIO_WRITE_FUA; + + return opflags; +} + static loff_t iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, struct iomap_dio *dio, struct iomap *iomap) @@ -208,6 +236,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, unsigned int blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev)); unsigned int fs_block_size = i_blocksize(inode), pad; unsigned int align = iov_iter_alignment(dio->submit.iter); + unsigned int bio_opf; struct bio *bio; bool need_zeroout = false; bool use_fua = false; @@ -263,6 +292,13 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, iomap_dio_zero(dio, iomap, pos - pad, pad); } + /* + * Set the operation flags early so that bio_iov_iter_get_pages + * can set up the page vector appropriately for a ZONE_APPEND + * operation. + */ + bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua); + do { size_t n; if (dio->error) { @@ -278,6 +314,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, bio->bi_ioprio = dio->iocb->ki_ioprio; bio->bi_private = dio; bio->bi_end_io = iomap_dio_bio_end_io; + bio->bi_opf = bio_opf; ret = bio_iov_iter_get_pages(bio, dio->submit.iter); if (unlikely(ret)) { @@ -293,14 +330,8 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, n = bio->bi_iter.bi_size; if (dio->flags & IOMAP_DIO_WRITE) { - bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; - if (use_fua) - bio->bi_opf |= REQ_FUA; - else - dio->flags &= ~IOMAP_DIO_WRITE_FUA; task_io_account_write(n); } else { - bio->bi_opf = REQ_OP_READ; if (dio->flags & IOMAP_DIO_DIRTY) bio_set_pages_dirty(bio); } diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 5bd3cac4df9c..8ebb1fa6f3b7 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -55,6 +55,7 @@ struct vm_fault; #define IOMAP_F_SHARED 0x04 #define IOMAP_F_MERGED 0x08 #define IOMAP_F_BUFFER_HEAD 0x10 +#define IOMAP_F_ZONE_APPEND 0x20 /* * Flags set by the core iomap code during operations: -- cgit v1.2.3 From 7365104236ade0bf22edd7724c8fd438b0342ee4 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:42 +0900 Subject: btrfs: zoned: defer loading zone info after opening trees This is a preparation patch to implement zone emulation on a regular device. To emulate a zoned filesystem on a regular (non-zoned) device, we need to decide an emulated zone size. Instead of making it a compile-time static value, we'll make it configurable at mkfs time. Since we have one zone == one device extent restriction, we can determine the emulated zone size from the size of a device extent. We can extend btrfs_get_dev_zone_info() to show a regular device filled with conventional zones once the zone size is decided. The current call site of btrfs_get_dev_zone_info() during the mount process is earlier than loading the file system trees so that we don't know the size of a device extent at this point. Thus we can't slice a regular device to conventional zones. This patch introduces btrfs_get_dev_zone_info_all_devices to load the zone info for all the devices. And, it places this function in open_ctree() after loading the trees. Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 13 +++++++++++++ fs/btrfs/volumes.c | 4 ---- fs/btrfs/zoned.c | 25 +++++++++++++++++++++++++ fs/btrfs/zoned.h | 6 ++++++ 4 files changed, 44 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 71fab77873a5..2b6a3df765cd 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -3333,6 +3333,19 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device if (ret) goto fail_tree_roots; + /* + * Get zone type information of zoned block devices. This will also + * handle emulation of a zoned filesystem if a regular device has the + * zoned incompat feature flag set. + */ + ret = btrfs_get_dev_zone_info_all_devices(fs_info); + if (ret) { + btrfs_err(fs_info, + "zoned: failed to read device zone info: %d", + ret); + goto fail_block_groups; + } + /* * If we have a uuid root and we're not being told to rescan we need to * check the generation here so we can set the diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 3948f5b50d11..07cd4742c123 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -669,10 +669,6 @@ static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices, clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); device->mode = flags; - ret = btrfs_get_dev_zone_info(device); - if (ret != 0) - goto error_free_page; - fs_devices->open_devices++; if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && device->devid != BTRFS_DEV_REPLACE_DEVID) { diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 41d27fefd306..0b1b1f38a196 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -143,6 +143,31 @@ static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos, return 0; } +int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + int ret = 0; + + /* fs_info->zone_size might not set yet. Use the incomapt flag here. */ + if (!btrfs_fs_incompat(fs_info, ZONED)) + return 0; + + mutex_lock(&fs_devices->device_list_mutex); + list_for_each_entry(device, &fs_devices->devices, dev_list) { + /* We can skip reading of zone info for missing devices */ + if (!device->bdev) + continue; + + ret = btrfs_get_dev_zone_info(device); + if (ret) + break; + } + mutex_unlock(&fs_devices->device_list_mutex); + + return ret; +} + int btrfs_get_dev_zone_info(struct btrfs_device *device) { struct btrfs_zoned_device_info *zone_info = NULL; diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 8abe2f83272b..eb47b7ad9ab1 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -25,6 +25,7 @@ struct btrfs_zoned_device_info { #ifdef CONFIG_BLK_DEV_ZONED int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone); +int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info); int btrfs_get_dev_zone_info(struct btrfs_device *device); void btrfs_destroy_dev_zone_info(struct btrfs_device *device); int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info); @@ -42,6 +43,11 @@ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, return 0; } +static inline int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) +{ + return 0; +} + static inline int btrfs_get_dev_zone_info(struct btrfs_device *device) { return 0; -- cgit v1.2.3 From d6639b35da2d742f9cbcdf8f49f87f2bde9fd479 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:43 +0900 Subject: btrfs: zoned: use regular super block location on zone emulation A zoned filesystem currently has a superblock at the beginning of the superblock logging zones if the zones are conventional. This difference in superblock position causes a chicken-and-egg problem for filesystems with emulated zones. Since the device is a regular (non-zoned) device, we cannot know if the filesystem is regular or zoned while reading the superblock. But, to load the superblock, we need to see if it is emulated zoned or not. Place the superblocks at the same location as they are on regular filesystem on regular devices to solve the problem. It is possible because it's ensured that all the superblock locations are at an (emulated) conventional zone on regular devices. Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/zoned.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 0b1b1f38a196..8b3868088c5e 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -552,7 +552,13 @@ int btrfs_sb_log_location(struct btrfs_device *device, int mirror, int rw, struct btrfs_zoned_device_info *zinfo = device->zone_info; u32 zone_num; - if (!zinfo) { + /* + * For a zoned filesystem on a non-zoned block device, use the same + * super block locations as regular filesystem. Doing so, the super + * block can always be retrieved and the zoned flag of the volume + * detected from the super block information. + */ + if (!bdev_is_zoned(device->bdev)) { *bytenr_ret = btrfs_sb_offset(mirror); return 0; } -- cgit v1.2.3 From 4afd2fe835a0ff87fb88cba7a7daa881d8e14233 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Thu, 4 Feb 2021 19:21:44 +0900 Subject: btrfs: release path before calling to btrfs_load_block_group_zone_info Since we have no write pointer in conventional zones, we cannot determine the allocation offset from it. Instead, we set the allocation offset after the highest addressed extent. This is done by reading the extent tree in btrfs_load_block_group_zone_info(). However, this function is called from btrfs_read_block_groups(), so the read lock for the tree node could be recursively taken. To avoid this unsafe locking scenario, release the path before reading the extent tree to get the allocation offset. Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 5fa6b3d540f4..b8fbee70a897 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1810,24 +1810,8 @@ static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info) return ret; } -static void read_block_group_item(struct btrfs_block_group *cache, - struct btrfs_path *path, - const struct btrfs_key *key) -{ - struct extent_buffer *leaf = path->nodes[0]; - struct btrfs_block_group_item bgi; - int slot = path->slots[0]; - - cache->length = key->offset; - - read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot), - sizeof(bgi)); - cache->used = btrfs_stack_block_group_used(&bgi); - cache->flags = btrfs_stack_block_group_flags(&bgi); -} - static int read_one_block_group(struct btrfs_fs_info *info, - struct btrfs_path *path, + struct btrfs_block_group_item *bgi, const struct btrfs_key *key, int need_clear) { @@ -1842,7 +1826,9 @@ static int read_one_block_group(struct btrfs_fs_info *info, if (!cache) return -ENOMEM; - read_block_group_item(cache, path, key); + cache->length = key->offset; + cache->used = btrfs_stack_block_group_used(bgi); + cache->flags = btrfs_stack_block_group_flags(bgi); set_free_space_tree_thresholds(cache); @@ -2001,19 +1987,29 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) need_clear = 1; while (1) { + struct btrfs_block_group_item bgi; + struct extent_buffer *leaf; + int slot; + ret = find_first_block_group(info, path, &key); if (ret > 0) break; if (ret != 0) goto error; - btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); - ret = read_one_block_group(info, path, &key, need_clear); + leaf = path->nodes[0]; + slot = path->slots[0]; + + read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot), + sizeof(bgi)); + + btrfs_item_key_to_cpu(leaf, &key, slot); + btrfs_release_path(path); + ret = read_one_block_group(info, &bgi, &key, need_clear); if (ret < 0) goto error; key.objectid += key.offset; key.offset = 0; - btrfs_release_path(path); } btrfs_release_path(path); -- cgit v1.2.3 From b53429bad3a3555fdbda190192c6e9dfef8e7787 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Thu, 4 Feb 2021 19:21:45 +0900 Subject: btrfs: zoned: do not load fs_info::zoned from incompat flag Don't set the zoned flag in fs_info as soon as we're encountering the incompat filesystem flag for a zoned filesystem on mount. The zoned flag in fs_info is in a union together with the zone_size, so setting it too early will result in setting an incorrect zone_size as well. Once the correct zone_size is read from the device, we can rely on the zoned flag in fs_info as well to determine if the filesystem is zoned. Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 2 -- fs/btrfs/zoned.c | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 2b6a3df765cd..8551b0fc1b22 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -3201,8 +3201,6 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA) btrfs_info(fs_info, "has skinny extents"); - fs_info->zoned = (features & BTRFS_FEATURE_INCOMPAT_ZONED); - /* * flag our filesystem as having big metadata blocks if * they are bigger than the page size diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 8b3868088c5e..c0840412ccb6 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -432,6 +432,14 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) fs_info->zone_size = zone_size; fs_info->max_zone_append_size = max_zone_append_size; + /* + * Check mount options here, because we might change fs_info->zoned + * from fs_info->zone_size. + */ + ret = btrfs_check_mountopts_zoned(fs_info); + if (ret) + goto out; + btrfs_info(fs_info, "zoned mode enabled with zone size %llu", zone_size); out: return ret; -- cgit v1.2.3 From 1cb3dc3f79153c2d7f9a4438381e1385dff09656 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:46 +0900 Subject: btrfs: zoned: disallow fitrim on zoned filesystems The implementation of fitrim depends on space cache, which is not used and disabled for zoned extent allocator. So the current code does not work with zoned filesystem. In the future, we can implement fitrim for zoned filesystems by enabling space cache (but, only for fitrim) or scanning the extent tree at fitrim time. For now, disallow fitrim on zoned filesystems. Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ioctl.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index e6a63f652235..a8c60d46d19c 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -527,6 +527,14 @@ static noinline int btrfs_ioctl_fitrim(struct btrfs_fs_info *fs_info, if (!capable(CAP_SYS_ADMIN)) return -EPERM; + /* + * btrfs_trim_block_group() depends on space cache, which is not + * available in zoned filesystem. So, disallow fitrim on a zoned + * filesystem for now. + */ + if (btrfs_is_zoned(fs_info)) + return -EOPNOTSUPP; + /* * If the fs is mounted with nologreplay, which requires it to be * mounted in RO mode as well, we can not allow discard on free space -- cgit v1.2.3 From 3c9daa09ccd43f68104634020b364d834c01738c Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Thu, 4 Feb 2021 19:21:47 +0900 Subject: btrfs: zoned: allow zoned filesystems on non-zoned block devices Run a zoned filesystem on non-zoned devices. This is done by "slicing up" the block device into static sized chunks and fake a conventional zone on each of them. The emulated zone size is determined from the size of device extent. This is mainly aimed at testing of zoned filesystems, i.e. the zoned chunk allocator, on regular block devices. Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/zoned.c | 150 +++++++++++++++++++++++++++++++++++++++++++++++++++---- fs/btrfs/zoned.h | 14 ++++-- 2 files changed, 148 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index c0840412ccb6..6699f626a86e 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -119,6 +119,36 @@ static inline u32 sb_zone_number(int shift, int mirror) return 0; } +/* + * Emulate blkdev_report_zones() for a non-zoned device. It slices up the block + * device into static sized chunks and fake a conventional zone on each of + * them. + */ +static int emulate_report_zones(struct btrfs_device *device, u64 pos, + struct blk_zone *zones, unsigned int nr_zones) +{ + const sector_t zone_sectors = device->fs_info->zone_size >> SECTOR_SHIFT; + sector_t bdev_size = bdev_nr_sectors(device->bdev); + unsigned int i; + + pos >>= SECTOR_SHIFT; + for (i = 0; i < nr_zones; i++) { + zones[i].start = i * zone_sectors + pos; + zones[i].len = zone_sectors; + zones[i].capacity = zone_sectors; + zones[i].wp = zones[i].start + zone_sectors; + zones[i].type = BLK_ZONE_TYPE_CONVENTIONAL; + zones[i].cond = BLK_ZONE_COND_NOT_WP; + + if (zones[i].wp >= bdev_size) { + i++; + break; + } + } + + return i; +} + static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos, struct blk_zone *zones, unsigned int *nr_zones) { @@ -127,6 +157,12 @@ static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos, if (!*nr_zones) return 0; + if (!bdev_is_zoned(device->bdev)) { + ret = emulate_report_zones(device, pos, zones, *nr_zones); + *nr_zones = ret; + return 0; + } + ret = blkdev_report_zones(device->bdev, pos >> SECTOR_SHIFT, *nr_zones, copy_zone_info_cb, zones); if (ret < 0) { @@ -143,6 +179,50 @@ static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos, return 0; } +/* The emulated zone size is determined from the size of device extent */ +static int calculate_emulated_zone_size(struct btrfs_fs_info *fs_info) +{ + struct btrfs_path *path; + struct btrfs_root *root = fs_info->dev_root; + struct btrfs_key key; + struct extent_buffer *leaf; + struct btrfs_dev_extent *dext; + int ret = 0; + + key.objectid = 1; + key.type = BTRFS_DEV_EXTENT_KEY; + key.offset = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out; + + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_item(root, path); + if (ret < 0) + goto out; + /* No dev extents at all? Not good */ + if (ret > 0) { + ret = -EUCLEAN; + goto out; + } + } + + leaf = path->nodes[0]; + dext = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_extent); + fs_info->zone_size = btrfs_dev_extent_length(leaf, dext); + ret = 0; + +out: + btrfs_free_path(path); + + return ret; +} + int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) { struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; @@ -170,6 +250,7 @@ int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) int btrfs_get_dev_zone_info(struct btrfs_device *device) { + struct btrfs_fs_info *fs_info = device->fs_info; struct btrfs_zoned_device_info *zone_info = NULL; struct block_device *bdev = device->bdev; struct request_queue *queue = bdev_get_queue(bdev); @@ -178,9 +259,14 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device) struct blk_zone *zones = NULL; unsigned int i, nreported = 0, nr_zones; unsigned int zone_sectors; + char *model, *emulated; int ret; - if (!bdev_is_zoned(bdev)) + /* + * Cannot use btrfs_is_zoned here, since fs_info::zone_size might not + * yet be set. + */ + if (!btrfs_fs_incompat(fs_info, ZONED)) return 0; if (device->zone_info) @@ -190,8 +276,20 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device) if (!zone_info) return -ENOMEM; + if (!bdev_is_zoned(bdev)) { + if (!fs_info->zone_size) { + ret = calculate_emulated_zone_size(fs_info); + if (ret) + goto out; + } + + ASSERT(fs_info->zone_size); + zone_sectors = fs_info->zone_size >> SECTOR_SHIFT; + } else { + zone_sectors = bdev_zone_sectors(bdev); + } + nr_sectors = bdev_nr_sectors(bdev); - zone_sectors = bdev_zone_sectors(bdev); /* Check if it's power of 2 (see is_power_of_2) */ ASSERT(zone_sectors != 0 && (zone_sectors & (zone_sectors - 1)) == 0); zone_info->zone_size = zone_sectors << SECTOR_SHIFT; @@ -297,20 +395,42 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device) device->zone_info = zone_info; - /* device->fs_info is not safe to use for printing messages */ - btrfs_info_in_rcu(NULL, - "host-%s zoned block device %s, %u zones of %llu bytes", - bdev_zoned_model(bdev) == BLK_ZONED_HM ? "managed" : "aware", - rcu_str_deref(device->name), zone_info->nr_zones, - zone_info->zone_size); + switch (bdev_zoned_model(bdev)) { + case BLK_ZONED_HM: + model = "host-managed zoned"; + emulated = ""; + break; + case BLK_ZONED_HA: + model = "host-aware zoned"; + emulated = ""; + break; + case BLK_ZONED_NONE: + model = "regular"; + emulated = "emulated "; + break; + default: + /* Just in case */ + btrfs_err_in_rcu(fs_info, "zoned: unsupported model %d on %s", + bdev_zoned_model(bdev), + rcu_str_deref(device->name)); + ret = -EOPNOTSUPP; + goto out_free_zone_info; + } + + btrfs_info_in_rcu(fs_info, + "%s block device %s, %u %szones of %llu bytes", + model, rcu_str_deref(device->name), zone_info->nr_zones, + emulated, zone_info->zone_size); return 0; out: kfree(zones); +out_free_zone_info: bitmap_free(zone_info->empty_zones); bitmap_free(zone_info->seq_zones); kfree(zone_info); + device->zone_info = NULL; return ret; } @@ -349,7 +469,7 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) u64 nr_devices = 0; u64 zone_size = 0; u64 max_zone_append_size = 0; - const bool incompat_zoned = btrfs_is_zoned(fs_info); + const bool incompat_zoned = btrfs_fs_incompat(fs_info, ZONED); int ret = 0; /* Count zoned devices */ @@ -360,9 +480,17 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) continue; model = bdev_zoned_model(device->bdev); + /* + * A Host-Managed zoned device must be used as a zoned device. + * A Host-Aware zoned device and a non-zoned devices can be + * treated as a zoned device, if ZONED flag is enabled in the + * superblock. + */ if (model == BLK_ZONED_HM || - (model == BLK_ZONED_HA && incompat_zoned)) { - struct btrfs_zoned_device_info *zone_info; + (model == BLK_ZONED_HA && incompat_zoned) || + (model == BLK_ZONED_NONE && incompat_zoned)) { + struct btrfs_zoned_device_info *zone_info = + device->zone_info; zone_info = device->zone_info; zoned_devices++; diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index eb47b7ad9ab1..5e78786bb723 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -142,12 +142,16 @@ static inline void btrfs_dev_clear_zone_empty(struct btrfs_device *device, u64 p static inline bool btrfs_check_device_zone_type(const struct btrfs_fs_info *fs_info, struct block_device *bdev) { - u64 zone_size; - if (btrfs_is_zoned(fs_info)) { - zone_size = bdev_zone_sectors(bdev) << SECTOR_SHIFT; - /* Do not allow non-zoned device */ - return bdev_is_zoned(bdev) && fs_info->zone_size == zone_size; + /* + * We can allow a regular device on a zoned filesystem, because + * we will emulate the zoned capabilities. + */ + if (!bdev_is_zoned(bdev)) + return true; + + return fs_info->zone_size == + (bdev_zone_sectors(bdev) << SECTOR_SHIFT); } /* Do not allow Host Manged zoned device */ -- cgit v1.2.3 From 1cd6121f2a382a840f01f506694b54bf403fddc9 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:48 +0900 Subject: btrfs: zoned: implement zoned chunk allocator Implement a zoned chunk and device extent allocator. One device zone becomes a device extent so that a zone reset affects only this device extent and does not change the state of blocks in the neighbor device extents. To implement the allocator, we need to extend the following functions for a zoned filesystem. - init_alloc_chunk_ctl - dev_extent_search_start - dev_extent_hole_check - decide_stripe_size init_alloc_chunk_ctl_zoned() is mostly the same as regular one. It always set the stripe_size to the zone size and aligns the parameters to the zone size. dev_extent_search_start() only aligns the start offset to zone boundaries. We don't care about the first 1MB like in regular filesystem because we anyway reserve the first two zones for superblock logging. dev_extent_hole_check_zoned() checks if zones in given hole are either conventional or empty sequential zones. Also, it skips zones reserved for superblock logging. With the change to the hole, the new hole may now contain pending extents. So, in this case, loop again to check that. Finally, decide_stripe_size_zoned() should shrink the number of devices instead of stripe size because we need to honor stripe_size == zone_size. Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 171 +++++++++++++++++++++++++++++++++++++++++++++++------ fs/btrfs/volumes.h | 1 + fs/btrfs/zoned.c | 141 +++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/zoned.h | 25 ++++++++ 4 files changed, 321 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 07cd4742c123..ae2aeadad5a0 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1414,11 +1414,62 @@ static u64 dev_extent_search_start(struct btrfs_device *device, u64 start) * make sure to start at an offset of at least 1MB. */ return max_t(u64, start, SZ_1M); + case BTRFS_CHUNK_ALLOC_ZONED: + /* + * We don't care about the starting region like regular + * allocator, because we anyway use/reserve the first two zones + * for superblock logging. + */ + return ALIGN(start, device->zone_info->zone_size); default: BUG(); } } +static bool dev_extent_hole_check_zoned(struct btrfs_device *device, + u64 *hole_start, u64 *hole_size, + u64 num_bytes) +{ + u64 zone_size = device->zone_info->zone_size; + u64 pos; + int ret; + bool changed = false; + + ASSERT(IS_ALIGNED(*hole_start, zone_size)); + + while (*hole_size > 0) { + pos = btrfs_find_allocatable_zones(device, *hole_start, + *hole_start + *hole_size, + num_bytes); + if (pos != *hole_start) { + *hole_size = *hole_start + *hole_size - pos; + *hole_start = pos; + changed = true; + if (*hole_size < num_bytes) + break; + } + + ret = btrfs_ensure_empty_zones(device, pos, num_bytes); + + /* Range is ensured to be empty */ + if (!ret) + return changed; + + /* Given hole range was invalid (outside of device) */ + if (ret == -ERANGE) { + *hole_start += *hole_size; + *hole_size = 0; + return 1; + } + + *hole_start += zone_size; + *hole_size -= zone_size; + changed = true; + } + + return changed; +} + /** * dev_extent_hole_check - check if specified hole is suitable for allocation * @device: the device which we have the hole @@ -1426,7 +1477,7 @@ static u64 dev_extent_search_start(struct btrfs_device *device, u64 start) * @hole_size: the size of the hole * @num_bytes: the size of the free space that we need * - * This function may modify @hole_start and @hole_end to reflect the suitable + * This function may modify @hole_start and @hole_size to reflect the suitable * position for allocation. Returns 1 if hole position is updated, 0 otherwise. */ static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start, @@ -1435,24 +1486,39 @@ static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start, bool changed = false; u64 hole_end = *hole_start + *hole_size; - /* - * Check before we set max_hole_start, otherwise we could end up - * sending back this offset anyway. - */ - if (contains_pending_extent(device, hole_start, *hole_size)) { - if (hole_end >= *hole_start) - *hole_size = hole_end - *hole_start; - else - *hole_size = 0; - changed = true; - } + for (;;) { + /* + * Check before we set max_hole_start, otherwise we could end up + * sending back this offset anyway. + */ + if (contains_pending_extent(device, hole_start, *hole_size)) { + if (hole_end >= *hole_start) + *hole_size = hole_end - *hole_start; + else + *hole_size = 0; + changed = true; + } + + switch (device->fs_devices->chunk_alloc_policy) { + case BTRFS_CHUNK_ALLOC_REGULAR: + /* No extra check */ + break; + case BTRFS_CHUNK_ALLOC_ZONED: + if (dev_extent_hole_check_zoned(device, hole_start, + hole_size, num_bytes)) { + changed = true; + /* + * The changed hole can contain pending extent. + * Loop again to check that. + */ + continue; + } + break; + default: + BUG(); + } - switch (device->fs_devices->chunk_alloc_policy) { - case BTRFS_CHUNK_ALLOC_REGULAR: - /* No extra check */ break; - default: - BUG(); } return changed; @@ -1505,6 +1571,9 @@ static int find_free_dev_extent_start(struct btrfs_device *device, search_start = dev_extent_search_start(device, search_start); + WARN_ON(device->zone_info && + !IS_ALIGNED(num_bytes, device->zone_info->zone_size)); + path = btrfs_alloc_path(); if (!path) return -ENOMEM; @@ -4899,6 +4968,37 @@ static void init_alloc_chunk_ctl_policy_regular( ctl->dev_extent_min = BTRFS_STRIPE_LEN * ctl->dev_stripes; } +static void init_alloc_chunk_ctl_policy_zoned( + struct btrfs_fs_devices *fs_devices, + struct alloc_chunk_ctl *ctl) +{ + u64 zone_size = fs_devices->fs_info->zone_size; + u64 limit; + int min_num_stripes = ctl->devs_min * ctl->dev_stripes; + int min_data_stripes = (min_num_stripes - ctl->nparity) / ctl->ncopies; + u64 min_chunk_size = min_data_stripes * zone_size; + u64 type = ctl->type; + + ctl->max_stripe_size = zone_size; + if (type & BTRFS_BLOCK_GROUP_DATA) { + ctl->max_chunk_size = round_down(BTRFS_MAX_DATA_CHUNK_SIZE, + zone_size); + } else if (type & BTRFS_BLOCK_GROUP_METADATA) { + ctl->max_chunk_size = ctl->max_stripe_size; + } else if (type & BTRFS_BLOCK_GROUP_SYSTEM) { + ctl->max_chunk_size = 2 * ctl->max_stripe_size; + ctl->devs_max = min_t(int, ctl->devs_max, + BTRFS_MAX_DEVS_SYS_CHUNK); + } + + /* We don't want a chunk larger than 10% of writable space */ + limit = max(round_down(div_factor(fs_devices->total_rw_bytes, 1), + zone_size), + min_chunk_size); + ctl->max_chunk_size = min(limit, ctl->max_chunk_size); + ctl->dev_extent_min = zone_size * ctl->dev_stripes; +} + static void init_alloc_chunk_ctl(struct btrfs_fs_devices *fs_devices, struct alloc_chunk_ctl *ctl) { @@ -4919,6 +5019,9 @@ static void init_alloc_chunk_ctl(struct btrfs_fs_devices *fs_devices, case BTRFS_CHUNK_ALLOC_REGULAR: init_alloc_chunk_ctl_policy_regular(fs_devices, ctl); break; + case BTRFS_CHUNK_ALLOC_ZONED: + init_alloc_chunk_ctl_policy_zoned(fs_devices, ctl); + break; default: BUG(); } @@ -5045,6 +5148,38 @@ static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl, return 0; } +static int decide_stripe_size_zoned(struct alloc_chunk_ctl *ctl, + struct btrfs_device_info *devices_info) +{ + u64 zone_size = devices_info[0].dev->zone_info->zone_size; + /* Number of stripes that count for block group size */ + int data_stripes; + + /* + * It should hold because: + * dev_extent_min == dev_extent_want == zone_size * dev_stripes + */ + ASSERT(devices_info[ctl->ndevs - 1].max_avail == ctl->dev_extent_min); + + ctl->stripe_size = zone_size; + ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; + data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; + + /* stripe_size is fixed in zoned filesysmte. Reduce ndevs instead. */ + if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) { + ctl->ndevs = div_u64(div_u64(ctl->max_chunk_size * ctl->ncopies, + ctl->stripe_size) + ctl->nparity, + ctl->dev_stripes); + ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; + data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; + ASSERT(ctl->stripe_size * data_stripes <= ctl->max_chunk_size); + } + + ctl->chunk_size = ctl->stripe_size * data_stripes; + + return 0; +} + static int decide_stripe_size(struct btrfs_fs_devices *fs_devices, struct alloc_chunk_ctl *ctl, struct btrfs_device_info *devices_info) @@ -5072,6 +5207,8 @@ static int decide_stripe_size(struct btrfs_fs_devices *fs_devices, switch (fs_devices->chunk_alloc_policy) { case BTRFS_CHUNK_ALLOC_REGULAR: return decide_stripe_size_regular(ctl, devices_info); + case BTRFS_CHUNK_ALLOC_ZONED: + return decide_stripe_size_zoned(ctl, devices_info); default: BUG(); } diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 04e2b26823c2..598ac225176d 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -214,6 +214,7 @@ BTRFS_DEVICE_GETSET_FUNCS(bytes_used); enum btrfs_chunk_allocation_policy { BTRFS_CHUNK_ALLOC_REGULAR, + BTRFS_CHUNK_ALLOC_ZONED, }; /* diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 6699f626a86e..69fd0d078b9b 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1,11 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include "ctree.h" #include "volumes.h" #include "zoned.h" #include "rcu-string.h" +#include "disk-io.h" /* Maximum number of zones to report per blkdev_report_zones() call */ #define BTRFS_REPORT_NR_ZONES 4096 @@ -559,6 +561,7 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) fs_info->zone_size = zone_size; fs_info->max_zone_append_size = max_zone_append_size; + fs_info->fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_ZONED; /* * Check mount options here, because we might change fs_info->zoned @@ -779,3 +782,141 @@ int btrfs_reset_sb_log_zones(struct block_device *bdev, int mirror) sb_zone << zone_sectors_shift, zone_sectors * BTRFS_NR_SB_LOG_ZONES, GFP_NOFS); } + +/** + * btrfs_find_allocatable_zones - find allocatable zones within a given region + * + * @device: the device to allocate a region on + * @hole_start: the position of the hole to allocate the region + * @num_bytes: size of wanted region + * @hole_end: the end of the hole + * @return: position of allocatable zones + * + * Allocatable region should not contain any superblock locations. + */ +u64 btrfs_find_allocatable_zones(struct btrfs_device *device, u64 hole_start, + u64 hole_end, u64 num_bytes) +{ + struct btrfs_zoned_device_info *zinfo = device->zone_info; + const u8 shift = zinfo->zone_size_shift; + u64 nzones = num_bytes >> shift; + u64 pos = hole_start; + u64 begin, end; + bool have_sb; + int i; + + ASSERT(IS_ALIGNED(hole_start, zinfo->zone_size)); + ASSERT(IS_ALIGNED(num_bytes, zinfo->zone_size)); + + while (pos < hole_end) { + begin = pos >> shift; + end = begin + nzones; + + if (end > zinfo->nr_zones) + return hole_end; + + /* Check if zones in the region are all empty */ + if (btrfs_dev_is_sequential(device, pos) && + find_next_zero_bit(zinfo->empty_zones, end, begin) != end) { + pos += zinfo->zone_size; + continue; + } + + have_sb = false; + for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { + u32 sb_zone; + u64 sb_pos; + + sb_zone = sb_zone_number(shift, i); + if (!(end <= sb_zone || + sb_zone + BTRFS_NR_SB_LOG_ZONES <= begin)) { + have_sb = true; + pos = ((u64)sb_zone + BTRFS_NR_SB_LOG_ZONES) << shift; + break; + } + + /* We also need to exclude regular superblock positions */ + sb_pos = btrfs_sb_offset(i); + if (!(pos + num_bytes <= sb_pos || + sb_pos + BTRFS_SUPER_INFO_SIZE <= pos)) { + have_sb = true; + pos = ALIGN(sb_pos + BTRFS_SUPER_INFO_SIZE, + zinfo->zone_size); + break; + } + } + if (!have_sb) + break; + } + + return pos; +} + +int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical, + u64 length, u64 *bytes) +{ + int ret; + + *bytes = 0; + ret = blkdev_zone_mgmt(device->bdev, REQ_OP_ZONE_RESET, + physical >> SECTOR_SHIFT, length >> SECTOR_SHIFT, + GFP_NOFS); + if (ret) + return ret; + + *bytes = length; + while (length) { + btrfs_dev_set_zone_empty(device, physical); + physical += device->zone_info->zone_size; + length -= device->zone_info->zone_size; + } + + return 0; +} + +int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size) +{ + struct btrfs_zoned_device_info *zinfo = device->zone_info; + const u8 shift = zinfo->zone_size_shift; + unsigned long begin = start >> shift; + unsigned long end = (start + size) >> shift; + u64 pos; + int ret; + + ASSERT(IS_ALIGNED(start, zinfo->zone_size)); + ASSERT(IS_ALIGNED(size, zinfo->zone_size)); + + if (end > zinfo->nr_zones) + return -ERANGE; + + /* All the zones are conventional */ + if (find_next_bit(zinfo->seq_zones, begin, end) == end) + return 0; + + /* All the zones are sequential and empty */ + if (find_next_zero_bit(zinfo->seq_zones, begin, end) == end && + find_next_zero_bit(zinfo->empty_zones, begin, end) == end) + return 0; + + for (pos = start; pos < start + size; pos += zinfo->zone_size) { + u64 reset_bytes; + + if (!btrfs_dev_is_sequential(device, pos) || + btrfs_dev_is_empty_zone(device, pos)) + continue; + + /* Free regions should be empty */ + btrfs_warn_in_rcu( + device->fs_info, + "zoned: resetting device %s (devid %llu) zone %llu for allocation", + rcu_str_deref(device->name), device->devid, pos >> shift); + WARN_ON_ONCE(1); + + ret = btrfs_reset_device_zone(device, pos, zinfo->zone_size, + &reset_bytes); + if (ret) + return ret; + } + + return 0; +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 5e78786bb723..6c8f83c48c2e 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -36,6 +36,11 @@ int btrfs_sb_log_location(struct btrfs_device *device, int mirror, int rw, u64 *bytenr_ret); void btrfs_advance_sb_log(struct btrfs_device *device, int mirror); int btrfs_reset_sb_log_zones(struct block_device *bdev, int mirror); +u64 btrfs_find_allocatable_zones(struct btrfs_device *device, u64 hole_start, + u64 hole_end, u64 num_bytes); +int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical, + u64 length, u64 *bytes); +int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -91,6 +96,26 @@ static inline int btrfs_reset_sb_log_zones(struct block_device *bdev, int mirror return 0; } +static inline u64 btrfs_find_allocatable_zones(struct btrfs_device *device, + u64 hole_start, u64 hole_end, + u64 num_bytes) +{ + return hole_start; +} + +static inline int btrfs_reset_device_zone(struct btrfs_device *device, + u64 physical, u64 length, u64 *bytes) +{ + *bytes = 0; + return 0; +} + +static inline int btrfs_ensure_empty_zones(struct btrfs_device *device, + u64 start, u64 size) +{ + return 0; +} + #endif static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) -- cgit v1.2.3 From 381a696eb5f99189a2c8d0d99aae766767f9cb1e Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:49 +0900 Subject: btrfs: zoned: verify device extent is aligned to zone Add a check in verify_one_dev_extent() to ensure that a device extent on a zoned block device is aligned to the respective zone boundary. If it isn't, mark the filesystem as unclean. Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index ae2aeadad5a0..10401def16ef 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -7769,6 +7769,20 @@ static int verify_one_dev_extent(struct btrfs_fs_info *fs_info, ret = -EUCLEAN; goto out; } + + if (dev->zone_info) { + u64 zone_size = dev->zone_info->zone_size; + + if (!IS_ALIGNED(physical_offset, zone_size) || + !IS_ALIGNED(physical_len, zone_size)) { + btrfs_err(fs_info, +"zoned: dev extent devid %llu physical offset %llu len %llu is not aligned to device zone", + devid, physical_offset, physical_len); + ret = -EUCLEAN; + goto out; + } + } + out: free_extent_map(em); return ret; -- cgit v1.2.3 From 08e11a3db098f4ba0cfee46d7ab449cba43dea1b Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:50 +0900 Subject: btrfs: zoned: load zone's allocation offset A zoned filesystem must allocate blocks at the zones' write pointer. The device's write pointer position can be mapped to a logical address within a block group. To facilitate this, add an "alloc_offset" to the block-group to track the logical addresses of the write pointer. This logical address is populated in btrfs_load_block_group_zone_info() from the write pointers of corresponding zones. For now, zoned filesystems the single profile. Supporting non-single profile with zone append writing is not trivial. For example, in the DUP profile, we send a zone append writing IO to two zones on a device. The device reply with written LBAs for the IOs. If the offsets of the returned addresses from the beginning of the zone are different, then it results in different logical addresses. We need fine-grained logical to physical mapping to support such separated physical address issue. Since it should require additional metadata type, disable non-single profiles for now. This commit supports the case all the zones in a block group are sequential. The next patch will handle the case having a conventional zone. Reviewed-by: Josef Bacik Reviewed-by: Anand Jain Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 15 +++++ fs/btrfs/block-group.h | 6 ++ fs/btrfs/zoned.c | 151 +++++++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/zoned.h | 7 +++ 4 files changed, 179 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index b8fbee70a897..e6bf728496eb 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -15,6 +15,7 @@ #include "delalloc-space.h" #include "discard.h" #include "raid56.h" +#include "zoned.h" /* * Return target flags in extended format or 0 if restripe for this chunk_type @@ -1855,6 +1856,13 @@ static int read_one_block_group(struct btrfs_fs_info *info, goto error; } + ret = btrfs_load_block_group_zone_info(cache); + if (ret) { + btrfs_err(info, "zoned: failed to load zone info of bg %llu", + cache->start); + goto error; + } + /* * We need to exclude the super stripes now so that the space info has * super bytes accounted for, otherwise we'll think we have more space @@ -2141,6 +2149,13 @@ int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used, cache->cached = BTRFS_CACHE_FINISHED; if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) cache->needs_free_space = 1; + + ret = btrfs_load_block_group_zone_info(cache); + if (ret) { + btrfs_put_block_group(cache); + return ret; + } + ret = exclude_super_stripes(cache); if (ret) { /* We may have excluded something, so call this just in case */ diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 8f74a96074f7..224946fa9bed 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -183,6 +183,12 @@ struct btrfs_block_group { /* Record locked full stripes for RAID5/6 block group */ struct btrfs_full_stripe_locks_tree full_stripe_locks_root; + + /* + * Allocation offset for the block group to implement sequential + * allocation. This is used only on a zoned filesystem. + */ + u64 alloc_offset; }; static inline u64 btrfs_block_group_end(struct btrfs_block_group *block_group) diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 69fd0d078b9b..0a7cd00f405f 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -3,14 +3,20 @@ #include #include #include +#include #include "ctree.h" #include "volumes.h" #include "zoned.h" #include "rcu-string.h" #include "disk-io.h" +#include "block-group.h" /* Maximum number of zones to report per blkdev_report_zones() call */ #define BTRFS_REPORT_NR_ZONES 4096 +/* Invalid allocation pointer value for missing devices */ +#define WP_MISSING_DEV ((u64)-1) +/* Pseudo write pointer value for conventional zone */ +#define WP_CONVENTIONAL ((u64)-2) /* Number of superblock log zones */ #define BTRFS_NR_SB_LOG_ZONES 2 @@ -920,3 +926,148 @@ int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size) return 0; } + +int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct extent_map_tree *em_tree = &fs_info->mapping_tree; + struct extent_map *em; + struct map_lookup *map; + struct btrfs_device *device; + u64 logical = cache->start; + u64 length = cache->length; + u64 physical = 0; + int ret; + int i; + unsigned int nofs_flag; + u64 *alloc_offsets = NULL; + u32 num_sequential = 0, num_conventional = 0; + + if (!btrfs_is_zoned(fs_info)) + return 0; + + /* Sanity check */ + if (!IS_ALIGNED(length, fs_info->zone_size)) { + btrfs_err(fs_info, + "zoned: block group %llu len %llu unaligned to zone size %llu", + logical, length, fs_info->zone_size); + return -EIO; + } + + /* Get the chunk mapping */ + read_lock(&em_tree->lock); + em = lookup_extent_mapping(em_tree, logical, length); + read_unlock(&em_tree->lock); + + if (!em) + return -EINVAL; + + map = em->map_lookup; + + alloc_offsets = kcalloc(map->num_stripes, sizeof(*alloc_offsets), GFP_NOFS); + if (!alloc_offsets) { + free_extent_map(em); + return -ENOMEM; + } + + for (i = 0; i < map->num_stripes; i++) { + bool is_sequential; + struct blk_zone zone; + + device = map->stripes[i].dev; + physical = map->stripes[i].physical; + + if (device->bdev == NULL) { + alloc_offsets[i] = WP_MISSING_DEV; + continue; + } + + is_sequential = btrfs_dev_is_sequential(device, physical); + if (is_sequential) + num_sequential++; + else + num_conventional++; + + if (!is_sequential) { + alloc_offsets[i] = WP_CONVENTIONAL; + continue; + } + + /* + * This zone will be used for allocation, so mark this zone + * non-empty. + */ + btrfs_dev_clear_zone_empty(device, physical); + + /* + * The group is mapped to a sequential zone. Get the zone write + * pointer to determine the allocation offset within the zone. + */ + WARN_ON(!IS_ALIGNED(physical, fs_info->zone_size)); + nofs_flag = memalloc_nofs_save(); + ret = btrfs_get_dev_zone(device, physical, &zone); + memalloc_nofs_restore(nofs_flag); + if (ret == -EIO || ret == -EOPNOTSUPP) { + ret = 0; + alloc_offsets[i] = WP_MISSING_DEV; + continue; + } else if (ret) { + goto out; + } + + switch (zone.cond) { + case BLK_ZONE_COND_OFFLINE: + case BLK_ZONE_COND_READONLY: + btrfs_err(fs_info, + "zoned: offline/readonly zone %llu on device %s (devid %llu)", + physical >> device->zone_info->zone_size_shift, + rcu_str_deref(device->name), device->devid); + alloc_offsets[i] = WP_MISSING_DEV; + break; + case BLK_ZONE_COND_EMPTY: + alloc_offsets[i] = 0; + break; + case BLK_ZONE_COND_FULL: + alloc_offsets[i] = fs_info->zone_size; + break; + default: + /* Partially used zone */ + alloc_offsets[i] = + ((zone.wp - zone.start) << SECTOR_SHIFT); + break; + } + } + + if (num_conventional > 0) { + /* + * Since conventional zones do not have a write pointer, we + * cannot determine alloc_offset from the pointer + */ + ret = -EINVAL; + goto out; + } + + switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { + case 0: /* single */ + cache->alloc_offset = alloc_offsets[0]; + break; + case BTRFS_BLOCK_GROUP_DUP: + case BTRFS_BLOCK_GROUP_RAID1: + case BTRFS_BLOCK_GROUP_RAID0: + case BTRFS_BLOCK_GROUP_RAID10: + case BTRFS_BLOCK_GROUP_RAID5: + case BTRFS_BLOCK_GROUP_RAID6: + /* non-single profiles are not supported yet */ + default: + btrfs_err(fs_info, "zoned: profile %s not yet supported", + btrfs_bg_type_to_raid_name(map->type)); + ret = -EINVAL; + goto out; + } + +out: + kfree(alloc_offsets); + free_extent_map(em); + + return ret; +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 6c8f83c48c2e..4f3152d7b98f 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -41,6 +41,7 @@ u64 btrfs_find_allocatable_zones(struct btrfs_device *device, u64 hole_start, int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical, u64 length, u64 *bytes); int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size); +int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -116,6 +117,12 @@ static inline int btrfs_ensure_empty_zones(struct btrfs_device *device, return 0; } +static inline int btrfs_load_block_group_zone_info( + struct btrfs_block_group *cache) +{ + return 0; +} + #endif static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) -- cgit v1.2.3 From a94794d50d788d4735fd8f656ac8c0510117457d Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:51 +0900 Subject: btrfs: zoned: calculate allocation offset for conventional zones Conventional zones do not have a write pointer, so we cannot use it to determine the allocation offset for sequential allocation if a block group contains a conventional zone. But instead, we can consider the end of the highest addressed extent in the block group for the allocation offset. For new block group, we cannot calculate the allocation offset by consulting the extent tree, because it can cause deadlock by taking extent buffer lock after chunk mutex, which is already taken in btrfs_make_block_group(). Since it is a new block group anyways, we can simply set the allocation offset to 0. Reviewed-by: Josef Bacik Reviewed-by: Anand Jain Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 4 +- fs/btrfs/zoned.c | 99 +++++++++++++++++++++++++++++++++++++++++++++++--- fs/btrfs/zoned.h | 4 +- 3 files changed, 98 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index e6bf728496eb..6d10874189df 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1856,7 +1856,7 @@ static int read_one_block_group(struct btrfs_fs_info *info, goto error; } - ret = btrfs_load_block_group_zone_info(cache); + ret = btrfs_load_block_group_zone_info(cache, false); if (ret) { btrfs_err(info, "zoned: failed to load zone info of bg %llu", cache->start); @@ -2150,7 +2150,7 @@ int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used, if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) cache->needs_free_space = 1; - ret = btrfs_load_block_group_zone_info(cache); + ret = btrfs_load_block_group_zone_info(cache, true); if (ret) { btrfs_put_block_group(cache); return ret; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 0a7cd00f405f..b892566a1c93 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -927,7 +927,68 @@ int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size) return 0; } -int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache) +/* + * Calculate an allocation pointer from the extent allocation information + * for a block group consist of conventional zones. It is pointed to the + * end of the highest addressed extent in the block group as an allocation + * offset. + */ +static int calculate_alloc_pointer(struct btrfs_block_group *cache, + u64 *offset_ret) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_root *root = fs_info->extent_root; + struct btrfs_path *path; + struct btrfs_key key; + struct btrfs_key found_key; + int ret; + u64 length; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = cache->start + cache->length; + key.type = 0; + key.offset = 0; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + /* We should not find the exact match */ + if (!ret) + ret = -EUCLEAN; + if (ret < 0) + goto out; + + ret = btrfs_previous_extent_item(root, path, cache->start); + if (ret) { + if (ret == 1) { + ret = 0; + *offset_ret = 0; + } + goto out; + } + + btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]); + + if (found_key.type == BTRFS_EXTENT_ITEM_KEY) + length = found_key.offset; + else + length = fs_info->nodesize; + + if (!(found_key.objectid >= cache->start && + found_key.objectid + length <= cache->start + cache->length)) { + ret = -EUCLEAN; + goto out; + } + *offset_ret = found_key.objectid + length - cache->start; + ret = 0; + +out: + btrfs_free_path(path); + return ret; +} + +int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) { struct btrfs_fs_info *fs_info = cache->fs_info; struct extent_map_tree *em_tree = &fs_info->mapping_tree; @@ -941,6 +1002,7 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache) int i; unsigned int nofs_flag; u64 *alloc_offsets = NULL; + u64 last_alloc = 0; u32 num_sequential = 0, num_conventional = 0; if (!btrfs_is_zoned(fs_info)) @@ -1040,11 +1102,30 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache) if (num_conventional > 0) { /* - * Since conventional zones do not have a write pointer, we - * cannot determine alloc_offset from the pointer + * Avoid calling calculate_alloc_pointer() for new BG. It + * is no use for new BG. It must be always 0. + * + * Also, we have a lock chain of extent buffer lock -> + * chunk mutex. For new BG, this function is called from + * btrfs_make_block_group() which is already taking the + * chunk mutex. Thus, we cannot call + * calculate_alloc_pointer() which takes extent buffer + * locks to avoid deadlock. */ - ret = -EINVAL; - goto out; + if (new) { + cache->alloc_offset = 0; + goto out; + } + ret = calculate_alloc_pointer(cache, &last_alloc); + if (ret || map->num_stripes == num_conventional) { + if (!ret) + cache->alloc_offset = last_alloc; + else + btrfs_err(fs_info, + "zoned: failed to determine allocation offset of bg %llu", + cache->start); + goto out; + } } switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { @@ -1066,6 +1147,14 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache) } out: + /* An extent is allocated after the write pointer */ + if (!ret && num_conventional && last_alloc > cache->alloc_offset) { + btrfs_err(fs_info, + "zoned: got wrong write pointer in BG %llu: %llu > %llu", + logical, last_alloc, cache->alloc_offset); + ret = -EIO; + } + kfree(alloc_offsets); free_extent_map(em); diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 4f3152d7b98f..d27db3993e51 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -41,7 +41,7 @@ u64 btrfs_find_allocatable_zones(struct btrfs_device *device, u64 hole_start, int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical, u64 length, u64 *bytes); int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size); -int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache); +int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -118,7 +118,7 @@ static inline int btrfs_ensure_empty_zones(struct btrfs_device *device, } static inline int btrfs_load_block_group_zone_info( - struct btrfs_block_group *cache) + struct btrfs_block_group *cache, bool new) { return 0; } -- cgit v1.2.3 From 169e0da91a21a571093feb8ff84c7e9229e64c08 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:52 +0900 Subject: btrfs: zoned: track unusable bytes for zones In a zoned filesystem a once written then freed region is not usable until the underlying zone has been reset. So we need to distinguish such unusable space from usable free space. Therefore we need to introduce the "zone_unusable" field to the block group structure, and "bytes_zone_unusable" to the space_info structure to track the unusable space. Pinned bytes are always reclaimed to the unusable space. But, when an allocated region is returned before using e.g., the block group becomes read-only between allocation time and reservation time, we can safely return the region to the block group. For the situation, this commit introduces "btrfs_add_free_space_unused". This behaves the same as btrfs_add_free_space() on regular filesystem. On zoned filesystems, it rewinds the allocation offset. Because the read-only bytes tracks free but unusable bytes when the block group is read-only, we need to migrate the zone_unusable bytes to read-only bytes when a block group is marked read-only. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 51 ++++++++++++++++++++++++++-------- fs/btrfs/block-group.h | 1 + fs/btrfs/extent-tree.c | 5 ++++ fs/btrfs/free-space-cache.c | 67 +++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/free-space-cache.h | 2 ++ fs/btrfs/space-info.c | 13 +++++---- fs/btrfs/space-info.h | 4 ++- fs/btrfs/sysfs.c | 2 ++ fs/btrfs/zoned.c | 21 ++++++++++++++ fs/btrfs/zoned.h | 3 ++ 10 files changed, 151 insertions(+), 18 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 6d10874189df..e4444d4dd4b5 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1009,12 +1009,17 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, WARN_ON(block_group->space_info->total_bytes < block_group->length); WARN_ON(block_group->space_info->bytes_readonly - < block_group->length); + < block_group->length - block_group->zone_unusable); + WARN_ON(block_group->space_info->bytes_zone_unusable + < block_group->zone_unusable); WARN_ON(block_group->space_info->disk_total < block_group->length * factor); } block_group->space_info->total_bytes -= block_group->length; - block_group->space_info->bytes_readonly -= block_group->length; + block_group->space_info->bytes_readonly -= + (block_group->length - block_group->zone_unusable); + block_group->space_info->bytes_zone_unusable -= + block_group->zone_unusable; block_group->space_info->disk_total -= block_group->length * factor; spin_unlock(&block_group->space_info->lock); @@ -1158,7 +1163,7 @@ static int inc_block_group_ro(struct btrfs_block_group *cache, int force) } num_bytes = cache->length - cache->reserved - cache->pinned - - cache->bytes_super - cache->used; + cache->bytes_super - cache->zone_unusable - cache->used; /* * Data never overcommits, even in mixed mode, so do just the straight @@ -1189,6 +1194,12 @@ static int inc_block_group_ro(struct btrfs_block_group *cache, int force) if (!ret) { sinfo->bytes_readonly += num_bytes; + if (btrfs_is_zoned(cache->fs_info)) { + /* Migrate zone_unusable bytes to readonly */ + sinfo->bytes_readonly += cache->zone_unusable; + sinfo->bytes_zone_unusable -= cache->zone_unusable; + cache->zone_unusable = 0; + } cache->ro++; list_add_tail(&cache->ro_list, &sinfo->ro_bgs); } @@ -1876,12 +1887,20 @@ static int read_one_block_group(struct btrfs_fs_info *info, } /* - * Check for two cases, either we are full, and therefore don't need - * to bother with the caching work since we won't find any space, or we - * are empty, and we can just add all the space in and be done with it. - * This saves us _a_lot_ of time, particularly in the full case. + * For zoned filesystem, space after the allocation offset is the only + * free space for a block group. So, we don't need any caching work. + * btrfs_calc_zone_unusable() will set the amount of free space and + * zone_unusable space. + * + * For regular filesystem, check for two cases, either we are full, and + * therefore don't need to bother with the caching work since we won't + * find any space, or we are empty, and we can just add all the space + * in and be done with it. This saves us _a_lot_ of time, particularly + * in the full case. */ - if (cache->length == cache->used) { + if (btrfs_is_zoned(info)) { + btrfs_calc_zone_unusable(cache); + } else if (cache->length == cache->used) { cache->last_byte_to_unpin = (u64)-1; cache->cached = BTRFS_CACHE_FINISHED; btrfs_free_excluded_extents(cache); @@ -1900,7 +1919,8 @@ static int read_one_block_group(struct btrfs_fs_info *info, } trace_btrfs_add_block_group(info, cache, 0); btrfs_update_space_info(info, cache->flags, cache->length, - cache->used, cache->bytes_super, &space_info); + cache->used, cache->bytes_super, + cache->zone_unusable, &space_info); cache->space_info = space_info; @@ -1956,7 +1976,7 @@ static int fill_dummy_bgs(struct btrfs_fs_info *fs_info) break; } btrfs_update_space_info(fs_info, bg->flags, em->len, em->len, - 0, &space_info); + 0, 0, &space_info); bg->space_info = space_info; link_block_group(bg); @@ -2197,7 +2217,7 @@ int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used, */ trace_btrfs_add_block_group(fs_info, cache, 1); btrfs_update_space_info(fs_info, cache->flags, size, bytes_used, - cache->bytes_super, &cache->space_info); + cache->bytes_super, 0, &cache->space_info); btrfs_update_global_block_rsv(fs_info); link_block_group(cache); @@ -2305,8 +2325,15 @@ void btrfs_dec_block_group_ro(struct btrfs_block_group *cache) spin_lock(&cache->lock); if (!--cache->ro) { num_bytes = cache->length - cache->reserved - - cache->pinned - cache->bytes_super - cache->used; + cache->pinned - cache->bytes_super - + cache->zone_unusable - cache->used; sinfo->bytes_readonly -= num_bytes; + if (btrfs_is_zoned(cache->fs_info)) { + /* Migrate zone_unusable bytes back */ + cache->zone_unusable = cache->alloc_offset - cache->used; + sinfo->bytes_zone_unusable += cache->zone_unusable; + sinfo->bytes_readonly -= cache->zone_unusable; + } list_del_init(&cache->ro_list); } spin_unlock(&cache->lock); diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 224946fa9bed..0fd66febe115 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -189,6 +189,7 @@ struct btrfs_block_group { * allocation. This is used only on a zoned filesystem. */ u64 alloc_offset; + u64 zone_unusable; }; static inline u64 btrfs_block_group_end(struct btrfs_block_group *block_group) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 5476ab84e544..5c61c3f136f7 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -34,6 +34,7 @@ #include "block-group.h" #include "discard.h" #include "rcu-string.h" +#include "zoned.h" #undef SCRAMBLE_DELAYED_REFS @@ -2740,6 +2741,10 @@ static int unpin_extent_range(struct btrfs_fs_info *fs_info, if (cache->ro) { space_info->bytes_readonly += len; readonly = true; + } else if (btrfs_is_zoned(fs_info)) { + /* Need reset before reusing in a zoned block group */ + space_info->bytes_zone_unusable += len; + readonly = true; } spin_unlock(&cache->lock); if (!readonly && return_free_space && diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 6134e10a6e7f..b93ac31eca69 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -2477,6 +2477,8 @@ int __btrfs_add_free_space(struct btrfs_fs_info *fs_info, int ret = 0; u64 filter_bytes = bytes; + ASSERT(!btrfs_is_zoned(fs_info)); + info = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS); if (!info) return -ENOMEM; @@ -2534,11 +2536,49 @@ out: return ret; } +static int __btrfs_add_free_space_zoned(struct btrfs_block_group *block_group, + u64 bytenr, u64 size, bool used) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + u64 offset = bytenr - block_group->start; + u64 to_free, to_unusable; + + spin_lock(&ctl->tree_lock); + if (!used) + to_free = size; + else if (offset >= block_group->alloc_offset) + to_free = size; + else if (offset + size <= block_group->alloc_offset) + to_free = 0; + else + to_free = offset + size - block_group->alloc_offset; + to_unusable = size - to_free; + + ctl->free_space += to_free; + block_group->zone_unusable += to_unusable; + spin_unlock(&ctl->tree_lock); + if (!used) { + spin_lock(&block_group->lock); + block_group->alloc_offset -= size; + spin_unlock(&block_group->lock); + } + + /* All the region is now unusable. Mark it as unused and reclaim */ + if (block_group->zone_unusable == block_group->length) + btrfs_mark_bg_unused(block_group); + + return 0; +} + int btrfs_add_free_space(struct btrfs_block_group *block_group, u64 bytenr, u64 size) { enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + if (btrfs_is_zoned(block_group->fs_info)) + return __btrfs_add_free_space_zoned(block_group, bytenr, size, + true); + if (btrfs_test_opt(block_group->fs_info, DISCARD_SYNC)) trim_state = BTRFS_TRIM_STATE_TRIMMED; @@ -2547,6 +2587,16 @@ int btrfs_add_free_space(struct btrfs_block_group *block_group, bytenr, size, trim_state); } +int btrfs_add_free_space_unused(struct btrfs_block_group *block_group, + u64 bytenr, u64 size) +{ + if (btrfs_is_zoned(block_group->fs_info)) + return __btrfs_add_free_space_zoned(block_group, bytenr, size, + false); + + return btrfs_add_free_space(block_group, bytenr, size); +} + /* * This is a subtle distinction because when adding free space back in general, * we want it to be added as untrimmed for async. But in the case where we add @@ -2557,6 +2607,10 @@ int btrfs_add_free_space_async_trimmed(struct btrfs_block_group *block_group, { enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + if (btrfs_is_zoned(block_group->fs_info)) + return __btrfs_add_free_space_zoned(block_group, bytenr, size, + true); + if (btrfs_test_opt(block_group->fs_info, DISCARD_SYNC) || btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC)) trim_state = BTRFS_TRIM_STATE_TRIMMED; @@ -2574,6 +2628,9 @@ int btrfs_remove_free_space(struct btrfs_block_group *block_group, int ret; bool re_search = false; + if (btrfs_is_zoned(block_group->fs_info)) + return 0; + spin_lock(&ctl->tree_lock); again: @@ -2668,6 +2725,16 @@ void btrfs_dump_free_space(struct btrfs_block_group *block_group, struct rb_node *n; int count = 0; + /* + * Zoned btrfs does not use free space tree and cluster. Just print + * out the free space after the allocation offset. + */ + if (btrfs_is_zoned(fs_info)) { + btrfs_info(fs_info, "free space %llu", + block_group->length - block_group->alloc_offset); + return; + } + spin_lock(&ctl->tree_lock); for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { info = rb_entry(n, struct btrfs_free_space, offset_index); diff --git a/fs/btrfs/free-space-cache.h b/fs/btrfs/free-space-cache.h index ecb09a02d544..1f23088d43f9 100644 --- a/fs/btrfs/free-space-cache.h +++ b/fs/btrfs/free-space-cache.h @@ -107,6 +107,8 @@ int __btrfs_add_free_space(struct btrfs_fs_info *fs_info, enum btrfs_trim_state trim_state); int btrfs_add_free_space(struct btrfs_block_group *block_group, u64 bytenr, u64 size); +int btrfs_add_free_space_unused(struct btrfs_block_group *block_group, + u64 bytenr, u64 size); int btrfs_add_free_space_async_trimmed(struct btrfs_block_group *block_group, u64 bytenr, u64 size); int btrfs_remove_free_space(struct btrfs_block_group *block_group, diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index bccd98141a6e..2da6177f4b0b 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -169,6 +169,7 @@ u64 __pure btrfs_space_info_used(struct btrfs_space_info *s_info, ASSERT(s_info); return s_info->bytes_used + s_info->bytes_reserved + s_info->bytes_pinned + s_info->bytes_readonly + + s_info->bytes_zone_unusable + (may_use_included ? s_info->bytes_may_use : 0); } @@ -264,7 +265,7 @@ out: void btrfs_update_space_info(struct btrfs_fs_info *info, u64 flags, u64 total_bytes, u64 bytes_used, - u64 bytes_readonly, + u64 bytes_readonly, u64 bytes_zone_unusable, struct btrfs_space_info **space_info) { struct btrfs_space_info *found; @@ -280,6 +281,7 @@ void btrfs_update_space_info(struct btrfs_fs_info *info, u64 flags, found->bytes_used += bytes_used; found->disk_used += bytes_used * factor; found->bytes_readonly += bytes_readonly; + found->bytes_zone_unusable += bytes_zone_unusable; if (total_bytes > 0) found->full = 0; btrfs_try_granting_tickets(info, found); @@ -429,10 +431,10 @@ static void __btrfs_dump_space_info(struct btrfs_fs_info *fs_info, info->total_bytes - btrfs_space_info_used(info, true), info->full ? "" : "not "); btrfs_info(fs_info, - "space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu", + "space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu zone_unusable=%llu", info->total_bytes, info->bytes_used, info->bytes_pinned, info->bytes_reserved, info->bytes_may_use, - info->bytes_readonly); + info->bytes_readonly, info->bytes_zone_unusable); DUMP_BLOCK_RSV(fs_info, global_block_rsv); DUMP_BLOCK_RSV(fs_info, trans_block_rsv); @@ -461,9 +463,10 @@ again: list_for_each_entry(cache, &info->block_groups[index], list) { spin_lock(&cache->lock); btrfs_info(fs_info, - "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s", + "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %llu zone_unusable %s", cache->start, cache->length, cache->used, cache->pinned, - cache->reserved, cache->ro ? "[readonly]" : ""); + cache->reserved, cache->zone_unusable, + cache->ro ? "[readonly]" : ""); spin_unlock(&cache->lock); btrfs_dump_free_space(cache, bytes); } diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index e237156ce888..b1a8ffb03b3e 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -17,6 +17,8 @@ struct btrfs_space_info { u64 bytes_may_use; /* number of bytes that may be used for delalloc/allocations */ u64 bytes_readonly; /* total bytes that are read only */ + u64 bytes_zone_unusable; /* total bytes that are unusable until + resetting the device zone */ u64 max_extent_size; /* This will hold the maximum extent size of the space info if we had an ENOSPC in the @@ -123,7 +125,7 @@ DECLARE_SPACE_INFO_UPDATE(bytes_pinned, "pinned"); int btrfs_init_space_info(struct btrfs_fs_info *fs_info); void btrfs_update_space_info(struct btrfs_fs_info *info, u64 flags, u64 total_bytes, u64 bytes_used, - u64 bytes_readonly, + u64 bytes_readonly, u64 bytes_zone_unusable, struct btrfs_space_info **space_info); struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info, u64 flags); diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c index 19b9fffa2c9c..6eb1c50fa98c 100644 --- a/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c @@ -666,6 +666,7 @@ SPACE_INFO_ATTR(bytes_pinned); SPACE_INFO_ATTR(bytes_reserved); SPACE_INFO_ATTR(bytes_may_use); SPACE_INFO_ATTR(bytes_readonly); +SPACE_INFO_ATTR(bytes_zone_unusable); SPACE_INFO_ATTR(disk_used); SPACE_INFO_ATTR(disk_total); BTRFS_ATTR(space_info, total_bytes_pinned, @@ -679,6 +680,7 @@ static struct attribute *space_info_attrs[] = { BTRFS_ATTR_PTR(space_info, bytes_reserved), BTRFS_ATTR_PTR(space_info, bytes_may_use), BTRFS_ATTR_PTR(space_info, bytes_readonly), + BTRFS_ATTR_PTR(space_info, bytes_zone_unusable), BTRFS_ATTR_PTR(space_info, disk_used), BTRFS_ATTR_PTR(space_info, disk_total), BTRFS_ATTR_PTR(space_info, total_bytes_pinned), diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index b892566a1c93..c5f9f4c6f20b 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1160,3 +1160,24 @@ out: return ret; } + +void btrfs_calc_zone_unusable(struct btrfs_block_group *cache) +{ + u64 unusable, free; + + if (!btrfs_is_zoned(cache->fs_info)) + return; + + WARN_ON(cache->bytes_super != 0); + unusable = cache->alloc_offset - cache->used; + free = cache->length - cache->alloc_offset; + + /* We only need ->free_space in ALLOC_SEQ block groups */ + cache->last_byte_to_unpin = (u64)-1; + cache->cached = BTRFS_CACHE_FINISHED; + cache->free_space_ctl->free_space = free; + cache->zone_unusable = unusable; + + /* Should not have any excluded extents. Just in case, though */ + btrfs_free_excluded_extents(cache); +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index d27db3993e51..37304d1675e6 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -42,6 +42,7 @@ int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical, u64 length, u64 *bytes); int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size); int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new); +void btrfs_calc_zone_unusable(struct btrfs_block_group *cache); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -123,6 +124,8 @@ static inline int btrfs_load_block_group_zone_info( return 0; } +static inline void btrfs_calc_zone_unusable(struct btrfs_block_group *cache) { } + #endif static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) -- cgit v1.2.3 From 2eda57089ea31942e067d6ac37923c3154ef8a25 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:53 +0900 Subject: btrfs: zoned: implement sequential extent allocation Implement a sequential extent allocator for zoned filesystems. This allocator only needs to check if there is enough space in the block group after the allocation pointer to satisfy the extent allocation request. Therefore the allocator never manages bitmaps or clusters. Also, add assertions to the corresponding functions. As zone append writing is used, it would be unnecessary to track the allocation offset, as the allocator only needs to check available space. But by tracking and returning the offset as an allocated region, we can skip modification of ordered extents and checksum information when there is no IO reordering. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 4 ++ fs/btrfs/extent-tree.c | 90 ++++++++++++++++++++++++++++++++++++++++++--- fs/btrfs/free-space-cache.c | 6 +++ 3 files changed, 94 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index e4444d4dd4b5..63093cfb807e 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -725,6 +725,10 @@ int btrfs_cache_block_group(struct btrfs_block_group *cache, int load_cache_only struct btrfs_caching_control *caching_ctl = NULL; int ret = 0; + /* Allocator for zoned filesystems does not use the cache at all */ + if (btrfs_is_zoned(fs_info)) + return 0; + caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS); if (!caching_ctl) return -ENOMEM; diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 5c61c3f136f7..3f83ca503051 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -3429,6 +3429,7 @@ btrfs_release_block_group(struct btrfs_block_group *cache, enum btrfs_extent_allocation_policy { BTRFS_EXTENT_ALLOC_CLUSTERED, + BTRFS_EXTENT_ALLOC_ZONED, }; /* @@ -3681,6 +3682,65 @@ static int do_allocation_clustered(struct btrfs_block_group *block_group, return find_free_extent_unclustered(block_group, ffe_ctl); } +/* + * Simple allocator for sequential-only block group. It only allows sequential + * allocation. No need to play with trees. This function also reserves the + * bytes as in btrfs_add_reserved_bytes. + */ +static int do_allocation_zoned(struct btrfs_block_group *block_group, + struct find_free_extent_ctl *ffe_ctl, + struct btrfs_block_group **bg_ret) +{ + struct btrfs_space_info *space_info = block_group->space_info; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + u64 start = block_group->start; + u64 num_bytes = ffe_ctl->num_bytes; + u64 avail; + int ret = 0; + + ASSERT(btrfs_is_zoned(block_group->fs_info)); + + spin_lock(&space_info->lock); + spin_lock(&block_group->lock); + + if (block_group->ro) { + ret = 1; + goto out; + } + + avail = block_group->length - block_group->alloc_offset; + if (avail < num_bytes) { + if (ffe_ctl->max_extent_size < avail) { + /* + * With sequential allocator, free space is always + * contiguous + */ + ffe_ctl->max_extent_size = avail; + ffe_ctl->total_free_space = avail; + } + ret = 1; + goto out; + } + + ffe_ctl->found_offset = start + block_group->alloc_offset; + block_group->alloc_offset += num_bytes; + spin_lock(&ctl->tree_lock); + ctl->free_space -= num_bytes; + spin_unlock(&ctl->tree_lock); + + /* + * We do not check if found_offset is aligned to stripesize. The + * address is anyway rewritten when using zone append writing. + */ + + ffe_ctl->search_start = ffe_ctl->found_offset; + +out: + spin_unlock(&block_group->lock); + spin_unlock(&space_info->lock); + return ret; +} + static int do_allocation(struct btrfs_block_group *block_group, struct find_free_extent_ctl *ffe_ctl, struct btrfs_block_group **bg_ret) @@ -3688,6 +3748,8 @@ static int do_allocation(struct btrfs_block_group *block_group, switch (ffe_ctl->policy) { case BTRFS_EXTENT_ALLOC_CLUSTERED: return do_allocation_clustered(block_group, ffe_ctl, bg_ret); + case BTRFS_EXTENT_ALLOC_ZONED: + return do_allocation_zoned(block_group, ffe_ctl, bg_ret); default: BUG(); } @@ -3702,6 +3764,9 @@ static void release_block_group(struct btrfs_block_group *block_group, ffe_ctl->retry_clustered = false; ffe_ctl->retry_unclustered = false; break; + case BTRFS_EXTENT_ALLOC_ZONED: + /* Nothing to do */ + break; default: BUG(); } @@ -3730,6 +3795,9 @@ static void found_extent(struct find_free_extent_ctl *ffe_ctl, case BTRFS_EXTENT_ALLOC_CLUSTERED: found_extent_clustered(ffe_ctl, ins); break; + case BTRFS_EXTENT_ALLOC_ZONED: + /* Nothing to do */ + break; default: BUG(); } @@ -3745,6 +3813,9 @@ static int chunk_allocation_failed(struct find_free_extent_ctl *ffe_ctl) */ ffe_ctl->loop = LOOP_NO_EMPTY_SIZE; return 0; + case BTRFS_EXTENT_ALLOC_ZONED: + /* Give up here */ + return -ENOSPC; default: BUG(); } @@ -3913,6 +3984,9 @@ static int prepare_allocation(struct btrfs_fs_info *fs_info, case BTRFS_EXTENT_ALLOC_CLUSTERED: return prepare_allocation_clustered(fs_info, ffe_ctl, space_info, ins); + case BTRFS_EXTENT_ALLOC_ZONED: + /* Nothing to do */ + return 0; default: BUG(); } @@ -3976,6 +4050,9 @@ static noinline int find_free_extent(struct btrfs_root *root, ffe_ctl.last_ptr = NULL; ffe_ctl.use_cluster = true; + if (btrfs_is_zoned(fs_info)) + ffe_ctl.policy = BTRFS_EXTENT_ALLOC_ZONED; + ins->type = BTRFS_EXTENT_ITEM_KEY; ins->objectid = 0; ins->offset = 0; @@ -4118,20 +4195,21 @@ have_block_group: /* move on to the next group */ if (ffe_ctl.search_start + num_bytes > block_group->start + block_group->length) { - btrfs_add_free_space(block_group, ffe_ctl.found_offset, - num_bytes); + btrfs_add_free_space_unused(block_group, + ffe_ctl.found_offset, num_bytes); goto loop; } if (ffe_ctl.found_offset < ffe_ctl.search_start) - btrfs_add_free_space(block_group, ffe_ctl.found_offset, - ffe_ctl.search_start - ffe_ctl.found_offset); + btrfs_add_free_space_unused(block_group, + ffe_ctl.found_offset, + ffe_ctl.search_start - ffe_ctl.found_offset); ret = btrfs_add_reserved_bytes(block_group, ram_bytes, num_bytes, delalloc); if (ret == -EAGAIN) { - btrfs_add_free_space(block_group, ffe_ctl.found_offset, - num_bytes); + btrfs_add_free_space_unused(block_group, + ffe_ctl.found_offset, num_bytes); goto loop; } btrfs_inc_block_group_reservations(block_group); diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index b93ac31eca69..d2a43186cc7f 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -2928,6 +2928,8 @@ u64 btrfs_find_space_for_alloc(struct btrfs_block_group *block_group, u64 align_gap_len = 0; enum btrfs_trim_state align_gap_trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + ASSERT(!btrfs_is_zoned(block_group->fs_info)); + spin_lock(&ctl->tree_lock); entry = find_free_space(ctl, &offset, &bytes_search, block_group->full_stripe_len, max_extent_size); @@ -3059,6 +3061,8 @@ u64 btrfs_alloc_from_cluster(struct btrfs_block_group *block_group, struct rb_node *node; u64 ret = 0; + ASSERT(!btrfs_is_zoned(block_group->fs_info)); + spin_lock(&cluster->lock); if (bytes > cluster->max_size) goto out; @@ -3835,6 +3839,8 @@ int btrfs_trim_block_group(struct btrfs_block_group *block_group, int ret; u64 rem = 0; + ASSERT(!btrfs_is_zoned(block_group->fs_info)); + *trimmed = 0; spin_lock(&block_group->lock); -- cgit v1.2.3 From d3575156f6623eecf086a20bcf99a63f1598109c Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:54 +0900 Subject: btrfs: zoned: redirty released extent buffers Tree manipulating operations like merging nodes often release once-allocated tree nodes. Such nodes are cleaned so that pages in the node are not uselessly written out. On zoned volumes, however, such optimization blocks the following IOs as the cancellation of the write out of the freed blocks breaks the sequential write sequence expected by the device. Introduce a list of clean and unwritten extent buffers that have been released in a transaction. Redirty the buffers so that btree_write_cache_pages() can send proper bios to the devices. Besides it clears the entire content of the extent buffer not to confuse raw block scanners e.g. 'btrfs check'. By clearing the content, csum_dirty_buffer() complains about bytenr mismatch, so avoid the checking and checksum using newly introduced buffer flag EXTENT_BUFFER_NO_CHECK. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 8 ++++++++ fs/btrfs/extent-tree.c | 12 +++++++++++- fs/btrfs/extent_io.c | 4 ++++ fs/btrfs/extent_io.h | 2 ++ fs/btrfs/transaction.c | 10 ++++++++++ fs/btrfs/transaction.h | 3 +++ fs/btrfs/tree-log.c | 6 ++++++ fs/btrfs/zoned.c | 37 +++++++++++++++++++++++++++++++++++++ fs/btrfs/zoned.h | 7 +++++++ 9 files changed, 88 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 8551b0fc1b22..eb1afd7d89f7 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -459,6 +459,12 @@ static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct bio_vec *bvec return 0; found_start = btrfs_header_bytenr(eb); + + if (test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags)) { + WARN_ON(found_start != 0); + return 0; + } + /* * Please do not consolidate these warnings into a single if. * It is useful to know what went wrong. @@ -4774,6 +4780,8 @@ void btrfs_cleanup_one_transaction(struct btrfs_transaction *cur_trans, EXTENT_DIRTY); btrfs_destroy_pinned_extent(fs_info, &cur_trans->pinned_extents); + btrfs_free_redirty_list(cur_trans); + cur_trans->state =TRANS_STATE_COMPLETED; wake_up(&cur_trans->commit_wait); } diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 3f83ca503051..dddcb8513c77 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -3292,8 +3292,10 @@ void btrfs_free_tree_block(struct btrfs_trans_handle *trans, if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { ret = check_ref_cleanup(trans, buf->start); - if (!ret) + if (!ret) { + btrfs_redirty_list_add(trans->transaction, buf); goto out; + } } cache = btrfs_lookup_block_group(fs_info, buf->start); @@ -3304,6 +3306,13 @@ void btrfs_free_tree_block(struct btrfs_trans_handle *trans, goto out; } + if (btrfs_is_zoned(fs_info)) { + btrfs_redirty_list_add(trans->transaction, buf); + pin_down_extent(trans, cache, buf->start, buf->len, 1); + btrfs_put_block_group(cache); + goto out; + } + WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)); btrfs_add_free_space(cache, buf->start, buf->len); @@ -4635,6 +4644,7 @@ btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, __btrfs_tree_lock(buf, nest); btrfs_clean_tree_block(buf); clear_bit(EXTENT_BUFFER_STALE, &buf->bflags); + clear_bit(EXTENT_BUFFER_NO_CHECK, &buf->bflags); set_extent_buffer_uptodate(buf); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 4be117adda33..eedcfb40c356 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -25,6 +25,7 @@ #include "backref.h" #include "disk-io.h" #include "subpage.h" +#include "zoned.h" static struct kmem_cache *extent_state_cache; static struct kmem_cache *extent_buffer_cache; @@ -5182,6 +5183,7 @@ __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, btrfs_leak_debug_add(&fs_info->eb_leak_lock, &eb->leak_list, &fs_info->allocated_ebs); + INIT_LIST_HEAD(&eb->release_list); spin_lock_init(&eb->refs_lock); atomic_set(&eb->refs, 1); @@ -6111,6 +6113,8 @@ void write_extent_buffer(const struct extent_buffer *eb, const void *srcv, char *src = (char *)srcv; unsigned long i = get_eb_page_index(start); + WARN_ON(test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags)); + if (check_eb_range(eb, start, len)) return; diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 047b3e66897f..824640cb0ace 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -31,6 +31,7 @@ enum { EXTENT_BUFFER_IN_TREE, /* write IO error */ EXTENT_BUFFER_WRITE_ERR, + EXTENT_BUFFER_NO_CHECK, }; /* these are flags for __process_pages_contig */ @@ -93,6 +94,7 @@ struct extent_buffer { struct rw_semaphore lock; struct page *pages[INLINE_EXTENT_BUFFER_PAGES]; + struct list_head release_list; #ifdef CONFIG_BTRFS_DEBUG struct list_head leak_list; #endif diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 00c0680dac3a..acff6bb49a97 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -21,6 +21,7 @@ #include "qgroup.h" #include "block-group.h" #include "space-info.h" +#include "zoned.h" #define BTRFS_ROOT_TRANS_TAG 0 @@ -380,6 +381,8 @@ loop: spin_lock_init(&cur_trans->dirty_bgs_lock); INIT_LIST_HEAD(&cur_trans->deleted_bgs); spin_lock_init(&cur_trans->dropped_roots_lock); + INIT_LIST_HEAD(&cur_trans->releasing_ebs); + spin_lock_init(&cur_trans->releasing_ebs_lock); list_add_tail(&cur_trans->list, &fs_info->trans_list); extent_io_tree_init(fs_info, &cur_trans->dirty_pages, IO_TREE_TRANS_DIRTY_PAGES, fs_info->btree_inode); @@ -2350,6 +2353,13 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) goto scrub_continue; } + /* + * At this point, we should have written all the tree blocks allocated + * in this transaction. So it's now safe to free the redirtyied extent + * buffers. + */ + btrfs_free_redirty_list(cur_trans); + ret = write_all_supers(fs_info, 0); /* * the super is written, we can safely allow the tree-loggers diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 935bd6958a8a..6335716e513f 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -93,6 +93,9 @@ struct btrfs_transaction { */ atomic_t pending_ordered; wait_queue_head_t pending_wait; + + spinlock_t releasing_ebs_lock; + struct list_head releasing_ebs; }; #define __TRANS_FREEZABLE (1U << 0) diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 4c7b283ed2b2..c02eeeac439c 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -19,6 +19,7 @@ #include "qgroup.h" #include "block-group.h" #include "space-info.h" +#include "zoned.h" /* magic values for the inode_only field in btrfs_log_inode: * @@ -2752,6 +2753,8 @@ static noinline int walk_down_log_tree(struct btrfs_trans_handle *trans, free_extent_buffer(next); return ret; } + btrfs_redirty_list_add( + trans->transaction, next); } else { if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &next->bflags)) clear_extent_buffer_dirty(next); @@ -3296,6 +3299,9 @@ static void free_log_tree(struct btrfs_trans_handle *trans, clear_extent_bits(&log->dirty_log_pages, 0, (u64)-1, EXTENT_DIRTY | EXTENT_NEW | EXTENT_NEED_WAIT); extent_io_tree_release(&log->log_csum_range); + + if (trans && log->node) + btrfs_redirty_list_add(trans->transaction, log->node); btrfs_put_root(log); } diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index c5f9f4c6f20b..1de67d789b83 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -10,6 +10,7 @@ #include "rcu-string.h" #include "disk-io.h" #include "block-group.h" +#include "transaction.h" /* Maximum number of zones to report per blkdev_report_zones() call */ #define BTRFS_REPORT_NR_ZONES 4096 @@ -1181,3 +1182,39 @@ void btrfs_calc_zone_unusable(struct btrfs_block_group *cache) /* Should not have any excluded extents. Just in case, though */ btrfs_free_excluded_extents(cache); } + +void btrfs_redirty_list_add(struct btrfs_transaction *trans, + struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + + if (!btrfs_is_zoned(fs_info) || + btrfs_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN) || + !list_empty(&eb->release_list)) + return; + + set_extent_buffer_dirty(eb); + set_extent_bits_nowait(&trans->dirty_pages, eb->start, + eb->start + eb->len - 1, EXTENT_DIRTY); + memzero_extent_buffer(eb, 0, eb->len); + set_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags); + + spin_lock(&trans->releasing_ebs_lock); + list_add_tail(&eb->release_list, &trans->releasing_ebs); + spin_unlock(&trans->releasing_ebs_lock); + atomic_inc(&eb->refs); +} + +void btrfs_free_redirty_list(struct btrfs_transaction *trans) +{ + spin_lock(&trans->releasing_ebs_lock); + while (!list_empty(&trans->releasing_ebs)) { + struct extent_buffer *eb; + + eb = list_first_entry(&trans->releasing_ebs, + struct extent_buffer, release_list); + list_del_init(&eb->release_list); + free_extent_buffer(eb); + } + spin_unlock(&trans->releasing_ebs_lock); +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 37304d1675e6..b250a578e38c 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -43,6 +43,9 @@ int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical, int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size); int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new); void btrfs_calc_zone_unusable(struct btrfs_block_group *cache); +void btrfs_redirty_list_add(struct btrfs_transaction *trans, + struct extent_buffer *eb); +void btrfs_free_redirty_list(struct btrfs_transaction *trans); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -126,6 +129,10 @@ static inline int btrfs_load_block_group_zone_info( static inline void btrfs_calc_zone_unusable(struct btrfs_block_group *cache) { } +static inline void btrfs_redirty_list_add(struct btrfs_transaction *trans, + struct extent_buffer *eb) { } +static inline void btrfs_free_redirty_list(struct btrfs_transaction *trans) { } + #endif static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) -- cgit v1.2.3 From 011b41bffa3dd086de3f2c393b35cde6133a7140 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:55 +0900 Subject: btrfs: zoned: advance allocation pointer after tree log node Since the allocation info of a tree log node is not recorded in the extent tree, calculate_alloc_pointer() cannot detect this node, so the pointer can be over a tree node. Replaying the log calls btrfs_remove_free_space() for each node in the log tree. So, advance the pointer after the node to not allocate over it. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/free-space-cache.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index d2a43186cc7f..5400294bd271 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -2628,8 +2628,22 @@ int btrfs_remove_free_space(struct btrfs_block_group *block_group, int ret; bool re_search = false; - if (btrfs_is_zoned(block_group->fs_info)) + if (btrfs_is_zoned(block_group->fs_info)) { + /* + * This can happen with conventional zones when replaying log. + * Since the allocation info of tree-log nodes are not recorded + * to the extent-tree, calculate_alloc_pointer() failed to + * advance the allocation pointer after last allocated tree log + * node blocks. + * + * This function is called from + * btrfs_pin_extent_for_log_replay() when replaying the log. + * Advance the pointer not to overwrite the tree-log nodes. + */ + if (block_group->alloc_offset < offset + bytes) + block_group->alloc_offset = offset + bytes; return 0; + } spin_lock(&ctl->tree_lock); -- cgit v1.2.3 From dcba6e48b518e5e48522e9ea2b73b60827c93146 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:56 +0900 Subject: btrfs: zoned: reset zones of unused block groups We must reset the zones of a deleted unused block group to rewind the zones' write pointers to the zones' start. To do this, we can use the DISCARD_SYNC code to do the reset when the filesystem is running on zoned devices. Reviewed-by: Josef Bacik Reviewed-by: Anand Jain Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 8 ++++++-- fs/btrfs/extent-tree.c | 17 ++++++++++++----- fs/btrfs/zoned.h | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 63093cfb807e..70a0c0f8f99f 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1408,8 +1408,12 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) if (!async_trim_enabled && btrfs_test_opt(fs_info, DISCARD_ASYNC)) goto flip_async; - /* DISCARD can flip during remount */ - trimming = btrfs_test_opt(fs_info, DISCARD_SYNC); + /* + * DISCARD can flip during remount. On zoned filesystems, we + * need to reset sequential-required zones. + */ + trimming = btrfs_test_opt(fs_info, DISCARD_SYNC) || + btrfs_is_zoned(fs_info); /* Implicit trim during transaction commit. */ if (trimming) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index dddcb8513c77..d976c56b3a56 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -1298,6 +1298,9 @@ int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr, stripe = bbio->stripes; for (i = 0; i < bbio->num_stripes; i++, stripe++) { + struct btrfs_device *dev = stripe->dev; + u64 physical = stripe->physical; + u64 length = stripe->length; u64 bytes; struct request_queue *req_q; @@ -1305,14 +1308,18 @@ int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr, ASSERT(btrfs_test_opt(fs_info, DEGRADED)); continue; } + req_q = bdev_get_queue(stripe->dev->bdev); - if (!blk_queue_discard(req_q)) + /* Zone reset on zoned filesystems */ + if (btrfs_can_zone_reset(dev, physical, length)) + ret = btrfs_reset_device_zone(dev, physical, + length, &bytes); + else if (blk_queue_discard(req_q)) + ret = btrfs_issue_discard(dev->bdev, physical, + length, &bytes); + else continue; - ret = btrfs_issue_discard(stripe->dev->bdev, - stripe->physical, - stripe->length, - &bytes); if (!ret) { discarded_bytes += bytes; } else if (ret != -EOPNOTSUPP) { diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index b250a578e38c..c105641a6ad3 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -209,4 +209,19 @@ static inline bool btrfs_check_super_location(struct btrfs_device *device, u64 p return device->zone_info == NULL || !btrfs_dev_is_sequential(device, pos); } +static inline bool btrfs_can_zone_reset(struct btrfs_device *device, + u64 physical, u64 length) +{ + u64 zone_size; + + if (!btrfs_dev_is_sequential(device, physical)) + return false; + + zone_size = device->zone_info->zone_size; + if (!IS_ALIGNED(physical, zone_size) || !IS_ALIGNED(length, zone_size)) + return false; + + return true; +} + #endif -- cgit v1.2.3 From 953651eb308fb56cd1a2d916e3d3c8b242240651 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:57 +0900 Subject: btrfs: factor out helper adding a page to bio Factor out adding a page to a bio from submit_extent_page(). The page is added only when bio_flags are the same, contiguous and the added page fits in the same stripe as pages in the bio. Condition checks are reordered to allow early return to avoid possibly heavy btrfs_bio_fits_in_stripe() calling. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 60 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index eedcfb40c356..3b33e7afd8e4 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -3084,6 +3084,48 @@ struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size) return bio; } +/** + * Attempt to add a page to bio + * + * @bio: destination bio + * @page: page to add to the bio + * @disk_bytenr: offset of the new bio or to check whether we are adding + * a contiguous page to the previous one + * @pg_offset: starting offset in the page + * @size: portion of page that we want to write + * @prev_bio_flags: flags of previous bio to see if we can merge the current one + * @bio_flags: flags of the current bio to see if we can merge them + * @return: true if page was added, false otherwise + * + * Attempt to add a page to bio considering stripe alignment etc. + * + * Return true if successfully page added. Otherwise, return false. + */ +static bool btrfs_bio_add_page(struct bio *bio, struct page *page, + u64 disk_bytenr, unsigned int size, + unsigned int pg_offset, + unsigned long prev_bio_flags, + unsigned long bio_flags) +{ + const sector_t sector = disk_bytenr >> SECTOR_SHIFT; + bool contig; + + if (prev_bio_flags != bio_flags) + return false; + + if (prev_bio_flags & EXTENT_BIO_COMPRESSED) + contig = bio->bi_iter.bi_sector == sector; + else + contig = bio_end_sector(bio) == sector; + if (!contig) + return false; + + if (btrfs_bio_fits_in_stripe(page, size, bio, bio_flags)) + return false; + + return bio_add_page(bio, page, size, pg_offset) == size; +} + /* * @opf: bio REQ_OP_* and REQ_* flags as one value * @wbc: optional writeback control for io accounting @@ -3112,27 +3154,15 @@ static int submit_extent_page(unsigned int opf, int ret = 0; struct bio *bio; size_t io_size = min_t(size_t, size, PAGE_SIZE); - sector_t sector = disk_bytenr >> 9; struct extent_io_tree *tree = &BTRFS_I(page->mapping->host)->io_tree; ASSERT(bio_ret); if (*bio_ret) { - bool contig; - bool can_merge = true; - bio = *bio_ret; - if (prev_bio_flags & EXTENT_BIO_COMPRESSED) - contig = bio->bi_iter.bi_sector == sector; - else - contig = bio_end_sector(bio) == sector; - - if (btrfs_bio_fits_in_stripe(page, io_size, bio, bio_flags)) - can_merge = false; - - if (prev_bio_flags != bio_flags || !contig || !can_merge || - force_bio_submit || - bio_add_page(bio, page, io_size, pg_offset) < io_size) { + if (force_bio_submit || + !btrfs_bio_add_page(bio, page, disk_bytenr, io_size, + pg_offset, prev_bio_flags, bio_flags)) { ret = submit_one_bio(bio, mirror_num, prev_bio_flags); if (ret < 0) { *bio_ret = NULL; -- cgit v1.2.3 From e1326f0339fe0a3beecb0da4d1b8793443798e09 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:58 +0900 Subject: btrfs: zoned: use bio_add_zone_append_page A zoned device has its own hardware restrictions e.g. max_zone_append_size when using REQ_OP_ZONE_APPEND. To follow these restrictions, use bio_add_zone_append_page() instead of bio_add_page(). We need target device to use bio_add_zone_append_page(), so this commit reads the chunk information to cache the target device to btrfs_io_bio(bio)->device. Caching only the target device is sufficient here as zoned filesystems only supports the single profile at the moment. Once more profiles will be supported btrfs_io_bio can hold an extent_map to be able to check for the restrictions of all devices the btrfs_bio will be mapped to. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 3b33e7afd8e4..6b26777efb92 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -3109,6 +3109,7 @@ static bool btrfs_bio_add_page(struct bio *bio, struct page *page, { const sector_t sector = disk_bytenr >> SECTOR_SHIFT; bool contig; + int ret; if (prev_bio_flags != bio_flags) return false; @@ -3123,7 +3124,12 @@ static bool btrfs_bio_add_page(struct bio *bio, struct page *page, if (btrfs_bio_fits_in_stripe(page, size, bio, bio_flags)) return false; - return bio_add_page(bio, page, size, pg_offset) == size; + if (bio_op(bio) == REQ_OP_ZONE_APPEND) + ret = bio_add_zone_append_page(bio, page, size, pg_offset); + else + ret = bio_add_page(bio, page, size, pg_offset); + + return ret == size; } /* @@ -3154,7 +3160,9 @@ static int submit_extent_page(unsigned int opf, int ret = 0; struct bio *bio; size_t io_size = min_t(size_t, size, PAGE_SIZE); - struct extent_io_tree *tree = &BTRFS_I(page->mapping->host)->io_tree; + struct btrfs_inode *inode = BTRFS_I(page->mapping->host); + struct extent_io_tree *tree = &inode->io_tree; + struct btrfs_fs_info *fs_info = inode->root->fs_info; ASSERT(bio_ret); @@ -3185,11 +3193,26 @@ static int submit_extent_page(unsigned int opf, if (wbc) { struct block_device *bdev; - bdev = BTRFS_I(page->mapping->host)->root->fs_info->fs_devices->latest_bdev; + bdev = fs_info->fs_devices->latest_bdev; bio_set_dev(bio, bdev); wbc_init_bio(wbc, bio); wbc_account_cgroup_owner(wbc, page, io_size); } + if (btrfs_is_zoned(fs_info) && bio_op(bio) == REQ_OP_ZONE_APPEND) { + struct extent_map *em; + struct map_lookup *map; + + em = btrfs_get_chunk_map(fs_info, disk_bytenr, io_size); + if (IS_ERR(em)) + return PTR_ERR(em); + + map = em->map_lookup; + /* We only support single profile for now */ + ASSERT(map->num_stripes == 1); + btrfs_io_bio(bio)->device = map->stripes[0].dev; + + free_extent_map(em); + } *bio_ret = bio; -- cgit v1.2.3 From cfe94440d17404478771179150e6e4554f092dd5 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:59 +0900 Subject: btrfs: zoned: handle REQ_OP_ZONE_APPEND as writing Zoned filesystems use REQ_OP_ZONE_APPEND bios for writing to actual devices. Let btrfs_end_bio() and btrfs_op be aware of it, by mapping REQ_OP_ZONE_APPEND to BTRFS_MAP_WRITE and using btrfs_op() instead of bio_op(). Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/inode.c | 10 +++++----- fs/btrfs/volumes.c | 8 ++++---- fs/btrfs/volumes.h | 1 + 4 files changed, 12 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index eb1afd7d89f7..70621184a731 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -709,7 +709,7 @@ static void end_workqueue_bio(struct bio *bio) fs_info = end_io_wq->info; end_io_wq->status = bio->bi_status; - if (bio_op(bio) == REQ_OP_WRITE) { + if (btrfs_op(bio) == BTRFS_MAP_WRITE) { if (end_io_wq->metadata == BTRFS_WQ_ENDIO_METADATA) wq = fs_info->endio_meta_write_workers; else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_FREE_SPACE) @@ -885,7 +885,7 @@ blk_status_t btrfs_submit_metadata_bio(struct inode *inode, struct bio *bio, int async = check_async_write(fs_info, BTRFS_I(inode)); blk_status_t ret; - if (bio_op(bio) != REQ_OP_WRITE) { + if (btrfs_op(bio) != BTRFS_MAP_WRITE) { /* * called for a read, do the setup so that checksum validation * can happen in the async kernel threads diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 5522e9d09c8a..d7a9c770dc3b 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2250,7 +2250,7 @@ blk_status_t btrfs_submit_data_bio(struct inode *inode, struct bio *bio, if (btrfs_is_free_space_inode(BTRFS_I(inode))) metadata = BTRFS_WQ_ENDIO_FREE_SPACE; - if (bio_op(bio) != REQ_OP_WRITE) { + if (btrfs_op(bio) != BTRFS_MAP_WRITE) { ret = btrfs_bio_wq_end_io(fs_info, bio, metadata); if (ret) goto out; @@ -7681,7 +7681,7 @@ static void btrfs_dio_private_put(struct btrfs_dio_private *dip) if (!refcount_dec_and_test(&dip->refs)) return; - if (bio_op(dip->dio_bio) == REQ_OP_WRITE) { + if (btrfs_op(dip->dio_bio) == BTRFS_MAP_WRITE) { __endio_write_update_ordered(BTRFS_I(dip->inode), dip->logical_offset, dip->bytes, @@ -7847,7 +7847,7 @@ static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_dio_private *dip = bio->bi_private; - bool write = bio_op(bio) == REQ_OP_WRITE; + bool write = btrfs_op(bio) == BTRFS_MAP_WRITE; blk_status_t ret; /* Check btrfs_submit_bio_hook() for rules about async submit. */ @@ -7897,7 +7897,7 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, struct inode *inode, loff_t file_offset) { - const bool write = (bio_op(dio_bio) == REQ_OP_WRITE); + const bool write = (btrfs_op(dio_bio) == BTRFS_MAP_WRITE); const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); size_t dip_size; struct btrfs_dio_private *dip; @@ -7927,7 +7927,7 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap, struct bio *dio_bio, loff_t file_offset) { - const bool write = (bio_op(dio_bio) == REQ_OP_WRITE); + const bool write = (btrfs_op(dio_bio) == BTRFS_MAP_WRITE); struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); const bool raid56 = (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK); diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 10401def16ef..400375aaa197 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -6448,7 +6448,7 @@ static void btrfs_end_bio(struct bio *bio) struct btrfs_device *dev = btrfs_io_bio(bio)->device; ASSERT(dev->bdev); - if (bio_op(bio) == REQ_OP_WRITE) + if (btrfs_op(bio) == BTRFS_MAP_WRITE) btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS); else if (!(bio->bi_opf & REQ_RAHEAD)) @@ -6561,10 +6561,10 @@ blk_status_t btrfs_map_bio(struct btrfs_fs_info *fs_info, struct bio *bio, atomic_set(&bbio->stripes_pending, bbio->num_stripes); if ((bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) && - ((bio_op(bio) == REQ_OP_WRITE) || (mirror_num > 1))) { + ((btrfs_op(bio) == BTRFS_MAP_WRITE) || (mirror_num > 1))) { /* In this case, map_length has been set to the length of a single stripe; not the whole write */ - if (bio_op(bio) == REQ_OP_WRITE) { + if (btrfs_op(bio) == BTRFS_MAP_WRITE) { ret = raid56_parity_write(fs_info, bio, bbio, map_length); } else { @@ -6587,7 +6587,7 @@ blk_status_t btrfs_map_bio(struct btrfs_fs_info *fs_info, struct bio *bio, dev = bbio->stripes[dev_nr].dev; if (!dev || !dev->bdev || test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || - (bio_op(first_bio) == REQ_OP_WRITE && + (btrfs_op(first_bio) == BTRFS_MAP_WRITE && !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) { bbio_error(bbio, first_bio, logical); continue; diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 598ac225176d..d3bbdb4175df 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -424,6 +424,7 @@ static inline enum btrfs_map_op btrfs_op(struct bio *bio) case REQ_OP_DISCARD: return BTRFS_MAP_DISCARD; case REQ_OP_WRITE: + case REQ_OP_ZONE_APPEND: return BTRFS_MAP_WRITE; default: WARN_ON_ONCE(1); -- cgit v1.2.3 From d22002fd37bd970480c59754dfa448866a1f38bd Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:00 +0900 Subject: btrfs: zoned: split ordered extent when bio is sent For a zone append write, the device decides the location the data is being written to. Therefore we cannot ensure that two bios are written consecutively on the device. In order to ensure that an ordered extent maps to a contiguous region on disk, we need to maintain a "one bio == one ordered extent" rule. Implement splitting of an ordered extent and extent map on bio submission to adhere to the rule. extract_ordered_extent() hooks into btrfs_submit_data_bio() and splits the corresponding ordered extent so that the ordered extent's region fits into one bio and the corresponding device limits. Several sanity checks need to be done in extract_ordered_extent() e.g. - We cannot split once end_bio'd ordered extent because we cannot divide ordered->bytes_left for the split ones - We do not expect a compressed ordered extent - We should not have checksum list because we omit the list splitting. Since the function is called before btrfs_wq_submit_bio() or btrfs_csum_one_bio(), this should be always ensured. We also need to split an extent map by creating a new one. If not, unpin_extent_cache() complains about the difference between the start of the extent map and the file's logical offset. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/inode.c | 95 +++++++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/ordered-data.c | 78 ++++++++++++++++++++++++++++++++++++++++ fs/btrfs/ordered-data.h | 2 ++ 3 files changed, 175 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index d7a9c770dc3b..750482a06d67 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2215,6 +2215,92 @@ static blk_status_t btrfs_submit_bio_start(struct inode *inode, struct bio *bio, return btrfs_csum_one_bio(BTRFS_I(inode), bio, 0, 0); } +static blk_status_t extract_ordered_extent(struct btrfs_inode *inode, + struct bio *bio, loff_t file_offset) +{ + struct btrfs_ordered_extent *ordered; + struct extent_map *em = NULL, *em_new = NULL; + struct extent_map_tree *em_tree = &inode->extent_tree; + u64 start = (u64)bio->bi_iter.bi_sector << SECTOR_SHIFT; + u64 len = bio->bi_iter.bi_size; + u64 end = start + len; + u64 ordered_end; + u64 pre, post; + int ret = 0; + + ordered = btrfs_lookup_ordered_extent(inode, file_offset); + if (WARN_ON_ONCE(!ordered)) + return BLK_STS_IOERR; + + /* No need to split */ + if (ordered->disk_num_bytes == len) + goto out; + + /* We cannot split once end_bio'd ordered extent */ + if (WARN_ON_ONCE(ordered->bytes_left != ordered->disk_num_bytes)) { + ret = -EINVAL; + goto out; + } + + /* We cannot split a compressed ordered extent */ + if (WARN_ON_ONCE(ordered->disk_num_bytes != ordered->num_bytes)) { + ret = -EINVAL; + goto out; + } + + ordered_end = ordered->disk_bytenr + ordered->disk_num_bytes; + /* bio must be in one ordered extent */ + if (WARN_ON_ONCE(start < ordered->disk_bytenr || end > ordered_end)) { + ret = -EINVAL; + goto out; + } + + /* Checksum list should be empty */ + if (WARN_ON_ONCE(!list_empty(&ordered->list))) { + ret = -EINVAL; + goto out; + } + + pre = start - ordered->disk_bytenr; + post = ordered_end - end; + + ret = btrfs_split_ordered_extent(ordered, pre, post); + if (ret) + goto out; + + read_lock(&em_tree->lock); + em = lookup_extent_mapping(em_tree, ordered->file_offset, len); + if (!em) { + read_unlock(&em_tree->lock); + ret = -EIO; + goto out; + } + read_unlock(&em_tree->lock); + + ASSERT(!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)); + /* + * We cannot reuse em_new here but have to create a new one, as + * unpin_extent_cache() expects the start of the extent map to be the + * logical offset of the file, which does not hold true anymore after + * splitting. + */ + em_new = create_io_em(inode, em->start + pre, len, + em->start + pre, em->block_start + pre, len, + len, len, BTRFS_COMPRESS_NONE, + BTRFS_ORDERED_REGULAR); + if (IS_ERR(em_new)) { + ret = PTR_ERR(em_new); + goto out; + } + free_extent_map(em_new); + +out: + free_extent_map(em); + btrfs_put_ordered_extent(ordered); + + return errno_to_blk_status(ret); +} + /* * extent_io.c submission hook. This does the right thing for csum calculation * on write, or reading the csums from the tree before a read. @@ -2250,6 +2336,15 @@ blk_status_t btrfs_submit_data_bio(struct inode *inode, struct bio *bio, if (btrfs_is_free_space_inode(BTRFS_I(inode))) metadata = BTRFS_WQ_ENDIO_FREE_SPACE; + if (bio_op(bio) == REQ_OP_ZONE_APPEND) { + struct page *page = bio_first_bvec_all(bio)->bv_page; + loff_t file_offset = page_offset(page); + + ret = extract_ordered_extent(BTRFS_I(inode), bio, file_offset); + if (ret) + goto out; + } + if (btrfs_op(bio) != BTRFS_MAP_WRITE) { ret = btrfs_bio_wq_end_io(fs_info, bio, metadata); if (ret) diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index e8dee1578d4a..2dc707f02f00 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -920,6 +920,84 @@ void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start, } } +static int clone_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pos, + u64 len) +{ + struct inode *inode = ordered->inode; + u64 file_offset = ordered->file_offset + pos; + u64 disk_bytenr = ordered->disk_bytenr + pos; + u64 num_bytes = len; + u64 disk_num_bytes = len; + int type; + unsigned long flags_masked = ordered->flags & ~(1 << BTRFS_ORDERED_DIRECT); + int compress_type = ordered->compress_type; + unsigned long weight; + int ret; + + weight = hweight_long(flags_masked); + WARN_ON_ONCE(weight > 1); + if (!weight) + type = 0; + else + type = __ffs(flags_masked); + + if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered->flags)) { + WARN_ON_ONCE(1); + ret = btrfs_add_ordered_extent_compress(BTRFS_I(inode), + file_offset, disk_bytenr, num_bytes, + disk_num_bytes, compress_type); + } else if (test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) { + ret = btrfs_add_ordered_extent_dio(BTRFS_I(inode), file_offset, + disk_bytenr, num_bytes, disk_num_bytes, type); + } else { + ret = btrfs_add_ordered_extent(BTRFS_I(inode), file_offset, + disk_bytenr, num_bytes, disk_num_bytes, type); + } + + return ret; +} + +int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pre, + u64 post) +{ + struct inode *inode = ordered->inode; + struct btrfs_ordered_inode_tree *tree = &BTRFS_I(inode)->ordered_tree; + struct rb_node *node; + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + int ret = 0; + + spin_lock_irq(&tree->lock); + /* Remove from tree once */ + node = &ordered->rb_node; + rb_erase(node, &tree->tree); + RB_CLEAR_NODE(node); + if (tree->last == node) + tree->last = NULL; + + ordered->file_offset += pre; + ordered->disk_bytenr += pre; + ordered->num_bytes -= (pre + post); + ordered->disk_num_bytes -= (pre + post); + ordered->bytes_left -= (pre + post); + + /* Re-insert the node */ + node = tree_insert(&tree->tree, ordered->file_offset, &ordered->rb_node); + if (node) + btrfs_panic(fs_info, -EEXIST, + "zoned: inconsistency in ordered tree at offset %llu", + ordered->file_offset); + + spin_unlock_irq(&tree->lock); + + if (pre) + ret = clone_ordered_extent(ordered, 0, pre); + if (post) + ret = clone_ordered_extent(ordered, pre + ordered->disk_num_bytes, + post); + + return ret; +} + int __init ordered_data_init(void) { btrfs_ordered_extent_cache = kmem_cache_create("btrfs_ordered_extent", diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index cca3307807e8..c400be75a3f1 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -201,6 +201,8 @@ void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr, void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start, u64 end, struct extent_state **cached_state); +int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pre, + u64 post); int __init ordered_data_init(void); void __cold ordered_data_exit(void); -- cgit v1.2.3 From cacb2cea46382aacf0365dbe231bd1ac3349478e Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Thu, 4 Feb 2021 19:22:01 +0900 Subject: btrfs: zoned: check if bio spans across an ordered extent To ensure that an ordered extent maps to a contiguous region on disk, we need to maintain a "one bio == one ordered extent" rule. Ensure that constructing bio does not span more than an ordered extent. Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 2 ++ fs/btrfs/extent_io.c | 9 +++++++-- fs/btrfs/inode.c | 27 +++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index a9b0521d9e89..10da47ab093a 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -3120,6 +3120,8 @@ void btrfs_split_delalloc_extent(struct inode *inode, struct extent_state *orig, u64 split); int btrfs_bio_fits_in_stripe(struct page *page, size_t size, struct bio *bio, unsigned long bio_flags); +bool btrfs_bio_fits_in_ordered_extent(struct page *page, struct bio *bio, + unsigned int size); void btrfs_set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end); vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf); int btrfs_readpage(struct file *file, struct page *page); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 6b26777efb92..f64e2be5749e 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -3124,10 +3124,15 @@ static bool btrfs_bio_add_page(struct bio *bio, struct page *page, if (btrfs_bio_fits_in_stripe(page, size, bio, bio_flags)) return false; - if (bio_op(bio) == REQ_OP_ZONE_APPEND) + if (bio_op(bio) == REQ_OP_ZONE_APPEND) { + struct page *first_page = bio_first_bvec_all(bio)->bv_page; + + if (!btrfs_bio_fits_in_ordered_extent(first_page, bio, size)) + return false; ret = bio_add_zone_append_page(bio, page, size, pg_offset); - else + } else { ret = bio_add_page(bio, page, size, pg_offset); + } return ret == size; } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 750482a06d67..31545e503b9e 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2215,6 +2215,33 @@ static blk_status_t btrfs_submit_bio_start(struct inode *inode, struct bio *bio, return btrfs_csum_one_bio(BTRFS_I(inode), bio, 0, 0); } +bool btrfs_bio_fits_in_ordered_extent(struct page *page, struct bio *bio, + unsigned int size) +{ + struct btrfs_inode *inode = BTRFS_I(page->mapping->host); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_ordered_extent *ordered; + u64 len = bio->bi_iter.bi_size + size; + bool ret = true; + + ASSERT(btrfs_is_zoned(fs_info)); + ASSERT(fs_info->max_zone_append_size > 0); + ASSERT(bio_op(bio) == REQ_OP_ZONE_APPEND); + + /* Ordered extent not yet created, so we're good */ + ordered = btrfs_lookup_ordered_extent(inode, page_offset(page)); + if (!ordered) + return ret; + + if ((bio->bi_iter.bi_sector << SECTOR_SHIFT) + len > + ordered->disk_bytenr + ordered->disk_num_bytes) + ret = false; + + btrfs_put_ordered_extent(ordered); + + return ret; +} + static blk_status_t extract_ordered_extent(struct btrfs_inode *inode, struct bio *bio, loff_t file_offset) { -- cgit v1.2.3 From 138082f36610698e3fd00318f275d7f2159b8d26 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:02 +0900 Subject: btrfs: extend btrfs_rmap_block for specifying a device btrfs_rmap_block currently reverse-maps the physical addresses on all devices to the corresponding logical addresses. Extend the function to match to a specified device. The old functionality of querying all devices is left intact by specifying NULL as target device. A block_device instead of a btrfs_device is passed into btrfs_rmap_block, as this function is intended to reverse-map the result of a bio, which only has a block_device. Also export the function for later use. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 16 +++++++++++----- fs/btrfs/block-group.h | 8 +++----- fs/btrfs/tests/extent-map-tests.c | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 70a0c0f8f99f..f5e9f560ce6d 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1588,6 +1588,7 @@ static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) * * @fs_info: the filesystem * @chunk_start: logical address of block group + * @bdev: physical device to resolve, can be NULL to indicate any device * @physical: physical address to map to logical addresses * @logical: return array of logical addresses which map to @physical * @naddrs: length of @logical @@ -1597,9 +1598,9 @@ static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) * Used primarily to exclude those portions of a block group that contain super * block copies. */ -EXPORT_FOR_TESTS int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, - u64 physical, u64 **logical, int *naddrs, int *stripe_len) + struct block_device *bdev, u64 physical, u64 **logical, + int *naddrs, int *stripe_len) { struct extent_map *em; struct map_lookup *map; @@ -1617,6 +1618,7 @@ int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, map = em->map_lookup; data_stripe_length = em->orig_block_len; io_stripe_size = map->stripe_len; + chunk_start = em->start; /* For RAID5/6 adjust to a full IO stripe length */ if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) @@ -1631,14 +1633,18 @@ int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, for (i = 0; i < map->num_stripes; i++) { bool already_inserted = false; u64 stripe_nr; + u64 offset; int j; if (!in_range(physical, map->stripes[i].physical, data_stripe_length)) continue; + if (bdev && map->stripes[i].dev->bdev != bdev) + continue; + stripe_nr = physical - map->stripes[i].physical; - stripe_nr = div64_u64(stripe_nr, map->stripe_len); + stripe_nr = div64_u64_rem(stripe_nr, map->stripe_len, &offset); if (map->type & BTRFS_BLOCK_GROUP_RAID10) { stripe_nr = stripe_nr * map->num_stripes + i; @@ -1652,7 +1658,7 @@ int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, * instead of map->stripe_len */ - bytenr = chunk_start + stripe_nr * io_stripe_size; + bytenr = chunk_start + stripe_nr * io_stripe_size + offset; /* Ensure we don't add duplicate addresses */ for (j = 0; j < nr; j++) { @@ -1694,7 +1700,7 @@ static int exclude_super_stripes(struct btrfs_block_group *cache) for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { bytenr = btrfs_sb_offset(i); - ret = btrfs_rmap_block(fs_info, cache->start, + ret = btrfs_rmap_block(fs_info, cache->start, NULL, bytenr, &logical, &nr, &stripe_len); if (ret) return ret; diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 0fd66febe115..d14ac03bb93d 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -277,6 +277,9 @@ void btrfs_put_block_group_cache(struct btrfs_fs_info *info); int btrfs_free_block_groups(struct btrfs_fs_info *info); void btrfs_wait_space_cache_v1_finished(struct btrfs_block_group *cache, struct btrfs_caching_control *caching_ctl); +int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, + struct block_device *bdev, u64 physical, u64 **logical, + int *naddrs, int *stripe_len); static inline u64 btrfs_data_alloc_profile(struct btrfs_fs_info *fs_info) { @@ -303,9 +306,4 @@ static inline int btrfs_block_group_done(struct btrfs_block_group *cache) void btrfs_freeze_block_group(struct btrfs_block_group *cache); void btrfs_unfreeze_block_group(struct btrfs_block_group *cache); -#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS -int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, - u64 physical, u64 **logical, int *naddrs, int *stripe_len); -#endif - #endif /* BTRFS_BLOCK_GROUP_H */ diff --git a/fs/btrfs/tests/extent-map-tests.c b/fs/btrfs/tests/extent-map-tests.c index 57379e96ccc9..c0aefe6dee0b 100644 --- a/fs/btrfs/tests/extent-map-tests.c +++ b/fs/btrfs/tests/extent-map-tests.c @@ -507,7 +507,7 @@ static int test_rmap_block(struct btrfs_fs_info *fs_info, goto out_free; } - ret = btrfs_rmap_block(fs_info, em->start, btrfs_sb_offset(1), + ret = btrfs_rmap_block(fs_info, em->start, NULL, btrfs_sb_offset(1), &logical, &out_ndaddrs, &out_stripe_len); if (ret || (out_ndaddrs == 0 && test->expected_mapped_addr)) { test_err("didn't rmap anything but expected %d", -- cgit v1.2.3 From 08f455593fff701e103876d4db5d3f4f6d0ff871 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Thu, 4 Feb 2021 19:22:03 +0900 Subject: btrfs: zoned: cache if block group is on a sequential zone On a zoned filesystem, cache if a block group is on a sequential write only zone. On sequential write only zones, we can use REQ_OP_ZONE_APPEND for writing data, therefore provide btrfs_use_zone_append() to figure out if IO is targeting a sequential write only zone and we can use REQ_OP_ZONE_APPEND for data writing. Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.h | 3 +++ fs/btrfs/zoned.c | 29 +++++++++++++++++++++++++++++ fs/btrfs/zoned.h | 6 ++++++ 3 files changed, 38 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index d14ac03bb93d..31c7c5872b92 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -181,6 +181,9 @@ struct btrfs_block_group { */ int needs_free_space; + /* Flag indicating this block group is placed on a sequential zone */ + bool seq_zone; + /* Record locked full stripes for RAID5/6 block group */ struct btrfs_full_stripe_locks_tree full_stripe_locks_root; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 1de67d789b83..f6c68704c840 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1101,6 +1101,9 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) } } + if (num_sequential > 0) + cache->seq_zone = true; + if (num_conventional > 0) { /* * Avoid calling calculate_alloc_pointer() for new BG. It @@ -1218,3 +1221,29 @@ void btrfs_free_redirty_list(struct btrfs_transaction *trans) } spin_unlock(&trans->releasing_ebs_lock); } + +bool btrfs_use_zone_append(struct btrfs_inode *inode, struct extent_map *em) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_block_group *cache; + bool ret = false; + + if (!btrfs_is_zoned(fs_info)) + return false; + + if (!fs_info->max_zone_append_size) + return false; + + if (!is_data_inode(&inode->vfs_inode)) + return false; + + cache = btrfs_lookup_block_group(fs_info, em->block_start); + ASSERT(cache); + if (!cache) + return false; + + ret = cache->seq_zone; + btrfs_put_block_group(cache); + + return ret; +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index c105641a6ad3..14d578328cbe 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -46,6 +46,7 @@ void btrfs_calc_zone_unusable(struct btrfs_block_group *cache); void btrfs_redirty_list_add(struct btrfs_transaction *trans, struct extent_buffer *eb); void btrfs_free_redirty_list(struct btrfs_transaction *trans); +bool btrfs_use_zone_append(struct btrfs_inode *inode, struct extent_map *em); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -133,6 +134,11 @@ static inline void btrfs_redirty_list_add(struct btrfs_transaction *trans, struct extent_buffer *eb) { } static inline void btrfs_free_redirty_list(struct btrfs_transaction *trans) { } +static inline bool btrfs_use_zone_append(struct btrfs_inode *inode, + struct extent_map *em) +{ + return false; +} #endif static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) -- cgit v1.2.3 From 24533f6a9ad633d6ff0332844fadafb9ecf4a917 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Thu, 4 Feb 2021 19:22:04 +0900 Subject: btrfs: save irq flags when looking up an ordered extent A following patch will add another caller of btrfs_lookup_ordered_extent(), but from a bio's endio context. btrfs_lookup_ordered_extent() uses spin_lock_irq() which unconditionally disables interrupts. Change this to spin_lock_irqsave() so interrupts aren't disabled and re-enabled unconditionally. Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ordered-data.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 2dc707f02f00..fe235ab935d3 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -767,9 +767,10 @@ struct btrfs_ordered_extent *btrfs_lookup_ordered_extent(struct btrfs_inode *ino struct btrfs_ordered_inode_tree *tree; struct rb_node *node; struct btrfs_ordered_extent *entry = NULL; + unsigned long flags; tree = &inode->ordered_tree; - spin_lock_irq(&tree->lock); + spin_lock_irqsave(&tree->lock, flags); node = tree_search(tree, file_offset); if (!node) goto out; @@ -780,7 +781,7 @@ struct btrfs_ordered_extent *btrfs_lookup_ordered_extent(struct btrfs_inode *ino if (entry) refcount_inc(&entry->refs); out: - spin_unlock_irq(&tree->lock); + spin_unlock_irqrestore(&tree->lock, flags); return entry; } -- cgit v1.2.3 From d8e3fb106f393858b90b3befc4f6092a76c86d1c Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:05 +0900 Subject: btrfs: zoned: use ZONE_APPEND write for zoned mode Enable zone append writing for zoned mode. When using zone append, a bio is issued to the start of a target zone and the device decides to place it inside the zone. Upon completion the device reports the actual written position back to the host. Three parts are necessary to enable zone append mode. First, modify the bio to use REQ_OP_ZONE_APPEND in btrfs_submit_bio_hook() and adjust the bi_sector to point the beginning of the zone. Second, record the returned physical address (and disk/partno) to the ordered extent in end_bio_extent_writepage() after the bio has been completed. We cannot resolve the physical address to the logical address because we can neither take locks nor allocate a buffer in this end_bio context. So, we need to record the physical address to resolve it later in btrfs_finish_ordered_io(). And finally, rewrite the logical addresses of the extent mapping and checksum data according to the physical address using btrfs_rmap_block. If the returned address matches the originally allocated address, we can skip this rewriting process. Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 15 +++++++++-- fs/btrfs/file.c | 6 ++++- fs/btrfs/inode.c | 4 +++ fs/btrfs/ordered-data.c | 3 +++ fs/btrfs/ordered-data.h | 8 ++++++ fs/btrfs/volumes.c | 14 ++++++++++ fs/btrfs/zoned.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/zoned.h | 12 +++++++++ 8 files changed, 129 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index f64e2be5749e..14a68f3589cc 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2735,6 +2735,7 @@ static void end_bio_extent_writepage(struct bio *bio) u64 start; u64 end; struct bvec_iter_all iter_all; + bool first_bvec = true; ASSERT(!bio_flagged(bio, BIO_CLONED)); bio_for_each_segment_all(bvec, bio, iter_all) { @@ -2761,6 +2762,11 @@ static void end_bio_extent_writepage(struct bio *bio) start = page_offset(page); end = start + bvec->bv_offset + bvec->bv_len - 1; + if (first_bvec) { + btrfs_record_physical_zoned(inode, start, bio); + first_bvec = false; + } + end_extent_writepage(page, error, start, end); end_page_writeback(page); } @@ -3664,6 +3670,7 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, struct extent_map *em; int ret = 0; int nr = 0; + u32 opf = REQ_OP_WRITE; const unsigned int write_flags = wbc_to_write_flags(wbc); bool compressed; @@ -3710,6 +3717,10 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, /* Note that em_end from extent_map_end() is exclusive */ iosize = min(em_end, end + 1) - cur; + + if (btrfs_use_zone_append(inode, em)) + opf = REQ_OP_ZONE_APPEND; + free_extent_map(em); em = NULL; @@ -3735,8 +3746,8 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, page->index, cur, end); } - ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, - page, disk_bytenr, iosize, + ret = submit_extent_page(opf | write_flags, wbc, page, + disk_bytenr, iosize, cur - page_offset(page), &epd->bio, end_bio_extent_writepage, 0, 0, 0, false); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index bf52d7e85914..01a72f53fb5d 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -2168,8 +2168,12 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) * commit waits for their completion, to avoid data loss if we fsync, * the current transaction commits before the ordered extents complete * and a power failure happens right after that. + * + * For zoned filesystem, if a write IO uses a ZONE_APPEND command, the + * logical address recorded in the ordered extent may change. We need + * to wait for the IO to stabilize the logical address. */ - if (full_sync) { + if (full_sync || btrfs_is_zoned(fs_info)) { ret = btrfs_wait_ordered_range(inode, start, len); } else { /* diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 31545e503b9e..6dbab9293425 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -50,6 +50,7 @@ #include "delalloc-space.h" #include "block-group.h" #include "space-info.h" +#include "zoned.h" struct btrfs_iget_args { u64 ino; @@ -2874,6 +2875,9 @@ static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) goto out; } + if (ordered_extent->disk) + btrfs_rewrite_logical_zoned(ordered_extent); + btrfs_free_io_failure_record(inode, start, end); if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index fe235ab935d3..985a21558437 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -199,6 +199,9 @@ static int __btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset entry->compress_type = compress_type; entry->truncated_len = (u64)-1; entry->qgroup_rsv = ret; + entry->physical = (u64)-1; + entry->disk = NULL; + entry->partno = (u8)-1; ASSERT(type == BTRFS_ORDERED_REGULAR || type == BTRFS_ORDERED_NOCOW || diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index c400be75a3f1..99e0853e4d3b 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -139,6 +139,14 @@ struct btrfs_ordered_extent { struct completion completion; struct btrfs_work flush_work; struct list_head work_list; + + /* + * Used to reverse-map physical address returned from ZONE_APPEND write + * command in a workqueue context + */ + u64 physical; + struct gendisk *disk; + u8 partno; }; /* diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 400375aaa197..a4d47c6050f7 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -6500,6 +6500,20 @@ static void submit_stripe_bio(struct btrfs_bio *bbio, struct bio *bio, btrfs_io_bio(bio)->device = dev; bio->bi_end_io = btrfs_end_bio; bio->bi_iter.bi_sector = physical >> 9; + /* + * For zone append writing, bi_sector must point the beginning of the + * zone + */ + if (bio_op(bio) == REQ_OP_ZONE_APPEND) { + if (btrfs_dev_is_sequential(dev, physical)) { + u64 zone_start = round_down(physical, fs_info->zone_size); + + bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT; + } else { + bio->bi_opf &= ~REQ_OP_ZONE_APPEND; + bio->bi_opf |= REQ_OP_WRITE; + } + } btrfs_debug_in_rcu(fs_info, "btrfs_map_bio: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u", bio_op(bio), bio->bi_opf, bio->bi_iter.bi_sector, diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index f6c68704c840..f4e226bda9b0 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1247,3 +1247,73 @@ bool btrfs_use_zone_append(struct btrfs_inode *inode, struct extent_map *em) return ret; } + +void btrfs_record_physical_zoned(struct inode *inode, u64 file_offset, + struct bio *bio) +{ + struct btrfs_ordered_extent *ordered; + const u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + + if (bio_op(bio) != REQ_OP_ZONE_APPEND) + return; + + ordered = btrfs_lookup_ordered_extent(BTRFS_I(inode), file_offset); + if (WARN_ON(!ordered)) + return; + + ordered->physical = physical; + ordered->disk = bio->bi_disk; + ordered->partno = bio->bi_partno; + + btrfs_put_ordered_extent(ordered); +} + +void btrfs_rewrite_logical_zoned(struct btrfs_ordered_extent *ordered) +{ + struct btrfs_inode *inode = BTRFS_I(ordered->inode); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_map_tree *em_tree; + struct extent_map *em; + struct btrfs_ordered_sum *sum; + struct block_device *bdev; + u64 orig_logical = ordered->disk_bytenr; + u64 *logical = NULL; + int nr, stripe_len; + + /* Zoned devices should not have partitions. So, we can assume it is 0 */ + ASSERT(ordered->partno == 0); + bdev = bdgrab(ordered->disk->part0); + if (WARN_ON(!bdev)) + return; + + if (WARN_ON(btrfs_rmap_block(fs_info, orig_logical, bdev, + ordered->physical, &logical, &nr, + &stripe_len))) + goto out; + + WARN_ON(nr != 1); + + if (orig_logical == *logical) + goto out; + + ordered->disk_bytenr = *logical; + + em_tree = &inode->extent_tree; + write_lock(&em_tree->lock); + em = search_extent_mapping(em_tree, ordered->file_offset, + ordered->num_bytes); + em->block_start = *logical; + free_extent_map(em); + write_unlock(&em_tree->lock); + + list_for_each_entry(sum, &ordered->list, list) { + if (*logical < orig_logical) + sum->bytenr -= orig_logical - *logical; + else + sum->bytenr += *logical - orig_logical; + } + +out: + kfree(logical); + bdput(bdev); +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 14d578328cbe..04f7b21652b6 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -47,6 +47,9 @@ void btrfs_redirty_list_add(struct btrfs_transaction *trans, struct extent_buffer *eb); void btrfs_free_redirty_list(struct btrfs_transaction *trans); bool btrfs_use_zone_append(struct btrfs_inode *inode, struct extent_map *em); +void btrfs_record_physical_zoned(struct inode *inode, u64 file_offset, + struct bio *bio); +void btrfs_rewrite_logical_zoned(struct btrfs_ordered_extent *ordered); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -139,6 +142,15 @@ static inline bool btrfs_use_zone_append(struct btrfs_inode *inode, { return false; } + +static inline void btrfs_record_physical_zoned(struct inode *inode, + u64 file_offset, struct bio *bio) +{ +} + +static inline void btrfs_rewrite_logical_zoned( + struct btrfs_ordered_extent *ordered) { } + #endif static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) -- cgit v1.2.3 From 544d24f9de73642a65d50389b789a957b14ae3f6 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:06 +0900 Subject: btrfs: zoned: enable zone append writing for direct IO Likewise to buffered IO, enable zone append writing for direct IO when its used on a zoned block device. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 6dbab9293425..dd6fe8afd0e0 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7738,6 +7738,9 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, iomap->bdev = fs_info->fs_devices->latest_bdev; iomap->length = len; + if (write && btrfs_use_zone_append(BTRFS_I(inode), em)) + iomap->flags |= IOMAP_F_ZONE_APPEND; + free_extent_map(em); return 0; @@ -7964,6 +7967,8 @@ static void btrfs_end_dio_bio(struct bio *bio) if (err) dip->dio_bio->bi_status = err; + btrfs_record_physical_zoned(dip->inode, dip->logical_offset, bio); + bio_put(bio); btrfs_dio_private_put(dip); } @@ -8124,6 +8129,19 @@ static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap, bio->bi_end_io = btrfs_end_dio_bio; btrfs_io_bio(bio)->logical = file_offset; + WARN_ON_ONCE(write && btrfs_is_zoned(fs_info) && + fs_info->max_zone_append_size && + bio_op(bio) != REQ_OP_ZONE_APPEND); + + if (bio_op(bio) == REQ_OP_ZONE_APPEND) { + status = extract_ordered_extent(BTRFS_I(inode), bio, + file_offset); + if (status) { + bio_put(bio); + goto out_err; + } + } + ASSERT(submit_len >= clone_len); submit_len -= clone_len; -- cgit v1.2.3 From 42c011000963442ce533d92a492c4a057b2f5a46 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:07 +0900 Subject: btrfs: zoned: introduce dedicated data write path for zoned filesystems If more than one IO is issued for one file extent, these IO can be written to separate regions on a device. Since we cannot map one file extent to such a separate area on a zoned filesystem, we need to follow the "one IO == one ordered extent" rule. The normal buffered, uncompressed and not pre-allocated write path (used by cow_file_range()) sometimes does not follow this rule. It can write a part of an ordered extent when specified a region to write e.g., when its called from fdatasync(). Introduce a dedicated (uncompressed buffered) data write path for zoned filesystems, that will COW the region and write it at once. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index dd6fe8afd0e0..c4779cde83c6 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -1394,6 +1394,29 @@ static int cow_file_range_async(struct btrfs_inode *inode, return 0; } +static noinline int run_delalloc_zoned(struct btrfs_inode *inode, + struct page *locked_page, u64 start, + u64 end, int *page_started, + unsigned long *nr_written) +{ + int ret; + + ret = cow_file_range(inode, locked_page, start, end, page_started, + nr_written, 0); + if (ret) + return ret; + + if (*page_started) + return 0; + + __set_page_dirty_nobuffers(locked_page); + account_page_redirty(locked_page); + extent_write_locked_range(&inode->vfs_inode, start, end, WB_SYNC_ALL); + *page_started = 1; + + return 0; +} + static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, u64 bytenr, u64 num_bytes) { @@ -1871,17 +1894,24 @@ int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct page *locked_page { int ret; int force_cow = need_force_cow(inode, start, end); + const bool zoned = btrfs_is_zoned(inode->root->fs_info); if (inode->flags & BTRFS_INODE_NODATACOW && !force_cow) { + ASSERT(!zoned); ret = run_delalloc_nocow(inode, locked_page, start, end, page_started, 1, nr_written); } else if (inode->flags & BTRFS_INODE_PREALLOC && !force_cow) { + ASSERT(!zoned); ret = run_delalloc_nocow(inode, locked_page, start, end, page_started, 0, nr_written); } else if (!inode_can_compress(inode) || !inode_need_compress(inode, start, end)) { - ret = cow_file_range(inode, locked_page, start, end, - page_started, nr_written, 1); + if (zoned) + ret = run_delalloc_zoned(inode, locked_page, start, end, + page_started, nr_written); + else + ret = cow_file_range(inode, locked_page, start, end, + page_started, nr_written, 1); } else { set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags); ret = cow_file_range_async(inode, wbc, locked_page, start, end, -- cgit v1.2.3 From 0bc09ca12980db3ef1e55bfad25b1803d57628c9 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:08 +0900 Subject: btrfs: zoned: serialize metadata IO We cannot use zone append for writing metadata, because the B-tree nodes have references to each other using logical address. Without knowing the address in advance, we cannot construct the tree in the first place. So we need to serialize write IOs for metadata. We cannot add a mutex around allocation and submission because metadata blocks are allocated in an earlier stage to build up B-trees. Add a zoned_meta_io_lock and hold it during metadata IO submission in btree_write_cache_pages() to serialize IOs. Furthermore, this adds a per-block group metadata IO submission pointer "meta_write_pointer" to ensure sequential writing, which can break when attempting to write back blocks in an unfinished transaction. If the writing out failed because of a hole and the write out is for data integrity (WB_SYNC_ALL), it returns EAGAIN. A caller like fsync() code should handle this properly e.g. by falling back to a full transaction commit. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/block-group.h | 1 + fs/btrfs/ctree.h | 1 + fs/btrfs/disk-io.c | 1 + fs/btrfs/extent_io.c | 25 ++++++++++++++++++++++++- fs/btrfs/zoned.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/zoned.h | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 109 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 31c7c5872b92..a07108d65c44 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -193,6 +193,7 @@ struct btrfs_block_group { */ u64 alloc_offset; u64 zone_unusable; + u64 meta_write_pointer; }; static inline u64 btrfs_block_group_end(struct btrfs_block_group *block_group) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 10da47ab093a..1bb4f767966a 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -975,6 +975,7 @@ struct btrfs_fs_info { /* Max size to emit ZONE_APPEND write command */ u64 max_zone_append_size; + struct mutex zoned_meta_io_lock; #ifdef CONFIG_BTRFS_FS_REF_VERIFY spinlock_t ref_verify_lock; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 70621184a731..458bb27e0327 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -2769,6 +2769,7 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) mutex_init(&fs_info->delete_unused_bgs_mutex); mutex_init(&fs_info->reloc_mutex); mutex_init(&fs_info->delalloc_root_mutex); + mutex_init(&fs_info->zoned_meta_io_lock); seqlock_init(&fs_info->profiles_lock); INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 14a68f3589cc..dfa6c6106b94 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -26,6 +26,7 @@ #include "disk-io.h" #include "subpage.h" #include "zoned.h" +#include "block-group.h" static struct kmem_cache *extent_state_cache; static struct kmem_cache *extent_buffer_cache; @@ -4161,6 +4162,7 @@ static int submit_eb_page(struct page *page, struct writeback_control *wbc, struct extent_buffer **eb_context) { struct address_space *mapping = page->mapping; + struct btrfs_block_group *cache = NULL; struct extent_buffer *eb; int ret; @@ -4193,13 +4195,31 @@ static int submit_eb_page(struct page *page, struct writeback_control *wbc, if (!ret) return 0; + if (!btrfs_check_meta_write_pointer(eb->fs_info, eb, &cache)) { + /* + * If for_sync, this hole will be filled with + * trasnsaction commit. + */ + if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) + ret = -EAGAIN; + else + ret = 0; + free_extent_buffer(eb); + return ret; + } + *eb_context = eb; ret = lock_extent_buffer_for_io(eb, epd); if (ret <= 0) { + btrfs_revert_meta_write_pointer(cache, eb); + if (cache) + btrfs_put_block_group(cache); free_extent_buffer(eb); return ret; } + if (cache) + btrfs_put_block_group(cache); ret = write_one_eb(eb, wbc, epd); free_extent_buffer(eb); if (ret < 0) @@ -4245,6 +4265,7 @@ int btree_write_cache_pages(struct address_space *mapping, tag = PAGECACHE_TAG_TOWRITE; else tag = PAGECACHE_TAG_DIRTY; + btrfs_zoned_meta_io_lock(fs_info); retry: if (wbc->sync_mode == WB_SYNC_ALL) tag_pages_for_writeback(mapping, index, end); @@ -4285,7 +4306,7 @@ retry: } if (ret < 0) { end_write_bio(&epd, ret); - return ret; + goto out; } /* * If something went wrong, don't allow any metadata write bio to be @@ -4320,6 +4341,8 @@ retry: ret = -EROFS; end_write_bio(&epd, ret); } +out: + btrfs_zoned_meta_io_unlock(fs_info); return ret; } diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index f4e226bda9b0..b2a6553b2db0 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1159,6 +1159,9 @@ out: ret = -EIO; } + if (!ret) + cache->meta_write_pointer = cache->alloc_offset + cache->start; + kfree(alloc_offsets); free_extent_map(em); @@ -1317,3 +1320,50 @@ out: kfree(logical); bdput(bdev); } + +bool btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, + struct extent_buffer *eb, + struct btrfs_block_group **cache_ret) +{ + struct btrfs_block_group *cache; + bool ret = true; + + if (!btrfs_is_zoned(fs_info)) + return true; + + cache = *cache_ret; + + if (cache && (eb->start < cache->start || + cache->start + cache->length <= eb->start)) { + btrfs_put_block_group(cache); + cache = NULL; + *cache_ret = NULL; + } + + if (!cache) + cache = btrfs_lookup_block_group(fs_info, eb->start); + + if (cache) { + if (cache->meta_write_pointer != eb->start) { + btrfs_put_block_group(cache); + cache = NULL; + ret = false; + } else { + cache->meta_write_pointer = eb->start + eb->len; + } + + *cache_ret = cache; + } + + return ret; +} + +void btrfs_revert_meta_write_pointer(struct btrfs_block_group *cache, + struct extent_buffer *eb) +{ + if (!btrfs_is_zoned(eb->fs_info) || !cache) + return; + + ASSERT(cache->meta_write_pointer == eb->start + eb->len); + cache->meta_write_pointer = eb->start; +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 04f7b21652b6..0755a25d0f4c 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -50,6 +50,11 @@ bool btrfs_use_zone_append(struct btrfs_inode *inode, struct extent_map *em); void btrfs_record_physical_zoned(struct inode *inode, u64 file_offset, struct bio *bio); void btrfs_rewrite_logical_zoned(struct btrfs_ordered_extent *ordered); +bool btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, + struct extent_buffer *eb, + struct btrfs_block_group **cache_ret); +void btrfs_revert_meta_write_pointer(struct btrfs_block_group *cache, + struct extent_buffer *eb); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -151,6 +156,19 @@ static inline void btrfs_record_physical_zoned(struct inode *inode, static inline void btrfs_rewrite_logical_zoned( struct btrfs_ordered_extent *ordered) { } +static inline bool btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, + struct extent_buffer *eb, + struct btrfs_block_group **cache_ret) +{ + return true; +} + +static inline void btrfs_revert_meta_write_pointer( + struct btrfs_block_group *cache, + struct extent_buffer *eb) +{ +} + #endif static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) @@ -242,4 +260,18 @@ static inline bool btrfs_can_zone_reset(struct btrfs_device *device, return true; } +static inline void btrfs_zoned_meta_io_lock(struct btrfs_fs_info *fs_info) +{ + if (!btrfs_is_zoned(fs_info)) + return; + mutex_lock(&fs_info->zoned_meta_io_lock); +} + +static inline void btrfs_zoned_meta_io_unlock(struct btrfs_fs_info *fs_info) +{ + if (!btrfs_is_zoned(fs_info)) + return; + mutex_unlock(&fs_info->zoned_meta_io_lock); +} + #endif -- cgit v1.2.3 From 24c0a7227fdfa598badcfc0f735d16745d39e0c4 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:09 +0900 Subject: btrfs: zoned: wait for existing extents before truncating When truncating a file, file buffers which have already been allocated but not yet written may be truncated. Truncating these buffers could cause breakage of a sequential write pattern in a block group if the truncated blocks are for example followed by blocks allocated to another file. To avoid this problem, always wait for write out of all unwritten buffers before proceeding with the truncate execution. Signed-off-by: Naohiro Aota Reviewed-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/inode.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index c4779cde83c6..535abf898225 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -5169,6 +5169,15 @@ static int btrfs_setsize(struct inode *inode, struct iattr *attr) btrfs_drew_write_unlock(&root->snapshot_lock); btrfs_end_transaction(trans); } else { + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + + if (btrfs_is_zoned(fs_info)) { + ret = btrfs_wait_ordered_range(inode, + ALIGN(newsize, fs_info->sectorsize), + (u64)-1); + if (ret) + return ret; + } /* * We're truncating a file that used to have good data down to -- cgit v1.2.3 From 4eef29ef6360d9c3e4be111392e20b70e19171cc Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:10 +0900 Subject: btrfs: zoned: do not use async metadata checksum on zoned filesystems On zoned filesystems, btrfs uses per-fs zoned_meta_io_lock to serialize the metadata write IOs. Even with this serialization, write bios sent from btree_write_cache_pages can be reordered by async checksum workers as these workers are per CPU and not per zone. To preserve write bio ordering, we disable async metadata checksum on a zoned filesystem. This does not result in lower performance with HDDs as a single CPU core is fast enough to do checksum for a single zone write stream with the maximum possible bandwidth of the device. If multiple zones are being written simultaneously, HDD seek overhead lowers the achievable maximum bandwidth, resulting again in a per zone checksum serialization not affecting the performance. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 458bb27e0327..6e16f556ed75 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -871,6 +871,8 @@ static blk_status_t btree_submit_bio_start(struct inode *inode, struct bio *bio, static int check_async_write(struct btrfs_fs_info *fs_info, struct btrfs_inode *bi) { + if (btrfs_is_zoned(fs_info)) + return 0; if (atomic_read(&bi->sync_writers)) return 0; if (test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags)) -- cgit v1.2.3 From 78ce9fc269af6e69c1399ab910ba6bc81c934f67 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:11 +0900 Subject: btrfs: zoned: mark block groups to copy for device-replace This is the 1/4 patch to support device-replace on zoned filesystems. We have two types of IOs during the device replace process. One is an IO to "copy" (by the scrub functions) all the device extents from the source device to the destination device. The other one is an IO to "clone" (by handle_ops_on_dev_replace()) new incoming write IOs from users to the source device into the target device. Cloning incoming IOs can break the sequential write rule in on target device. When a write is mapped in the middle of a block group, the IO is directed to the middle of a target device zone, which breaks the sequential write requirement. However, the cloning function cannot be disabled since incoming IOs targeting already copied device extents must be cloned so that the IO is executed on the target device. We cannot use dev_replace->cursor_{left,right} to determine whether a bio is going to a not yet copied region. Since we have a time gap between finishing btrfs_scrub_dev() and rewriting the mapping tree in btrfs_dev_replace_finishing(), we can have a newly allocated device extent which is never cloned nor copied. So the point is to copy only already existing device extents. This patch introduces mark_block_group_to_copy() to mark existing block groups as a target of copying. Then, handle_ops_on_dev_replace() and dev-replace can check the flag to do their job. Also, btrfs_finish_block_group_to_copy() will check if the copied stripe is the last stripe in the block group. With the last stripe copied, the to_copy flag is finally disabled. Afterwards we can safely clone incoming IOs on this block group. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/block-group.h | 1 + fs/btrfs/dev-replace.c | 184 +++++++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/dev-replace.h | 3 + fs/btrfs/scrub.c | 16 +++++ 4 files changed, 204 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index a07108d65c44..d37ee576ac6e 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -95,6 +95,7 @@ struct btrfs_block_group { unsigned int iref:1; unsigned int has_caching_ctl:1; unsigned int removed:1; + unsigned int to_copy:1; int disk_cache_state; diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index bc73f798ce3a..3a9c1e046ebe 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -22,6 +22,7 @@ #include "dev-replace.h" #include "sysfs.h" #include "zoned.h" +#include "block-group.h" /* * Device replace overview @@ -459,6 +460,185 @@ static char* btrfs_dev_name(struct btrfs_device *device) return rcu_str_deref(device->name); } +static int mark_block_group_to_copy(struct btrfs_fs_info *fs_info, + struct btrfs_device *src_dev) +{ + struct btrfs_path *path; + struct btrfs_key key; + struct btrfs_key found_key; + struct btrfs_root *root = fs_info->dev_root; + struct btrfs_dev_extent *dev_extent = NULL; + struct btrfs_block_group *cache; + struct btrfs_trans_handle *trans; + int ret = 0; + u64 chunk_offset; + + /* Do not use "to_copy" on non zoned filesystem for now */ + if (!btrfs_is_zoned(fs_info)) + return 0; + + mutex_lock(&fs_info->chunk_mutex); + + /* Ensure we don't have pending new block group */ + spin_lock(&fs_info->trans_lock); + while (fs_info->running_transaction && + !list_empty(&fs_info->running_transaction->dev_update_list)) { + spin_unlock(&fs_info->trans_lock); + mutex_unlock(&fs_info->chunk_mutex); + trans = btrfs_attach_transaction(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + mutex_lock(&fs_info->chunk_mutex); + if (ret == -ENOENT) { + spin_lock(&fs_info->trans_lock); + continue; + } else { + goto unlock; + } + } + + ret = btrfs_commit_transaction(trans); + mutex_lock(&fs_info->chunk_mutex); + if (ret) + goto unlock; + + spin_lock(&fs_info->trans_lock); + } + spin_unlock(&fs_info->trans_lock); + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto unlock; + } + + path->reada = READA_FORWARD; + path->search_commit_root = 1; + path->skip_locking = 1; + + key.objectid = src_dev->devid; + key.type = BTRFS_DEV_EXTENT_KEY; + key.offset = 0; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto free_path; + if (ret > 0) { + if (path->slots[0] >= + btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto free_path; + if (ret > 0) { + ret = 0; + goto free_path; + } + } else { + ret = 0; + } + } + + while (1) { + struct extent_buffer *leaf = path->nodes[0]; + int slot = path->slots[0]; + + btrfs_item_key_to_cpu(leaf, &found_key, slot); + + if (found_key.objectid != src_dev->devid) + break; + + if (found_key.type != BTRFS_DEV_EXTENT_KEY) + break; + + if (found_key.offset < key.offset) + break; + + dev_extent = btrfs_item_ptr(leaf, slot, struct btrfs_dev_extent); + + chunk_offset = btrfs_dev_extent_chunk_offset(leaf, dev_extent); + + cache = btrfs_lookup_block_group(fs_info, chunk_offset); + if (!cache) + goto skip; + + spin_lock(&cache->lock); + cache->to_copy = 1; + spin_unlock(&cache->lock); + + btrfs_put_block_group(cache); + +skip: + ret = btrfs_next_item(root, path); + if (ret != 0) { + if (ret > 0) + ret = 0; + break; + } + } + +free_path: + btrfs_free_path(path); +unlock: + mutex_unlock(&fs_info->chunk_mutex); + + return ret; +} + +bool btrfs_finish_block_group_to_copy(struct btrfs_device *srcdev, + struct btrfs_block_group *cache, + u64 physical) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct extent_map *em; + struct map_lookup *map; + u64 chunk_offset = cache->start; + int num_extents, cur_extent; + int i; + + /* Do not use "to_copy" on non zoned filesystem for now */ + if (!btrfs_is_zoned(fs_info)) + return true; + + spin_lock(&cache->lock); + if (cache->removed) { + spin_unlock(&cache->lock); + return true; + } + spin_unlock(&cache->lock); + + em = btrfs_get_chunk_map(fs_info, chunk_offset, 1); + ASSERT(!IS_ERR(em)); + map = em->map_lookup; + + num_extents = cur_extent = 0; + for (i = 0; i < map->num_stripes; i++) { + /* We have more device extent to copy */ + if (srcdev != map->stripes[i].dev) + continue; + + num_extents++; + if (physical == map->stripes[i].physical) + cur_extent = i; + } + + free_extent_map(em); + + if (num_extents > 1 && cur_extent < num_extents - 1) { + /* + * Has more stripes on this device. Keep this block group + * readonly until we finish all the stripes. + */ + return false; + } + + /* Last stripe on this device */ + spin_lock(&cache->lock); + cache->to_copy = 0; + spin_unlock(&cache->lock); + + return true; +} + static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, const char *tgtdev_name, u64 srcdevid, const char *srcdev_name, int read_src) @@ -500,6 +680,10 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, if (ret) return ret; + ret = mark_block_group_to_copy(fs_info, src_device); + if (ret) + return ret; + down_write(&dev_replace->rwsem); switch (dev_replace->replace_state) { case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: diff --git a/fs/btrfs/dev-replace.h b/fs/btrfs/dev-replace.h index 60b70dacc299..3911049a5f23 100644 --- a/fs/btrfs/dev-replace.h +++ b/fs/btrfs/dev-replace.h @@ -18,5 +18,8 @@ int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info); void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info); int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info); int __pure btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace); +bool btrfs_finish_block_group_to_copy(struct btrfs_device *srcdev, + struct btrfs_block_group *cache, + u64 physical); #endif diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 5f4f88a4d2c8..da4f9c24e42d 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -3561,6 +3561,16 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, if (!cache) goto skip; + if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) { + spin_lock(&cache->lock); + if (!cache->to_copy) { + spin_unlock(&cache->lock); + ro_set = 0; + goto done; + } + spin_unlock(&cache->lock); + } + /* * Make sure that while we are scrubbing the corresponding block * group doesn't get its logical address and its device extents @@ -3692,6 +3702,12 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, scrub_pause_off(fs_info); + if (sctx->is_dev_replace && + !btrfs_finish_block_group_to_copy(dev_replace->srcdev, + cache, found_key.offset)) + ro_set = 0; + +done: down_write(&dev_replace->rwsem); dev_replace->cursor_left = dev_replace->cursor_right; dev_replace->item_needs_writeback = 1; -- cgit v1.2.3 From 6143c23ccced762d21a87ef5fa421ba876231131 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:12 +0900 Subject: btrfs: zoned: implement cloning for zoned device-replace This is 2/4 patch to implement device replace for zoned filesystems. In zoned mode, a block group must be either copied (from the source device to the target device) or cloned (to both devices). Implement the cloning part. If a block group targeted by an IO is marked to copy, we should not clone the IO to the destination device, because the block group is eventually copied by the replace process. This commit also handles cloning of device reset. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 57 +++++++++++++++++++++++++++++++++++++------------- fs/btrfs/volumes.c | 31 +++++++++++++++++++++++++-- fs/btrfs/zoned.c | 9 ++++++++ 3 files changed, 80 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index d976c56b3a56..cd308bf3a220 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -35,6 +35,7 @@ #include "discard.h" #include "rcu-string.h" #include "zoned.h" +#include "dev-replace.h" #undef SCRAMBLE_DELAYED_REFS @@ -1265,6 +1266,46 @@ static int btrfs_issue_discard(struct block_device *bdev, u64 start, u64 len, return ret; } +static int do_discard_extent(struct btrfs_bio_stripe *stripe, u64 *bytes) +{ + struct btrfs_device *dev = stripe->dev; + struct btrfs_fs_info *fs_info = dev->fs_info; + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + u64 phys = stripe->physical; + u64 len = stripe->length; + u64 discarded = 0; + int ret = 0; + + /* Zone reset on a zoned filesystem */ + if (btrfs_can_zone_reset(dev, phys, len)) { + u64 src_disc; + + ret = btrfs_reset_device_zone(dev, phys, len, &discarded); + if (ret) + goto out; + + if (!btrfs_dev_replace_is_ongoing(dev_replace) || + dev != dev_replace->srcdev) + goto out; + + src_disc = discarded; + + /* Send to replace target as well */ + ret = btrfs_reset_device_zone(dev_replace->tgtdev, phys, len, + &discarded); + discarded += src_disc; + } else if (blk_queue_discard(bdev_get_queue(stripe->dev->bdev))) { + ret = btrfs_issue_discard(dev->bdev, phys, len, &discarded); + } else { + ret = 0; + *bytes = 0; + } + +out: + *bytes = discarded; + return ret; +} + int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr, u64 num_bytes, u64 *actual_bytes) { @@ -1298,28 +1339,14 @@ int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr, stripe = bbio->stripes; for (i = 0; i < bbio->num_stripes; i++, stripe++) { - struct btrfs_device *dev = stripe->dev; - u64 physical = stripe->physical; - u64 length = stripe->length; u64 bytes; - struct request_queue *req_q; if (!stripe->dev->bdev) { ASSERT(btrfs_test_opt(fs_info, DEGRADED)); continue; } - req_q = bdev_get_queue(stripe->dev->bdev); - /* Zone reset on zoned filesystems */ - if (btrfs_can_zone_reset(dev, physical, length)) - ret = btrfs_reset_device_zone(dev, physical, - length, &bytes); - else if (blk_queue_discard(req_q)) - ret = btrfs_issue_discard(dev->bdev, physical, - length, &bytes); - else - continue; - + ret = do_discard_extent(stripe, &bytes); if (!ret) { discarded_bytes += bytes; } else if (ret != -EOPNOTSUPP) { diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index a4d47c6050f7..52ec6721ada2 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -5973,9 +5973,29 @@ static int get_extra_mirror_from_replace(struct btrfs_fs_info *fs_info, return ret; } +static bool is_block_group_to_copy(struct btrfs_fs_info *fs_info, u64 logical) +{ + struct btrfs_block_group *cache; + bool ret; + + /* Non-ZONED mode does not use "to_copy" flag */ + if (!btrfs_is_zoned(fs_info)) + return false; + + cache = btrfs_lookup_block_group(fs_info, logical); + + spin_lock(&cache->lock); + ret = cache->to_copy; + spin_unlock(&cache->lock); + + btrfs_put_block_group(cache); + return ret; +} + static void handle_ops_on_dev_replace(enum btrfs_map_op op, struct btrfs_bio **bbio_ret, struct btrfs_dev_replace *dev_replace, + u64 logical, int *num_stripes_ret, int *max_errors_ret) { struct btrfs_bio *bbio = *bbio_ret; @@ -5988,6 +6008,13 @@ static void handle_ops_on_dev_replace(enum btrfs_map_op op, if (op == BTRFS_MAP_WRITE) { int index_where_to_add; + /* + * A block group which have "to_copy" set will eventually + * copied by dev-replace process. We can avoid cloning IO here. + */ + if (is_block_group_to_copy(dev_replace->srcdev->fs_info, logical)) + return; + /* * duplicate the write operations while the dev replace * procedure is running. Since the copying of the old disk to @@ -6376,8 +6403,8 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL && need_full_stripe(op)) { - handle_ops_on_dev_replace(op, &bbio, dev_replace, &num_stripes, - &max_errors); + handle_ops_on_dev_replace(op, &bbio, dev_replace, logical, + &num_stripes, &max_errors); } *bbio_ret = bbio; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index b2a6553b2db0..15288f766842 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -11,6 +11,7 @@ #include "disk-io.h" #include "block-group.h" #include "transaction.h" +#include "dev-replace.h" /* Maximum number of zones to report per blkdev_report_zones() call */ #define BTRFS_REPORT_NR_ZONES 4096 @@ -1036,6 +1037,8 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) for (i = 0; i < map->num_stripes; i++) { bool is_sequential; struct blk_zone zone; + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + int dev_replace_is_ongoing = 0; device = map->stripes[i].dev; physical = map->stripes[i].physical; @@ -1062,6 +1065,12 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) */ btrfs_dev_clear_zone_empty(device, physical); + down_read(&dev_replace->rwsem); + dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace); + if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL) + btrfs_dev_clear_zone_empty(dev_replace->tgtdev, physical); + up_read(&dev_replace->rwsem); + /* * The group is mapped to a sequential zone. Get the zone write * pointer to determine the allocation offset within the zone. -- cgit v1.2.3 From de17addce7a20db311c020fa91497a7341782d2d Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:13 +0900 Subject: btrfs: zoned: implement copying for zoned device-replace This is 3/4 patch to implement device-replace on zoned filesystems. This commit implements copying. To do this, it tracks the write pointer during the device replace process. As device-replace's copy process is smart enough to only copy used extents on the source device, we have to fill the gap to honor the sequential write requirement in the target device. The device-replace process on zoned filesystems must copy or clone all the extents in the source device exactly once. So, we need to ensure allocations started just before the dev-replace process to have their corresponding extent information in the B-trees. finish_extent_writes_for_zoned() implements that functionality, which basically is the removed code in the commit 042528f8d840 ("Btrfs: fix block group remaining RO forever after error during device replace"). Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/scrub.c | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/volumes.c | 2 +- fs/btrfs/zoned.c | 9 ++++++ fs/btrfs/zoned.h | 7 +++++ 4 files changed, 101 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index da4f9c24e42d..92904902d160 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -166,6 +166,7 @@ struct scrub_ctx { int pages_per_rd_bio; int is_dev_replace; + u64 write_pointer; struct scrub_bio *wr_curr_bio; struct mutex wr_lock; @@ -1619,6 +1620,25 @@ static int scrub_write_page_to_dev_replace(struct scrub_block *sblock, return scrub_add_page_to_wr_bio(sblock->sctx, spage); } +static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical) +{ + int ret = 0; + u64 length; + + if (!btrfs_is_zoned(sctx->fs_info)) + return 0; + + if (sctx->write_pointer < physical) { + length = physical - sctx->write_pointer; + + ret = btrfs_zoned_issue_zeroout(sctx->wr_tgtdev, + sctx->write_pointer, length); + if (!ret) + sctx->write_pointer = physical; + } + return ret; +} + static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, struct scrub_page *spage) { @@ -1641,6 +1661,13 @@ again: if (sbio->page_count == 0) { struct bio *bio; + ret = fill_writer_pointer_gap(sctx, + spage->physical_for_dev_replace); + if (ret) { + mutex_unlock(&sctx->wr_lock); + return ret; + } + sbio->physical = spage->physical_for_dev_replace; sbio->logical = spage->logical; sbio->dev = sctx->wr_tgtdev; @@ -1702,6 +1729,9 @@ static void scrub_wr_submit(struct scrub_ctx *sctx) * doubled the write performance on spinning disks when measured * with Linux 3.5 */ btrfsic_submit_bio(sbio->bio); + + if (btrfs_is_zoned(sctx->fs_info)) + sctx->write_pointer = sbio->physical + sbio->page_count * PAGE_SIZE; } static void scrub_wr_bio_end_io(struct bio *bio) @@ -3025,6 +3055,20 @@ out: return ret < 0 ? ret : 0; } +static void sync_replace_for_zoned(struct scrub_ctx *sctx) +{ + if (!btrfs_is_zoned(sctx->fs_info)) + return; + + sctx->flush_all_writes = true; + scrub_submit(sctx); + mutex_lock(&sctx->wr_lock); + scrub_wr_submit(sctx); + mutex_unlock(&sctx->wr_lock); + + wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); +} + static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, struct map_lookup *map, struct btrfs_device *scrub_dev, @@ -3165,6 +3209,14 @@ static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, */ blk_start_plug(&plug); + if (sctx->is_dev_replace && + btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) { + mutex_lock(&sctx->wr_lock); + sctx->write_pointer = physical; + mutex_unlock(&sctx->wr_lock); + sctx->flush_all_writes = true; + } + /* * now find all extents for each stripe and scrub them */ @@ -3353,6 +3405,9 @@ again: if (ret) goto out; + if (sctx->is_dev_replace) + sync_replace_for_zoned(sctx); + if (extent_logical + extent_len < key.objectid + bytes) { if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { @@ -3475,6 +3530,25 @@ out: return ret; } +static int finish_extent_writes_for_zoned(struct btrfs_root *root, + struct btrfs_block_group *cache) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_trans_handle *trans; + + if (!btrfs_is_zoned(fs_info)) + return 0; + + btrfs_wait_block_group_reservations(cache); + btrfs_wait_nocow_writers(cache); + btrfs_wait_ordered_roots(fs_info, U64_MAX, cache->start, cache->length); + + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) + return PTR_ERR(trans); + return btrfs_commit_transaction(trans); +} + static noinline_for_stack int scrub_enumerate_chunks(struct scrub_ctx *sctx, struct btrfs_device *scrub_dev, u64 start, u64 end) @@ -3629,6 +3703,16 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, * group is not RO. */ ret = btrfs_inc_block_group_ro(cache, sctx->is_dev_replace); + if (!ret && sctx->is_dev_replace) { + ret = finish_extent_writes_for_zoned(root, cache); + if (ret) { + btrfs_dec_block_group_ro(cache); + scrub_pause_off(fs_info); + btrfs_put_block_group(cache); + break; + } + } + if (ret == 0) { ro_set = 1; } else if (ret == -ENOSPC && !sctx->is_dev_replace) { diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 52ec6721ada2..1312b17a6b49 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -5978,7 +5978,7 @@ static bool is_block_group_to_copy(struct btrfs_fs_info *fs_info, u64 logical) struct btrfs_block_group *cache; bool ret; - /* Non-ZONED mode does not use "to_copy" flag */ + /* Non zoned filesystem does not use "to_copy" flag */ if (!btrfs_is_zoned(fs_info)) return false; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 15288f766842..a4707bab6073 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1376,3 +1376,12 @@ void btrfs_revert_meta_write_pointer(struct btrfs_block_group *cache, ASSERT(cache->meta_write_pointer == eb->start + eb->len); cache->meta_write_pointer = eb->start; } + +int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 length) +{ + if (!btrfs_dev_is_sequential(device, physical)) + return -EOPNOTSUPP; + + return blkdev_issue_zeroout(device->bdev, physical >> SECTOR_SHIFT, + length >> SECTOR_SHIFT, GFP_NOFS, 0); +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 0755a25d0f4c..5ed1ea2009ea 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -55,6 +55,7 @@ bool btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, struct btrfs_block_group **cache_ret); void btrfs_revert_meta_write_pointer(struct btrfs_block_group *cache, struct extent_buffer *eb); +int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 length); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -169,6 +170,12 @@ static inline void btrfs_revert_meta_write_pointer( { } +static inline int btrfs_zoned_issue_zeroout(struct btrfs_device *device, + u64 physical, u64 length) +{ + return -EOPNOTSUPP; +} + #endif static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) -- cgit v1.2.3 From 7db1c5d14dcd521bef1780b79dcc68b3968447a9 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:14 +0900 Subject: btrfs: zoned: support dev-replace in zoned filesystems This is 4/4 patch to implement device-replace on zoned filesystems. Even after the copying is done, the write pointers of the source device and the destination device may not be synchronized. For example, when the last allocated extent is freed before device-replace process, the extent is not copied, leaving a hole there. Synchronize the write pointers by writing zeroes to the destination device. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/scrub.c | 40 ++++++++++++++++++++++++++++++ fs/btrfs/zoned.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/zoned.h | 9 +++++++ 3 files changed, 123 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 92904902d160..e0c3ec01e324 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -1628,6 +1628,9 @@ static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical) if (!btrfs_is_zoned(sctx->fs_info)) return 0; + if (!btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) + return 0; + if (sctx->write_pointer < physical) { length = physical - sctx->write_pointer; @@ -3069,6 +3072,32 @@ static void sync_replace_for_zoned(struct scrub_ctx *sctx) wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); } +static int sync_write_pointer_for_zoned(struct scrub_ctx *sctx, u64 logical, + u64 physical, u64 physical_end) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + int ret = 0; + + if (!btrfs_is_zoned(fs_info)) + return 0; + + wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); + + mutex_lock(&sctx->wr_lock); + if (sctx->write_pointer < physical_end) { + ret = btrfs_sync_zone_write_pointer(sctx->wr_tgtdev, logical, + physical, + sctx->write_pointer); + if (ret) + btrfs_err(fs_info, + "zoned: failed to recover write pointer"); + } + mutex_unlock(&sctx->wr_lock); + btrfs_dev_clear_zone_empty(sctx->wr_tgtdev, physical); + + return ret; +} + static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, struct map_lookup *map, struct btrfs_device *scrub_dev, @@ -3475,6 +3504,17 @@ out: blk_finish_plug(&plug); btrfs_free_path(path); btrfs_free_path(ppath); + + if (sctx->is_dev_replace && ret >= 0) { + int ret2; + + ret2 = sync_write_pointer_for_zoned(sctx, base + offset, + map->stripes[num].physical, + physical_end); + if (ret2) + ret = ret2; + } + return ret < 0 ? ret : 0; } diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index a4707bab6073..9a5cf153da89 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -12,6 +12,7 @@ #include "block-group.h" #include "transaction.h" #include "dev-replace.h" +#include "space-info.h" /* Maximum number of zones to report per blkdev_report_zones() call */ #define BTRFS_REPORT_NR_ZONES 4096 @@ -1385,3 +1386,76 @@ int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 len return blkdev_issue_zeroout(device->bdev, physical >> SECTOR_SHIFT, length >> SECTOR_SHIFT, GFP_NOFS, 0); } + +static int read_zone_info(struct btrfs_fs_info *fs_info, u64 logical, + struct blk_zone *zone) +{ + struct btrfs_bio *bbio = NULL; + u64 mapped_length = PAGE_SIZE; + unsigned int nofs_flag; + int nmirrors; + int i, ret; + + ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical, + &mapped_length, &bbio); + if (ret || !bbio || mapped_length < PAGE_SIZE) { + btrfs_put_bbio(bbio); + return -EIO; + } + + if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) + return -EINVAL; + + nofs_flag = memalloc_nofs_save(); + nmirrors = (int)bbio->num_stripes; + for (i = 0; i < nmirrors; i++) { + u64 physical = bbio->stripes[i].physical; + struct btrfs_device *dev = bbio->stripes[i].dev; + + /* Missing device */ + if (!dev->bdev) + continue; + + ret = btrfs_get_dev_zone(dev, physical, zone); + /* Failing device */ + if (ret == -EIO || ret == -EOPNOTSUPP) + continue; + break; + } + memalloc_nofs_restore(nofs_flag); + + return ret; +} + +/* + * Synchronize write pointer in a zone at @physical_start on @tgt_dev, by + * filling zeros between @physical_pos to a write pointer of dev-replace + * source device. + */ +int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, + u64 physical_start, u64 physical_pos) +{ + struct btrfs_fs_info *fs_info = tgt_dev->fs_info; + struct blk_zone zone; + u64 length; + u64 wp; + int ret; + + if (!btrfs_dev_is_sequential(tgt_dev, physical_pos)) + return 0; + + ret = read_zone_info(fs_info, logical, &zone); + if (ret) + return ret; + + wp = physical_start + ((zone.wp - zone.start) << SECTOR_SHIFT); + + if (physical_pos == wp) + return 0; + + if (physical_pos > wp) + return -EUCLEAN; + + length = wp - physical_pos; + return btrfs_zoned_issue_zeroout(tgt_dev, physical_pos, length); +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 5ed1ea2009ea..932ad9bc0de6 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -56,6 +56,8 @@ bool btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, void btrfs_revert_meta_write_pointer(struct btrfs_block_group *cache, struct extent_buffer *eb); int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 length); +int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, + u64 physical_start, u64 physical_pos); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -176,6 +178,13 @@ static inline int btrfs_zoned_issue_zeroout(struct btrfs_device *device, return -EOPNOTSUPP; } +static inline int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, + u64 logical, u64 physical_start, + u64 physical_pos) +{ + return -EOPNOTSUPP; +} + #endif static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) -- cgit v1.2.3 From 32430c614844169a5e5554dcbb307735ddd1f780 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:15 +0900 Subject: btrfs: zoned: enable relocation on a zoned filesystem Currently fallocate() is disabled on a zoned filesystem. Since current relocation process relies on preallocation to move file data extents, it must be handled differently. On a zoned filesystem, we just truncate the inode to the size that we wanted to pre-allocate. Then, we flush dirty pages on the file before finishing the relocation process. run_delalloc_zoned() will handle all the allocations and submit IOs to the underlying layers. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 473b78874844..232d5da7b7be 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -2553,6 +2553,31 @@ static noinline_for_stack int prealloc_file_extent_cluster( if (ret) return ret; + /* + * On a zoned filesystem, we cannot preallocate the file region. + * Instead, we dirty and fiemap_write the region. + */ + if (btrfs_is_zoned(inode->root->fs_info)) { + struct btrfs_root *root = inode->root; + struct btrfs_trans_handle *trans; + + end = cluster->end - offset + 1; + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + inode->vfs_inode.i_ctime = current_time(&inode->vfs_inode); + i_size_write(&inode->vfs_inode, end); + ret = btrfs_update_inode(trans, root, inode); + if (ret) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + + return btrfs_end_transaction(trans); + } + inode_lock(&inode->vfs_inode); for (nr = 0; nr < cluster->nr; nr++) { start = cluster->boundary[nr] - offset; @@ -2756,6 +2781,8 @@ static int relocate_file_extent_cluster(struct inode *inode, } } WARN_ON(nr != cluster->nr); + if (btrfs_is_zoned(fs_info) && !ret) + ret = btrfs_wait_ordered_range(inode, 0, (u64)-1); out: kfree(ra); return ret; @@ -3434,8 +3461,12 @@ static int __insert_orphan_inode(struct btrfs_trans_handle *trans, struct btrfs_path *path; struct btrfs_inode_item *item; struct extent_buffer *leaf; + u64 flags = BTRFS_INODE_NOCOMPRESS | BTRFS_INODE_PREALLOC; int ret; + if (btrfs_is_zoned(trans->fs_info)) + flags &= ~BTRFS_INODE_PREALLOC; + path = btrfs_alloc_path(); if (!path) return -ENOMEM; @@ -3450,8 +3481,7 @@ static int __insert_orphan_inode(struct btrfs_trans_handle *trans, btrfs_set_inode_generation(leaf, item, 1); btrfs_set_inode_size(leaf, item, 0); btrfs_set_inode_mode(leaf, item, S_IFREG | 0600); - btrfs_set_inode_flags(leaf, item, BTRFS_INODE_NOCOMPRESS | - BTRFS_INODE_PREALLOC); + btrfs_set_inode_flags(leaf, item, flags); btrfs_mark_buffer_dirty(leaf); out: btrfs_free_path(path); -- cgit v1.2.3 From f7ef5287a63d644e62a52893af8c6cfcb5043213 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:16 +0900 Subject: btrfs: zoned: relocate block group to repair IO failure in zoned filesystems When a bad checksum is found and if the filesystem has a mirror of the damaged data, we read the correct data from the mirror and writes it to damaged blocks. This however, violates the sequential write constraints of a zoned block device. We can consider three methods to repair an IO failure in zoned filesystems: (1) Reset and rewrite the damaged zone (2) Allocate new device extent and replace the damaged device extent to the new extent (3) Relocate the corresponding block group Method (1) is most similar to a behavior done with regular devices. However, it also wipes non-damaged data in the same device extent, and so it unnecessary degrades non-damaged data. Method (2) is much like device replacing but done in the same device. It is safe because it keeps the device extent until the replacing finish. However, extending device replacing is non-trivial. It assumes "src_dev->physical == dst_dev->physical". Also, the extent mapping replacing function should be extended to support replacing device extent position in one device. Method (3) invokes relocation of the damaged block group and is straightforward to implement. It relocates all the mirrored device extents, so it potentially is a more costly operation than method (1) or (2). But it relocates only used extents which reduce the total IO size. Let's apply method (3) for now. In the future, we can extend device-replace and apply method (2). For protecting a block group gets relocated multiple time with multiple IO errors, this commit introduces "relocating_repair" bit to show it's now relocating to repair IO failures. Also it uses a new kthread "btrfs-relocating-repair", not to block IO path with relocating process. This commit also supports repairing in the scrub process. Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/block-group.h | 1 + fs/btrfs/extent_io.c | 3 +++ fs/btrfs/scrub.c | 3 +++ fs/btrfs/volumes.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/volumes.h | 1 + 5 files changed, 80 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index d37ee576ac6e..29678426247d 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -96,6 +96,7 @@ struct btrfs_block_group { unsigned int has_caching_ctl:1; unsigned int removed:1; unsigned int to_copy:1; + unsigned int relocating_repair:1; int disk_cache_state; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index dfa6c6106b94..4dfb3ead1175 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2260,6 +2260,9 @@ int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, ASSERT(!(fs_info->sb->s_flags & SB_RDONLY)); BUG_ON(!mirror_num); + if (btrfs_is_zoned(fs_info)) + return btrfs_repair_one_zone(fs_info, logical); + bio = btrfs_io_bio_alloc(1); bio->bi_iter.bi_size = 0; map_length = length; diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index e0c3ec01e324..310fce00fcda 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -857,6 +857,9 @@ static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) have_csum = sblock_to_check->pagev[0]->have_csum; dev = sblock_to_check->pagev[0]->dev; + if (btrfs_is_zoned(fs_info) && !sctx->is_dev_replace) + return btrfs_repair_one_zone(fs_info, logical); + /* * We must use GFP_NOFS because the scrub task might be waiting for a * worker task executing this function and in turn a transaction commit diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 1312b17a6b49..b8fab44394f5 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -7980,3 +7980,75 @@ bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr) spin_unlock(&fs_info->swapfile_pins_lock); return node != NULL; } + +static int relocating_repair_kthread(void *data) +{ + struct btrfs_block_group *cache = (struct btrfs_block_group *)data; + struct btrfs_fs_info *fs_info = cache->fs_info; + u64 target; + int ret = 0; + + target = cache->start; + btrfs_put_block_group(cache); + + if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) { + btrfs_info(fs_info, + "zoned: skip relocating block group %llu to repair: EBUSY", + target); + return -EBUSY; + } + + mutex_lock(&fs_info->delete_unused_bgs_mutex); + + /* Ensure block group still exists */ + cache = btrfs_lookup_block_group(fs_info, target); + if (!cache) + goto out; + + if (!cache->relocating_repair) + goto out; + + ret = btrfs_may_alloc_data_chunk(fs_info, target); + if (ret < 0) + goto out; + + btrfs_info(fs_info, + "zoned: relocating block group %llu to repair IO failure", + target); + ret = btrfs_relocate_chunk(fs_info, target); + +out: + if (cache) + btrfs_put_block_group(cache); + mutex_unlock(&fs_info->delete_unused_bgs_mutex); + btrfs_exclop_finish(fs_info); + + return ret; +} + +int btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical) +{ + struct btrfs_block_group *cache; + + /* Do not attempt to repair in degraded state */ + if (btrfs_test_opt(fs_info, DEGRADED)) + return 0; + + cache = btrfs_lookup_block_group(fs_info, logical); + if (!cache) + return 0; + + spin_lock(&cache->lock); + if (cache->relocating_repair) { + spin_unlock(&cache->lock); + btrfs_put_block_group(cache); + return 0; + } + cache->relocating_repair = 1; + spin_unlock(&cache->lock); + + kthread_run(relocating_repair_kthread, cache, + "btrfs-relocating-repair"); + + return 0; +} diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index d3bbdb4175df..d4c3e0dd32b8 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -599,5 +599,6 @@ void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info, int btrfs_bg_type_to_factor(u64 flags); const char *btrfs_bg_type_to_raid_name(u64 flags); int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info); +int btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical); #endif -- cgit v1.2.3 From 6ab6ebb76042d3d94a7c6c447f770a28a412c68c Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:17 +0900 Subject: btrfs: split alloc_log_tree() This is a preparation patch for the next patch. Split alloc_log_tree() into two parts. The first one allocating the tree structure, remains in alloc_log_tree() and the second part allocating the tree node, which is moved into btrfs_alloc_log_tree_node(). Also export the latter part is to be used in the next patch. Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 33 +++++++++++++++++++++++++++------ fs/btrfs/disk-io.h | 2 ++ 2 files changed, 29 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 6e16f556ed75..d2fa92526b3b 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1254,7 +1254,6 @@ static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info) { struct btrfs_root *root; - struct extent_buffer *leaf; root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID, GFP_NOFS); if (!root) @@ -1264,6 +1263,14 @@ static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans, root->root_key.type = BTRFS_ROOT_ITEM_KEY; root->root_key.offset = BTRFS_TREE_LOG_OBJECTID; + return root; +} + +int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + struct extent_buffer *leaf; + /* * DON'T set SHAREABLE bit for log trees. * @@ -1276,26 +1283,33 @@ static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans, leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID, NULL, 0, 0, 0, BTRFS_NESTING_NORMAL); - if (IS_ERR(leaf)) { - btrfs_put_root(root); - return ERR_CAST(leaf); - } + if (IS_ERR(leaf)) + return PTR_ERR(leaf); root->node = leaf; btrfs_mark_buffer_dirty(root->node); btrfs_tree_unlock(root->node); - return root; + + return 0; } int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info) { struct btrfs_root *log_root; + int ret; log_root = alloc_log_tree(trans, fs_info); if (IS_ERR(log_root)) return PTR_ERR(log_root); + + ret = btrfs_alloc_log_tree_node(trans, log_root); + if (ret) { + btrfs_put_root(log_root); + return ret; + } + WARN_ON(fs_info->log_root_tree); fs_info->log_root_tree = log_root; return 0; @@ -1307,11 +1321,18 @@ int btrfs_add_log_tree(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *log_root; struct btrfs_inode_item *inode_item; + int ret; log_root = alloc_log_tree(trans, fs_info); if (IS_ERR(log_root)) return PTR_ERR(log_root); + ret = btrfs_alloc_log_tree_node(trans, log_root); + if (ret) { + btrfs_put_root(log_root); + return ret; + } + log_root->last_trans = trans->transid; log_root->root_key.offset = root->root_key.objectid; diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index 9f4a2a1e3d36..0e7e9526b6a8 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -120,6 +120,8 @@ blk_status_t btrfs_wq_submit_bio(struct inode *inode, struct bio *bio, extent_submit_bio_start_t *submit_bio_start); blk_status_t btrfs_submit_bio_done(void *private_data, struct bio *bio, int mirror_num); +int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans, + struct btrfs_root *root); int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info); int btrfs_add_log_tree(struct btrfs_trans_handle *trans, -- cgit v1.2.3 From 40ab3be102f0a61dbb93093f330b432324a793f1 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:18 +0900 Subject: btrfs: zoned: extend zoned allocator to use dedicated tree-log block group This is the 1/3 patch to enable tree log on zoned filesystems. The tree-log feature does not work on a zoned filesystem as is. Blocks for a tree-log tree are allocated mixed with other metadata blocks and btrfs writes and syncs the tree-log blocks to devices at the time of fsync(), which has a different timing than a global transaction commit. As a result, both writing tree-log blocks and writing other metadata blocks become non-sequential writes that zoned filesystems must avoid. Introduce a dedicated block group for tree-log blocks, so that tree-log blocks and other metadata blocks can be separate write streams. As a result, each write stream can now be written to devices separately. "fs_info->treelog_bg" tracks the dedicated block group and assigns "treelog_bg" on-demand on tree-log block allocation time. This commit extends the zoned block allocator to use the block group. Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 2 ++ fs/btrfs/ctree.h | 2 ++ fs/btrfs/disk-io.c | 1 + fs/btrfs/extent-tree.c | 75 +++++++++++++++++++++++++++++++++++++++++++++++--- fs/btrfs/zoned.h | 14 ++++++++++ 5 files changed, 90 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index f5e9f560ce6d..5064be59dac5 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -901,6 +901,8 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, btrfs_return_cluster_to_free_space(block_group, cluster); spin_unlock(&cluster->refill_lock); + btrfs_clear_treelog_bg(block_group); + path = btrfs_alloc_path(); if (!path) { ret = -ENOMEM; diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 1bb4f767966a..6f4b493625ef 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -976,6 +976,8 @@ struct btrfs_fs_info { /* Max size to emit ZONE_APPEND write command */ u64 max_zone_append_size; struct mutex zoned_meta_io_lock; + spinlock_t treelog_bg_lock; + u64 treelog_bg; #ifdef CONFIG_BTRFS_FS_REF_VERIFY spinlock_t ref_verify_lock; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index d2fa92526b3b..84c6650d5ef7 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -2787,6 +2787,7 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) spin_lock_init(&fs_info->super_lock); spin_lock_init(&fs_info->buffer_lock); spin_lock_init(&fs_info->unused_bgs_lock); + spin_lock_init(&fs_info->treelog_bg_lock); rwlock_init(&fs_info->tree_mod_log_lock); mutex_init(&fs_info->unused_bg_unpin_mutex); mutex_init(&fs_info->delete_unused_bgs_mutex); diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index cd308bf3a220..78ad31a59e59 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -3497,6 +3497,9 @@ struct find_free_extent_ctl { bool have_caching_bg; bool orig_have_caching_bg; + /* Allocation is called for tree-log */ + bool for_treelog; + /* RAID index, converted from flags */ int index; @@ -3725,6 +3728,22 @@ static int do_allocation_clustered(struct btrfs_block_group *block_group, return find_free_extent_unclustered(block_group, ffe_ctl); } +/* + * Tree-log block group locking + * ============================ + * + * fs_info::treelog_bg_lock protects the fs_info::treelog_bg which + * indicates the starting address of a block group, which is reserved only + * for tree-log metadata. + * + * Lock nesting + * ============ + * + * space_info::lock + * block_group::lock + * fs_info::treelog_bg_lock + */ + /* * Simple allocator for sequential-only block group. It only allows sequential * allocation. No need to play with trees. This function also reserves the @@ -3734,23 +3753,54 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group, struct find_free_extent_ctl *ffe_ctl, struct btrfs_block_group **bg_ret) { + struct btrfs_fs_info *fs_info = block_group->fs_info; struct btrfs_space_info *space_info = block_group->space_info; struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; u64 start = block_group->start; u64 num_bytes = ffe_ctl->num_bytes; u64 avail; + u64 bytenr = block_group->start; + u64 log_bytenr; int ret = 0; + bool skip; ASSERT(btrfs_is_zoned(block_group->fs_info)); + /* + * Do not allow non-tree-log blocks in the dedicated tree-log block + * group, and vice versa. + */ + spin_lock(&fs_info->treelog_bg_lock); + log_bytenr = fs_info->treelog_bg; + skip = log_bytenr && ((ffe_ctl->for_treelog && bytenr != log_bytenr) || + (!ffe_ctl->for_treelog && bytenr == log_bytenr)); + spin_unlock(&fs_info->treelog_bg_lock); + if (skip) + return 1; + spin_lock(&space_info->lock); spin_lock(&block_group->lock); + spin_lock(&fs_info->treelog_bg_lock); + + ASSERT(!ffe_ctl->for_treelog || + block_group->start == fs_info->treelog_bg || + fs_info->treelog_bg == 0); if (block_group->ro) { ret = 1; goto out; } + /* + * Do not allow currently using block group to be tree-log dedicated + * block group. + */ + if (ffe_ctl->for_treelog && !fs_info->treelog_bg && + (block_group->used || block_group->reserved)) { + ret = 1; + goto out; + } + avail = block_group->length - block_group->alloc_offset; if (avail < num_bytes) { if (ffe_ctl->max_extent_size < avail) { @@ -3765,6 +3815,9 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group, goto out; } + if (ffe_ctl->for_treelog && !fs_info->treelog_bg) + fs_info->treelog_bg = block_group->start; + ffe_ctl->found_offset = start + block_group->alloc_offset; block_group->alloc_offset += num_bytes; spin_lock(&ctl->tree_lock); @@ -3779,6 +3832,9 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group, ffe_ctl->search_start = ffe_ctl->found_offset; out: + if (ret && ffe_ctl->for_treelog) + fs_info->treelog_bg = 0; + spin_unlock(&fs_info->treelog_bg_lock); spin_unlock(&block_group->lock); spin_unlock(&space_info->lock); return ret; @@ -4028,7 +4084,12 @@ static int prepare_allocation(struct btrfs_fs_info *fs_info, return prepare_allocation_clustered(fs_info, ffe_ctl, space_info, ins); case BTRFS_EXTENT_ALLOC_ZONED: - /* Nothing to do */ + if (ffe_ctl->for_treelog) { + spin_lock(&fs_info->treelog_bg_lock); + if (fs_info->treelog_bg) + ffe_ctl->hint_byte = fs_info->treelog_bg; + spin_unlock(&fs_info->treelog_bg_lock); + } return 0; default: BUG(); @@ -4072,6 +4133,7 @@ static noinline int find_free_extent(struct btrfs_root *root, struct find_free_extent_ctl ffe_ctl = {0}; struct btrfs_space_info *space_info; bool full_search = false; + bool for_treelog = (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID); WARN_ON(num_bytes < fs_info->sectorsize); @@ -4085,6 +4147,7 @@ static noinline int find_free_extent(struct btrfs_root *root, ffe_ctl.orig_have_caching_bg = false; ffe_ctl.found_offset = 0; ffe_ctl.hint_byte = hint_byte_orig; + ffe_ctl.for_treelog = for_treelog; ffe_ctl.policy = BTRFS_EXTENT_ALLOC_CLUSTERED; /* For clustered allocation */ @@ -4159,8 +4222,11 @@ search: struct btrfs_block_group *bg_ret; /* If the block group is read-only, we can skip it entirely. */ - if (unlikely(block_group->ro)) + if (unlikely(block_group->ro)) { + if (for_treelog) + btrfs_clear_treelog_bg(block_group); continue; + } btrfs_grab_block_group(block_group, delalloc); ffe_ctl.search_start = block_group->start; @@ -4346,6 +4412,7 @@ int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes, bool final_tried = num_bytes == min_alloc_size; u64 flags; int ret; + bool for_treelog = (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID); flags = get_alloc_profile_by_root(root, is_data); again: @@ -4369,8 +4436,8 @@ again: sinfo = btrfs_find_space_info(fs_info, flags); btrfs_err(fs_info, - "allocation failed flags %llu, wanted %llu", - flags, num_bytes); + "allocation failed flags %llu, wanted %llu tree-log %d", + flags, num_bytes, for_treelog); if (sinfo) btrfs_dump_space_info(fs_info, sinfo, num_bytes, 1); diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 932ad9bc0de6..61e969652fe1 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -7,6 +7,7 @@ #include #include "volumes.h" #include "disk-io.h" +#include "block-group.h" struct btrfs_zoned_device_info { /* @@ -290,4 +291,17 @@ static inline void btrfs_zoned_meta_io_unlock(struct btrfs_fs_info *fs_info) mutex_unlock(&fs_info->zoned_meta_io_lock); } +static inline void btrfs_clear_treelog_bg(struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + + if (!btrfs_is_zoned(fs_info)) + return; + + spin_lock(&fs_info->treelog_bg_lock); + if (fs_info->treelog_bg == bg->start) + fs_info->treelog_bg = 0; + spin_unlock(&fs_info->treelog_bg_lock); +} + #endif -- cgit v1.2.3 From fa1a0f42a0356846fb1acd1d53061d53413a4c45 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:19 +0900 Subject: btrfs: zoned: serialize log transaction on zoned filesystems This is the 2/3 patch to enable tree-log on zoned filesystems. Since we can start more than one log transactions per subvolume simultaneously, nodes from multiple transactions can be allocated interleaved. Such mixed allocation results in non-sequential writes at the time of a log transaction commit. The nodes of the global log root tree (fs_info->log_root_tree), also have the same problem with mixed allocation. Serializes log transactions by waiting for a committing transaction when someone tries to start a new transaction, to avoid the mixed allocation problem. We must also wait for running log transactions from another subvolume, but there is no easy way to detect which subvolume root is running a log transaction. So, this patch forbids starting a new log transaction when other subvolumes already allocated the global log root tree. Reviewed-by: Josef Bacik Reviewed-by: Filipe Manana Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index c02eeeac439c..4e72794342c0 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -105,6 +105,7 @@ static noinline int replay_dir_deletes(struct btrfs_trans_handle *trans, struct btrfs_root *log, struct btrfs_path *path, u64 dirid, int del_all); +static void wait_log_commit(struct btrfs_root *root, int transid); /* * tree logging is a special write ahead log used to make sure that @@ -140,7 +141,9 @@ static int start_log_trans(struct btrfs_trans_handle *trans, { struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *tree_root = fs_info->tree_root; + const bool zoned = btrfs_is_zoned(fs_info); int ret = 0; + bool created = false; /* * First check if the log root tree was already created. If not, create @@ -150,8 +153,10 @@ static int start_log_trans(struct btrfs_trans_handle *trans, mutex_lock(&tree_root->log_mutex); if (!fs_info->log_root_tree) { ret = btrfs_init_log_root_tree(trans, fs_info); - if (!ret) + if (!ret) { set_bit(BTRFS_ROOT_HAS_LOG_TREE, &tree_root->state); + created = true; + } } mutex_unlock(&tree_root->log_mutex); if (ret) @@ -160,12 +165,20 @@ static int start_log_trans(struct btrfs_trans_handle *trans, mutex_lock(&root->log_mutex); +again: if (root->log_root) { + int index = (root->log_transid + 1) % 2; + if (btrfs_need_log_full_commit(trans)) { ret = -EAGAIN; goto out; } + if (zoned && atomic_read(&root->log_commit[index])) { + wait_log_commit(root, root->log_transid - 1); + goto again; + } + if (!root->log_start_pid) { clear_bit(BTRFS_ROOT_MULTI_LOG_TASKS, &root->state); root->log_start_pid = current->pid; @@ -173,6 +186,17 @@ static int start_log_trans(struct btrfs_trans_handle *trans, set_bit(BTRFS_ROOT_MULTI_LOG_TASKS, &root->state); } } else { + /* + * This means fs_info->log_root_tree was already created + * for some other FS trees. Do the full commit not to mix + * nodes from multiple log transactions to do sequential + * writing. + */ + if (zoned && !created) { + ret = -EAGAIN; + goto out; + } + ret = btrfs_add_log_tree(trans, root); if (ret) goto out; @@ -201,14 +225,22 @@ out: */ static int join_running_log_trans(struct btrfs_root *root) { + const bool zoned = btrfs_is_zoned(root->fs_info); int ret = -ENOENT; if (!test_bit(BTRFS_ROOT_HAS_LOG_TREE, &root->state)) return ret; mutex_lock(&root->log_mutex); +again: if (root->log_root) { + int index = (root->log_transid + 1) % 2; + ret = 0; + if (zoned && atomic_read(&root->log_commit[index])) { + wait_log_commit(root, root->log_transid - 1); + goto again; + } atomic_inc(&root->log_writers); } mutex_unlock(&root->log_mutex); -- cgit v1.2.3 From 3ddebf27fcd3a910989c85a3bfc9085225038c5b Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:20 +0900 Subject: btrfs: zoned: reorder log node allocation on zoned filesystem This is the 3/3 patch to enable tree-log on zoned filesystems. The allocation order of nodes of "fs_info->log_root_tree" and nodes of "root->log_root" is not the same as the writing order of them. So, the writing causes unaligned write errors. Reorder the allocation of them by delaying allocation of the root node of "fs_info->log_root_tree," so that the node buffers can go out sequentially to devices. Cc: Filipe Manana Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 12 +++++++----- fs/btrfs/tree-log.c | 27 +++++++++++++++++++++------ 2 files changed, 28 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 84c6650d5ef7..c2576c5fe62e 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1298,16 +1298,18 @@ int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info) { struct btrfs_root *log_root; - int ret; log_root = alloc_log_tree(trans, fs_info); if (IS_ERR(log_root)) return PTR_ERR(log_root); - ret = btrfs_alloc_log_tree_node(trans, log_root); - if (ret) { - btrfs_put_root(log_root); - return ret; + if (!btrfs_is_zoned(fs_info)) { + int ret = btrfs_alloc_log_tree_node(trans, log_root); + + if (ret) { + btrfs_put_root(log_root); + return ret; + } } WARN_ON(fs_info->log_root_tree); diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 4e72794342c0..fc04625cbbd1 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3162,6 +3162,19 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, list_add_tail(&root_log_ctx.list, &log_root_tree->log_ctxs[index2]); root_log_ctx.log_transid = log_root_tree->log_transid; + if (btrfs_is_zoned(fs_info)) { + mutex_lock(&fs_info->tree_root->log_mutex); + if (!log_root_tree->node) { + ret = btrfs_alloc_log_tree_node(trans, log_root_tree); + if (ret) { + mutex_unlock(&fs_info->tree_log_mutex); + mutex_unlock(&log_root_tree->log_mutex); + goto out; + } + } + mutex_unlock(&fs_info->tree_root->log_mutex); + } + /* * Now we are safe to update the log_root_tree because we're under the * log_mutex, and we're a current writer so we're holding the commit @@ -3320,12 +3333,14 @@ static void free_log_tree(struct btrfs_trans_handle *trans, .process_func = process_one_buffer }; - ret = walk_log_tree(trans, log, &wc); - if (ret) { - if (trans) - btrfs_abort_transaction(trans, ret); - else - btrfs_handle_fs_error(log->fs_info, ret, NULL); + if (log->node) { + ret = walk_log_tree(trans, log, &wc); + if (ret) { + if (trans) + btrfs_abort_transaction(trans, ret); + else + btrfs_handle_fs_error(log->fs_info, ret, NULL); + } } clear_extent_bits(&log->dirty_log_pages, 0, (u64)-1, -- cgit v1.2.3 From b528f467132713a03984b0f9592073d75677c501 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Fri, 5 Feb 2021 23:58:36 +0900 Subject: btrfs: zoned: deal with holes writing out tree-log pages Since the zoned filesystem requires sequential write out of metadata, we cannot proceed with a hole in tree-log pages. When such a hole exists, btree_write_cache_pages() will return -EAGAIN. This happens when someone, e.g., a concurrent transaction commit, writes a dirty extent in this tree-log commit. If we are not going to wait for the extents, we can hope the concurrent writing fills the hole for us. So, we can ignore the error in this case and hope the next write will succeed. If we want to wait for them and got the error, we cannot wait for them because it will cause a deadlock. So, let's bail out to a full commit in this case. Reviewed-by: Filipe Manana Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index fc04625cbbd1..d90695c1ab6c 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3120,6 +3120,17 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, */ blk_start_plug(&plug); ret = btrfs_write_marked_extents(fs_info, &log->dirty_log_pages, mark); + /* + * -EAGAIN happens when someone, e.g., a concurrent transaction + * commit, writes a dirty extent in this tree-log commit. This + * concurrent write will create a hole writing out the extents, + * and we cannot proceed on a zoned filesystem, requiring + * sequential writing. While we can bail out to a full commit + * here, but we can continue hoping the concurrent writing fills + * the hole. + */ + if (ret == -EAGAIN && btrfs_is_zoned(fs_info)) + ret = 0; if (ret) { blk_finish_plug(&plug); btrfs_abort_transaction(trans, ret); @@ -3242,7 +3253,17 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, &log_root_tree->dirty_log_pages, EXTENT_DIRTY | EXTENT_NEW); blk_finish_plug(&plug); - if (ret) { + /* + * As described above, -EAGAIN indicates a hole in the extents. We + * cannot wait for these write outs since the waiting cause a + * deadlock. Bail out to the full commit instead. + */ + if (ret == -EAGAIN && btrfs_is_zoned(fs_info)) { + btrfs_set_log_full_commit(trans); + btrfs_wait_tree_log_extents(log, mark); + mutex_unlock(&log_root_tree->log_mutex); + goto out_wake_log_root; + } else if (ret) { btrfs_set_log_full_commit(trans); btrfs_abort_transaction(trans, ret); mutex_unlock(&log_root_tree->log_mutex); -- cgit v1.2.3 From 9d294a685fbcb256ce8c5f7fd88a7596d0f52a8a Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:22:21 +0900 Subject: btrfs: zoned: enable to mount ZONED incompat flag This final patch adds the ZONED incompat flag to the supported flags and enables to mount ZONED flagged file system. Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 6f4b493625ef..3bc00aed13b2 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -298,7 +298,8 @@ struct btrfs_super_block { BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \ BTRFS_FEATURE_INCOMPAT_NO_HOLES | \ BTRFS_FEATURE_INCOMPAT_METADATA_UUID | \ - BTRFS_FEATURE_INCOMPAT_RAID1C34) + BTRFS_FEATURE_INCOMPAT_RAID1C34 | \ + BTRFS_FEATURE_INCOMPAT_ZONED) #define BTRFS_FEATURE_INCOMPAT_SAFE_SET \ (BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF) -- cgit v1.2.3 From 9fd6dad1261a541b3f5fa7dc5b152222306e6702 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 5 Feb 2021 05:07:11 -0500 Subject: mm: provide a saner PTE walking API for modules Currently, the follow_pfn function is exported for modules but follow_pte is not. However, follow_pfn is very easy to misuse, because it does not provide protections (so most of its callers assume the page is writable!) and because it returns after having already unlocked the page table lock. Provide instead a simplified version of follow_pte that does not have the pmdpp and range arguments. The older version survives as follow_invalidate_pte() for use by fs/dax.c. Reviewed-by: Jason Gunthorpe Signed-off-by: Paolo Bonzini --- arch/s390/pci/pci_mmio.c | 4 ++-- fs/dax.c | 5 +++-- include/linux/mm.h | 6 ++++-- mm/memory.c | 41 ++++++++++++++++++++++++++++++++++++----- virt/kvm/kvm_main.c | 4 ++-- 5 files changed, 47 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 18f2d10c3176..474617b88648 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -170,7 +170,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, if (!(vma->vm_flags & VM_WRITE)) goto out_unlock_mmap; - ret = follow_pte(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl); + ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); if (ret) goto out_unlock_mmap; @@ -311,7 +311,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, if (!(vma->vm_flags & VM_WRITE)) goto out_unlock_mmap; - ret = follow_pte(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl); + ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); if (ret) goto out_unlock_mmap; diff --git a/fs/dax.c b/fs/dax.c index 26d5dcd2d69e..b3d27fdc6775 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -810,11 +810,12 @@ static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index, address = pgoff_address(index, vma); /* - * Note because we provide range to follow_pte it will call + * follow_invalidate_pte() will use the range to call * mmu_notifier_invalidate_range_start() on our behalf before * taking any lock. */ - if (follow_pte(vma->vm_mm, address, &range, &ptep, &pmdp, &ptl)) + if (follow_invalidate_pte(vma->vm_mm, address, &range, &ptep, + &pmdp, &ptl)) continue; /* diff --git a/include/linux/mm.h b/include/linux/mm.h index ecdf8a8cd6ae..24b292fce8e5 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1658,9 +1658,11 @@ void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling); int copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); +int follow_invalidate_pte(struct mm_struct *mm, unsigned long address, + struct mmu_notifier_range *range, pte_t **ptepp, + pmd_t **pmdpp, spinlock_t **ptlp); int follow_pte(struct mm_struct *mm, unsigned long address, - struct mmu_notifier_range *range, pte_t **ptepp, pmd_t **pmdpp, - spinlock_t **ptlp); + pte_t **ptepp, spinlock_t **ptlp); int follow_pfn(struct vm_area_struct *vma, unsigned long address, unsigned long *pfn); int follow_phys(struct vm_area_struct *vma, unsigned long address, diff --git a/mm/memory.c b/mm/memory.c index feff48e1465a..985dac0958dc 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4709,9 +4709,9 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) } #endif /* __PAGETABLE_PMD_FOLDED */ -int follow_pte(struct mm_struct *mm, unsigned long address, - struct mmu_notifier_range *range, pte_t **ptepp, pmd_t **pmdpp, - spinlock_t **ptlp) +int follow_invalidate_pte(struct mm_struct *mm, unsigned long address, + struct mmu_notifier_range *range, pte_t **ptepp, + pmd_t **pmdpp, spinlock_t **ptlp) { pgd_t *pgd; p4d_t *p4d; @@ -4776,6 +4776,34 @@ out: return -EINVAL; } +/** + * follow_pte - look up PTE at a user virtual address + * @mm: the mm_struct of the target address space + * @address: user virtual address + * @ptepp: location to store found PTE + * @ptlp: location to store the lock for the PTE + * + * On a successful return, the pointer to the PTE is stored in @ptepp; + * the corresponding lock is taken and its location is stored in @ptlp. + * The contents of the PTE are only stable until @ptlp is released; + * any further use, if any, must be protected against invalidation + * with MMU notifiers. + * + * Only IO mappings and raw PFN mappings are allowed. The mmap semaphore + * should be taken for read. + * + * KVM uses this function. While it is arguably less bad than ``follow_pfn``, + * it is not a good general-purpose API. + * + * Return: zero on success, -ve otherwise. + */ +int follow_pte(struct mm_struct *mm, unsigned long address, + pte_t **ptepp, spinlock_t **ptlp) +{ + return follow_invalidate_pte(mm, address, NULL, ptepp, NULL, ptlp); +} +EXPORT_SYMBOL_GPL(follow_pte); + /** * follow_pfn - look up PFN at a user virtual address * @vma: memory mapping @@ -4784,6 +4812,9 @@ out: * * Only IO mappings and raw PFN mappings are allowed. * + * This function does not allow the caller to read the permissions + * of the PTE. Do not use it. + * * Return: zero and the pfn at @pfn on success, -ve otherwise. */ int follow_pfn(struct vm_area_struct *vma, unsigned long address, @@ -4796,7 +4827,7 @@ int follow_pfn(struct vm_area_struct *vma, unsigned long address, if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) return ret; - ret = follow_pte(vma->vm_mm, address, NULL, &ptep, NULL, &ptl); + ret = follow_pte(vma->vm_mm, address, &ptep, &ptl); if (ret) return ret; *pfn = pte_pfn(*ptep); @@ -4817,7 +4848,7 @@ int follow_phys(struct vm_area_struct *vma, if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) goto out; - if (follow_pte(vma->vm_mm, address, NULL, &ptep, NULL, &ptl)) + if (follow_pte(vma->vm_mm, address, &ptep, &ptl)) goto out; pte = *ptep; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 48ccdf4e3d04..ee4ac2618ec5 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1911,7 +1911,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, spinlock_t *ptl; int r; - r = follow_pte(vma->vm_mm, addr, NULL, &ptep, NULL, &ptl); + r = follow_pte(vma->vm_mm, addr, &ptep, &ptl); if (r) { /* * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does @@ -1926,7 +1926,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, if (r) return r; - r = follow_pte(vma->vm_mm, addr, NULL, &ptep, NULL, &ptl); + r = follow_pte(vma->vm_mm, addr, &ptep, &ptl); if (r) return r; } -- cgit v1.2.3 From 848fdd62399c638e65a1512616acaa5de7d5c5e8 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 8 Feb 2021 16:45:49 -0500 Subject: NFS: Don't set NFS_INO_INVALID_XATTR if there is no xattr cache Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/inode.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'fs') diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 2533053d764a..1575e3e1dda9 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -195,6 +195,18 @@ bool nfs_check_cache_invalid(struct inode *inode, unsigned long flags) } EXPORT_SYMBOL_GPL(nfs_check_cache_invalid); +#ifdef CONFIG_NFS_V4_2 +static bool nfs_has_xattr_cache(const struct nfs_inode *nfsi) +{ + return nfsi->xattr_cache != NULL; +} +#else +static bool nfs_has_xattr_cache(const struct nfs_inode *nfsi) +{ + return false; +} +#endif + static void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) { struct nfs_inode *nfsi = NFS_I(inode); @@ -209,6 +221,8 @@ static void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) | NFS_INO_INVALID_XATTR); } + if (!nfs_has_xattr_cache(nfsi)) + flags &= ~NFS_INO_INVALID_XATTR; if (inode->i_mapping->nrpages == 0) flags &= ~(NFS_INO_INVALID_DATA|NFS_INO_DATA_INVAL_DEFER); nfsi->cache_validity |= flags; -- cgit v1.2.3 From 70779b897395b330ba5a47bed84f94178da599f9 Mon Sep 17 00:00:00 2001 From: Pan Bian Date: Wed, 20 Jan 2021 00:51:13 -0800 Subject: fs/affs: release old buffer head on error path The reference count of the old buffer head should be decremented on path that fails to get the new buffer head. Fixes: 6b4657667ba0 ("fs/affs: add rename exchange") CC: stable@vger.kernel.org # 4.14+ Signed-off-by: Pan Bian Signed-off-by: David Sterba --- fs/affs/namei.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/affs/namei.c b/fs/affs/namei.c index 41c5749f4db7..5400a876d73f 100644 --- a/fs/affs/namei.c +++ b/fs/affs/namei.c @@ -460,8 +460,10 @@ affs_xrename(struct inode *old_dir, struct dentry *old_dentry, return -EIO; bh_new = affs_bread(sb, d_inode(new_dentry)->i_ino); - if (!bh_new) + if (!bh_new) { + affs_brelse(bh_old); return -EIO; + } /* Remove old header from its parent directory. */ affs_lock_dir(old_dir); -- cgit v1.2.3 From abd4af47d3fb64157133bd35de25d39a21a31122 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Mon, 8 Feb 2021 16:48:31 +1000 Subject: cifs: fix dfs-links This fixes a regression following dfs links that was introduced in the patch series for the new mount api. Signed-off-by: Ronnie Sahlberg Reviewed-by: Paulo Alcantara (SUSE) Signed-off-by: Steve French --- fs/cifs/connect.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'fs') diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 10fe6d6d2dee..76e4d8d8b3a6 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -2983,6 +2983,14 @@ expand_dfs_referral(const unsigned int xid, struct cifs_ses *ses, rc = PTR_ERR(mdata); mdata = NULL; } else { + /* + * We can not clear out the whole structure since we + * no longer have an explicit function to parse + * a mount-string. Instead we need to clear out the + * individual fields that are no longer valid. + */ + kfree(ctx->prepath); + ctx->prepath = NULL; rc = cifs_setup_volume_info(ctx, mdata, fake_devname); } kfree(fake_devname); -- cgit v1.2.3 From e812cbbbbbb15adbbbee176baa1e8bda53059bf0 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Tue, 9 Feb 2021 13:41:50 -0800 Subject: squashfs: avoid out of bounds writes in decompressors Patch series "Squashfs: fix BIO migration regression and add sanity checks". Patch [1/4] fixes a regression introduced by the "migrate from ll_rw_block usage to BIO" patch, which has produced a number of Sysbot/Syzkaller reports. Patches [2/4], [3/4], and [4/4] fix a number of filesystem corruption issues which have produced Sysbot reports in the id, inode and xattr lookup code. Each patch has been tested against the Sysbot reproducers using the given kernel configuration. They have the appropriate "Reported-by:" lines added. Additionally, all of the reproducer filesystems are indirectly fixed by patch [4/4] due to the fact they all have xattr corruption which is now detected there. Additional testing with other configurations and architectures (32bit, big endian), and normal filesystems has also been done to trap any inadvertent regressions caused by the additional sanity checks. This patch (of 4): This is a regression introduced by the patch "migrate from ll_rw_block usage to BIO". Sysbot/Syskaller has reported a number of "out of bounds writes" and "unable to handle kernel paging request in squashfs_decompress" errors which have been identified as a regression introduced by the above patch. Specifically, the patch removed the following sanity check if (length < 0 || length > output->length || (index + length) > msblk->bytes_used) This check did two things: 1. It ensured any reads were not beyond the end of the filesystem 2. It ensured that the "length" field read from the filesystem was within the expected maximum length. Without this any corrupted values can over-run allocated buffers. Link: https://lkml.kernel.org/r/20210204130249.4495-1-phillip@squashfs.org.uk Link: https://lkml.kernel.org/r/20210204130249.4495-2-phillip@squashfs.org.uk Fixes: 93e72b3c612adc ("squashfs: migrate from ll_rw_block usage to BIO") Reported-by: syzbot+6fba78f99b9afd4b5634@syzkaller.appspotmail.com Signed-off-by: Phillip Lougher Cc: Philippe Liard Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/squashfs/block.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/squashfs/block.c b/fs/squashfs/block.c index 8a19773b5a0b..45f44425d856 100644 --- a/fs/squashfs/block.c +++ b/fs/squashfs/block.c @@ -196,9 +196,15 @@ int squashfs_read_data(struct super_block *sb, u64 index, int length, length = SQUASHFS_COMPRESSED_SIZE(length); index += 2; - TRACE("Block @ 0x%llx, %scompressed size %d\n", index, + TRACE("Block @ 0x%llx, %scompressed size %d\n", index - 2, compressed ? "" : "un", length); } + if (length < 0 || length > output->length || + (index + length) > msblk->bytes_used) { + res = -EIO; + goto out; + } + if (next_index) *next_index = index + length; -- cgit v1.2.3 From f37aa4c7366e23f91b81d00bafd6a7ab54e4a381 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Tue, 9 Feb 2021 13:41:53 -0800 Subject: squashfs: add more sanity checks in id lookup Sysbot has reported a number of "slab-out-of-bounds reads" and "use-after-free read" errors which has been identified as being caused by a corrupted index value read from the inode. This could be because the metadata block is uncompressed, or because the "compression" bit has been corrupted (turning a compressed block into an uncompressed block). This patch adds additional sanity checks to detect this, and the following corruption. 1. It checks against corruption of the ids count. This can either lead to a larger table to be read, or a smaller than expected table to be read. In the case of a too large ids count, this would often have been trapped by the existing sanity checks, but this patch introduces a more exact check, which can identify too small values. 2. It checks the contents of the index table for corruption. Link: https://lkml.kernel.org/r/20210204130249.4495-3-phillip@squashfs.org.uk Signed-off-by: Phillip Lougher Reported-by: syzbot+b06d57ba83f604522af2@syzkaller.appspotmail.com Reported-by: syzbot+c021ba012da41ee9807c@syzkaller.appspotmail.com Reported-by: syzbot+5024636e8b5fd19f0f19@syzkaller.appspotmail.com Reported-by: syzbot+bcbc661df46657d0fa4f@syzkaller.appspotmail.com Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/squashfs/id.c | 40 ++++++++++++++++++++++++++++++++-------- fs/squashfs/squashfs_fs_sb.h | 1 + fs/squashfs/super.c | 6 +++--- fs/squashfs/xattr.h | 10 +++++++++- 4 files changed, 45 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/squashfs/id.c b/fs/squashfs/id.c index 6be5afe7287d..11581bf31af4 100644 --- a/fs/squashfs/id.c +++ b/fs/squashfs/id.c @@ -35,10 +35,15 @@ int squashfs_get_id(struct super_block *sb, unsigned int index, struct squashfs_sb_info *msblk = sb->s_fs_info; int block = SQUASHFS_ID_BLOCK(index); int offset = SQUASHFS_ID_BLOCK_OFFSET(index); - u64 start_block = le64_to_cpu(msblk->id_table[block]); + u64 start_block; __le32 disk_id; int err; + if (index >= msblk->ids) + return -EINVAL; + + start_block = le64_to_cpu(msblk->id_table[block]); + err = squashfs_read_metadata(sb, &disk_id, &start_block, &offset, sizeof(disk_id)); if (err < 0) @@ -56,7 +61,10 @@ __le64 *squashfs_read_id_index_table(struct super_block *sb, u64 id_table_start, u64 next_table, unsigned short no_ids) { unsigned int length = SQUASHFS_ID_BLOCK_BYTES(no_ids); + unsigned int indexes = SQUASHFS_ID_BLOCKS(no_ids); + int n; __le64 *table; + u64 start, end; TRACE("In read_id_index_table, length %d\n", length); @@ -67,20 +75,36 @@ __le64 *squashfs_read_id_index_table(struct super_block *sb, return ERR_PTR(-EINVAL); /* - * length bytes should not extend into the next table - this check - * also traps instances where id_table_start is incorrectly larger - * than the next table start + * The computed size of the index table (length bytes) should exactly + * match the table start and end points */ - if (id_table_start + length > next_table) + if (length != (next_table - id_table_start)) return ERR_PTR(-EINVAL); table = squashfs_read_table(sb, id_table_start, length); + if (IS_ERR(table)) + return table; /* - * table[0] points to the first id lookup table metadata block, this - * should be less than id_table_start + * table[0], table[1], ... table[indexes - 1] store the locations + * of the compressed id blocks. Each entry should be less than + * the next (i.e. table[0] < table[1]), and the difference between them + * should be SQUASHFS_METADATA_SIZE or less. table[indexes - 1] + * should be less than id_table_start, and again the difference + * should be SQUASHFS_METADATA_SIZE or less */ - if (!IS_ERR(table) && le64_to_cpu(table[0]) >= id_table_start) { + for (n = 0; n < (indexes - 1); n++) { + start = le64_to_cpu(table[n]); + end = le64_to_cpu(table[n + 1]); + + if (start >= end || (end - start) > SQUASHFS_METADATA_SIZE) { + kfree(table); + return ERR_PTR(-EINVAL); + } + } + + start = le64_to_cpu(table[indexes - 1]); + if (start >= id_table_start || (id_table_start - start) > SQUASHFS_METADATA_SIZE) { kfree(table); return ERR_PTR(-EINVAL); } diff --git a/fs/squashfs/squashfs_fs_sb.h b/fs/squashfs/squashfs_fs_sb.h index 34c21ffb6df3..166e98806265 100644 --- a/fs/squashfs/squashfs_fs_sb.h +++ b/fs/squashfs/squashfs_fs_sb.h @@ -64,5 +64,6 @@ struct squashfs_sb_info { unsigned int inodes; unsigned int fragments; int xattr_ids; + unsigned int ids; }; #endif diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c index d6c6593ec169..88cc94be1076 100644 --- a/fs/squashfs/super.c +++ b/fs/squashfs/super.c @@ -166,6 +166,7 @@ static int squashfs_fill_super(struct super_block *sb, struct fs_context *fc) msblk->directory_table = le64_to_cpu(sblk->directory_table_start); msblk->inodes = le32_to_cpu(sblk->inodes); msblk->fragments = le32_to_cpu(sblk->fragments); + msblk->ids = le16_to_cpu(sblk->no_ids); flags = le16_to_cpu(sblk->flags); TRACE("Found valid superblock on %pg\n", sb->s_bdev); @@ -177,7 +178,7 @@ static int squashfs_fill_super(struct super_block *sb, struct fs_context *fc) TRACE("Block size %d\n", msblk->block_size); TRACE("Number of inodes %d\n", msblk->inodes); TRACE("Number of fragments %d\n", msblk->fragments); - TRACE("Number of ids %d\n", le16_to_cpu(sblk->no_ids)); + TRACE("Number of ids %d\n", msblk->ids); TRACE("sblk->inode_table_start %llx\n", msblk->inode_table); TRACE("sblk->directory_table_start %llx\n", msblk->directory_table); TRACE("sblk->fragment_table_start %llx\n", @@ -236,8 +237,7 @@ static int squashfs_fill_super(struct super_block *sb, struct fs_context *fc) allocate_id_index_table: /* Allocate and read id index table */ msblk->id_table = squashfs_read_id_index_table(sb, - le64_to_cpu(sblk->id_table_start), next_table, - le16_to_cpu(sblk->no_ids)); + le64_to_cpu(sblk->id_table_start), next_table, msblk->ids); if (IS_ERR(msblk->id_table)) { errorf(fc, "unable to read id index table"); err = PTR_ERR(msblk->id_table); diff --git a/fs/squashfs/xattr.h b/fs/squashfs/xattr.h index 184129afd456..d8a270d3ac4c 100644 --- a/fs/squashfs/xattr.h +++ b/fs/squashfs/xattr.h @@ -17,8 +17,16 @@ extern int squashfs_xattr_lookup(struct super_block *, unsigned int, int *, static inline __le64 *squashfs_read_xattr_id_table(struct super_block *sb, u64 start, u64 *xattr_table_start, int *xattr_ids) { + struct squashfs_xattr_id_table *id_table; + + id_table = squashfs_read_table(sb, start, sizeof(*id_table)); + if (IS_ERR(id_table)) + return (__le64 *) id_table; + + *xattr_table_start = le64_to_cpu(id_table->xattr_table_start); + kfree(id_table); + ERROR("Xattrs in filesystem, these will be ignored\n"); - *xattr_table_start = start; return ERR_PTR(-ENOTSUPP); } -- cgit v1.2.3 From eabac19e40c095543def79cb6ffeb3a8588aaff4 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Tue, 9 Feb 2021 13:41:56 -0800 Subject: squashfs: add more sanity checks in inode lookup Sysbot has reported an "slab-out-of-bounds read" error which has been identified as being caused by a corrupted "ino_num" value read from the inode. This could be because the metadata block is uncompressed, or because the "compression" bit has been corrupted (turning a compressed block into an uncompressed block). This patch adds additional sanity checks to detect this, and the following corruption. 1. It checks against corruption of the inodes count. This can either lead to a larger table to be read, or a smaller than expected table to be read. In the case of a too large inodes count, this would often have been trapped by the existing sanity checks, but this patch introduces a more exact check, which can identify too small values. 2. It checks the contents of the index table for corruption. [phillip@squashfs.org.uk: fix checkpatch issue] Link: https://lkml.kernel.org/r/527909353.754618.1612769948607@webmail.123-reg.co.uk Link: https://lkml.kernel.org/r/20210204130249.4495-4-phillip@squashfs.org.uk Signed-off-by: Phillip Lougher Reported-by: syzbot+04419e3ff19d2970ea28@syzkaller.appspotmail.com Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/squashfs/export.c | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/squashfs/export.c b/fs/squashfs/export.c index ae2c87bb0fbe..eb02072d28dd 100644 --- a/fs/squashfs/export.c +++ b/fs/squashfs/export.c @@ -41,12 +41,17 @@ static long long squashfs_inode_lookup(struct super_block *sb, int ino_num) struct squashfs_sb_info *msblk = sb->s_fs_info; int blk = SQUASHFS_LOOKUP_BLOCK(ino_num - 1); int offset = SQUASHFS_LOOKUP_BLOCK_OFFSET(ino_num - 1); - u64 start = le64_to_cpu(msblk->inode_lookup_table[blk]); + u64 start; __le64 ino; int err; TRACE("Entered squashfs_inode_lookup, inode_number = %d\n", ino_num); + if (ino_num == 0 || (ino_num - 1) >= msblk->inodes) + return -EINVAL; + + start = le64_to_cpu(msblk->inode_lookup_table[blk]); + err = squashfs_read_metadata(sb, &ino, &start, &offset, sizeof(ino)); if (err < 0) return err; @@ -111,7 +116,10 @@ __le64 *squashfs_read_inode_lookup_table(struct super_block *sb, u64 lookup_table_start, u64 next_table, unsigned int inodes) { unsigned int length = SQUASHFS_LOOKUP_BLOCK_BYTES(inodes); + unsigned int indexes = SQUASHFS_LOOKUP_BLOCKS(inodes); + int n; __le64 *table; + u64 start, end; TRACE("In read_inode_lookup_table, length %d\n", length); @@ -121,20 +129,37 @@ __le64 *squashfs_read_inode_lookup_table(struct super_block *sb, if (inodes == 0) return ERR_PTR(-EINVAL); - /* length bytes should not extend into the next table - this check - * also traps instances where lookup_table_start is incorrectly larger - * than the next table start + /* + * The computed size of the lookup table (length bytes) should exactly + * match the table start and end points */ - if (lookup_table_start + length > next_table) + if (length != (next_table - lookup_table_start)) return ERR_PTR(-EINVAL); table = squashfs_read_table(sb, lookup_table_start, length); + if (IS_ERR(table)) + return table; /* - * table[0] points to the first inode lookup table metadata block, - * this should be less than lookup_table_start + * table0], table[1], ... table[indexes - 1] store the locations + * of the compressed inode lookup blocks. Each entry should be + * less than the next (i.e. table[0] < table[1]), and the difference + * between them should be SQUASHFS_METADATA_SIZE or less. + * table[indexes - 1] should be less than lookup_table_start, and + * again the difference should be SQUASHFS_METADATA_SIZE or less */ - if (!IS_ERR(table) && le64_to_cpu(table[0]) >= lookup_table_start) { + for (n = 0; n < (indexes - 1); n++) { + start = le64_to_cpu(table[n]); + end = le64_to_cpu(table[n + 1]); + + if (start >= end || (end - start) > SQUASHFS_METADATA_SIZE) { + kfree(table); + return ERR_PTR(-EINVAL); + } + } + + start = le64_to_cpu(table[indexes - 1]); + if (start >= lookup_table_start || (lookup_table_start - start) > SQUASHFS_METADATA_SIZE) { kfree(table); return ERR_PTR(-EINVAL); } -- cgit v1.2.3 From 506220d2ba21791314af569211ffd8870b8208fa Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Tue, 9 Feb 2021 13:42:00 -0800 Subject: squashfs: add more sanity checks in xattr id lookup Sysbot has reported a warning where a kmalloc() attempt exceeds the maximum limit. This has been identified as corruption of the xattr_ids count when reading the xattr id lookup table. This patch adds a number of additional sanity checks to detect this corruption and others. 1. It checks for a corrupted xattr index read from the inode. This could be because the metadata block is uncompressed, or because the "compression" bit has been corrupted (turning a compressed block into an uncompressed block). This would cause an out of bounds read. 2. It checks against corruption of the xattr_ids count. This can either lead to the above kmalloc failure, or a smaller than expected table to be read. 3. It checks the contents of the index table for corruption. [phillip@squashfs.org.uk: fix checkpatch issue] Link: https://lkml.kernel.org/r/270245655.754655.1612770082682@webmail.123-reg.co.uk Link: https://lkml.kernel.org/r/20210204130249.4495-5-phillip@squashfs.org.uk Signed-off-by: Phillip Lougher Reported-by: syzbot+2ccea6339d368360800d@syzkaller.appspotmail.com Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/squashfs/xattr_id.c | 66 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/squashfs/xattr_id.c b/fs/squashfs/xattr_id.c index d99e08464554..ead66670b41a 100644 --- a/fs/squashfs/xattr_id.c +++ b/fs/squashfs/xattr_id.c @@ -31,10 +31,15 @@ int squashfs_xattr_lookup(struct super_block *sb, unsigned int index, struct squashfs_sb_info *msblk = sb->s_fs_info; int block = SQUASHFS_XATTR_BLOCK(index); int offset = SQUASHFS_XATTR_BLOCK_OFFSET(index); - u64 start_block = le64_to_cpu(msblk->xattr_id_table[block]); + u64 start_block; struct squashfs_xattr_id id; int err; + if (index >= msblk->xattr_ids) + return -EINVAL; + + start_block = le64_to_cpu(msblk->xattr_id_table[block]); + err = squashfs_read_metadata(sb, &id, &start_block, &offset, sizeof(id)); if (err < 0) @@ -50,13 +55,17 @@ int squashfs_xattr_lookup(struct super_block *sb, unsigned int index, /* * Read uncompressed xattr id lookup table indexes from disk into memory */ -__le64 *squashfs_read_xattr_id_table(struct super_block *sb, u64 start, +__le64 *squashfs_read_xattr_id_table(struct super_block *sb, u64 table_start, u64 *xattr_table_start, int *xattr_ids) { - unsigned int len; + struct squashfs_sb_info *msblk = sb->s_fs_info; + unsigned int len, indexes; struct squashfs_xattr_id_table *id_table; + __le64 *table; + u64 start, end; + int n; - id_table = squashfs_read_table(sb, start, sizeof(*id_table)); + id_table = squashfs_read_table(sb, table_start, sizeof(*id_table)); if (IS_ERR(id_table)) return (__le64 *) id_table; @@ -70,13 +79,52 @@ __le64 *squashfs_read_xattr_id_table(struct super_block *sb, u64 start, if (*xattr_ids == 0) return ERR_PTR(-EINVAL); - /* xattr_table should be less than start */ - if (*xattr_table_start >= start) + len = SQUASHFS_XATTR_BLOCK_BYTES(*xattr_ids); + indexes = SQUASHFS_XATTR_BLOCKS(*xattr_ids); + + /* + * The computed size of the index table (len bytes) should exactly + * match the table start and end points + */ + start = table_start + sizeof(*id_table); + end = msblk->bytes_used; + + if (len != (end - start)) return ERR_PTR(-EINVAL); - len = SQUASHFS_XATTR_BLOCK_BYTES(*xattr_ids); + table = squashfs_read_table(sb, start, len); + if (IS_ERR(table)) + return table; + + /* table[0], table[1], ... table[indexes - 1] store the locations + * of the compressed xattr id blocks. Each entry should be less than + * the next (i.e. table[0] < table[1]), and the difference between them + * should be SQUASHFS_METADATA_SIZE or less. table[indexes - 1] + * should be less than table_start, and again the difference + * shouls be SQUASHFS_METADATA_SIZE or less. + * + * Finally xattr_table_start should be less than table[0]. + */ + for (n = 0; n < (indexes - 1); n++) { + start = le64_to_cpu(table[n]); + end = le64_to_cpu(table[n + 1]); + + if (start >= end || (end - start) > SQUASHFS_METADATA_SIZE) { + kfree(table); + return ERR_PTR(-EINVAL); + } + } + + start = le64_to_cpu(table[indexes - 1]); + if (start >= table_start || (table_start - start) > SQUASHFS_METADATA_SIZE) { + kfree(table); + return ERR_PTR(-EINVAL); + } - TRACE("In read_xattr_index_table, length %d\n", len); + if (*xattr_table_start >= le64_to_cpu(table[0])) { + kfree(table); + return ERR_PTR(-EINVAL); + } - return squashfs_read_table(sb, start + sizeof(*id_table), len); + return table; } -- cgit v1.2.3 From b85a7a8bb5736998b8a681937a9749b350c17988 Mon Sep 17 00:00:00 2001 From: Seth Forshee Date: Tue, 9 Feb 2021 13:42:14 -0800 Subject: tmpfs: disallow CONFIG_TMPFS_INODE64 on s390 Currently there is an assumption in tmpfs that 64-bit architectures also have a 64-bit ino_t. This is not true on s390 which has a 32-bit ino_t. With CONFIG_TMPFS_INODE64=y tmpfs mounts will get 64-bit inode numbers and display "inode64" in the mount options, but passing the "inode64" mount option will fail. This leads to the following behavior: # mkdir mnt # mount -t tmpfs nodev mnt # mount -o remount,rw mnt mount: /home/ubuntu/mnt: mount point not mounted or bad option. As mount sees "inode64" in the mount options and thus passes it in the options for the remount. So prevent CONFIG_TMPFS_INODE64 from being selected on s390. Link: https://lkml.kernel.org/r/20210205230620.518245-1-seth.forshee@canonical.com Fixes: ea3271f7196c ("tmpfs: support 64-bit inums per-sb") Signed-off-by: Seth Forshee Acked-by: Hugh Dickins Cc: Chris Down Cc: Hugh Dickins Cc: Amir Goldstein Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Christian Borntraeger Cc: [5.9+] Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/Kconfig b/fs/Kconfig index aa4c12282301..3347ec7bd837 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -203,7 +203,7 @@ config TMPFS_XATTR config TMPFS_INODE64 bool "Use 64-bit ino_t by default in tmpfs" - depends on TMPFS && 64BIT + depends on TMPFS && 64BIT && !S390 default n help tmpfs has historically used only inode numbers as wide as an unsigned -- cgit v1.2.3 From ad69c389ec110ea54f8b0c0884b255340ef1c736 Mon Sep 17 00:00:00 2001 From: Seth Forshee Date: Tue, 9 Feb 2021 13:42:17 -0800 Subject: tmpfs: disallow CONFIG_TMPFS_INODE64 on alpha As with s390, alpha is a 64-bit architecture with a 32-bit ino_t. With CONFIG_TMPFS_INODE64=y tmpfs mounts will get 64-bit inode numbers and display "inode64" in the mount options, whereas passing "inode64" in the mount options will fail. This leads to erroneous behaviours such as this: # mkdir mnt # mount -t tmpfs nodev mnt # mount -o remount,rw mnt mount: /home/ubuntu/mnt: mount point not mounted or bad option. Prevent CONFIG_TMPFS_INODE64 from being selected on alpha. Link: https://lkml.kernel.org/r/20210208215726.608197-1-seth.forshee@canonical.com Fixes: ea3271f7196c ("tmpfs: support 64-bit inums per-sb") Signed-off-by: Seth Forshee Acked-by: Hugh Dickins Cc: Chris Down Cc: Amir Goldstein Cc: Richard Henderson Cc: Ivan Kokshaysky Cc: Matt Turner Cc: [5.9+] Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/Kconfig b/fs/Kconfig index 3347ec7bd837..da524c4d7b7e 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -203,7 +203,7 @@ config TMPFS_XATTR config TMPFS_INODE64 bool "Use 64-bit ino_t by default in tmpfs" - depends on TMPFS && 64BIT && !S390 + depends on TMPFS && 64BIT && !(S390 || ALPHA) default n help tmpfs has historically used only inode numbers as wide as an unsigned -- cgit v1.2.3 From 45d189c6062922ffe272e98013ba464b355dede7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:07 +0000 Subject: io_uring: replace force_nonblock with flags Replace bool force_nonblock with flags. It has a long standing goal of differentiating context from which we execute. Currently we have some subtle places where some invariants, like holding of uring_lock, are subtly inferred. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 178 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 96 insertions(+), 82 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 9c77fbc0c395..862121c48cee 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -187,6 +187,10 @@ struct io_rings { struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp; }; +enum io_uring_cmd_flags { + IO_URING_F_NONBLOCK = 1, +}; + struct io_mapped_ubuf { u64 ubuf; size_t len; @@ -3477,7 +3481,7 @@ static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) return -EINVAL; } -static int io_read(struct io_kiocb *req, bool force_nonblock, +static int io_read(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; @@ -3485,6 +3489,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock, struct iov_iter __iter, *iter = &__iter; struct io_async_rw *rw = req->async_data; ssize_t io_size, ret, ret2; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; if (rw) { iter = &rw->iter; @@ -3588,7 +3593,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return io_rw_prep_async(req, WRITE); } -static int io_write(struct io_kiocb *req, bool force_nonblock, +static int io_write(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; @@ -3596,6 +3601,7 @@ static int io_write(struct io_kiocb *req, bool force_nonblock, struct iov_iter __iter, *iter = &__iter; struct io_async_rw *rw = req->async_data; ssize_t ret, ret2, io_size; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; if (rw) { iter = &rw->iter; @@ -3706,12 +3712,12 @@ static int io_renameat_prep(struct io_kiocb *req, return 0; } -static int io_renameat(struct io_kiocb *req, bool force_nonblock) +static int io_renameat(struct io_kiocb *req, unsigned int issue_flags) { struct io_rename *ren = &req->rename; int ret; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd, @@ -3748,12 +3754,12 @@ static int io_unlinkat_prep(struct io_kiocb *req, return 0; } -static int io_unlinkat(struct io_kiocb *req, bool force_nonblock) +static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) { struct io_unlink *un = &req->unlink; int ret; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; if (un->flags & AT_REMOVEDIR) @@ -3785,13 +3791,13 @@ static int io_shutdown_prep(struct io_kiocb *req, #endif } -static int io_shutdown(struct io_kiocb *req, bool force_nonblock) +static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) { #if defined(CONFIG_NET) struct socket *sock; int ret; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; sock = sock_from_file(req->file); @@ -3850,7 +3856,7 @@ static int io_tee_prep(struct io_kiocb *req, return __io_splice_prep(req, sqe); } -static int io_tee(struct io_kiocb *req, bool force_nonblock) +static int io_tee(struct io_kiocb *req, unsigned int issue_flags) { struct io_splice *sp = &req->splice; struct file *in = sp->file_in; @@ -3858,7 +3864,7 @@ static int io_tee(struct io_kiocb *req, bool force_nonblock) unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; long ret = 0; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; if (sp->len) ret = do_tee(in, out, sp->len, flags); @@ -3881,7 +3887,7 @@ static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return __io_splice_prep(req, sqe); } -static int io_splice(struct io_kiocb *req, bool force_nonblock) +static int io_splice(struct io_kiocb *req, unsigned int issue_flags) { struct io_splice *sp = &req->splice; struct file *in = sp->file_in; @@ -3890,7 +3896,7 @@ static int io_splice(struct io_kiocb *req, bool force_nonblock) loff_t *poff_in, *poff_out; long ret = 0; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; poff_in = (sp->off_in == -1) ? NULL : &sp->off_in; @@ -3943,13 +3949,13 @@ static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_fsync(struct io_kiocb *req, bool force_nonblock) +static int io_fsync(struct io_kiocb *req, unsigned int issue_flags) { loff_t end = req->sync.off + req->sync.len; int ret; /* fsync always requires a blocking context */ - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; ret = vfs_fsync_range(req->file, req->sync.off, @@ -3975,12 +3981,12 @@ static int io_fallocate_prep(struct io_kiocb *req, return 0; } -static int io_fallocate(struct io_kiocb *req, bool force_nonblock) +static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags) { int ret; /* fallocate always requiring blocking context */ - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off, req->sync.len); @@ -4050,7 +4056,7 @@ static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return __io_openat_prep(req, sqe); } -static int io_openat2(struct io_kiocb *req, bool force_nonblock) +static int io_openat2(struct io_kiocb *req, unsigned int issue_flags) { struct open_flags op; struct file *file; @@ -4063,7 +4069,7 @@ static int io_openat2(struct io_kiocb *req, bool force_nonblock) goto err; nonblock_set = op.open_flag & O_NONBLOCK; resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED; - if (force_nonblock) { + if (issue_flags & IO_URING_F_NONBLOCK) { /* * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, * it'll always -EAGAIN @@ -4080,7 +4086,8 @@ static int io_openat2(struct io_kiocb *req, bool force_nonblock) file = do_filp_open(req->open.dfd, req->open.filename, &op); /* only retry if RESOLVE_CACHED wasn't already set by application */ - if ((!resolve_nonblock && force_nonblock) && file == ERR_PTR(-EAGAIN)) { + if ((!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)) && + file == ERR_PTR(-EAGAIN)) { /* * We could hang on to this 'fd', but seems like marginal * gain for something that is now known to be a slower path. @@ -4094,7 +4101,7 @@ static int io_openat2(struct io_kiocb *req, bool force_nonblock) put_unused_fd(ret); ret = PTR_ERR(file); } else { - if (force_nonblock && !nonblock_set) + if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set) file->f_flags &= ~O_NONBLOCK; fsnotify_open(file); fd_install(ret, file); @@ -4108,9 +4115,9 @@ err: return 0; } -static int io_openat(struct io_kiocb *req, bool force_nonblock) +static int io_openat(struct io_kiocb *req, unsigned int issue_flags) { - return io_openat2(req, force_nonblock); + return io_openat2(req, issue_flags & IO_URING_F_NONBLOCK); } static int io_remove_buffers_prep(struct io_kiocb *req, @@ -4158,13 +4165,14 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, return i; } -static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock, +static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_provide_buf *p = &req->pbuf; struct io_ring_ctx *ctx = req->ctx; struct io_buffer *head; int ret = 0; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; io_ring_submit_lock(ctx, !force_nonblock); @@ -4242,13 +4250,14 @@ static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head) return i ? i : -ENOMEM; } -static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock, +static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_provide_buf *p = &req->pbuf; struct io_ring_ctx *ctx = req->ctx; struct io_buffer *head, *list; int ret = 0; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; io_ring_submit_lock(ctx, !force_nonblock); @@ -4310,12 +4319,13 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, #endif } -static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock, +static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { #if defined(CONFIG_EPOLL) struct io_epoll *ie = &req->epoll; int ret; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock); if (force_nonblock && ret == -EAGAIN) @@ -4347,13 +4357,13 @@ static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) #endif } -static int io_madvise(struct io_kiocb *req, bool force_nonblock) +static int io_madvise(struct io_kiocb *req, unsigned int issue_flags) { #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) struct io_madvise *ma = &req->madvise; int ret; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice); @@ -4379,12 +4389,12 @@ static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_fadvise(struct io_kiocb *req, bool force_nonblock) +static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags) { struct io_fadvise *fa = &req->fadvise; int ret; - if (force_nonblock) { + if (issue_flags & IO_URING_F_NONBLOCK) { switch (fa->advice) { case POSIX_FADV_NORMAL: case POSIX_FADV_RANDOM: @@ -4420,12 +4430,12 @@ static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_statx(struct io_kiocb *req, bool force_nonblock) +static int io_statx(struct io_kiocb *req, unsigned int issue_flags) { struct io_statx *ctx = &req->statx; int ret; - if (force_nonblock) { + if (issue_flags & IO_URING_F_NONBLOCK) { /* only need file table for an actual valid fd */ if (ctx->dfd == -1 || ctx->dfd == AT_FDCWD) req->flags |= REQ_F_NO_FILE_TABLE; @@ -4455,7 +4465,7 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_close(struct io_kiocb *req, bool force_nonblock, +static int io_close(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct files_struct *files = current->files; @@ -4485,7 +4495,7 @@ static int io_close(struct io_kiocb *req, bool force_nonblock, } /* if the file has a flush method, be safe and punt to async */ - if (file->f_op->flush && force_nonblock) { + if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) { spin_unlock(&files->file_lock); return -EAGAIN; } @@ -4527,12 +4537,12 @@ static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock) +static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) { int ret; /* sync_file_range always requires a blocking context */ - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; ret = sync_file_range(req->file, req->sync.off, req->sync.len, @@ -4601,7 +4611,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return ret; } -static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, +static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_async_msghdr iomsg, *kmsg; @@ -4624,11 +4634,11 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, flags = req->sr_msg.msg_flags; if (flags & MSG_DONTWAIT) req->flags |= REQ_F_NOWAIT; - else if (force_nonblock) + else if (issue_flags & IO_URING_F_NONBLOCK) flags |= MSG_DONTWAIT; ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); - if (force_nonblock && ret == -EAGAIN) + if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN) return io_setup_async_msg(req, kmsg); if (ret == -ERESTARTSYS) ret = -EINTR; @@ -4643,7 +4653,7 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, return 0; } -static int io_send(struct io_kiocb *req, bool force_nonblock, +static int io_send(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_sr_msg *sr = &req->sr_msg; @@ -4669,12 +4679,12 @@ static int io_send(struct io_kiocb *req, bool force_nonblock, flags = req->sr_msg.msg_flags; if (flags & MSG_DONTWAIT) req->flags |= REQ_F_NOWAIT; - else if (force_nonblock) + else if (issue_flags & IO_URING_F_NONBLOCK) flags |= MSG_DONTWAIT; msg.msg_flags = flags; ret = sock_sendmsg(sock, &msg); - if (force_nonblock && ret == -EAGAIN) + if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN) return -EAGAIN; if (ret == -ERESTARTSYS) ret = -EINTR; @@ -4822,7 +4832,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, return ret; } -static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, +static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_async_msghdr iomsg, *kmsg; @@ -4830,6 +4840,7 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, struct io_buffer *kbuf; unsigned flags; int ret, cflags = 0; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; sock = sock_from_file(req->file); if (unlikely(!sock)) @@ -4878,7 +4889,7 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, return 0; } -static int io_recv(struct io_kiocb *req, bool force_nonblock, +static int io_recv(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_buffer *kbuf; @@ -4889,6 +4900,7 @@ static int io_recv(struct io_kiocb *req, bool force_nonblock, struct iovec iov; unsigned flags; int ret, cflags = 0; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; sock = sock_from_file(req->file); if (unlikely(!sock)) @@ -4948,10 +4960,11 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_accept(struct io_kiocb *req, bool force_nonblock, +static int io_accept(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_accept *accept = &req->accept; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; int ret; @@ -4992,12 +5005,13 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) &io->address); } -static int io_connect(struct io_kiocb *req, bool force_nonblock, +static int io_connect(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_async_connect __io, *io; unsigned file_flags; int ret; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; if (req->async_data) { io = req->async_data; @@ -5039,13 +5053,13 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_sendmsg(struct io_kiocb *req, bool force_nonblock, +static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; } -static int io_send(struct io_kiocb *req, bool force_nonblock, +static int io_send(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; @@ -5057,13 +5071,13 @@ static int io_recvmsg_prep(struct io_kiocb *req, return -EOPNOTSUPP; } -static int io_recvmsg(struct io_kiocb *req, bool force_nonblock, +static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; } -static int io_recv(struct io_kiocb *req, bool force_nonblock, +static int io_recv(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; @@ -5074,7 +5088,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_accept(struct io_kiocb *req, bool force_nonblock, +static int io_accept(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; @@ -5085,7 +5099,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_connect(struct io_kiocb *req, bool force_nonblock, +static int io_connect(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { return -EOPNOTSUPP; @@ -5963,14 +5977,14 @@ static int io_rsrc_update_prep(struct io_kiocb *req, return 0; } -static int io_files_update(struct io_kiocb *req, bool force_nonblock, +static int io_files_update(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_ring_ctx *ctx = req->ctx; struct io_uring_rsrc_update up; int ret; - if (force_nonblock) + if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; up.offset = req->rsrc_update.offset; @@ -6189,7 +6203,7 @@ static void __io_clean_op(struct io_kiocb *req) } } -static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, +static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, struct io_comp_state *cs) { struct io_ring_ctx *ctx = req->ctx; @@ -6202,15 +6216,15 @@ static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, case IORING_OP_READV: case IORING_OP_READ_FIXED: case IORING_OP_READ: - ret = io_read(req, force_nonblock, cs); + ret = io_read(req, issue_flags, cs); break; case IORING_OP_WRITEV: case IORING_OP_WRITE_FIXED: case IORING_OP_WRITE: - ret = io_write(req, force_nonblock, cs); + ret = io_write(req, issue_flags, cs); break; case IORING_OP_FSYNC: - ret = io_fsync(req, force_nonblock); + ret = io_fsync(req, issue_flags); break; case IORING_OP_POLL_ADD: ret = io_poll_add(req); @@ -6219,19 +6233,19 @@ static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, ret = io_poll_remove(req); break; case IORING_OP_SYNC_FILE_RANGE: - ret = io_sync_file_range(req, force_nonblock); + ret = io_sync_file_range(req, issue_flags); break; case IORING_OP_SENDMSG: - ret = io_sendmsg(req, force_nonblock, cs); + ret = io_sendmsg(req, issue_flags, cs); break; case IORING_OP_SEND: - ret = io_send(req, force_nonblock, cs); + ret = io_send(req, issue_flags, cs); break; case IORING_OP_RECVMSG: - ret = io_recvmsg(req, force_nonblock, cs); + ret = io_recvmsg(req, issue_flags, cs); break; case IORING_OP_RECV: - ret = io_recv(req, force_nonblock, cs); + ret = io_recv(req, issue_flags, cs); break; case IORING_OP_TIMEOUT: ret = io_timeout(req); @@ -6240,61 +6254,61 @@ static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, ret = io_timeout_remove(req); break; case IORING_OP_ACCEPT: - ret = io_accept(req, force_nonblock, cs); + ret = io_accept(req, issue_flags, cs); break; case IORING_OP_CONNECT: - ret = io_connect(req, force_nonblock, cs); + ret = io_connect(req, issue_flags, cs); break; case IORING_OP_ASYNC_CANCEL: ret = io_async_cancel(req); break; case IORING_OP_FALLOCATE: - ret = io_fallocate(req, force_nonblock); + ret = io_fallocate(req, issue_flags); break; case IORING_OP_OPENAT: - ret = io_openat(req, force_nonblock); + ret = io_openat(req, issue_flags); break; case IORING_OP_CLOSE: - ret = io_close(req, force_nonblock, cs); + ret = io_close(req, issue_flags, cs); break; case IORING_OP_FILES_UPDATE: - ret = io_files_update(req, force_nonblock, cs); + ret = io_files_update(req, issue_flags, cs); break; case IORING_OP_STATX: - ret = io_statx(req, force_nonblock); + ret = io_statx(req, issue_flags); break; case IORING_OP_FADVISE: - ret = io_fadvise(req, force_nonblock); + ret = io_fadvise(req, issue_flags); break; case IORING_OP_MADVISE: - ret = io_madvise(req, force_nonblock); + ret = io_madvise(req, issue_flags); break; case IORING_OP_OPENAT2: - ret = io_openat2(req, force_nonblock); + ret = io_openat2(req, issue_flags); break; case IORING_OP_EPOLL_CTL: - ret = io_epoll_ctl(req, force_nonblock, cs); + ret = io_epoll_ctl(req, issue_flags, cs); break; case IORING_OP_SPLICE: - ret = io_splice(req, force_nonblock); + ret = io_splice(req, issue_flags); break; case IORING_OP_PROVIDE_BUFFERS: - ret = io_provide_buffers(req, force_nonblock, cs); + ret = io_provide_buffers(req, issue_flags, cs); break; case IORING_OP_REMOVE_BUFFERS: - ret = io_remove_buffers(req, force_nonblock, cs); + ret = io_remove_buffers(req, issue_flags, cs); break; case IORING_OP_TEE: - ret = io_tee(req, force_nonblock); + ret = io_tee(req, issue_flags); break; case IORING_OP_SHUTDOWN: - ret = io_shutdown(req, force_nonblock); + ret = io_shutdown(req, issue_flags); break; case IORING_OP_RENAMEAT: - ret = io_renameat(req, force_nonblock); + ret = io_renameat(req, issue_flags); break; case IORING_OP_UNLINKAT: - ret = io_unlinkat(req, force_nonblock); + ret = io_unlinkat(req, issue_flags); break; default: ret = -EINVAL; @@ -6336,7 +6350,7 @@ static void io_wq_submit_work(struct io_wq_work *work) if (!ret) { do { - ret = io_issue_sqe(req, false, NULL); + ret = io_issue_sqe(req, 0, NULL); /* * We can get EAGAIN for polled IO even though we're * forcing a sync submission from here, since we can't @@ -6499,7 +6513,7 @@ again: old_creds = override_creds(req->work.identity->creds); } - ret = io_issue_sqe(req, true, cs); + ret = io_issue_sqe(req, IO_URING_F_NONBLOCK, cs); /* * We async punt it if the file wasn't marked NOWAIT, or if the file -- cgit v1.2.3 From 61e98203047983fd959cfef889b328a57315847c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:08 +0000 Subject: io_uring: make op handlers always take issue flags Make opcode handler interfaces a bit more consistent by always passing in issue flags. Bulky but pretty easy and mechanical change. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 862121c48cee..ac233d04ee71 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3917,7 +3917,8 @@ static int io_splice(struct io_kiocb *req, unsigned int issue_flags) /* * IORING_OP_NOP just posts a completion event, nothing else. */ -static int io_nop(struct io_kiocb *req, struct io_comp_state *cs) +static int io_nop(struct io_kiocb *req, unsigned int issue_flags, + struct io_comp_state *cs) { struct io_ring_ctx *ctx = req->ctx; @@ -5581,7 +5582,7 @@ static int io_poll_remove_prep(struct io_kiocb *req, * Find a running poll command that matches one specified in sqe->addr, * and remove it if found. */ -static int io_poll_remove(struct io_kiocb *req) +static int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; int ret; @@ -5632,7 +5633,7 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe return 0; } -static int io_poll_add(struct io_kiocb *req) +static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) { struct io_poll_iocb *poll = &req->poll; struct io_ring_ctx *ctx = req->ctx; @@ -5772,7 +5773,7 @@ static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) /* * Remove or update an existing timeout command */ -static int io_timeout_remove(struct io_kiocb *req) +static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) { struct io_timeout_rem *tr = &req->timeout_rem; struct io_ring_ctx *ctx = req->ctx; @@ -5828,7 +5829,7 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, return 0; } -static int io_timeout(struct io_kiocb *req) +static int io_timeout(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; struct io_timeout_data *data = req->async_data; @@ -5951,7 +5952,7 @@ static int io_async_cancel_prep(struct io_kiocb *req, return 0; } -static int io_async_cancel(struct io_kiocb *req) +static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; @@ -6211,7 +6212,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, switch (req->opcode) { case IORING_OP_NOP: - ret = io_nop(req, cs); + ret = io_nop(req, issue_flags, cs); break; case IORING_OP_READV: case IORING_OP_READ_FIXED: @@ -6227,10 +6228,10 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_fsync(req, issue_flags); break; case IORING_OP_POLL_ADD: - ret = io_poll_add(req); + ret = io_poll_add(req, issue_flags); break; case IORING_OP_POLL_REMOVE: - ret = io_poll_remove(req); + ret = io_poll_remove(req, issue_flags); break; case IORING_OP_SYNC_FILE_RANGE: ret = io_sync_file_range(req, issue_flags); @@ -6248,10 +6249,10 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_recv(req, issue_flags, cs); break; case IORING_OP_TIMEOUT: - ret = io_timeout(req); + ret = io_timeout(req, issue_flags); break; case IORING_OP_TIMEOUT_REMOVE: - ret = io_timeout_remove(req); + ret = io_timeout_remove(req, issue_flags); break; case IORING_OP_ACCEPT: ret = io_accept(req, issue_flags, cs); @@ -6260,7 +6261,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_connect(req, issue_flags, cs); break; case IORING_OP_ASYNC_CANCEL: - ret = io_async_cancel(req); + ret = io_async_cancel(req, issue_flags); break; case IORING_OP_FALLOCATE: ret = io_fallocate(req, issue_flags); -- cgit v1.2.3 From a0f85e38a3c23a334d9ea111cfa4fcfa4b3f9b64 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Wed, 10 Feb 2021 11:55:47 +1000 Subject: cifs: do not disable noperm if multiuser mount option is not provided Fixes small regression in implementation of new mount API. Signed-off-by: Ronnie Sahlberg Reported-by: Hyunchul Lee Tested-by: Hyunchul Lee Signed-off-by: Steve French --- fs/cifs/fs_context.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 5111aadfdb6b..1b1c56e52395 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -1533,8 +1533,8 @@ void smb3_update_mnt_flags(struct cifs_sb_info *cifs_sb) cifs_sb->mnt_cifs_flags |= (CIFS_MOUNT_MULTIUSER | CIFS_MOUNT_NO_PERM); else - cifs_sb->mnt_cifs_flags &= ~(CIFS_MOUNT_MULTIUSER | - CIFS_MOUNT_NO_PERM); + cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_MULTIUSER; + if (ctx->strict_io) cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_STRICT_IO; -- cgit v1.2.3 From 7009fa9cd9a5262944b30eb7efb1f0561d074b68 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Tue, 9 Feb 2021 18:32:32 +0100 Subject: gfs2: Recursive gfs2_quota_hold in gfs2_iomap_end When starting an iomap write, gfs2_quota_lock_check -> gfs2_quota_lock -> gfs2_quota_hold is called from gfs2_iomap_begin. At the end of the write, before unlocking the quotas, punch_hole -> gfs2_quota_hold can be called again in gfs2_iomap_end, which is incorrect and leads to a failed assertion. Instead, move the call to gfs2_quota_unlock before the call to punch_hole to fix that. Fixes: 64bc06bb32ee ("gfs2: iomap buffered write support") Cc: stable@vger.kernel.org # v4.19+ Signed-off-by: Andreas Gruenbacher --- fs/gfs2/bmap.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index 62d9081d1e26..a1f9dde33058 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1230,6 +1230,9 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length, gfs2_inplace_release(ip); + if (ip->i_qadata && ip->i_qadata->qa_qd_num) + gfs2_quota_unlock(ip); + if (length != written && (iomap->flags & IOMAP_F_NEW)) { /* Deallocate blocks that were just allocated. */ loff_t blockmask = i_blocksize(inode) - 1; @@ -1242,9 +1245,6 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length, } } - if (ip->i_qadata && ip->i_qadata->qa_qd_num) - gfs2_quota_unlock(ip); - if (unlikely(!written)) goto out_unlock; -- cgit v1.2.3 From 889fca73287b0ae21c9d8712379c9ae5a3b27d08 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:09 +0000 Subject: io_uring: don't propagate io_comp_state There is no reason to drag io_comp_state into opcode handlers, we just need a flag and the actual work will be done in __io_queue_sqe(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 164 ++++++++++++++++++++++++++-------------------------------- 1 file changed, 73 insertions(+), 91 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index ac233d04ee71..273ebbac0654 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -189,6 +189,7 @@ struct io_rings { enum io_uring_cmd_flags { IO_URING_F_NONBLOCK = 1, + IO_URING_F_COMPLETE_DEFER = 2, }; struct io_mapped_ubuf { @@ -1018,7 +1019,7 @@ static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, struct fixed_rsrc_ref_node *ref_node); static void __io_complete_rw(struct io_kiocb *req, long res, long res2, - struct io_comp_state *cs); + unsigned int issue_flags); static void io_cqring_fill_event(struct io_kiocb *req, long res); static void io_put_req(struct io_kiocb *req); static void io_put_req_deferred(struct io_kiocb *req, int nr); @@ -1957,7 +1958,7 @@ static void io_submit_flush_completions(struct io_comp_state *cs) } static void io_req_complete_state(struct io_kiocb *req, long res, - unsigned int cflags, struct io_comp_state *cs) + unsigned int cflags) { io_clean_op(req); req->result = res; @@ -1965,18 +1966,18 @@ static void io_req_complete_state(struct io_kiocb *req, long res, req->flags |= REQ_F_COMPLETE_INLINE; } -static inline void __io_req_complete(struct io_kiocb *req, long res, - unsigned cflags, struct io_comp_state *cs) +static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, + long res, unsigned cflags) { - if (!cs) - io_req_complete_nostate(req, res, cflags); + if (issue_flags & IO_URING_F_COMPLETE_DEFER) + io_req_complete_state(req, res, cflags); else - io_req_complete_state(req, res, cflags, cs); + io_req_complete_nostate(req, res, cflags); } static inline void io_req_complete(struct io_kiocb *req, long res) { - __io_req_complete(req, res, 0, NULL); + __io_req_complete(req, 0, res, 0); } static inline bool io_is_fallback_req(struct io_kiocb *req) @@ -2449,7 +2450,7 @@ static void io_iopoll_queue(struct list_head *again) do { req = list_first_entry(again, struct io_kiocb, inflight_entry); list_del(&req->inflight_entry); - __io_complete_rw(req, -EAGAIN, 0, NULL); + __io_complete_rw(req, -EAGAIN, 0, 0); } while (!list_empty(again)); } @@ -2662,7 +2663,7 @@ static void kiocb_end_write(struct io_kiocb *req) } static void io_complete_rw_common(struct kiocb *kiocb, long res, - struct io_comp_state *cs) + unsigned int issue_flags) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); int cflags = 0; @@ -2674,7 +2675,7 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res, req_set_fail_links(req); if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_rw_kbuf(req); - __io_req_complete(req, res, cflags, cs); + __io_req_complete(req, issue_flags, res, cflags); } #ifdef CONFIG_BLOCK @@ -2741,17 +2742,17 @@ static bool io_rw_reissue(struct io_kiocb *req, long res) } static void __io_complete_rw(struct io_kiocb *req, long res, long res2, - struct io_comp_state *cs) + unsigned int issue_flags) { if (!io_rw_reissue(req, res)) - io_complete_rw_common(&req->rw.kiocb, res, cs); + io_complete_rw_common(&req->rw.kiocb, res, issue_flags); } static void io_complete_rw(struct kiocb *kiocb, long res, long res2) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); - __io_complete_rw(req, res, res2, NULL); + __io_complete_rw(req, res, res2, 0); } static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) @@ -2970,7 +2971,7 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) } static void kiocb_done(struct kiocb *kiocb, ssize_t ret, - struct io_comp_state *cs) + unsigned int issue_flags) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); struct io_async_rw *io = req->async_data; @@ -2986,7 +2987,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, if (req->flags & REQ_F_CUR_POS) req->file->f_pos = kiocb->ki_pos; if (ret >= 0 && kiocb->ki_complete == io_complete_rw) - __io_complete_rw(req, ret, 0, cs); + __io_complete_rw(req, ret, 0, issue_flags); else io_rw_done(kiocb, ret); } @@ -3481,8 +3482,7 @@ static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) return -EINVAL; } -static int io_read(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_read(struct io_kiocb *req, unsigned int issue_flags) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct kiocb *kiocb = &req->rw.kiocb; @@ -3572,7 +3572,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags, /* we got some bytes, but not all. retry. */ } while (ret > 0 && ret < io_size); done: - kiocb_done(kiocb, ret, cs); + kiocb_done(kiocb, ret, issue_flags); return 0; } @@ -3593,8 +3593,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return io_rw_prep_async(req, WRITE); } -static int io_write(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_write(struct io_kiocb *req, unsigned int issue_flags) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct kiocb *kiocb = &req->rw.kiocb; @@ -3668,7 +3667,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags, if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) goto copy_iov; done: - kiocb_done(kiocb, ret2, cs); + kiocb_done(kiocb, ret2, issue_flags); } else { copy_iov: /* some cases will consume bytes even on error returns */ @@ -3917,15 +3916,14 @@ static int io_splice(struct io_kiocb *req, unsigned int issue_flags) /* * IORING_OP_NOP just posts a completion event, nothing else. */ -static int io_nop(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_nop(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; - __io_req_complete(req, 0, 0, cs); + __io_req_complete(req, issue_flags, 0, 0); return 0; } @@ -4166,8 +4164,7 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, return i; } -static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) { struct io_provide_buf *p = &req->pbuf; struct io_ring_ctx *ctx = req->ctx; @@ -4188,11 +4185,11 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags, /* need to hold the lock to complete IOPOLL requests */ if (ctx->flags & IORING_SETUP_IOPOLL) { - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); io_ring_submit_unlock(ctx, !force_nonblock); } else { io_ring_submit_unlock(ctx, !force_nonblock); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); } return 0; } @@ -4251,8 +4248,7 @@ static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head) return i ? i : -ENOMEM; } -static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) { struct io_provide_buf *p = &req->pbuf; struct io_ring_ctx *ctx = req->ctx; @@ -4284,11 +4280,11 @@ out: /* need to hold the lock to complete IOPOLL requests */ if (ctx->flags & IORING_SETUP_IOPOLL) { - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); io_ring_submit_unlock(ctx, !force_nonblock); } else { io_ring_submit_unlock(ctx, !force_nonblock); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); } return 0; } @@ -4320,8 +4316,7 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, #endif } -static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags) { #if defined(CONFIG_EPOLL) struct io_epoll *ie = &req->epoll; @@ -4334,7 +4329,7 @@ static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags, if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; #else return -EOPNOTSUPP; @@ -4466,8 +4461,7 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_close(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_close(struct io_kiocb *req, unsigned int issue_flags) { struct files_struct *files = current->files; struct io_close *close = &req->close; @@ -4516,7 +4510,7 @@ err: req_set_fail_links(req); if (file) fput(file); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -4612,8 +4606,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return ret; } -static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) { struct io_async_msghdr iomsg, *kmsg; struct socket *sock; @@ -4650,12 +4643,11 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } -static int io_send(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_send(struct io_kiocb *req, unsigned int issue_flags) { struct io_sr_msg *sr = &req->sr_msg; struct msghdr msg; @@ -4692,7 +4684,7 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags, if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -4833,8 +4825,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, return ret; } -static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) { struct io_async_msghdr iomsg, *kmsg; struct socket *sock; @@ -4886,12 +4877,11 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, cflags, cs); + __io_req_complete(req, issue_flags, ret, cflags); return 0; } -static int io_recv(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_recv(struct io_kiocb *req, unsigned int issue_flags) { struct io_buffer *kbuf; struct io_sr_msg *sr = &req->sr_msg; @@ -4941,7 +4931,7 @@ out_free: cflags = io_put_recv_kbuf(req); if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, cflags, cs); + __io_req_complete(req, issue_flags, ret, cflags); return 0; } @@ -4961,8 +4951,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int io_accept(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_accept(struct io_kiocb *req, unsigned int issue_flags) { struct io_accept *accept = &req->accept; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; @@ -4982,7 +4971,7 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags, ret = -EINTR; req_set_fail_links(req); } - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -5006,8 +4995,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) &io->address); } -static int io_connect(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_connect(struct io_kiocb *req, unsigned int issue_flags) { struct io_async_connect __io, *io; unsigned file_flags; @@ -5045,7 +5033,7 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags, out: if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } #else /* !CONFIG_NET */ @@ -5054,14 +5042,12 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } -static int io_send(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_send(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } @@ -5072,14 +5058,12 @@ static int io_recvmsg_prep(struct io_kiocb *req, return -EOPNOTSUPP; } -static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } -static int io_recv(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_recv(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } @@ -5089,8 +5073,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_accept(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_accept(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } @@ -5100,8 +5083,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EOPNOTSUPP; } -static int io_connect(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_connect(struct io_kiocb *req, unsigned int issue_flags) { return -EOPNOTSUPP; } @@ -5978,8 +5960,7 @@ static int io_rsrc_update_prep(struct io_kiocb *req, return 0; } -static int io_files_update(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; struct io_uring_rsrc_update up; @@ -5997,7 +5978,7 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags, if (ret < 0) req_set_fail_links(req); - __io_req_complete(req, ret, 0, cs); + __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -6204,25 +6185,24 @@ static void __io_clean_op(struct io_kiocb *req) } } -static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, - struct io_comp_state *cs) +static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; int ret; switch (req->opcode) { case IORING_OP_NOP: - ret = io_nop(req, issue_flags, cs); + ret = io_nop(req, issue_flags); break; case IORING_OP_READV: case IORING_OP_READ_FIXED: case IORING_OP_READ: - ret = io_read(req, issue_flags, cs); + ret = io_read(req, issue_flags); break; case IORING_OP_WRITEV: case IORING_OP_WRITE_FIXED: case IORING_OP_WRITE: - ret = io_write(req, issue_flags, cs); + ret = io_write(req, issue_flags); break; case IORING_OP_FSYNC: ret = io_fsync(req, issue_flags); @@ -6237,16 +6217,16 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_sync_file_range(req, issue_flags); break; case IORING_OP_SENDMSG: - ret = io_sendmsg(req, issue_flags, cs); + ret = io_sendmsg(req, issue_flags); break; case IORING_OP_SEND: - ret = io_send(req, issue_flags, cs); + ret = io_send(req, issue_flags); break; case IORING_OP_RECVMSG: - ret = io_recvmsg(req, issue_flags, cs); + ret = io_recvmsg(req, issue_flags); break; case IORING_OP_RECV: - ret = io_recv(req, issue_flags, cs); + ret = io_recv(req, issue_flags); break; case IORING_OP_TIMEOUT: ret = io_timeout(req, issue_flags); @@ -6255,10 +6235,10 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_timeout_remove(req, issue_flags); break; case IORING_OP_ACCEPT: - ret = io_accept(req, issue_flags, cs); + ret = io_accept(req, issue_flags); break; case IORING_OP_CONNECT: - ret = io_connect(req, issue_flags, cs); + ret = io_connect(req, issue_flags); break; case IORING_OP_ASYNC_CANCEL: ret = io_async_cancel(req, issue_flags); @@ -6270,10 +6250,10 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_openat(req, issue_flags); break; case IORING_OP_CLOSE: - ret = io_close(req, issue_flags, cs); + ret = io_close(req, issue_flags); break; case IORING_OP_FILES_UPDATE: - ret = io_files_update(req, issue_flags, cs); + ret = io_files_update(req, issue_flags); break; case IORING_OP_STATX: ret = io_statx(req, issue_flags); @@ -6288,16 +6268,16 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags, ret = io_openat2(req, issue_flags); break; case IORING_OP_EPOLL_CTL: - ret = io_epoll_ctl(req, issue_flags, cs); + ret = io_epoll_ctl(req, issue_flags); break; case IORING_OP_SPLICE: ret = io_splice(req, issue_flags); break; case IORING_OP_PROVIDE_BUFFERS: - ret = io_provide_buffers(req, issue_flags, cs); + ret = io_provide_buffers(req, issue_flags); break; case IORING_OP_REMOVE_BUFFERS: - ret = io_remove_buffers(req, issue_flags, cs); + ret = io_remove_buffers(req, issue_flags); break; case IORING_OP_TEE: ret = io_tee(req, issue_flags); @@ -6351,7 +6331,7 @@ static void io_wq_submit_work(struct io_wq_work *work) if (!ret) { do { - ret = io_issue_sqe(req, 0, NULL); + ret = io_issue_sqe(req, 0); /* * We can get EAGAIN for polled IO even though we're * forcing a sync submission from here, since we can't @@ -6498,8 +6478,10 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) { struct io_kiocb *linked_timeout; const struct cred *old_creds = NULL; - int ret; + int ret, issue_flags = IO_URING_F_NONBLOCK; + if (cs) + issue_flags |= IO_URING_F_COMPLETE_DEFER; again: linked_timeout = io_prep_linked_timeout(req); @@ -6514,7 +6496,7 @@ again: old_creds = override_creds(req->work.identity->creds); } - ret = io_issue_sqe(req, IO_URING_F_NONBLOCK, cs); + ret = io_issue_sqe(req, issue_flags); /* * We async punt it if the file wasn't marked NOWAIT, or if the file -- cgit v1.2.3 From 258b29a93bfe74a57c01e1b10b698d5b62e173fe Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:10 +0000 Subject: io_uring: don't keep submit_state on stack struct io_submit_state is quite big (168 bytes) and going to grow. It's better to not keep it on stack as it is now. Move it to context, it's always protected by uring_lock, so it's fine to have only one instance of it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 90 ++++++++++++++++++++++++++++++----------------------------- 1 file changed, 46 insertions(+), 44 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 273ebbac0654..743090364890 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -264,6 +264,39 @@ struct io_sq_data { unsigned sq_thread_idle; }; +#define IO_IOPOLL_BATCH 8 + +struct io_comp_state { + unsigned int nr; + struct list_head list; + struct io_ring_ctx *ctx; +}; + +struct io_submit_state { + struct blk_plug plug; + + /* + * io_kiocb alloc cache + */ + void *reqs[IO_IOPOLL_BATCH]; + unsigned int free_reqs; + + bool plug_started; + + /* + * Batch completion logic + */ + struct io_comp_state comp; + + /* + * File reference cache + */ + struct file *file; + unsigned int fd; + unsigned int file_refs; + unsigned int ios_left; +}; + struct io_ring_ctx { struct { struct percpu_ref refs; @@ -406,6 +439,7 @@ struct io_ring_ctx { struct work_struct exit_work; struct io_restriction restrictions; + struct io_submit_state submit_state; }; /* @@ -758,39 +792,6 @@ struct io_defer_entry { u32 seq; }; -#define IO_IOPOLL_BATCH 8 - -struct io_comp_state { - unsigned int nr; - struct list_head list; - struct io_ring_ctx *ctx; -}; - -struct io_submit_state { - struct blk_plug plug; - - /* - * io_kiocb alloc cache - */ - void *reqs[IO_IOPOLL_BATCH]; - unsigned int free_reqs; - - bool plug_started; - - /* - * Batch completion logic - */ - struct io_comp_state comp; - - /* - * File reference cache - */ - struct file *file; - unsigned int fd; - unsigned int file_refs; - unsigned int ios_left; -}; - struct io_op_def { /* needs req->file assigned */ unsigned needs_file : 1; @@ -1997,9 +1998,10 @@ static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx) return NULL; } -static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx, - struct io_submit_state *state) +static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { + struct io_submit_state *state = &ctx->submit_state; + if (!state->free_reqs) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; size_t sz; @@ -6758,9 +6760,9 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx, IOSQE_BUFFER_SELECT) static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, - const struct io_uring_sqe *sqe, - struct io_submit_state *state) + const struct io_uring_sqe *sqe) { + struct io_submit_state *state; unsigned int sqe_flags; int id, ret; @@ -6812,6 +6814,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, /* same numerical values with corresponding REQ_F_*, safe to copy */ req->flags |= sqe_flags; + state = &ctx->submit_state; /* * Plug now if we have more than 1 IO left after this, and the target @@ -6838,7 +6841,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) { - struct io_submit_state state; struct io_submit_link link; int i, submitted = 0; @@ -6857,7 +6859,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) percpu_counter_add(¤t->io_uring->inflight, nr); refcount_add(nr, ¤t->usage); - io_submit_state_start(&state, ctx, nr); + io_submit_state_start(&ctx->submit_state, ctx, nr); link.head = NULL; for (i = 0; i < nr; i++) { @@ -6870,7 +6872,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) io_consume_sqe(ctx); break; } - req = io_alloc_req(ctx, &state); + req = io_alloc_req(ctx); if (unlikely(!req)) { if (!submitted) submitted = -EAGAIN; @@ -6880,7 +6882,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) /* will complete beyond this point, count as submitted */ submitted++; - err = io_init_req(ctx, req, sqe, &state); + err = io_init_req(ctx, req, sqe); if (unlikely(err)) { fail_req: io_put_req(req); @@ -6890,7 +6892,7 @@ fail_req: trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, true, ctx->flags & IORING_SETUP_SQPOLL); - err = io_submit_sqe(req, sqe, &link, &state.comp); + err = io_submit_sqe(req, sqe, &link, &ctx->submit_state.comp); if (err) goto fail_req; } @@ -6905,8 +6907,8 @@ fail_req: put_task_struct_many(current, unused); } if (link.head) - io_queue_link_head(link.head, &state.comp); - io_submit_state_end(&state); + io_queue_link_head(link.head, &ctx->submit_state.comp); + io_submit_state_end(&ctx->submit_state); /* Commit SQ ring head once we've consumed and submitted all SQEs */ io_commit_sqring(ctx); -- cgit v1.2.3 From ba88ff112bdfde8103a8143f867bcdc46bc0e50f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:11 +0000 Subject: io_uring: remove ctx from comp_state completion state is closely bound to ctx, we don't need to store ctx inside as we always have it around to pass to flush. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 743090364890..6e800f9df292 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -269,7 +269,6 @@ struct io_sq_data { struct io_comp_state { unsigned int nr; struct list_head list; - struct io_ring_ctx *ctx; }; struct io_submit_state { @@ -1924,10 +1923,9 @@ static inline void io_req_complete_nostate(struct io_kiocb *req, long res, io_put_req(req); } -static void io_submit_flush_completions(struct io_comp_state *cs) +static void io_submit_flush_completions(struct io_comp_state *cs, + struct io_ring_ctx *ctx) { - struct io_ring_ctx *ctx = cs->ctx; - spin_lock_irq(&ctx->completion_lock); while (!list_empty(&cs->list)) { struct io_kiocb *req; @@ -6520,7 +6518,7 @@ again: if (req->flags & REQ_F_COMPLETE_INLINE) { list_add_tail(&req->compl.list, &cs->list); if (++cs->nr >= 32) - io_submit_flush_completions(cs); + io_submit_flush_completions(cs, req->ctx); req = NULL; } else { req = io_put_req_find_next(req); @@ -6655,10 +6653,11 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, /* * Batched submission is done, ensure local IO is flushed out. */ -static void io_submit_state_end(struct io_submit_state *state) +static void io_submit_state_end(struct io_submit_state *state, + struct io_ring_ctx *ctx) { if (!list_empty(&state->comp.list)) - io_submit_flush_completions(&state->comp); + io_submit_flush_completions(&state->comp, ctx); if (state->plug_started) blk_finish_plug(&state->plug); io_state_file_put(state); @@ -6670,12 +6669,11 @@ static void io_submit_state_end(struct io_submit_state *state) * Start submission side cache. */ static void io_submit_state_start(struct io_submit_state *state, - struct io_ring_ctx *ctx, unsigned int max_ios) + unsigned int max_ios) { state->plug_started = false; state->comp.nr = 0; INIT_LIST_HEAD(&state->comp.list); - state->comp.ctx = ctx; state->free_reqs = 0; state->file_refs = 0; state->ios_left = max_ios; @@ -6859,7 +6857,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) percpu_counter_add(¤t->io_uring->inflight, nr); refcount_add(nr, ¤t->usage); - io_submit_state_start(&ctx->submit_state, ctx, nr); + io_submit_state_start(&ctx->submit_state, nr); link.head = NULL; for (i = 0; i < nr; i++) { @@ -6908,7 +6906,7 @@ fail_req: } if (link.head) io_queue_link_head(link.head, &ctx->submit_state.comp); - io_submit_state_end(&ctx->submit_state); + io_submit_state_end(&ctx->submit_state, ctx); /* Commit SQ ring head once we've consumed and submitted all SQEs */ io_commit_sqring(ctx); -- cgit v1.2.3 From 5087275dba02943179720bd95d1d6c7047007550 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:12 +0000 Subject: io_uring: don't reinit submit state every time As now submit_state is retained across syscalls, we can save ourself from initialising it from ground up for each io_submit_sqes(). Set some fields during ctx allocation, and just keep them always consistent. Signed-off-by: Pavel Begunkov [axboe: remove unnecessary zeroing of ctx members] Signed-off-by: Jens Axboe --- fs/io_uring.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6e800f9df292..6bce9094280c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1345,6 +1345,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->rsrc_ref_list); INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); + INIT_LIST_HEAD(&submit_state->comp.list); return ctx; err: if (ctx->fallback_req) @@ -6661,8 +6662,10 @@ static void io_submit_state_end(struct io_submit_state *state, if (state->plug_started) blk_finish_plug(&state->plug); io_state_file_put(state); - if (state->free_reqs) + if (state->free_reqs) { kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs); + state->free_reqs = 0; + } } /* @@ -6672,10 +6675,6 @@ static void io_submit_state_start(struct io_submit_state *state, unsigned int max_ios) { state->plug_started = false; - state->comp.nr = 0; - INIT_LIST_HEAD(&state->comp.list); - state->free_reqs = 0; - state->file_refs = 0; state->ios_left = max_ios; } -- cgit v1.2.3 From 6dd0be1e2481b32c39870e187840ade6c2a11a72 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:13 +0000 Subject: io_uring: replace list with array for compl batch Reincarnation of an old patch that replaces a list in struct io_compl_batch with an array. It's needed to avoid hooking requests via their compl.list, because it won't be always available in the future. It's also nice to split io_submit_flush_completions() to avoid free under locks and remove unlock/lock with a long comment describing when it can be done. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6bce9094280c..aef640616edb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -265,10 +265,11 @@ struct io_sq_data { }; #define IO_IOPOLL_BATCH 8 +#define IO_COMPL_BATCH 32 struct io_comp_state { unsigned int nr; - struct list_head list; + struct io_kiocb *reqs[IO_COMPL_BATCH]; }; struct io_submit_state { @@ -1345,7 +1346,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->rsrc_ref_list); INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); - INIT_LIST_HEAD(&submit_state->comp.list); return ctx; err: if (ctx->fallback_req) @@ -1927,33 +1927,20 @@ static inline void io_req_complete_nostate(struct io_kiocb *req, long res, static void io_submit_flush_completions(struct io_comp_state *cs, struct io_ring_ctx *ctx) { + int i, nr = cs->nr; + spin_lock_irq(&ctx->completion_lock); - while (!list_empty(&cs->list)) { - struct io_kiocb *req; + for (i = 0; i < nr; i++) { + struct io_kiocb *req = cs->reqs[i]; - req = list_first_entry(&cs->list, struct io_kiocb, compl.list); - list_del(&req->compl.list); __io_cqring_fill_event(req, req->result, req->compl.cflags); - - /* - * io_free_req() doesn't care about completion_lock unless one - * of these flags is set. REQ_F_WORK_INITIALIZED is in the list - * because of a potential deadlock with req->work.fs->lock - * We defer both, completion and submission refs. - */ - if (req->flags & (REQ_F_FAIL_LINK|REQ_F_LINK_TIMEOUT - |REQ_F_WORK_INITIALIZED)) { - spin_unlock_irq(&ctx->completion_lock); - io_double_put_req(req); - spin_lock_irq(&ctx->completion_lock); - } else { - io_double_put_req(req); - } } io_commit_cqring(ctx); spin_unlock_irq(&ctx->completion_lock); io_cqring_ev_posted(ctx); + for (i = 0; i < nr; i++) + io_double_put_req(cs->reqs[i]); cs->nr = 0; } @@ -6517,8 +6504,8 @@ again: } else if (likely(!ret)) { /* drop submission reference */ if (req->flags & REQ_F_COMPLETE_INLINE) { - list_add_tail(&req->compl.list, &cs->list); - if (++cs->nr >= 32) + cs->reqs[cs->nr++] = req; + if (cs->nr == IO_COMPL_BATCH) io_submit_flush_completions(cs, req->ctx); req = NULL; } else { @@ -6657,7 +6644,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, static void io_submit_state_end(struct io_submit_state *state, struct io_ring_ctx *ctx) { - if (!list_empty(&state->comp.list)) + if (state->comp.nr) io_submit_flush_completions(&state->comp, ctx); if (state->plug_started) blk_finish_plug(&state->plug); -- cgit v1.2.3 From 905c172f32c56f0740630b639ca5c10ba3689da0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:14 +0000 Subject: io_uring: submit-completion free batching io_submit_flush_completions() does completion batching, but may also use free batching as iopoll does. The main beneficiaries should be buffered reads/writes and send/recv. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 49 +++++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 20 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index aef640616edb..885dd16893c0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1924,26 +1924,6 @@ static inline void io_req_complete_nostate(struct io_kiocb *req, long res, io_put_req(req); } -static void io_submit_flush_completions(struct io_comp_state *cs, - struct io_ring_ctx *ctx) -{ - int i, nr = cs->nr; - - spin_lock_irq(&ctx->completion_lock); - for (i = 0; i < nr; i++) { - struct io_kiocb *req = cs->reqs[i]; - - __io_cqring_fill_event(req, req->result, req->compl.cflags); - } - io_commit_cqring(ctx); - spin_unlock_irq(&ctx->completion_lock); - - io_cqring_ev_posted(ctx); - for (i = 0; i < nr; i++) - io_double_put_req(cs->reqs[i]); - cs->nr = 0; -} - static void io_req_complete_state(struct io_kiocb *req, long res, unsigned int cflags) { @@ -2329,6 +2309,35 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) __io_req_free_batch_flush(req->ctx, rb); } +static void io_submit_flush_completions(struct io_comp_state *cs, + struct io_ring_ctx *ctx) +{ + int i, nr = cs->nr; + struct io_kiocb *req; + struct req_batch rb; + + io_init_req_batch(&rb); + spin_lock_irq(&ctx->completion_lock); + for (i = 0; i < nr; i++) { + req = cs->reqs[i]; + __io_cqring_fill_event(req, req->result, req->compl.cflags); + } + io_commit_cqring(ctx); + spin_unlock_irq(&ctx->completion_lock); + + io_cqring_ev_posted(ctx); + for (i = 0; i < nr; i++) { + req = cs->reqs[i]; + + /* submission and completion refs */ + if (refcount_sub_and_test(2, &req->refs)) + io_req_free_batch(&rb, req); + } + + io_req_free_batch_finish(ctx, &rb); + cs->nr = 0; +} + /* * Drop reference to request, return next in chain (if there is one) if this * was the last reference to this request. -- cgit v1.2.3 From 3893f39f2245eec04b8052cd441c2cb8a9ea3447 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:15 +0000 Subject: io_uring: remove fallback_req Remove fallback_req for now, it gets in the way of other changes. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 38 ++------------------------------------ 1 file changed, 2 insertions(+), 36 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 885dd16893c0..a0b5f2c6d8ea 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -386,9 +386,6 @@ struct io_ring_ctx { struct completion ref_comp; struct completion sq_thread_comp; - /* if all else fails... */ - struct io_kiocb *fallback_req; - #if defined(CONFIG_UNIX) struct socket *ring_sock; #endif @@ -1302,10 +1299,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) if (!ctx) return NULL; - ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL); - if (!ctx->fallback_req) - goto err; - /* * Use 5 bits less than the max cq entries, that should give us around * 32 entries per hash list if totally full and uniformly spread. @@ -1348,8 +1341,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) init_llist_head(&ctx->rsrc_put_llist); return ctx; err: - if (ctx->fallback_req) - kmem_cache_free(req_cachep, ctx->fallback_req); kfree(ctx->cancel_hash); kfree(ctx); return NULL; @@ -1947,23 +1938,6 @@ static inline void io_req_complete(struct io_kiocb *req, long res) __io_req_complete(req, 0, res, 0); } -static inline bool io_is_fallback_req(struct io_kiocb *req) -{ - return req == (struct io_kiocb *) - ((unsigned long) req->ctx->fallback_req & ~1UL); -} - -static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx) -{ - struct io_kiocb *req; - - req = ctx->fallback_req; - if (!test_and_set_bit_lock(0, (unsigned long *) &ctx->fallback_req)) - return req; - - return NULL; -} - static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; @@ -1983,7 +1957,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) if (unlikely(ret <= 0)) { state->reqs[0] = kmem_cache_alloc(req_cachep, gfp); if (!state->reqs[0]) - return io_get_fallback_req(ctx); + return NULL; ret = 1; } state->free_reqs = ret; @@ -2030,10 +2004,7 @@ static void __io_free_req(struct io_kiocb *req) io_dismantle_req(req); io_put_task(req->task, 1); - if (likely(!io_is_fallback_req(req))) - kmem_cache_free(req_cachep, req); - else - clear_bit_unlock(0, (unsigned long *) &ctx->fallback_req); + kmem_cache_free(req_cachep, req); percpu_ref_put(&ctx->refs); } @@ -2289,10 +2260,6 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx, static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) { - if (unlikely(io_is_fallback_req(req))) { - io_free_req(req); - return; - } io_queue_next(req); if (req->task != rb->task) { @@ -8695,7 +8662,6 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) free_uid(ctx->user); put_cred(ctx->creds); kfree(ctx->cancel_hash); - kmem_cache_free(req_cachep, ctx->fallback_req); kfree(ctx); } -- cgit v1.2.3 From 9ae7246321d2b735867f6767e0fab96dd248c555 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:16 +0000 Subject: io_uring: count ctx refs separately from reqs Currently batch free handles request memory freeing and ctx ref putting together. Separate them and use different counters, that will be needed for reusing reqs memory. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index a0b5f2c6d8ea..9b84d6314c11 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2227,6 +2227,7 @@ static void io_free_req(struct io_kiocb *req) struct req_batch { void *reqs[IO_IOPOLL_BATCH]; int to_free; + int ctx_refs; struct task_struct *task; int task_refs; @@ -2236,6 +2237,7 @@ static inline void io_init_req_batch(struct req_batch *rb) { rb->to_free = 0; rb->task_refs = 0; + rb->ctx_refs = 0; rb->task = NULL; } @@ -2243,7 +2245,6 @@ static void __io_req_free_batch_flush(struct io_ring_ctx *ctx, struct req_batch *rb) { kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs); - percpu_ref_put_many(&ctx->refs, rb->to_free); rb->to_free = 0; } @@ -2256,6 +2257,8 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx, io_put_task(rb->task, rb->task_refs); rb->task = NULL; } + if (rb->ctx_refs) + percpu_ref_put_many(&ctx->refs, rb->ctx_refs); } static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) @@ -2269,6 +2272,7 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) rb->task_refs = 0; } rb->task_refs++; + rb->ctx_refs++; io_dismantle_req(req); rb->reqs[rb->to_free++] = req; -- cgit v1.2.3 From bf019da7fcbe7e42372582cc339fd1fb8e1e4fa5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:17 +0000 Subject: io_uring: persistent req cache Don't free batch-allocated requests across syscalls. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 9b84d6314c11..1f0b3b332d32 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -266,6 +266,8 @@ struct io_sq_data { #define IO_IOPOLL_BATCH 8 #define IO_COMPL_BATCH 32 +#define IO_REQ_CACHE_SIZE 8 +#define IO_REQ_ALLOC_BATCH 8 struct io_comp_state { unsigned int nr; @@ -278,7 +280,7 @@ struct io_submit_state { /* * io_kiocb alloc cache */ - void *reqs[IO_IOPOLL_BATCH]; + void *reqs[IO_REQ_CACHE_SIZE]; unsigned int free_reqs; bool plug_started; @@ -1942,13 +1944,14 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; + BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs)); + if (!state->free_reqs) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; - size_t sz; int ret; - sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs)); - ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs); + ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH, + state->reqs); /* * Bulk alloc is all-or-nothing. If we fail to get a batch, @@ -6629,10 +6632,6 @@ static void io_submit_state_end(struct io_submit_state *state, if (state->plug_started) blk_finish_plug(&state->plug); io_state_file_put(state); - if (state->free_reqs) { - kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs); - state->free_reqs = 0; - } } /* @@ -8632,6 +8631,8 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) static void io_ring_ctx_free(struct io_ring_ctx *ctx) { + struct io_submit_state *submit_state = &ctx->submit_state; + io_finish_async(ctx); io_sqe_buffers_unregister(ctx); @@ -8642,6 +8643,10 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) ctx->mm_account = NULL; } + if (submit_state->free_reqs) + kmem_cache_free_bulk(req_cachep, submit_state->free_reqs, + submit_state->reqs); + #ifdef CONFIG_BLK_CGROUP if (ctx->sqo_blkcg_css) css_put(ctx->sqo_blkcg_css); -- cgit v1.2.3 From 6ff119a6e4c3fe900e75e6667930dc086f185f2b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:18 +0000 Subject: io_uring: feed reqs back into alloc cache Make io_req_free_batch(), which is used for inline executed requests and IOPOLL, to return requests back into the allocation cache, so avoid most of kmalloc()/kfree() for those cases. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 1f0b3b332d32..fe07af756186 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -266,7 +266,7 @@ struct io_sq_data { #define IO_IOPOLL_BATCH 8 #define IO_COMPL_BATCH 32 -#define IO_REQ_CACHE_SIZE 8 +#define IO_REQ_CACHE_SIZE 32 #define IO_REQ_ALLOC_BATCH 8 struct io_comp_state { @@ -2264,7 +2264,8 @@ static void io_req_free_batch_finish(struct io_ring_ctx *ctx, percpu_ref_put_many(&ctx->refs, rb->ctx_refs); } -static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) +static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, + struct io_submit_state *state) { io_queue_next(req); @@ -2278,9 +2279,13 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req) rb->ctx_refs++; io_dismantle_req(req); - rb->reqs[rb->to_free++] = req; - if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs))) - __io_req_free_batch_flush(req->ctx, rb); + if (state->free_reqs != ARRAY_SIZE(state->reqs)) { + state->reqs[state->free_reqs++] = req; + } else { + rb->reqs[rb->to_free++] = req; + if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs))) + __io_req_free_batch_flush(req->ctx, rb); + } } static void io_submit_flush_completions(struct io_comp_state *cs, @@ -2305,7 +2310,7 @@ static void io_submit_flush_completions(struct io_comp_state *cs, /* submission and completion refs */ if (refcount_sub_and_test(2, &req->refs)) - io_req_free_batch(&rb, req); + io_req_free_batch(&rb, req, &ctx->submit_state); } io_req_free_batch_finish(ctx, &rb); @@ -2458,7 +2463,7 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, (*nr_events)++; if (refcount_dec_and_test(&req->refs)) - io_req_free_batch(&rb, req); + io_req_free_batch(&rb, req, &ctx->submit_state); } io_commit_cqring(ctx); -- cgit v1.2.3 From 1b4c351f6eb7467c77fc19e0cd7e5f0083ecd847 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 10 Feb 2021 00:03:19 +0000 Subject: io_uring: use persistent request cache Now that we have the submit_state in the ring itself, we can have io_kiocb allocations that are persistent across invocations. This reduces the time spent doing slab allocations and frees. [sil: rebased] Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index fe07af756186..87a4b727fe1c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -270,8 +270,9 @@ struct io_sq_data { #define IO_REQ_ALLOC_BATCH 8 struct io_comp_state { - unsigned int nr; struct io_kiocb *reqs[IO_COMPL_BATCH]; + unsigned int nr; + struct list_head free_list; }; struct io_submit_state { @@ -1341,6 +1342,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->rsrc_ref_list); INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); + INIT_LIST_HEAD(&ctx->submit_state.comp.free_list); return ctx; err: kfree(ctx->cancel_hash); @@ -1946,6 +1948,15 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs)); + if (!list_empty(&state->comp.free_list)) { + struct io_kiocb *req; + + req = list_first_entry(&state->comp.free_list, struct io_kiocb, + compl.list); + list_del(&req->compl.list); + return req; + } + if (!state->free_reqs) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; int ret; @@ -2228,34 +2239,21 @@ static void io_free_req(struct io_kiocb *req) } struct req_batch { - void *reqs[IO_IOPOLL_BATCH]; - int to_free; - int ctx_refs; - struct task_struct *task; int task_refs; + int ctx_refs; }; static inline void io_init_req_batch(struct req_batch *rb) { - rb->to_free = 0; rb->task_refs = 0; rb->ctx_refs = 0; rb->task = NULL; } -static void __io_req_free_batch_flush(struct io_ring_ctx *ctx, - struct req_batch *rb) -{ - kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs); - rb->to_free = 0; -} - static void io_req_free_batch_finish(struct io_ring_ctx *ctx, struct req_batch *rb) { - if (rb->to_free) - __io_req_free_batch_flush(ctx, rb); if (rb->task) { io_put_task(rb->task, rb->task_refs); rb->task = NULL; @@ -2282,9 +2280,9 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, if (state->free_reqs != ARRAY_SIZE(state->reqs)) { state->reqs[state->free_reqs++] = req; } else { - rb->reqs[rb->to_free++] = req; - if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs))) - __io_req_free_batch_flush(req->ctx, rb); + struct io_comp_state *cs = &req->ctx->submit_state.comp; + + list_add(&req->compl.list, &cs->free_list); } } @@ -8634,6 +8632,19 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) idr_destroy(&ctx->io_buffer_idr); } +static void io_req_cache_free(struct io_ring_ctx *ctx) +{ + struct io_comp_state *cs = &ctx->submit_state.comp; + + while (!list_empty(&cs->free_list)) { + struct io_kiocb *req; + + req = list_first_entry(&cs->free_list, struct io_kiocb, compl.list); + list_del(&req->compl.list); + kmem_cache_free(req_cachep, req); + } +} + static void io_ring_ctx_free(struct io_ring_ctx *ctx) { struct io_submit_state *submit_state = &ctx->submit_state; @@ -8676,6 +8687,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) free_uid(ctx->user); put_cred(ctx->creds); kfree(ctx->cancel_hash); + io_req_cache_free(ctx); kfree(ctx); } -- cgit v1.2.3 From 7cbf1722d5fc5779946ee8f338e9e38b5de15856 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 10 Feb 2021 00:03:20 +0000 Subject: io_uring: provide FIFO ordering for task_work task_work is a LIFO list, due to how it's implemented as a lockless list. For long chains of task_work, this can be problematic as the first entry added is the last one processed. Similarly, we'd waste a lot of CPU cycles reversing this list. Wrap the task_work so we have a single task_work entry per task per ctx, and use that to run it in the right order. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io-wq.h | 9 ----- fs/io_uring.c | 101 +++++++++++++++++++++++++++++++++++++++++++---- include/linux/io_uring.h | 14 +++++++ 3 files changed, 108 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.h b/fs/io-wq.h index e37a0f217cc8..096f1021018e 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -27,15 +27,6 @@ enum io_wq_cancel { IO_WQ_CANCEL_NOTFOUND, /* work not found */ }; -struct io_wq_work_node { - struct io_wq_work_node *next; -}; - -struct io_wq_work_list { - struct io_wq_work_node *first; - struct io_wq_work_node *last; -}; - static inline void wq_list_add_after(struct io_wq_work_node *node, struct io_wq_work_node *pos, struct io_wq_work_list *list) diff --git a/fs/io_uring.c b/fs/io_uring.c index 87a4b727fe1c..bfc8fcd93504 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -721,6 +721,11 @@ struct async_poll { struct io_poll_iocb *double_poll; }; +struct io_task_work { + struct io_wq_work_node node; + task_work_func_t func; +}; + /* * NOTE! Each of the iocb union members has the file pointer * as the first entry in their struct definition. So you can @@ -779,7 +784,10 @@ struct io_kiocb { * 2. to track reqs with ->files (see io_op_def::file_table) */ struct list_head inflight_entry; - struct callback_head task_work; + union { + struct io_task_work io_task_work; + struct callback_head task_work; + }; /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ struct hlist_node hash_node; struct async_poll *apoll; @@ -2129,6 +2137,81 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) return __io_req_find_next(req); } +static bool __tctx_task_work(struct io_uring_task *tctx) +{ + struct io_wq_work_list list; + struct io_wq_work_node *node; + + if (wq_list_empty(&tctx->task_list)) + return false; + + spin_lock(&tctx->task_lock); + list = tctx->task_list; + INIT_WQ_LIST(&tctx->task_list); + spin_unlock(&tctx->task_lock); + + node = list.first; + while (node) { + struct io_wq_work_node *next = node->next; + struct io_kiocb *req; + + req = container_of(node, struct io_kiocb, io_task_work.node); + req->task_work.func(&req->task_work); + node = next; + } + + return list.first != NULL; +} + +static void tctx_task_work(struct callback_head *cb) +{ + struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work); + + while (__tctx_task_work(tctx)) + cond_resched(); + + clear_bit(0, &tctx->task_state); +} + +static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req, + enum task_work_notify_mode notify) +{ + struct io_uring_task *tctx = tsk->io_uring; + struct io_wq_work_node *node, *prev; + int ret; + + WARN_ON_ONCE(!tctx); + + spin_lock(&tctx->task_lock); + wq_list_add_tail(&req->io_task_work.node, &tctx->task_list); + spin_unlock(&tctx->task_lock); + + /* task_work already pending, we're done */ + if (test_bit(0, &tctx->task_state) || + test_and_set_bit(0, &tctx->task_state)) + return 0; + + if (!task_work_add(tsk, &tctx->task_work, notify)) + return 0; + + /* + * Slow path - we failed, find and delete work. if the work is not + * in the list, it got run and we're fine. + */ + ret = 0; + spin_lock(&tctx->task_lock); + wq_list_for_each(node, prev, &tctx->task_list) { + if (&req->io_task_work.node == node) { + wq_list_del(&tctx->task_list, node, prev); + ret = 1; + break; + } + } + spin_unlock(&tctx->task_lock); + clear_bit(0, &tctx->task_state); + return ret; +} + static int io_req_task_work_add(struct io_kiocb *req) { struct task_struct *tsk = req->task; @@ -2149,7 +2232,7 @@ static int io_req_task_work_add(struct io_kiocb *req) if (!(ctx->flags & IORING_SETUP_SQPOLL)) notify = TWA_SIGNAL; - ret = task_work_add(tsk, &req->task_work, notify); + ret = io_task_work_add(tsk, req, notify); if (!ret) wake_up_process(tsk); @@ -2157,7 +2240,7 @@ static int io_req_task_work_add(struct io_kiocb *req) } static void io_req_task_work_add_fallback(struct io_kiocb *req, - void (*cb)(struct callback_head *)) + task_work_func_t cb) { struct task_struct *tsk = io_wq_get_task(req->ctx->io_wq); @@ -2216,7 +2299,7 @@ static void io_req_task_queue(struct io_kiocb *req) { int ret; - init_task_work(&req->task_work, io_req_task_submit); + req->task_work.func = io_req_task_submit; percpu_ref_get(&req->ctx->refs); ret = io_req_task_work_add(req); @@ -2347,7 +2430,7 @@ static void io_free_req_deferred(struct io_kiocb *req) { int ret; - init_task_work(&req->task_work, io_put_req_deferred_cb); + req->task_work.func = io_put_req_deferred_cb; ret = io_req_task_work_add(req); if (unlikely(ret)) io_req_task_work_add_fallback(req, io_put_req_deferred_cb); @@ -3392,7 +3475,7 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, req->rw.kiocb.ki_flags &= ~IOCB_WAITQ; list_del_init(&wait->entry); - init_task_work(&req->task_work, io_req_task_submit); + req->task_work.func = io_req_task_submit; percpu_ref_get(&req->ctx->refs); /* submit ref gets dropped, acquire a new one */ @@ -5083,7 +5166,7 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, list_del_init(&poll->wait.entry); req->result = mask; - init_task_work(&req->task_work, func); + req->task_work.func = func; percpu_ref_get(&req->ctx->refs); /* @@ -8086,6 +8169,10 @@ static int io_uring_alloc_task_context(struct task_struct *task) io_init_identity(&tctx->__identity); tctx->identity = &tctx->__identity; task->io_uring = tctx; + spin_lock_init(&tctx->task_lock); + INIT_WQ_LIST(&tctx->task_list); + tctx->task_state = 0; + init_task_work(&tctx->task_work, tctx_task_work); return 0; } diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 35b2d845704d..2eb6d19de336 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -22,6 +22,15 @@ struct io_identity { refcount_t count; }; +struct io_wq_work_node { + struct io_wq_work_node *next; +}; + +struct io_wq_work_list { + struct io_wq_work_node *first; + struct io_wq_work_node *last; +}; + struct io_uring_task { /* submission side */ struct xarray xa; @@ -32,6 +41,11 @@ struct io_uring_task { struct io_identity *identity; atomic_t in_idle; bool sqpoll; + + spinlock_t task_lock; + struct io_wq_work_list task_list; + unsigned long task_state; + struct callback_head task_work; }; #if defined(CONFIG_IO_URING) -- cgit v1.2.3 From 65453d1efbd20f3825beba2a9c93ffb2ec729ece Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 10 Feb 2021 00:03:21 +0000 Subject: io_uring: enable req cache for task_work items task_work is run without utilizing the req alloc cache, so any deferred items don't get to take advantage of either the alloc or free side of it. With task_work now being wrapped by io_uring, we can use the ctx completion state to both use the req cache and the completion flush batching. With this, the only request type that cannot take advantage of the req cache is IRQ driven IO for regular files / block devices. Anything else, including IOPOLL polled IO to those same tyes, will take advantage of it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index bfc8fcd93504..fe8921a728b0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1051,6 +1051,8 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, const struct iovec *fast_iov, struct iov_iter *iter, bool force); static void io_req_task_queue(struct io_kiocb *req); +static void io_submit_flush_completions(struct io_comp_state *cs, + struct io_ring_ctx *ctx); static struct kmem_cache *req_cachep; @@ -2139,6 +2141,7 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) static bool __tctx_task_work(struct io_uring_task *tctx) { + struct io_ring_ctx *ctx = NULL; struct io_wq_work_list list; struct io_wq_work_node *node; @@ -2153,11 +2156,28 @@ static bool __tctx_task_work(struct io_uring_task *tctx) node = list.first; while (node) { struct io_wq_work_node *next = node->next; + struct io_ring_ctx *this_ctx; struct io_kiocb *req; req = container_of(node, struct io_kiocb, io_task_work.node); + this_ctx = req->ctx; req->task_work.func(&req->task_work); node = next; + + if (!ctx) { + ctx = this_ctx; + } else if (ctx != this_ctx) { + mutex_lock(&ctx->uring_lock); + io_submit_flush_completions(&ctx->submit_state.comp, ctx); + mutex_unlock(&ctx->uring_lock); + ctx = this_ctx; + } + } + + if (ctx && ctx->submit_state.comp.nr) { + mutex_lock(&ctx->uring_lock); + io_submit_flush_completions(&ctx->submit_state.comp, ctx); + mutex_unlock(&ctx->uring_lock); } return list.first != NULL; @@ -2280,7 +2300,7 @@ static void __io_req_task_submit(struct io_kiocb *req) if (!ctx->sqo_dead && !__io_sq_thread_acquire_mm(ctx) && !__io_sq_thread_acquire_files(ctx)) - __io_queue_sqe(req, NULL); + __io_queue_sqe(req, &ctx->submit_state.comp); else __io_req_task_cancel(req, -EFAULT); mutex_unlock(&ctx->uring_lock); -- cgit v1.2.3 From c5eef2b9449ba267f53bfa7cf63d2bc93acbee32 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:22 +0000 Subject: io_uring: take comp_state from ctx __io_queue_sqe() is always called with a non-NULL comp_state, which is taken directly from context. Don't pass it around but infer from ctx. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index fe8921a728b0..bff5bc4a2b6e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1042,7 +1042,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, static void __io_clean_op(struct io_kiocb *req); static struct file *io_file_get(struct io_submit_state *state, struct io_kiocb *req, int fd, bool fixed); -static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs); +static void __io_queue_sqe(struct io_kiocb *req); static void io_rsrc_put_work(struct work_struct *work); static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, @@ -2300,7 +2300,7 @@ static void __io_req_task_submit(struct io_kiocb *req) if (!ctx->sqo_dead && !__io_sq_thread_acquire_mm(ctx) && !__io_sq_thread_acquire_files(ctx)) - __io_queue_sqe(req, &ctx->submit_state.comp); + __io_queue_sqe(req); else __io_req_task_cancel(req, -EFAULT); mutex_unlock(&ctx->uring_lock); @@ -6551,14 +6551,12 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) return nxt; } -static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs) +static void __io_queue_sqe(struct io_kiocb *req) { struct io_kiocb *linked_timeout; const struct cred *old_creds = NULL; - int ret, issue_flags = IO_URING_F_NONBLOCK; + int ret; - if (cs) - issue_flags |= IO_URING_F_COMPLETE_DEFER; again: linked_timeout = io_prep_linked_timeout(req); @@ -6573,7 +6571,7 @@ again: old_creds = override_creds(req->work.identity->creds); } - ret = io_issue_sqe(req, issue_flags); + ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); /* * We async punt it if the file wasn't marked NOWAIT, or if the file @@ -6593,9 +6591,12 @@ again: } else if (likely(!ret)) { /* drop submission reference */ if (req->flags & REQ_F_COMPLETE_INLINE) { + struct io_ring_ctx *ctx = req->ctx; + struct io_comp_state *cs = &ctx->submit_state.comp; + cs->reqs[cs->nr++] = req; if (cs->nr == IO_COMPL_BATCH) - io_submit_flush_completions(cs, req->ctx); + io_submit_flush_completions(cs, ctx); req = NULL; } else { req = io_put_req_find_next(req); @@ -6621,8 +6622,7 @@ again: revert_creds(old_creds); } -static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, - struct io_comp_state *cs) +static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe) { int ret; @@ -6647,18 +6647,17 @@ fail_req: if (unlikely(ret)) goto fail_req; } - __io_queue_sqe(req, cs); + __io_queue_sqe(req); } } -static inline void io_queue_link_head(struct io_kiocb *req, - struct io_comp_state *cs) +static inline void io_queue_link_head(struct io_kiocb *req) { if (unlikely(req->flags & REQ_F_FAIL_LINK)) { io_put_req(req); io_req_complete(req, -ECANCELED); } else - io_queue_sqe(req, NULL, cs); + io_queue_sqe(req, NULL); } struct io_submit_link { @@ -6667,7 +6666,7 @@ struct io_submit_link { }; static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, - struct io_submit_link *link, struct io_comp_state *cs) + struct io_submit_link *link) { struct io_ring_ctx *ctx = req->ctx; int ret; @@ -6705,7 +6704,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, /* last request of a link, enqueue the link */ if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { - io_queue_link_head(head, cs); + io_queue_link_head(head); link->head = NULL; } } else { @@ -6720,7 +6719,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, link->head = req; link->last = req; } else { - io_queue_sqe(req, sqe, cs); + io_queue_sqe(req, sqe); } } @@ -6961,7 +6960,7 @@ fail_req: trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, true, ctx->flags & IORING_SETUP_SQPOLL); - err = io_submit_sqe(req, sqe, &link, &ctx->submit_state.comp); + err = io_submit_sqe(req, sqe, &link); if (err) goto fail_req; } @@ -6976,7 +6975,7 @@ fail_req: put_task_struct_many(current, unused); } if (link.head) - io_queue_link_head(link.head, &ctx->submit_state.comp); + io_queue_link_head(link.head); io_submit_state_end(&ctx->submit_state, ctx); /* Commit SQ ring head once we've consumed and submitted all SQEs */ -- cgit v1.2.3 From e5d1bc0a91f16959aa279aa3ee9fdc246d4bb382 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 00:03:23 +0000 Subject: io_uring: defer flushing cached reqs Awhile there are requests in the allocation cache -- use them, only if those ended go for the stashed memory in comp.free_list. As list manipulation are generally heavy and are not good for caches, flush them all or as much as can in one go. Signed-off-by: Pavel Begunkov [axboe: return success/failure from io_flush_cached_reqs()] Signed-off-by: Jens Axboe --- fs/io_uring.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index bff5bc4a2b6e..4a28032ba35b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1952,25 +1952,35 @@ static inline void io_req_complete(struct io_kiocb *req, long res) __io_req_complete(req, 0, res, 0); } -static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) +static bool io_flush_cached_reqs(struct io_submit_state *state) { - struct io_submit_state *state = &ctx->submit_state; - - BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs)); - - if (!list_empty(&state->comp.free_list)) { - struct io_kiocb *req; + struct io_kiocb *req = NULL; + while (!list_empty(&state->comp.free_list)) { req = list_first_entry(&state->comp.free_list, struct io_kiocb, compl.list); list_del(&req->compl.list); - return req; + state->reqs[state->free_reqs++] = req; + if (state->free_reqs == ARRAY_SIZE(state->reqs)) + break; } + return req != NULL; +} + +static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) +{ + struct io_submit_state *state = &ctx->submit_state; + + BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs)); + if (!state->free_reqs) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; int ret; + if (io_flush_cached_reqs(state)) + goto got_req; + ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH, state->reqs); @@ -1986,7 +1996,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) } state->free_reqs = ret; } - +got_req: state->free_reqs--; return state->reqs[state->free_reqs]; } -- cgit v1.2.3 From ed670c3f90a67d9e16ab6d8893be6f072d79cd4c Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Fri, 5 Feb 2021 16:34:21 +0800 Subject: io_uring: fix possible deadlock in io_uring_poll Abaci reported follow issue: [ 30.615891] ====================================================== [ 30.616648] WARNING: possible circular locking dependency detected [ 30.617423] 5.11.0-rc3-next-20210115 #1 Not tainted [ 30.618035] ------------------------------------------------------ [ 30.618914] a.out/1128 is trying to acquire lock: [ 30.619520] ffff88810b063868 (&ep->mtx){+.+.}-{3:3}, at: __ep_eventpoll_poll+0x9f/0x220 [ 30.620505] [ 30.620505] but task is already holding lock: [ 30.621218] ffff88810e952be8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_enter+0x3f0/0x5b0 [ 30.622349] [ 30.622349] which lock already depends on the new lock. [ 30.622349] [ 30.623289] [ 30.623289] the existing dependency chain (in reverse order) is: [ 30.624243] [ 30.624243] -> #1 (&ctx->uring_lock){+.+.}-{3:3}: [ 30.625263] lock_acquire+0x2c7/0x390 [ 30.625868] __mutex_lock+0xae/0x9f0 [ 30.626451] io_cqring_overflow_flush.part.95+0x6d/0x70 [ 30.627278] io_uring_poll+0xcb/0xd0 [ 30.627890] ep_item_poll.isra.14+0x4e/0x90 [ 30.628531] do_epoll_ctl+0xb7e/0x1120 [ 30.629122] __x64_sys_epoll_ctl+0x70/0xb0 [ 30.629770] do_syscall_64+0x2d/0x40 [ 30.630332] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 30.631187] [ 30.631187] -> #0 (&ep->mtx){+.+.}-{3:3}: [ 30.631985] check_prevs_add+0x226/0xb00 [ 30.632584] __lock_acquire+0x1237/0x13a0 [ 30.633207] lock_acquire+0x2c7/0x390 [ 30.633740] __mutex_lock+0xae/0x9f0 [ 30.634258] __ep_eventpoll_poll+0x9f/0x220 [ 30.634879] __io_arm_poll_handler+0xbf/0x220 [ 30.635462] io_issue_sqe+0xa6b/0x13e0 [ 30.635982] __io_queue_sqe+0x10b/0x550 [ 30.636648] io_queue_sqe+0x235/0x470 [ 30.637281] io_submit_sqes+0xcce/0xf10 [ 30.637839] __x64_sys_io_uring_enter+0x3fb/0x5b0 [ 30.638465] do_syscall_64+0x2d/0x40 [ 30.638999] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 30.639643] [ 30.639643] other info that might help us debug this: [ 30.639643] [ 30.640618] Possible unsafe locking scenario: [ 30.640618] [ 30.641402] CPU0 CPU1 [ 30.641938] ---- ---- [ 30.642664] lock(&ctx->uring_lock); [ 30.643425] lock(&ep->mtx); [ 30.644498] lock(&ctx->uring_lock); [ 30.645668] lock(&ep->mtx); [ 30.646321] [ 30.646321] *** DEADLOCK *** [ 30.646321] [ 30.647642] 1 lock held by a.out/1128: [ 30.648424] #0: ffff88810e952be8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_enter+0x3f0/0x5b0 [ 30.649954] [ 30.649954] stack backtrace: [ 30.650592] CPU: 1 PID: 1128 Comm: a.out Not tainted 5.11.0-rc3-next-20210115 #1 [ 30.651554] Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011 [ 30.652290] Call Trace: [ 30.652688] dump_stack+0xac/0xe3 [ 30.653164] check_noncircular+0x11e/0x130 [ 30.653747] ? check_prevs_add+0x226/0xb00 [ 30.654303] check_prevs_add+0x226/0xb00 [ 30.654845] ? add_lock_to_list.constprop.49+0xac/0x1d0 [ 30.655564] __lock_acquire+0x1237/0x13a0 [ 30.656262] lock_acquire+0x2c7/0x390 [ 30.656788] ? __ep_eventpoll_poll+0x9f/0x220 [ 30.657379] ? __io_queue_proc.isra.88+0x180/0x180 [ 30.658014] __mutex_lock+0xae/0x9f0 [ 30.658524] ? __ep_eventpoll_poll+0x9f/0x220 [ 30.659112] ? mark_held_locks+0x5a/0x80 [ 30.659648] ? __ep_eventpoll_poll+0x9f/0x220 [ 30.660229] ? _raw_spin_unlock_irqrestore+0x2d/0x40 [ 30.660885] ? trace_hardirqs_on+0x46/0x110 [ 30.661471] ? __io_queue_proc.isra.88+0x180/0x180 [ 30.662102] ? __ep_eventpoll_poll+0x9f/0x220 [ 30.662696] __ep_eventpoll_poll+0x9f/0x220 [ 30.663273] ? __ep_eventpoll_poll+0x220/0x220 [ 30.663875] __io_arm_poll_handler+0xbf/0x220 [ 30.664463] io_issue_sqe+0xa6b/0x13e0 [ 30.664984] ? __lock_acquire+0x782/0x13a0 [ 30.665544] ? __io_queue_proc.isra.88+0x180/0x180 [ 30.666170] ? __io_queue_sqe+0x10b/0x550 [ 30.666725] __io_queue_sqe+0x10b/0x550 [ 30.667252] ? __fget_files+0x131/0x260 [ 30.667791] ? io_req_prep+0xd8/0x1090 [ 30.668316] ? io_queue_sqe+0x235/0x470 [ 30.668868] io_queue_sqe+0x235/0x470 [ 30.669398] io_submit_sqes+0xcce/0xf10 [ 30.669931] ? xa_load+0xe4/0x1c0 [ 30.670425] __x64_sys_io_uring_enter+0x3fb/0x5b0 [ 30.671051] ? lockdep_hardirqs_on_prepare+0xde/0x180 [ 30.671719] ? syscall_enter_from_user_mode+0x2b/0x80 [ 30.672380] do_syscall_64+0x2d/0x40 [ 30.672901] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 30.673503] RIP: 0033:0x7fd89c813239 [ 30.673962] Code: 01 00 48 81 c4 80 00 00 00 e9 f1 fe ff ff 0f 1f 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 3d 01 f0 ff ff 73 01 c3 48 8b 0d 27 ec 2c 00 f7 d8 64 89 01 48 [ 30.675920] RSP: 002b:00007ffc65a7c628 EFLAGS: 00000217 ORIG_RAX: 00000000000001aa [ 30.676791] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007fd89c813239 [ 30.677594] RDX: 0000000000000000 RSI: 0000000000000014 RDI: 0000000000000003 [ 30.678678] RBP: 00007ffc65a7c720 R08: 0000000000000000 R09: 0000000003000000 [ 30.679492] R10: 0000000000000000 R11: 0000000000000217 R12: 0000000000400ff0 [ 30.680282] R13: 00007ffc65a7c840 R14: 0000000000000000 R15: 0000000000000000 This might happen if we do epoll_wait on a uring fd while reading/writing the former epoll fd in a sqe in the former uring instance. So let's don't flush cqring overflow list, just do a simple check. Reported-by: Abaci Fixes: 6c503150ae33 ("io_uring: patch up IOPOLL overflow_flush sync") Signed-off-by: Hao Xu Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 4a28032ba35b..e73ca37c6a3b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8820,8 +8820,21 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) smp_rmb(); if (!io_sqring_full(ctx)) mask |= EPOLLOUT | EPOLLWRNORM; - io_cqring_overflow_flush(ctx, false, NULL, NULL); - if (io_cqring_events(ctx)) + + /* + * Don't flush cqring overflow list here, just do a simple check. + * Otherwise there could possible be ABBA deadlock: + * CPU0 CPU1 + * ---- ---- + * lock(&ctx->uring_lock); + * lock(&ep->mtx); + * lock(&ctx->uring_lock); + * lock(&ep->mtx); + * + * Users may get EPOLLIN meanwhile seeing nothing in cqring, this + * pushs them to do the flush. + */ + if (io_cqring_events(ctx) || test_bit(0, &ctx->cq_check_overflow)) mask |= EPOLLIN | EPOLLRDNORM; return mask; -- cgit v1.2.3 From c7dae4ba46c9d7d56430b800907b708711995414 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 9 Feb 2021 19:53:37 -0700 Subject: io_uring: enable req cache for IRQ driven IO This is the last class of requests that cannot utilize the req alloc cache. Add a per-ctx req cache that is protected by the completion_lock, and refill our submit side cache when it gets over our batch count. Signed-off-by: Jens Axboe --- fs/io_uring.c | 71 ++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 20 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index e73ca37c6a3b..2c7ff0b1b086 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -272,7 +272,11 @@ struct io_sq_data { struct io_comp_state { struct io_kiocb *reqs[IO_COMPL_BATCH]; unsigned int nr; + unsigned int locked_free_nr; + /* inline/task_work completion list, under ->uring_lock */ struct list_head free_list; + /* IRQ completion list, under ->completion_lock */ + struct list_head locked_free_list; }; struct io_submit_state { @@ -1033,6 +1037,9 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res); static void io_put_req(struct io_kiocb *req); static void io_put_req_deferred(struct io_kiocb *req, int nr); static void io_double_put_req(struct io_kiocb *req); +static void io_dismantle_req(struct io_kiocb *req); +static void io_put_task(struct task_struct *task, int nr); +static void io_queue_next(struct io_kiocb *req); static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req); static void __io_queue_linked_timeout(struct io_kiocb *req); static void io_queue_linked_timeout(struct io_kiocb *req); @@ -1353,6 +1360,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); INIT_LIST_HEAD(&ctx->submit_state.comp.free_list); + INIT_LIST_HEAD(&ctx->submit_state.comp.locked_free_list); return ctx; err: kfree(ctx->cancel_hash); @@ -1908,8 +1916,8 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) __io_cqring_fill_event(req, res, 0); } -static void io_req_complete_post(struct io_kiocb *req, long res, - unsigned int cflags) +static inline void io_req_complete_post(struct io_kiocb *req, long res, + unsigned int cflags) { struct io_ring_ctx *ctx = req->ctx; unsigned long flags; @@ -1917,16 +1925,26 @@ static void io_req_complete_post(struct io_kiocb *req, long res, spin_lock_irqsave(&ctx->completion_lock, flags); __io_cqring_fill_event(req, res, cflags); io_commit_cqring(ctx); + /* + * If we're the last reference to this request, add to our locked + * free_list cache. + */ + if (refcount_dec_and_test(&req->refs)) { + struct io_comp_state *cs = &ctx->submit_state.comp; + + io_dismantle_req(req); + io_put_task(req->task, 1); + list_add(&req->compl.list, &cs->locked_free_list); + cs->locked_free_nr++; + } else + req = NULL; spin_unlock_irqrestore(&ctx->completion_lock, flags); io_cqring_ev_posted(ctx); -} - -static inline void io_req_complete_nostate(struct io_kiocb *req, long res, - unsigned int cflags) -{ - io_req_complete_post(req, res, cflags); - io_put_req(req); + if (req) { + io_queue_next(req); + percpu_ref_put(&ctx->refs); + } } static void io_req_complete_state(struct io_kiocb *req, long res, @@ -1944,7 +1962,7 @@ static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, if (issue_flags & IO_URING_F_COMPLETE_DEFER) io_req_complete_state(req, res, cflags); else - io_req_complete_nostate(req, res, cflags); + io_req_complete_post(req, res, cflags); } static inline void io_req_complete(struct io_kiocb *req, long res) @@ -1952,12 +1970,26 @@ static inline void io_req_complete(struct io_kiocb *req, long res) __io_req_complete(req, 0, res, 0); } -static bool io_flush_cached_reqs(struct io_submit_state *state) +static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) { + struct io_submit_state *state = &ctx->submit_state; + struct io_comp_state *cs = &state->comp; struct io_kiocb *req = NULL; - while (!list_empty(&state->comp.free_list)) { - req = list_first_entry(&state->comp.free_list, struct io_kiocb, + /* + * If we have more than a batch's worth of requests in our IRQ side + * locked cache, grab the lock and move them over to our submission + * side cache. + */ + if (READ_ONCE(cs->locked_free_nr) > IO_COMPL_BATCH) { + spin_lock_irq(&ctx->completion_lock); + list_splice_init(&cs->locked_free_list, &cs->free_list); + cs->locked_free_nr = 0; + spin_unlock_irq(&ctx->completion_lock); + } + + while (!list_empty(&cs->free_list)) { + req = list_first_entry(&cs->free_list, struct io_kiocb, compl.list); list_del(&req->compl.list); state->reqs[state->free_reqs++] = req; @@ -1978,7 +2010,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; int ret; - if (io_flush_cached_reqs(state)) + if (io_flush_cached_reqs(ctx)) goto got_req; ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH, @@ -8748,14 +8780,12 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) idr_destroy(&ctx->io_buffer_idr); } -static void io_req_cache_free(struct io_ring_ctx *ctx) +static void io_req_cache_free(struct list_head *list) { - struct io_comp_state *cs = &ctx->submit_state.comp; - - while (!list_empty(&cs->free_list)) { + while (!list_empty(list)) { struct io_kiocb *req; - req = list_first_entry(&cs->free_list, struct io_kiocb, compl.list); + req = list_first_entry(list, struct io_kiocb, compl.list); list_del(&req->compl.list); kmem_cache_free(req_cachep, req); } @@ -8803,7 +8833,8 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) free_uid(ctx->user); put_cred(ctx->creds); kfree(ctx->cancel_hash); - io_req_cache_free(ctx); + io_req_cache_free(&ctx->submit_state.comp.free_list); + io_req_cache_free(&ctx->submit_state.comp.locked_free_list); kfree(ctx); } -- cgit v1.2.3 From 91f245d5d5de0802428a478802ec051f7de2f5d6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 9 Feb 2021 13:48:50 -0700 Subject: io_uring: enable kmemcg account for io_uring requests This puts io_uring under the memory cgroups accounting and limits for requests. Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 2c7ff0b1b086..bffed6aa5722 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -10350,7 +10350,8 @@ static int __init io_uring_init(void) BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int)); - req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC); + req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC | + SLAB_ACCOUNT); return 0; }; __initcall(io_uring_init); -- cgit v1.2.3 From 26bfa89e25f42d2b26fe951bbcf04bb13937fbba Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 9 Feb 2021 20:14:12 -0700 Subject: io_uring: place ring SQ/CQ arrays under memcg memory limits Instead of imposing rlimit memlock limits for the rings themselves, ensure that we account them properly under memcg with __GFP_ACCOUNT. We retain rlimit memlock for registered buffers, this is just for the ring arrays themselves. Signed-off-by: Jens Axboe --- fs/io_uring.c | 85 +++++++---------------------------------------------------- 1 file changed, 10 insertions(+), 75 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index bffed6aa5722..7a1e4ecf5f94 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1017,11 +1017,6 @@ static const struct io_op_def io_op_defs[] = { }, }; -enum io_mem_account { - ACCT_LOCKED, - ACCT_PINNED, -}; - static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, struct files_struct *files); @@ -8355,25 +8350,16 @@ static inline int __io_account_mem(struct user_struct *user, return 0; } -static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages, - enum io_mem_account acct) +static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) { if (ctx->limit_mem) __io_unaccount_mem(ctx->user, nr_pages); - if (ctx->mm_account) { - if (acct == ACCT_LOCKED) { - mmap_write_lock(ctx->mm_account); - ctx->mm_account->locked_vm -= nr_pages; - mmap_write_unlock(ctx->mm_account); - }else if (acct == ACCT_PINNED) { - atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm); - } - } + if (ctx->mm_account) + atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm); } -static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages, - enum io_mem_account acct) +static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) { int ret; @@ -8383,15 +8369,8 @@ static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages, return ret; } - if (ctx->mm_account) { - if (acct == ACCT_LOCKED) { - mmap_write_lock(ctx->mm_account); - ctx->mm_account->locked_vm += nr_pages; - mmap_write_unlock(ctx->mm_account); - } else if (acct == ACCT_PINNED) { - atomic64_add(nr_pages, &ctx->mm_account->pinned_vm); - } - } + if (ctx->mm_account) + atomic64_add(nr_pages, &ctx->mm_account->pinned_vm); return 0; } @@ -8411,7 +8390,7 @@ static void io_mem_free(void *ptr) static void *io_mem_alloc(size_t size) { gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP | - __GFP_NORETRY; + __GFP_NORETRY | __GFP_ACCOUNT; return (void *) __get_free_pages(gfp_flags, get_order(size)); } @@ -8445,18 +8424,6 @@ static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries, return off; } -static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries) -{ - size_t pages; - - pages = (size_t)1 << get_order( - rings_size(sq_entries, cq_entries, NULL)); - pages += (size_t)1 << get_order( - array_size(sizeof(struct io_uring_sqe), sq_entries)); - - return pages; -} - static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) { int i, j; @@ -8471,7 +8438,7 @@ static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) unpin_user_page(imu->bvec[j].bv_page); if (imu->acct_pages) - io_unaccount_mem(ctx, imu->acct_pages, ACCT_PINNED); + io_unaccount_mem(ctx, imu->acct_pages); kvfree(imu->bvec); imu->nr_bvecs = 0; } @@ -8569,7 +8536,7 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages, if (!imu->acct_pages) return 0; - ret = io_account_mem(ctx, imu->acct_pages, ACCT_PINNED); + ret = io_account_mem(ctx, imu->acct_pages); if (ret) imu->acct_pages = 0; return ret; @@ -8949,14 +8916,6 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) /* if we failed setting up the ctx, we might not have any rings */ io_iopoll_try_reap_events(ctx); - /* - * Do this upfront, so we won't have a grace period where the ring - * is closed but resources aren't reaped yet. This can cause - * spurious failure in setting up a new ring. - */ - io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries), - ACCT_LOCKED); - INIT_WORK(&ctx->exit_work, io_ring_exit_work); /* * Use system_unbound_wq to avoid spawning tons of event kworkers @@ -9780,7 +9739,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, struct user_struct *user = NULL; struct io_ring_ctx *ctx; struct file *file; - bool limit_mem; int ret; if (!entries) @@ -9821,26 +9779,14 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, } user = get_uid(current_user()); - limit_mem = !capable(CAP_IPC_LOCK); - - if (limit_mem) { - ret = __io_account_mem(user, - ring_pages(p->sq_entries, p->cq_entries)); - if (ret) { - free_uid(user); - return ret; - } - } ctx = io_ring_ctx_alloc(p); if (!ctx) { - if (limit_mem) - __io_unaccount_mem(user, ring_pages(p->sq_entries, - p->cq_entries)); free_uid(user); return -ENOMEM; } ctx->compat = in_compat_syscall(); + ctx->limit_mem = !capable(CAP_IPC_LOCK); ctx->user = user; ctx->creds = get_current_cred(); #ifdef CONFIG_AUDIT @@ -9876,17 +9822,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, goto err; } #endif - - /* - * Account memory _before_ installing the file descriptor. Once - * the descriptor is installed, it can get closed at any time. Also - * do this before hitting the general error path, as ring freeing - * will un-account as well. - */ - io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries), - ACCT_LOCKED); - ctx->limit_mem = limit_mem; - ret = io_allocate_scq_urings(ctx, p); if (ret) goto err; -- cgit v1.2.3 From 0f1ba5f5d80f84b605e70cf4661cb1a44a3c02a6 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Thu, 28 Jan 2021 13:47:31 +0900 Subject: zonefs: use zone write granularity as block size Zoned block devices have different granularity constraints for write operations into sequential zones. E.g. ZBC and ZAC devices require that writes be aligned to the device physical block size while NVMe ZNS devices allow logical block size aligned write operations. To correctly handle such difference, use the device zone write granularity limit to set the block size of a zonefs volume, thus allowing the smallest possible write unit for all zoned device types. Signed-off-by: Damien Le Moal Reviewed-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Reviewed-by: Chaitanya Kulkarni Reviewed-by: Johannes Thumshirn Signed-off-by: Jens Axboe --- fs/zonefs/super.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index ab68e27bb322..b9fb55b250ae 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -1581,12 +1581,11 @@ static int zonefs_fill_super(struct super_block *sb, void *data, int silent) sb->s_time_gran = 1; /* - * The block size is set to the device physical sector size to ensure - * that write operations on 512e devices (512B logical block and 4KB - * physical block) are always aligned to the device physical blocks, - * as mandated by the ZBC/ZAC specifications. + * The block size is set to the device zone write granularity to ensure + * that write operations are always aligned according to the device + * interface constraints. */ - sb_set_blocksize(sb, bdev_physical_block_size(sb->s_bdev)); + sb_set_blocksize(sb, bdev_zone_write_granularity(sb->s_bdev)); sbi->s_zone_sectors_shift = ilog2(bdev_zone_sectors(sb->s_bdev)); sbi->s_uid = GLOBAL_ROOT_UID; sbi->s_gid = GLOBAL_ROOT_GID; -- cgit v1.2.3 From a35d8f016e0b68634035217d06d1c53863456b50 Mon Sep 17 00:00:00 2001 From: Joachim Henke Date: Tue, 9 Feb 2021 13:42:36 -0800 Subject: nilfs2: make splice write available again Since 5.10, splice() or sendfile() to NILFS2 return EINVAL. This was caused by commit 36e2c7421f02 ("fs: don't allow splice read/write without explicit ops"). This patch initializes the splice_write field in file_operations, like most file systems do, to restore the functionality. Link: https://lkml.kernel.org/r/1612784101-14353-1-git-send-email-konishi.ryusuke@gmail.com Signed-off-by: Joachim Henke Signed-off-by: Ryusuke Konishi Tested-by: Ryusuke Konishi Cc: [5.10+] Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/nilfs2/file.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/nilfs2/file.c b/fs/nilfs2/file.c index 64bc81363c6c..e1bd592ce700 100644 --- a/fs/nilfs2/file.c +++ b/fs/nilfs2/file.c @@ -141,6 +141,7 @@ const struct file_operations nilfs_file_operations = { /* .release = nilfs_release_file, */ .fsync = nilfs_sync_file, .splice_read = generic_file_splice_read, + .splice_write = iter_file_splice_write, }; const struct inode_operations nilfs_file_inode_operations = { -- cgit v1.2.3 From 92c75f7594d5060a4cb240f0e987a802f8486b11 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 10 Feb 2021 12:37:58 -0700 Subject: Revert "io_uring: don't take fs for recvmsg/sendmsg" This reverts commit 10cad2c40dcb04bb46b2bf399e00ca5ea93d36b0. Petr reports that with this commit in place, io_uring fails the chroot test (CVE-202-29373). We do need to retain ->fs for send/recvmsg, so revert this commit. Reported-by: Petr Vorel Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 1f68105a41ed..931671082e61 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -857,7 +857,8 @@ static const struct io_op_def io_op_defs[] = { .pollout = 1, .needs_async_data = 1, .async_size = sizeof(struct io_async_msghdr), - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, + .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | + IO_WQ_WORK_FS, }, [IORING_OP_RECVMSG] = { .needs_file = 1, @@ -866,7 +867,8 @@ static const struct io_op_def io_op_defs[] = { .buffer_select = 1, .needs_async_data = 1, .async_size = sizeof(struct io_async_msghdr), - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, + .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | + IO_WQ_WORK_FS, }, [IORING_OP_TIMEOUT] = { .needs_async_data = 1, -- cgit v1.2.3 From 34343786ecc5ff493ca4d1f873b4386759ba52ee Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Feb 2021 11:45:42 +0000 Subject: io_uring: unpark SQPOLL thread for cancelation We park SQPOLL task before going into io_uring_cancel_files(), so the task won't run task_works including those that might be important for the cancellation passes. In this case it's io_poll_remove_one(), which frees requests via io_put_req_deferred(). Unpark it for while waiting, it's ok as we disable submissions beforehand, so no new requests will be generated. INFO: task syz-executor893:8493 blocked for more than 143 seconds. Call Trace: context_switch kernel/sched/core.c:4327 [inline] __schedule+0x90c/0x21a0 kernel/sched/core.c:5078 schedule+0xcf/0x270 kernel/sched/core.c:5157 io_uring_cancel_files fs/io_uring.c:8912 [inline] io_uring_cancel_task_requests+0xe70/0x11a0 fs/io_uring.c:8979 __io_uring_files_cancel+0x110/0x1b0 fs/io_uring.c:9067 io_uring_files_cancel include/linux/io_uring.h:51 [inline] do_exit+0x2fe/0x2ae0 kernel/exit.c:780 do_group_exit+0x125/0x310 kernel/exit.c:922 __do_sys_exit_group kernel/exit.c:933 [inline] __se_sys_exit_group kernel/exit.c:931 [inline] __x64_sys_exit_group+0x3a/0x50 kernel/exit.c:931 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9 Cc: stable@vger.kernel.org # 5.5+ Reported-by: syzbot+695b03d82fa8e4901b06@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 7a1e4ecf5f94..9ed79509f389 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9047,11 +9047,16 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, break; io_uring_try_cancel_requests(ctx, task, files); + + if (ctx->sq_data) + io_sq_thread_unpark(ctx->sq_data); prepare_to_wait(&task->io_uring->wait, &wait, TASK_UNINTERRUPTIBLE); if (inflight == io_uring_count_inflight(ctx, task, files)) schedule(); finish_wait(&task->io_uring->wait, &wait); + if (ctx->sq_data) + io_sq_thread_park(ctx->sq_data); } } -- cgit v1.2.3 From 4a245479c2312e6b51862c21af134d4191ab9cf7 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 10 Feb 2021 20:00:07 +0000 Subject: io_uring: remove redundant initialization of variable ret The variable ret is being initialized with a value that is never read and it is being updated later with a new value. The initialization is redundant and can be removed. Addresses-Coverity: ("Unused value") Fixes: b63534c41e20 ("io_uring: re-issue block requests that failed because of resources") Signed-off-by: Colin Ian King Reviewed-by: Chaitanya Kulkarni Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 9ed79509f389..f730af32c17a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2796,7 +2796,7 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res, static bool io_resubmit_prep(struct io_kiocb *req) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; - int rw, ret = -ECANCELED; + int rw, ret; struct iov_iter iter; /* already prepared */ -- cgit v1.2.3 From e4826691cc7e5458bcb659935d0092bcf3f08c20 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 10 Feb 2021 17:27:20 -0800 Subject: xfs: restore shutdown check in mapped write fault path XFS triggers an iomap warning in the write fault path due to a !PageUptodate() page if a write fault happens to occur on a page that recently failed writeback. The iomap writeback error handling code can clear the Uptodate flag if no portion of the page is submitted for I/O. This is reproduced by fstest generic/019, which combines various forms of I/O with simulated disk failures that inevitably lead to filesystem shutdown (which then unconditionally fails page writeback). This is a regression introduced by commit f150b4234397 ("xfs: split the iomap ops for buffered vs direct writes") due to the removal of a shutdown check and explicit error return in the ->iomap_begin() path used by the write fault path. The explicit error return historically translated to a SIGBUS, but now carries on with iomap processing where it complains about the unexpected state. Restore the shutdown check to xfs_buffered_write_iomap_begin() to restore historical behavior. Fixes: f150b4234397 ("xfs: split the iomap ops for buffered vs direct writes") Signed-off-by: Brian Foster Reviewed-by: Eric Sandeen Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_iomap.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 70c341658c01..6594f572096e 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -860,6 +860,9 @@ xfs_buffered_write_iomap_begin( int allocfork = XFS_DATA_FORK; int error = 0; + if (XFS_FORCED_SHUTDOWN(mp)) + return -EIO; + /* we can't use delayed allocations when using extent size hints */ if (xfs_get_extsz_hint(ip)) return xfs_direct_write_iomap_begin(inode, offset, count, -- cgit v1.2.3 From 8646b982baf7d389a140ca3974974a4cbbc3f171 Mon Sep 17 00:00:00 2001 From: kernel test robot Date: Wed, 10 Feb 2021 17:27:31 -0800 Subject: xfs: fix boolreturn.cocci warnings fs/xfs/xfs_log.c:1062:9-10: WARNING: return of 0/1 in function 'xfs_log_need_covered' with return type bool Return statements in functions returning bool should use true/false instead of 1/0. Generated by: scripts/coccinelle/misc/boolreturn.cocci Fixes: 37444fc4cc39 ("xfs: lift writable fs check up into log worker task") CC: Brian Foster Reported-by: kernel test robot Signed-off-by: kernel test robot Reviewed-by: Brian Foster Reviewed-by: Chaitanya Kulkarni Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 9aa30e7cd314..06041834daa3 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -1069,7 +1069,7 @@ xfs_log_need_covered( bool needed = false; if (!xlog_cil_empty(log)) - return 0; + return false; spin_lock(&log->l_icloglock); switch (log->l_covered_state) { -- cgit v1.2.3 From bde545295b710bdd13a0fcd4b9fddd2383eeeb3a Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Wed, 20 Jan 2021 09:30:16 +0800 Subject: erofs: fix shift-out-of-bounds of blkszbits syzbot generated a crafted bitszbits which can be shifted out-of-bounds[1]. So directly print unsupported blkszbits instead of blksize. [1] https://lore.kernel.org/r/000000000000c72ddd05b9444d2f@google.com Link: https://lore.kernel.org/r/20210120013016.14071-1-hsiangkao@aol.com Reported-by: syzbot+c68f467cd7c45860e8d4@syzkaller.appspotmail.com Reviewed-by: Chao Yu Signed-off-by: Gao Xiang --- fs/erofs/super.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/erofs/super.c b/fs/erofs/super.c index be10b16ea66e..d5a6b9b888a5 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -158,8 +158,8 @@ static int erofs_read_superblock(struct super_block *sb) blkszbits = dsb->blkszbits; /* 9(512 bytes) + LOG_SECTORS_PER_BLOCK == LOG_BLOCK_SIZE */ if (blkszbits != LOG_BLOCK_SIZE) { - erofs_err(sb, "blksize %u isn't supported on this platform", - 1 << blkszbits); + erofs_err(sb, "blkszbits %u isn't supported on this platform", + blkszbits); goto out; } -- cgit v1.2.3 From ce063129181312f8781a047a50be439c5859747b Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Tue, 9 Feb 2021 21:06:18 +0800 Subject: erofs: initialized fields can only be observed after bit is set Currently, although set_bit() & test_bit() pairs are used as a fast- path for initialized configurations. However, these atomic ops are actually relaxed forms. Instead, load-acquire & store-release form is needed to make sure uninitialized fields won't be observed in advance here (yet no such corresponding bitops so use full barriers instead.) Link: https://lore.kernel.org/r/20210209130618.15838-1-hsiangkao@aol.com Fixes: 62dc45979f3f ("staging: erofs: fix race of initializing xattrs of a inode at the same time") Fixes: 152a333a5895 ("staging: erofs: add compacted compression indexes support") Cc: # 5.3+ Reported-by: Huang Jianan Reviewed-by: Chao Yu Signed-off-by: Gao Xiang --- fs/erofs/xattr.c | 10 +++++++++- fs/erofs/zmap.c | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c index 5bde77d70852..47314a26767a 100644 --- a/fs/erofs/xattr.c +++ b/fs/erofs/xattr.c @@ -48,8 +48,14 @@ static int init_inode_xattrs(struct inode *inode) int ret = 0; /* the most case is that xattrs of this inode are initialized. */ - if (test_bit(EROFS_I_EA_INITED_BIT, &vi->flags)) + if (test_bit(EROFS_I_EA_INITED_BIT, &vi->flags)) { + /* + * paired with smp_mb() at the end of the function to ensure + * fields will only be observed after the bit is set. + */ + smp_mb(); return 0; + } if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_XATTR_BIT, TASK_KILLABLE)) return -ERESTARTSYS; @@ -137,6 +143,8 @@ static int init_inode_xattrs(struct inode *inode) } xattr_iter_end(&it, atomic_map); + /* paired with smp_mb() at the beginning of the function. */ + smp_mb(); set_bit(EROFS_I_EA_INITED_BIT, &vi->flags); out_unlock: diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index ae325541884e..14d2de35110c 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -36,8 +36,14 @@ static int z_erofs_fill_inode_lazy(struct inode *inode) void *kaddr; struct z_erofs_map_header *h; - if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) + if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { + /* + * paired with smp_mb() at the end of the function to ensure + * fields will only be observed after the bit is set. + */ + smp_mb(); return 0; + } if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) return -ERESTARTSYS; @@ -83,6 +89,8 @@ static int z_erofs_fill_inode_lazy(struct inode *inode) vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits + ((h->h_clusterbits >> 5) & 7); + /* paired with smp_mb() at the beginning of the function */ + smp_mb(); set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); unmap_done: kunmap_atomic(kaddr); -- cgit v1.2.3 From e68a3ff8c342b655f01f74a577c15605eec9aa12 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 11 Feb 2021 07:45:08 -0700 Subject: io_uring: assign file_slot prior to calling io_sqe_file_register() We use the assigned slot in io_sqe_file_register(), and a previous patch moved the assignment to after we have called it. This isn't super pretty, and will get cleaned up in the future. For now, fix the regression by restoring the previous assignment/clear of the file_slot. Fixes: ea64ec02b31d ("io_uring: deduplicate file table slot calculation") Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index f730af32c17a..cd9c4c05f6f5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8112,12 +8112,13 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EBADF; break; } + *file_slot = file; err = io_sqe_file_register(ctx, file, i); if (err) { + *file_slot = NULL; fput(file); break; } - *file_slot = file; } } -- cgit v1.2.3 From 1cd738b13ae9b29e03d6149f0246c61f76e81fcf Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Thu, 11 Feb 2021 08:46:38 -0800 Subject: xfs: consider shutdown in bmapbt cursor delete assert The assert in xfs_btree_del_cursor() checks that the bmapbt block allocation field has been handled correctly before the cursor is freed. This field is used for accurate calculation of indirect block reservation requirements (for delayed allocations), for example. generic/019 reproduces a scenario where this assert fails because the filesystem has shutdown while in the middle of a bmbt record insertion. This occurs after a bmbt block has been allocated via the cursor but before the higher level bmap function (i.e. xfs_bmap_add_extent_hole_real()) completes and resets the field. Update the assert to accommodate the transient state if the filesystem has shutdown. While here, clean up the indentation and comments in the function. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_btree.c | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c index c4d7a9241dc3..b56ff451adce 100644 --- a/fs/xfs/libxfs/xfs_btree.c +++ b/fs/xfs/libxfs/xfs_btree.c @@ -353,20 +353,17 @@ xfs_btree_free_block( */ void xfs_btree_del_cursor( - xfs_btree_cur_t *cur, /* btree cursor */ - int error) /* del because of error */ + struct xfs_btree_cur *cur, /* btree cursor */ + int error) /* del because of error */ { - int i; /* btree level */ + int i; /* btree level */ /* - * Clear the buffer pointers, and release the buffers. - * If we're doing this in the face of an error, we - * need to make sure to inspect all of the entries - * in the bc_bufs array for buffers to be unlocked. - * This is because some of the btree code works from - * level n down to 0, and if we get an error along - * the way we won't have initialized all the entries - * down to 0. + * Clear the buffer pointers and release the buffers. If we're doing + * this because of an error, inspect all of the entries in the bc_bufs + * array for buffers to be unlocked. This is because some of the btree + * code works from level n down to 0, and if we get an error along the + * way we won't have initialized all the entries down to 0. */ for (i = 0; i < cur->bc_nlevels; i++) { if (cur->bc_bufs[i]) @@ -374,17 +371,11 @@ xfs_btree_del_cursor( else if (!error) break; } - /* - * Can't free a bmap cursor without having dealt with the - * allocated indirect blocks' accounting. - */ - ASSERT(cur->bc_btnum != XFS_BTNUM_BMAP || - cur->bc_ino.allocated == 0); - /* - * Free the cursor. - */ + + ASSERT(cur->bc_btnum != XFS_BTNUM_BMAP || cur->bc_ino.allocated == 0 || + XFS_FORCED_SHUTDOWN(cur->bc_mp)); if (unlikely(cur->bc_flags & XFS_BTREE_STAGING)) - kmem_free((void *)cur->bc_ops); + kmem_free(cur->bc_ops); kmem_cache_free(xfs_btree_cur_zone, cur); } -- cgit v1.2.3 From af1a3d2ba9543e99d78914d8fb88b61d0531d9a1 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Thu, 11 Feb 2021 16:06:16 +1000 Subject: cifs: In the new mount api we get the full devname as source= so we no longer need to handle or parse the UNC= and prefixpath= options that mount.cifs are generating. This also fixes a bug in the mount command option where the devname would be truncated into just //server/share because we were looking at the truncated UNC value and not the full path. I.e. in the mount command output the devive //server/share/path would show up as just //server/share Signed-off-by: Ronnie Sahlberg Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/cifsfs.c | 2 +- fs/cifs/fs_context.c | 16 +++++++++++++++- fs/cifs/fs_context.h | 1 + 3 files changed, 17 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index e46da536ed33..ab883e84e116 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -469,7 +469,7 @@ cifs_show_cache_flavor(struct seq_file *s, struct cifs_sb_info *cifs_sb) static int cifs_show_devname(struct seq_file *m, struct dentry *root) { struct cifs_sb_info *cifs_sb = CIFS_SB(root->d_sb); - char *devname = kstrdup(cifs_sb->ctx->UNC, GFP_KERNEL); + char *devname = kstrdup(cifs_sb->ctx->source, GFP_KERNEL); if (devname == NULL) seq_puts(m, "none"); diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 1b1c56e52395..12a5da0230b5 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -148,7 +148,6 @@ const struct fs_parameter_spec smb3_fs_parameters[] = { /* Mount options which take string value */ fsparam_string("source", Opt_source), - fsparam_string("unc", Opt_source), fsparam_string("user", Opt_user), fsparam_string("username", Opt_user), fsparam_string("pass", Opt_pass), @@ -178,6 +177,11 @@ const struct fs_parameter_spec smb3_fs_parameters[] = { fsparam_flag_no("auto", Opt_ignore), fsparam_string("cred", Opt_ignore), fsparam_string("credentials", Opt_ignore), + /* + * UNC and prefixpath is now extracted from Opt_source + * in the new mount API so we can just ignore them going forward. + */ + fsparam_string("unc", Opt_ignore), fsparam_string("prefixpath", Opt_ignore), {} }; @@ -313,6 +317,7 @@ smb3_fs_context_dup(struct smb3_fs_context *new_ctx, struct smb3_fs_context *ctx new_ctx->password = NULL; new_ctx->domainname = NULL; new_ctx->UNC = NULL; + new_ctx->source = NULL; new_ctx->iocharset = NULL; /* @@ -323,6 +328,7 @@ smb3_fs_context_dup(struct smb3_fs_context *new_ctx, struct smb3_fs_context *ctx DUP_CTX_STR(username); DUP_CTX_STR(password); DUP_CTX_STR(UNC); + DUP_CTX_STR(source); DUP_CTX_STR(domainname); DUP_CTX_STR(nodename); DUP_CTX_STR(iocharset); @@ -732,6 +738,7 @@ static int smb3_reconfigure(struct fs_context *fc) * just use what we already have in cifs_sb->ctx. */ STEAL_STRING(cifs_sb, ctx, UNC); + STEAL_STRING(cifs_sb, ctx, source); STEAL_STRING(cifs_sb, ctx, username); STEAL_STRING(cifs_sb, ctx, password); STEAL_STRING(cifs_sb, ctx, domainname); @@ -974,6 +981,11 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, cifs_dbg(VFS, "Unknown error parsing devname\n"); goto cifs_parse_mount_err; } + ctx->source = kstrdup(param->string, GFP_KERNEL); + if (ctx->source == NULL) { + cifs_dbg(VFS, "OOM when copying UNC string\n"); + goto cifs_parse_mount_err; + } fc->source = kstrdup(param->string, GFP_KERNEL); if (fc->source == NULL) { cifs_dbg(VFS, "OOM when copying UNC string\n"); @@ -1396,6 +1408,8 @@ smb3_cleanup_fs_context_contents(struct smb3_fs_context *ctx) ctx->password = NULL; kfree(ctx->UNC); ctx->UNC = NULL; + kfree(ctx->source); + ctx->source = NULL; kfree(ctx->domainname); ctx->domainname = NULL; kfree(ctx->nodename); diff --git a/fs/cifs/fs_context.h b/fs/cifs/fs_context.h index 3358b33abcd0..1c44a460e2c0 100644 --- a/fs/cifs/fs_context.h +++ b/fs/cifs/fs_context.h @@ -159,6 +159,7 @@ struct smb3_fs_context { char *username; char *password; char *domainname; + char *source; char *UNC; char *nodename; char *iocharset; /* local code page for mapping to and from Unicode */ -- cgit v1.2.3 From a738c93fb1c17e386a09304b517b1c6b2a6a5a8b Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Thu, 11 Feb 2021 03:26:54 -0800 Subject: cifs: Set CIFS_MOUNT_USE_PREFIX_PATH flag on setting cifs_sb->prepath. While debugging another issue today, Steve and I noticed that if a subdir for a file share is already mounted on the client, any new mount of any other subdir (or the file share root) of the same share results in sharing the cifs superblock, which e.g. can result in incorrect device name. While setting prefix path for the root of a cifs_sb, CIFS_MOUNT_USE_PREFIX_PATH flag should also be set. Without it, prepath is not even considered in some places, and output of "mount" and various /proc/<>/*mount* related options can be missing part of the device name. Signed-off-by: Shyam Prasad N Reviewed-by: Aurelien Aptel Signed-off-by: Steve French --- fs/cifs/connect.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 76e4d8d8b3a6..4bb9decbbf27 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -2756,6 +2756,7 @@ int cifs_setup_cifs_sb(struct cifs_sb_info *cifs_sb) cifs_sb->prepath = kstrdup(ctx->prepath, GFP_KERNEL); if (cifs_sb->prepath == NULL) return -ENOMEM; + cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_USE_PREFIX_PATH; } return 0; -- cgit v1.2.3 From 4208c398aae4c2290864ba15c3dab7111f32bec1 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 11 Feb 2021 13:01:08 +0000 Subject: fs/jfs: fix potential integer overflow on shift of a int The left shift of int 32 bit integer constant 1 is evaluated using 32 bit arithmetic and then assigned to a signed 64 bit integer. In the case where l2nb is 32 or more this can lead to an overflow. Avoid this by shifting the value 1LL instead. Addresses-Coverity: ("Uninitentional integer overflow") Fixes: b40c2e665cd5 ("fs/jfs: TRIM support for JFS Filesystem") Signed-off-by: Colin Ian King Signed-off-by: Dave Kleikamp --- fs/jfs/jfs_dmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/jfs/jfs_dmap.c b/fs/jfs/jfs_dmap.c index 94b7c1cb5ceb..7aee15608619 100644 --- a/fs/jfs/jfs_dmap.c +++ b/fs/jfs/jfs_dmap.c @@ -1656,7 +1656,7 @@ s64 dbDiscardAG(struct inode *ip, int agno, s64 minlen) } else if (rc == -ENOSPC) { /* search for next smaller log2 block */ l2nb = BLKSTOL2(nblocks) - 1; - nblocks = 1 << l2nb; + nblocks = 1LL << l2nb; } else { /* Trim any already allocated blocks */ jfs_error(bmp->db_ipbmap->i_sb, "-EIO\n"); -- cgit v1.2.3 From 3c1a2ead915c1bcb7b1f9e902469ea0ee1f7857f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 11 Feb 2021 10:48:03 -0700 Subject: io_uring: move submit side state closer in the ring We recently added the submit side req cache, but it was placed at the end of the struct. Move it near the other submission state for better memory placement, and reshuffle a few other members at the same time. Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index cd9c4c05f6f5..8be7a24aa10e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -346,6 +346,13 @@ struct io_ring_ctx { struct io_uring_sqe *sq_sqes; } ____cacheline_aligned_in_smp; + struct { + struct mutex uring_lock; + wait_queue_head_t wait; + } ____cacheline_aligned_in_smp; + + struct io_submit_state submit_state; + struct io_rings *rings; /* IO offload */ @@ -413,11 +420,6 @@ struct io_ring_ctx { struct eventfd_ctx *cq_ev_fd; } ____cacheline_aligned_in_smp; - struct { - struct mutex uring_lock; - wait_queue_head_t wait; - } ____cacheline_aligned_in_smp; - struct { spinlock_t completion_lock; @@ -441,9 +443,10 @@ struct io_ring_ctx { struct list_head rsrc_ref_list; spinlock_t rsrc_ref_lock; - struct work_struct exit_work; struct io_restriction restrictions; - struct io_submit_state submit_state; + + /* Keep this last, we don't need it for the fast path */ + struct work_struct exit_work; }; /* -- cgit v1.2.3 From 6e833d538b3123767393c987d11c40b7728b3f79 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Feb 2021 18:28:20 +0000 Subject: io_uring: clean up io_req_free_batch_finish() io_req_free_batch_finish() is final and does not permit struct req_batch to be reused without re-init. To be more consistent don't clear ->task there. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 8be7a24aa10e..fe06ca43e832 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2397,10 +2397,8 @@ static inline void io_init_req_batch(struct req_batch *rb) static void io_req_free_batch_finish(struct io_ring_ctx *ctx, struct req_batch *rb) { - if (rb->task) { + if (rb->task) io_put_task(rb->task, rb->task_refs); - rb->task = NULL; - } if (rb->ctx_refs) percpu_ref_put_many(&ctx->refs, rb->ctx_refs); } -- cgit v1.2.3 From f161340d9e85b9038031b497b32383e50ff00ca1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Feb 2021 18:28:21 +0000 Subject: io_uring: simplify iopoll reissuing Don't stash -EAGAIN'ed iopoll requests into a list to reissue it later, do it eagerly. It removes overhead on keeping and checking that list, and allows in case of failure for these requests to be completed through normal iopoll completion path. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index fe06ca43e832..18e449b5d632 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1029,8 +1029,7 @@ static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, struct fixed_rsrc_ref_node *ref_node); -static void __io_complete_rw(struct io_kiocb *req, long res, long res2, - unsigned int issue_flags); +static bool io_rw_reissue(struct io_kiocb *req, long res); static void io_cqring_fill_event(struct io_kiocb *req, long res); static void io_put_req(struct io_kiocb *req); static void io_put_req_deferred(struct io_kiocb *req, int nr); @@ -2558,17 +2557,6 @@ static inline bool io_run_task_work(void) return false; } -static void io_iopoll_queue(struct list_head *again) -{ - struct io_kiocb *req; - - do { - req = list_first_entry(again, struct io_kiocb, inflight_entry); - list_del(&req->inflight_entry); - __io_complete_rw(req, -EAGAIN, 0, 0); - } while (!list_empty(again)); -} - /* * Find and free completed poll iocbs */ @@ -2577,7 +2565,6 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, { struct req_batch rb; struct io_kiocb *req; - LIST_HEAD(again); /* order with ->result store in io_complete_rw_iopoll() */ smp_rmb(); @@ -2587,13 +2574,13 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, int cflags = 0; req = list_first_entry(done, struct io_kiocb, inflight_entry); + list_del(&req->inflight_entry); + if (READ_ONCE(req->result) == -EAGAIN) { - req->result = 0; req->iopoll_completed = 0; - list_move_tail(&req->inflight_entry, &again); - continue; + if (io_rw_reissue(req, -EAGAIN)) + continue; } - list_del(&req->inflight_entry); if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_rw_kbuf(req); @@ -2608,9 +2595,6 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, io_commit_cqring(ctx); io_cqring_ev_posted_iopoll(ctx); io_req_free_batch_finish(ctx, &rb); - - if (!list_empty(&again)) - io_iopoll_queue(&again); } static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, -- cgit v1.2.3 From 23faba36ce287e4af9018dea51893a1067701508 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Feb 2021 18:28:22 +0000 Subject: io_uring: move res check out of io_rw_reissue() We pass return code into io_rw_reissue() only to be able to check if it's -EAGAIN. That's not the cleanest approach and may prevent inlining of the non-EAGAIN fast path, so do it at call sites. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 18e449b5d632..c873ec113bcc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1029,7 +1029,7 @@ static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, struct fixed_rsrc_ref_node *ref_node); -static bool io_rw_reissue(struct io_kiocb *req, long res); +static bool io_rw_reissue(struct io_kiocb *req); static void io_cqring_fill_event(struct io_kiocb *req, long res); static void io_put_req(struct io_kiocb *req); static void io_put_req_deferred(struct io_kiocb *req, int nr); @@ -2578,7 +2578,7 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, if (READ_ONCE(req->result) == -EAGAIN) { req->iopoll_completed = 0; - if (io_rw_reissue(req, -EAGAIN)) + if (io_rw_reissue(req)) continue; } @@ -2812,15 +2812,12 @@ static bool io_resubmit_prep(struct io_kiocb *req) } #endif -static bool io_rw_reissue(struct io_kiocb *req, long res) +static bool io_rw_reissue(struct io_kiocb *req) { #ifdef CONFIG_BLOCK - umode_t mode; + umode_t mode = file_inode(req->file)->i_mode; int ret; - if (res != -EAGAIN && res != -EOPNOTSUPP) - return false; - mode = file_inode(req->file)->i_mode; if (!S_ISBLK(mode) && !S_ISREG(mode)) return false; if ((req->flags & REQ_F_NOWAIT) || io_wq_current_is_worker()) @@ -2843,8 +2840,10 @@ static bool io_rw_reissue(struct io_kiocb *req, long res) static void __io_complete_rw(struct io_kiocb *req, long res, long res2, unsigned int issue_flags) { - if (!io_rw_reissue(req, res)) - io_complete_rw_common(&req->rw.kiocb, res, issue_flags); + if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_reissue(req)) + return; + + io_complete_rw_common(&req->rw.kiocb, res, issue_flags); } static void io_complete_rw(struct kiocb *kiocb, long res, long res2) -- cgit v1.2.3 From 2f8e45f16c57360dd4d8b1310c2952a29a8fa890 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Feb 2021 18:28:23 +0000 Subject: io_uring: inline io_complete_rw_common() __io_complete_rw() casts request to kiocb for it to be immediately container_of()'ed by io_complete_rw_common(). And the last function's name doesn't do a great job of illuminating its purposes, so just inline it in its only user. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index c873ec113bcc..7b1979624320 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2761,22 +2761,6 @@ static void kiocb_end_write(struct io_kiocb *req) file_end_write(req->file); } -static void io_complete_rw_common(struct kiocb *kiocb, long res, - unsigned int issue_flags) -{ - struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); - int cflags = 0; - - if (kiocb->ki_flags & IOCB_WRITE) - kiocb_end_write(req); - - if (res != req->result) - req_set_fail_links(req); - if (req->flags & REQ_F_BUFFER_SELECTED) - cflags = io_put_rw_kbuf(req); - __io_req_complete(req, issue_flags, res, cflags); -} - #ifdef CONFIG_BLOCK static bool io_resubmit_prep(struct io_kiocb *req) { @@ -2840,10 +2824,18 @@ static bool io_rw_reissue(struct io_kiocb *req) static void __io_complete_rw(struct io_kiocb *req, long res, long res2, unsigned int issue_flags) { + int cflags = 0; + if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_reissue(req)) return; + if (res != req->result) + req_set_fail_links(req); - io_complete_rw_common(&req->rw.kiocb, res, issue_flags); + if (req->rw.kiocb.ki_flags & IOCB_WRITE) + kiocb_end_write(req); + if (req->flags & REQ_F_BUFFER_SELECTED) + cflags = io_put_rw_kbuf(req); + __io_req_complete(req, issue_flags, res, cflags); } static void io_complete_rw(struct kiocb *kiocb, long res, long res2) -- cgit v1.2.3 From 302fdadeafe4be539f247abf25f61822e4a5a577 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 22 Jan 2021 12:02:34 +0100 Subject: ext: EXT4_KUNIT_TESTS should depend on EXT4_FS instead of selecting it EXT4_KUNIT_TESTS selects EXT4_FS, thus enabling an optional feature the user may not want to enable. Fix this by making the test depend on EXT4_FS instead. Fixes: 1cbeab1b242d16fd ("ext4: add kunit test for decoding extended timestamps") Reviewed-by: Randy Dunlap Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20210122110234.2825685-1-geert@linux-m68k.org Signed-off-by: Theodore Ts'o --- fs/ext4/Kconfig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/ext4/Kconfig b/fs/ext4/Kconfig index 619dd35ddd48..86699c8cab28 100644 --- a/fs/ext4/Kconfig +++ b/fs/ext4/Kconfig @@ -103,8 +103,7 @@ config EXT4_DEBUG config EXT4_KUNIT_TESTS tristate "KUnit tests for ext4" if !KUNIT_ALL_TESTS - select EXT4_FS - depends on KUNIT + depends on EXT4_FS && KUNIT default KUNIT_ALL_TESTS help This builds the ext4 KUnit tests. -- cgit v1.2.3 From 0a76945fd1ba2ab44da7b578b311efdfedf92e6c Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Tue, 9 Feb 2021 17:32:06 -0800 Subject: ext4: add .kunitconfig fragment to enable ext4-specific tests As of [1], we no longer want EXT4_KUNIT_TESTS and others to `select` their deps. This means it can get harder to get all the right things selected as we gain more tests w/ more deps over time. This patch (and [2]) proposes we store kunitconfig fragments in-tree to represent sets of tests. (N.B. right now we only have one ext4 test). There's still a discussion to be had about how to have a hierarchy of these files (e.g. if one wanted to test all of fs/, not just fs/ext4). But this fragment would likely be a leaf node and isn't blocked on deciding if we want `import` statements and the like. Usage ===== Before [2] (on its way to being merged): $ cp fs/ext4/.kunitconfig .kunit/ $ ./tools/testing/kunit/kunit.py run After [2]: $ ./tools/testing/kunit/kunit.py run --kunitconfig=fs/ext4/.kunitconfig ".kunitconfig" vs "kunitconfig" =============================== See also: commit 14ee5cfd4512 ("kunit: Rename 'kunitconfig' to '.kunitconfig'"). * The bit about .gitignore exluding it by default is now a con, however. * But there are a lot of directories with files that begin with "k" and so this could cause some annoyance w/ tab completion* * This is the name kunit.py expects right now, so some people are used to .kunitconfig over "kunitconfig" [1] https://lore.kernel.org/linux-ext4/20210122110234.2825685-1-geert@linux-m68k.org/ [2] https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git/commit/?h=kunit&id=243180f5924ed27ea417db39feb7f9691777688e * 372/5556 directories isn't too much, but still not a small number: $ find -type f -name 'k*' | xargs dirname | sort -u | wc -l 372 Signed-off-by: Daniel Latypov Link: https://lore.kernel.org/r/20210210013206.136227-1-dlatypov@google.com Signed-off-by: Theodore Ts'o --- fs/ext4/.kunitconfig | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 fs/ext4/.kunitconfig (limited to 'fs') diff --git a/fs/ext4/.kunitconfig b/fs/ext4/.kunitconfig new file mode 100644 index 000000000000..bf51da7cd9fc --- /dev/null +++ b/fs/ext4/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_KUNIT_TESTS=y -- cgit v1.2.3 From bd75904590de1c2bbdff55180cef209b13bd50fa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 03:23:50 +0000 Subject: io_uring: take compl state from submit state Completion and submission states are now coupled together, it's weird to get one from argument and another from ctx, do it consistently for io_req_free_batch(). It's also faster as we already have @state cached in registers. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 7b1979624320..8c2613bf54d3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2417,13 +2417,10 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, rb->ctx_refs++; io_dismantle_req(req); - if (state->free_reqs != ARRAY_SIZE(state->reqs)) { + if (state->free_reqs != ARRAY_SIZE(state->reqs)) state->reqs[state->free_reqs++] = req; - } else { - struct io_comp_state *cs = &req->ctx->submit_state.comp; - - list_add(&req->compl.list, &cs->free_list); - } + else + list_add(&req->compl.list, &state->comp.free_list); } static void io_submit_flush_completions(struct io_comp_state *cs, -- cgit v1.2.3 From d3d7298d05cb026305b0f5033acc9c9c4f281e14 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 03:23:51 +0000 Subject: io_uring: optimise out unlikely link queue __io_queue_sqe() tries to issue as much requests of a link as it can, and uses io_put_req_find_next() to extract a next one, targeting inline completed requests. As now __io_queue_sqe() is always used together with struct io_comp_state, it leaves next propagation only a small window and only for async reqs, that doesn't justify its existence. Remove it, make __io_queue_sqe() to issue only a head request. It simplifies the code and will allow other optimisations. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 42 ++++++++++-------------------------------- 1 file changed, 10 insertions(+), 32 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 8c2613bf54d3..26d1080217e5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6563,26 +6563,20 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) static void __io_queue_sqe(struct io_kiocb *req) { - struct io_kiocb *linked_timeout; + struct io_kiocb *linked_timeout = io_prep_linked_timeout(req); const struct cred *old_creds = NULL; int ret; -again: - linked_timeout = io_prep_linked_timeout(req); - if ((req->flags & REQ_F_WORK_INITIALIZED) && (req->work.flags & IO_WQ_WORK_CREDS) && - req->work.identity->creds != current_cred()) { - if (old_creds) - revert_creds(old_creds); - if (old_creds == req->work.identity->creds) - old_creds = NULL; /* restored original creds */ - else - old_creds = override_creds(req->work.identity->creds); - } + req->work.identity->creds != current_cred()) + old_creds = override_creds(req->work.identity->creds); ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); + if (old_creds) + revert_creds(old_creds); + /* * We async punt it if the file wasn't marked NOWAIT, or if the file * doesn't support non-blocking read/write attempts @@ -6595,9 +6589,6 @@ again: */ io_queue_async_work(req); } - - if (linked_timeout) - io_queue_linked_timeout(linked_timeout); } else if (likely(!ret)) { /* drop submission reference */ if (req->flags & REQ_F_COMPLETE_INLINE) { @@ -6605,31 +6596,18 @@ again: struct io_comp_state *cs = &ctx->submit_state.comp; cs->reqs[cs->nr++] = req; - if (cs->nr == IO_COMPL_BATCH) + if (cs->nr == ARRAY_SIZE(cs->reqs)) io_submit_flush_completions(cs, ctx); - req = NULL; } else { - req = io_put_req_find_next(req); - } - - if (linked_timeout) - io_queue_linked_timeout(linked_timeout); - - if (req) { - if (!(req->flags & REQ_F_FORCE_ASYNC)) - goto again; - io_queue_async_work(req); + io_put_req(req); } } else { - /* un-prep timeout, so it'll be killed as any other linked */ - req->flags &= ~REQ_F_LINK_TIMEOUT; req_set_fail_links(req); io_put_req(req); io_req_complete(req, ret); } - - if (old_creds) - revert_creds(old_creds); + if (linked_timeout) + io_queue_linked_timeout(linked_timeout); } static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe) -- cgit v1.2.3 From 4e32635834a30b8aa9583d3899a8ecc6416023fb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 03:23:52 +0000 Subject: io_uring: optimise SQPOLL mm/files grabbing There are two reasons for this. First is to optimise io_sq_thread_acquire_mm_files() for non-SQPOLL case, which currently do too many checks and function calls in the hot path, e.g. in io_init_req(). The second is to not grab mm/files when there are not needed. As __io_queue_sqe() issues only one request now, we can reuse io_sq_thread_acquire_mm_files() instead of unconditional acquire mm/files. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 26d1080217e5..813d1ccd7a69 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1145,9 +1145,6 @@ static void io_sq_thread_drop_mm_files(void) static int __io_sq_thread_acquire_files(struct io_ring_ctx *ctx) { - if (current->flags & PF_EXITING) - return -EFAULT; - if (!current->files) { struct files_struct *files; struct nsproxy *nsproxy; @@ -1175,15 +1172,9 @@ static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx) { struct mm_struct *mm; - if (current->flags & PF_EXITING) - return -EFAULT; if (current->mm) return 0; - /* Should never happen */ - if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL))) - return -EFAULT; - task_lock(ctx->sqo_task); mm = ctx->sqo_task->mm; if (unlikely(!mm || !mmget_not_zero(mm))) @@ -1198,8 +1189,8 @@ static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx) return -EFAULT; } -static int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, - struct io_kiocb *req) +static int __io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, + struct io_kiocb *req) { const struct io_op_def *def = &io_op_defs[req->opcode]; int ret; @@ -1219,6 +1210,16 @@ static int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, return 0; } +static inline int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, + struct io_kiocb *req) +{ + if (unlikely(current->flags & PF_EXITING)) + return -EFAULT; + if (!(ctx->flags & IORING_SETUP_SQPOLL)) + return 0; + return __io_sq_thread_acquire_mm_files(ctx, req); +} + static void io_sq_thread_associate_blkcg(struct io_ring_ctx *ctx, struct cgroup_subsys_state **cur_css) @@ -2336,9 +2337,7 @@ static void __io_req_task_submit(struct io_kiocb *req) struct io_ring_ctx *ctx = req->ctx; mutex_lock(&ctx->uring_lock); - if (!ctx->sqo_dead && - !__io_sq_thread_acquire_mm(ctx) && - !__io_sq_thread_acquire_files(ctx)) + if (!ctx->sqo_dead && !io_sq_thread_acquire_mm_files(ctx, req)) __io_queue_sqe(req); else __io_req_task_cancel(req, -EFAULT); -- cgit v1.2.3 From 921b9054e0c4c443c479c21800f6c4c8b43fa1b0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 03:23:53 +0000 Subject: io_uring: don't duplicate io_req_task_queue() Don't hand code io_req_task_queue() inside of io_async_buf_func(), just call it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 813d1ccd7a69..5c0b1a7dba80 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3494,7 +3494,6 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, struct wait_page_queue *wpq; struct io_kiocb *req = wait->private; struct wait_page_key *key = arg; - int ret; wpq = container_of(wait, struct wait_page_queue, wait); @@ -3504,14 +3503,9 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, req->rw.kiocb.ki_flags &= ~IOCB_WAITQ; list_del_init(&wait->entry); - req->task_work.func = io_req_task_submit; - percpu_ref_get(&req->ctx->refs); - /* submit ref gets dropped, acquire a new one */ refcount_inc(&req->refs); - ret = io_req_task_work_add(req); - if (unlikely(ret)) - io_req_task_work_add_fallback(req, io_req_task_cancel); + io_req_task_queue(req); return 1; } -- cgit v1.2.3 From 04fc6c802dfacba800f5a5d00bea0ebfcc60f840 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 03:23:54 +0000 Subject: io_uring: save ctx put/get for task_work submit Do a little trick in io_ring_ctx_free() briefly taking uring_lock, that will wait for everyone currently holding it, so we can skip pinning ctx with ctx->refs for __io_req_task_submit(), which is executed and loses its refs/reqs while holding the lock. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5c0b1a7dba80..87f2f8e660e8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2336,6 +2336,7 @@ static void __io_req_task_submit(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; + /* ctx stays valid until unlock, even if we drop all ours ctx->refs */ mutex_lock(&ctx->uring_lock); if (!ctx->sqo_dead && !io_sq_thread_acquire_mm_files(ctx, req)) __io_queue_sqe(req); @@ -2347,10 +2348,8 @@ static void __io_req_task_submit(struct io_kiocb *req) static void io_req_task_submit(struct callback_head *cb) { struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); - struct io_ring_ctx *ctx = req->ctx; __io_req_task_submit(req); - percpu_ref_put(&ctx->refs); } static void io_req_task_queue(struct io_kiocb *req) @@ -2358,11 +2357,11 @@ static void io_req_task_queue(struct io_kiocb *req) int ret; req->task_work.func = io_req_task_submit; - percpu_ref_get(&req->ctx->refs); - ret = io_req_task_work_add(req); - if (unlikely(ret)) + if (unlikely(ret)) { + percpu_ref_get(&req->ctx->refs); io_req_task_work_add_fallback(req, io_req_task_cancel); + } } static inline void io_queue_next(struct io_kiocb *req) @@ -8707,6 +8706,14 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) { struct io_submit_state *submit_state = &ctx->submit_state; + /* + * Some may use context even when all refs and requests have been put, + * and they are free to do so while still holding uring_lock, see + * __io_req_task_submit(). Wait for them to finish. + */ + mutex_lock(&ctx->uring_lock); + mutex_unlock(&ctx->uring_lock); + io_finish_async(ctx); io_sqe_buffers_unregister(ctx); -- cgit v1.2.3 From 4fccfcbb733794634d4e873e7973c1847beca5bf Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 11:55:17 +0000 Subject: io_uring: don't split out consume out of SQE get Remove io_consume_sqe() and inline it back into io_get_sqe(). It requires req dealloc on error, but in exchange we get cleaner io_submit_sqes() and better locality for cached_sq_head. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 87f2f8e660e8..9c58be0579f3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6762,7 +6762,7 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) * 2) allows the kernel side to track the head on its own, even * though the application is the one updating it. */ - head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]); + head = READ_ONCE(sq_array[ctx->cached_sq_head++ & ctx->sq_mask]); if (likely(head < ctx->sq_entries)) return &ctx->sq_sqes[head]; @@ -6772,11 +6772,6 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) return NULL; } -static inline void io_consume_sqe(struct io_ring_ctx *ctx) -{ - ctx->cached_sq_head++; -} - /* * Check SQE restrictions (opcode and flags). * @@ -6915,18 +6910,17 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) struct io_kiocb *req; int err; - sqe = io_get_sqe(ctx); - if (unlikely(!sqe)) { - io_consume_sqe(ctx); - break; - } req = io_alloc_req(ctx); if (unlikely(!req)) { if (!submitted) submitted = -EAGAIN; break; } - io_consume_sqe(ctx); + sqe = io_get_sqe(ctx); + if (unlikely(!sqe)) { + kmem_cache_free(req_cachep, req); + break; + } /* will complete beyond this point, count as submitted */ submitted++; -- cgit v1.2.3 From 83c68bbcb6ac2dbbcaf12e2281a29a9f73b97d0f Mon Sep 17 00:00:00 2001 From: Su Yue Date: Thu, 11 Feb 2021 16:38:28 +0800 Subject: btrfs: initialize fs_info::csum_size earlier in open_ctree User reported that btrfs-progs misc-tests/028-superblock-recover fails: [TEST/misc] 028-superblock-recover unexpected success: mounted fs with corrupted superblock test failed for case 028-superblock-recover The test case expects that a broken image with bad superblock will be rejected to be mounted. However, the test image just passed csum check of superblock and was successfully mounted. Commit 55fc29bed8dd ("btrfs: use cached value of fs_info::csum_size everywhere") replaces all calls to btrfs_super_csum_size by fs_info::csum_size. The calls include the place where fs_info->csum_size is not initialized. So btrfs_check_super_csum() passes because memcmp() with len 0 always returns 0. Fix it by caching csum size in btrfs_fs_info::csum_size once we know the csum type in superblock is valid in open_ctree(). Link: https://github.com/kdave/btrfs-progs/issues/250 Fixes: 55fc29bed8dd ("btrfs: use cached value of fs_info::csum_size everywhere") Signed-off-by: Su Yue Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 6b35b7e88136..07a2b4f69b10 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -3044,6 +3044,8 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device goto fail_alloc; } + fs_info->csum_size = btrfs_super_csum_size(disk_super); + ret = btrfs_init_csum_hash(fs_info, csum_type); if (ret) { err = ret; @@ -3161,7 +3163,6 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device fs_info->nodesize = nodesize; fs_info->sectorsize = sectorsize; fs_info->sectorsize_bits = ilog2(sectorsize); - fs_info->csum_size = btrfs_super_csum_size(disk_super); fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size; fs_info->stripesize = stripesize; -- cgit v1.2.3 From dc0eced5d92052a84d58df03a3bc6382f64fecfa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 18:41:15 +0000 Subject: io_uring: don't check PF_EXITING from syscall io_sq_thread_acquire_mm_files() can find a PF_EXITING task only when it's called from task_work context. Don't check it in all other cases, that are when we're in io_uring_enter(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 9c58be0579f3..66bbb0dc50af 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1213,8 +1213,6 @@ static int __io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, static inline int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, struct io_kiocb *req) { - if (unlikely(current->flags & PF_EXITING)) - return -EFAULT; if (!(ctx->flags & IORING_SETUP_SQPOLL)) return 0; return __io_sq_thread_acquire_mm_files(ctx, req); @@ -2338,7 +2336,8 @@ static void __io_req_task_submit(struct io_kiocb *req) /* ctx stays valid until unlock, even if we drop all ours ctx->refs */ mutex_lock(&ctx->uring_lock); - if (!ctx->sqo_dead && !io_sq_thread_acquire_mm_files(ctx, req)) + if (!ctx->sqo_dead && !(current->flags & PF_EXITING) && + !io_sq_thread_acquire_mm_files(ctx, req)) __io_queue_sqe(req); else __io_req_task_cancel(req, -EFAULT); -- cgit v1.2.3 From cdbff98223330cdb6c57ead1533ce066dddd61b7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 18:41:16 +0000 Subject: io_uring: clean io_req_find_next() fast check Indirectly io_req_find_next() is called for every request, optimise the check by testing flags as it was long before -- __io_req_find_next() tolerates false-positives well (i.e. link==NULL), and those should be really rare. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 66bbb0dc50af..776531f6e18b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2172,7 +2172,7 @@ static struct io_kiocb *__io_req_find_next(struct io_kiocb *req) static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) { - if (likely(!(req->link) && !(req->flags & REQ_F_LINK_TIMEOUT))) + if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK)))) return NULL; return __io_req_find_next(req); } -- cgit v1.2.3 From 5be9ad1e4287e1742fd8d253267c86446441bdaf Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Feb 2021 18:41:17 +0000 Subject: io_uring: optimise io_init_req() flags setting Invalid req->flags are tolerated by free/put well, avoid this dancing needlessly presetting it to zero, and then not even resetting but modifying it, i.e. "|=". Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 776531f6e18b..2e8cb739c835 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6806,14 +6806,15 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, { struct io_submit_state *state; unsigned int sqe_flags; - int id, ret; + int id, ret = 0; req->opcode = READ_ONCE(sqe->opcode); + /* same numerical values with corresponding REQ_F_*, safe to copy */ + req->flags = sqe_flags = READ_ONCE(sqe->flags); req->user_data = READ_ONCE(sqe->user_data); req->async_data = NULL; req->file = NULL; req->ctx = ctx; - req->flags = 0; req->link = NULL; req->fixed_rsrc_refs = NULL; /* one is dropped after submission, the other at completion */ @@ -6821,17 +6822,16 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->task = current; req->result = 0; + /* enforce forwards compatibility on users */ + if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) + return -EINVAL; + if (unlikely(req->opcode >= IORING_OP_LAST)) return -EINVAL; if (unlikely(io_sq_thread_acquire_mm_files(ctx, req))) return -EFAULT; - sqe_flags = READ_ONCE(sqe->flags); - /* enforce forwards compatibility on users */ - if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) - return -EINVAL; - if (unlikely(!io_check_restriction(ctx, req, sqe_flags))) return -EACCES; @@ -6854,8 +6854,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->work.flags |= IO_WQ_WORK_CREDS; } - /* same numerical values with corresponding REQ_F_*, safe to copy */ - req->flags |= sqe_flags; state = &ctx->submit_state; /* @@ -6868,7 +6866,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, state->plug_started = true; } - ret = 0; if (io_op_defs[req->opcode].needs_file) { bool fixed = req->flags & REQ_F_FIXED_FILE; -- cgit v1.2.3 From a15f1e41fbf59d987365018d7439f24aa5801269 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Wed, 13 Jan 2021 21:31:55 +0100 Subject: um: hostfs: use a kmem cache for inodes This collects all of them together and makes it possible to e.g. exclude it from slub debugging. Signed-off-by: Johannes Berg Signed-off-by: Richard Weinberger --- fs/hostfs/hostfs_kern.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c index aea35459d390..4a5beca6eaa8 100644 --- a/fs/hostfs/hostfs_kern.c +++ b/fs/hostfs/hostfs_kern.c @@ -34,6 +34,8 @@ static inline struct hostfs_inode_info *HOSTFS_I(struct inode *inode) #define FILE_HOSTFS_I(file) HOSTFS_I(file_inode(file)) +static struct kmem_cache *hostfs_inode_cache; + /* Changed in hostfs_args before the kernel starts running */ static char *root_ino = ""; static int append = 0; @@ -221,7 +223,7 @@ static struct inode *hostfs_alloc_inode(struct super_block *sb) { struct hostfs_inode_info *hi; - hi = kmalloc(sizeof(*hi), GFP_KERNEL_ACCOUNT); + hi = kmem_cache_alloc(hostfs_inode_cache, GFP_KERNEL_ACCOUNT); if (hi == NULL) return NULL; hi->fd = -1; @@ -243,7 +245,7 @@ static void hostfs_evict_inode(struct inode *inode) static void hostfs_free_inode(struct inode *inode) { - kfree(HOSTFS_I(inode)); + kmem_cache_free(hostfs_inode_cache, HOSTFS_I(inode)); } static int hostfs_show_options(struct seq_file *seq, struct dentry *root) @@ -986,12 +988,16 @@ MODULE_ALIAS_FS("hostfs"); static int __init init_hostfs(void) { + hostfs_inode_cache = KMEM_CACHE(hostfs_inode_info, 0); + if (!hostfs_inode_cache) + return -ENOMEM; return register_filesystem(&hostfs_type); } static void __exit exit_hostfs(void) { unregister_filesystem(&hostfs_type); + kmem_cache_destroy(hostfs_inode_cache); } module_init(init_hostfs) -- cgit v1.2.3 From 19646447ad3a680d2ab08c097585b7d96a66126b Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Wed, 30 Dec 2020 06:56:04 -0800 Subject: jffs2: fix use after free in jffs2_sum_write_data() clang static analysis reports this problem fs/jffs2/summary.c:794:31: warning: Use of memory after it is freed c->summary->sum_list_head = temp->u.next; ^~~~~~~~~~~~ In jffs2_sum_write_data(), in a loop summary data is handles a node at a time. When it has written out the node it is removed the summary list, and the node is deleted. In the corner case when a JFFS2_FEATURE_RWCOMPAT_COPY is seen, a call is made to jffs2_sum_disable_collecting(). jffs2_sum_disable_collecting() deletes the whole list which conflicts with the loop's deleting the list by parts. To preserve the old behavior of stopping the write midway, bail out of the loop after disabling summary collection. Fixes: 6171586a7ae5 ("[JFFS2] Correct handling of JFFS2_FEATURE_RWCOMPAT_COPY nodes.") Signed-off-by: Tom Rix Reviewed-by: Nathan Chancellor Signed-off-by: Richard Weinberger --- fs/jffs2/summary.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/jffs2/summary.c b/fs/jffs2/summary.c index be7c8a6a5748..4fe64519870f 100644 --- a/fs/jffs2/summary.c +++ b/fs/jffs2/summary.c @@ -783,6 +783,8 @@ static int jffs2_sum_write_data(struct jffs2_sb_info *c, struct jffs2_eraseblock dbg_summary("Writing unknown RWCOMPAT_COPY node type %x\n", je16_to_cpu(temp->u.nodetype)); jffs2_sum_disable_collecting(c->summary); + /* The above call removes the list, nothing more to do */ + goto bail_rwcompat; } else { BUG(); /* unknown node in summary information */ } @@ -794,6 +796,7 @@ static int jffs2_sum_write_data(struct jffs2_sb_info *c, struct jffs2_eraseblock c->summary->sum_num--; } + bail_rwcompat: jffs2_sum_reset_collected(c->summary); -- cgit v1.2.3 From 11b8ab3836454a2600e396f34731e491b661f9d5 Mon Sep 17 00:00:00 2001 From: Dinghao Liu Date: Tue, 5 Jan 2021 14:03:40 +0800 Subject: ubifs: Fix memleak in ubifs_init_authentication When crypto_shash_digestsize() fails, c->hmac_tfm has not been freed before returning, which leads to memleak. Fixes: 49525e5eecca5 ("ubifs: Add helper functions for authentication support") Signed-off-by: Dinghao Liu Reviewed-by: Zhihao Cheng Signed-off-by: Richard Weinberger --- fs/ubifs/auth.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c index 51a7c8c2c3f0..e564d5ff8781 100644 --- a/fs/ubifs/auth.c +++ b/fs/ubifs/auth.c @@ -327,7 +327,7 @@ int ubifs_init_authentication(struct ubifs_info *c) ubifs_err(c, "hmac %s is bigger than maximum allowed hmac size (%d > %d)", hmac_name, c->hmac_desc_len, UBIFS_HMAC_ARR_SZ); err = -EINVAL; - goto out_free_hash; + goto out_free_hmac; } err = crypto_shash_setkey(c->hmac_tfm, ukp->data, ukp->datalen); -- cgit v1.2.3 From 410b6de702ef84fea6e7abcb6620ef8bfc112fae Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 14 Jan 2021 22:30:11 +0100 Subject: ubifs: replay: Fix high stack usage, again An earlier commit moved out some functions to not be inlined by gcc, but after some other rework to remove one of those, clang started inlining the other one and ran into the same problem as gcc did before: fs/ubifs/replay.c:1174:5: error: stack frame size of 1152 bytes in function 'ubifs_replay_journal' [-Werror,-Wframe-larger-than=] Mark the function as noinline_for_stack to ensure it doesn't happen again. Fixes: f80df3851246 ("ubifs: use crypto_shash_tfm_digest()") Fixes: eb66eff6636d ("ubifs: replay: Fix high stack usage") Signed-off-by: Arnd Bergmann Reviewed-by: Nathan Chancellor Signed-off-by: Richard Weinberger --- fs/ubifs/replay.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ubifs/replay.c b/fs/ubifs/replay.c index 79801c9a5b87..0f8a6a16421b 100644 --- a/fs/ubifs/replay.c +++ b/fs/ubifs/replay.c @@ -559,7 +559,9 @@ static int is_last_bud(struct ubifs_info *c, struct ubifs_bud *bud) } /* authenticate_sleb_hash is split out for stack usage */ -static int authenticate_sleb_hash(struct ubifs_info *c, struct shash_desc *log_hash, u8 *hash) +static int noinline_for_stack +authenticate_sleb_hash(struct ubifs_info *c, + struct shash_desc *log_hash, u8 *hash) { SHASH_DESC_ON_STACK(hash_desc, c->hash_tfm); -- cgit v1.2.3 From d984bcf5766dbdbe95d325bb8a1b49a996fecfd4 Mon Sep 17 00:00:00 2001 From: Sascha Hauer Date: Wed, 20 Jan 2021 14:12:53 +0100 Subject: ubifs: Fix off-by-one error An inode is allowed to have ubifs_xattr_max_cnt() xattrs, so we must complain only when an inode has more xattrs, having exactly ubifs_xattr_max_cnt() xattrs is fine. With this the maximum number of xattrs can be created without hitting the "has too many xattrs" warning when removing it. Signed-off-by: Sascha Hauer Signed-off-by: Richard Weinberger --- fs/ubifs/journal.c | 2 +- fs/ubifs/xattr.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/ubifs/journal.c b/fs/ubifs/journal.c index 03410ae0813a..2857e64d673d 100644 --- a/fs/ubifs/journal.c +++ b/fs/ubifs/journal.c @@ -881,7 +881,7 @@ int ubifs_jnl_write_inode(struct ubifs_info *c, const struct inode *inode) struct inode *xino; struct ubifs_dent_node *xent, *pxent = NULL; - if (ui->xattr_cnt >= ubifs_xattr_max_cnt(c)) { + if (ui->xattr_cnt > ubifs_xattr_max_cnt(c)) { ubifs_err(c, "Cannot delete inode, it has too much xattrs!"); goto out_release; } diff --git a/fs/ubifs/xattr.c b/fs/ubifs/xattr.c index a0b9b349efe6..842d5f14545d 100644 --- a/fs/ubifs/xattr.c +++ b/fs/ubifs/xattr.c @@ -498,7 +498,7 @@ int ubifs_purge_xattrs(struct inode *host) struct fscrypt_name nm = {0}; int err; - if (ubifs_inode(host)->xattr_cnt < ubifs_xattr_max_cnt(c)) + if (ubifs_inode(host)->xattr_cnt <= ubifs_xattr_max_cnt(c)) return 0; ubifs_warn(c, "inode %lu has too many xattrs, doing a non-atomic deletion", -- cgit v1.2.3 From 90ada91f4610c5ef11bc52576516d96c496fc3f1 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Thu, 28 Jan 2021 02:55:35 -0800 Subject: jffs2: check the validity of dstlen in jffs2_zlib_compress() KASAN reports a BUG when download file in jffs2 filesystem.It is because when dstlen == 1, cpage_out will write array out of bounds. Actually, data will not be compressed in jffs2_zlib_compress() if data's length less than 4. [ 393.799778] BUG: KASAN: slab-out-of-bounds in jffs2_rtime_compress+0x214/0x2f0 at addr ffff800062e3b281 [ 393.809166] Write of size 1 by task tftp/2918 [ 393.813526] CPU: 3 PID: 2918 Comm: tftp Tainted: G B 4.9.115-rt93-EMBSYS-CGEL-6.1.R6-dirty #1 [ 393.823173] Hardware name: LS1043A RDB Board (DT) [ 393.827870] Call trace: [ 393.830322] [] dump_backtrace+0x0/0x2f0 [ 393.835721] [] show_stack+0x14/0x20 [ 393.840774] [] dump_stack+0x90/0xb0 [ 393.845829] [] kasan_object_err+0x24/0x80 [ 393.851402] [] kasan_report_error+0x1b4/0x4d8 [ 393.857323] [] kasan_report+0x38/0x40 [ 393.862548] [] __asan_store1+0x4c/0x58 [ 393.867859] [] jffs2_rtime_compress+0x214/0x2f0 [ 393.873955] [] jffs2_selected_compress+0x178/0x2a0 [ 393.880308] [] jffs2_compress+0x58/0x478 [ 393.885796] [] jffs2_write_inode_range+0x13c/0x450 [ 393.892150] [] jffs2_write_end+0x2a8/0x4a0 [ 393.897811] [] generic_perform_write+0x1c0/0x280 [ 393.903990] [] __generic_file_write_iter+0x1c4/0x228 [ 393.910517] [] generic_file_write_iter+0x138/0x288 [ 393.916870] [] __vfs_write+0x1b4/0x238 [ 393.922181] [] vfs_write+0xd0/0x238 [ 393.927232] [] SyS_write+0xa0/0x110 [ 393.932283] [] __sys_trace_return+0x0/0x4 [ 393.937851] Object at ffff800062e3b280, in cache kmalloc-64 size: 64 [ 393.944197] Allocated: [ 393.946552] PID = 2918 [ 393.948913] save_stack_trace_tsk+0x0/0x220 [ 393.953096] save_stack_trace+0x18/0x20 [ 393.956932] kasan_kmalloc+0xd8/0x188 [ 393.960594] __kmalloc+0x144/0x238 [ 393.963994] jffs2_selected_compress+0x48/0x2a0 [ 393.968524] jffs2_compress+0x58/0x478 [ 393.972273] jffs2_write_inode_range+0x13c/0x450 [ 393.976889] jffs2_write_end+0x2a8/0x4a0 [ 393.980810] generic_perform_write+0x1c0/0x280 [ 393.985251] __generic_file_write_iter+0x1c4/0x228 [ 393.990040] generic_file_write_iter+0x138/0x288 [ 393.994655] __vfs_write+0x1b4/0x238 [ 393.998228] vfs_write+0xd0/0x238 [ 394.001543] SyS_write+0xa0/0x110 [ 394.004856] __sys_trace_return+0x0/0x4 [ 394.008684] Freed: [ 394.010691] PID = 2918 [ 394.013051] save_stack_trace_tsk+0x0/0x220 [ 394.017233] save_stack_trace+0x18/0x20 [ 394.021069] kasan_slab_free+0x88/0x188 [ 394.024902] kfree+0x6c/0x1d8 [ 394.027868] jffs2_sum_write_sumnode+0x2c4/0x880 [ 394.032486] jffs2_do_reserve_space+0x198/0x598 [ 394.037016] jffs2_reserve_space+0x3f8/0x4d8 [ 394.041286] jffs2_write_inode_range+0xf0/0x450 [ 394.045816] jffs2_write_end+0x2a8/0x4a0 [ 394.049737] generic_perform_write+0x1c0/0x280 [ 394.054179] __generic_file_write_iter+0x1c4/0x228 [ 394.058968] generic_file_write_iter+0x138/0x288 [ 394.063583] __vfs_write+0x1b4/0x238 [ 394.067157] vfs_write+0xd0/0x238 [ 394.070470] SyS_write+0xa0/0x110 [ 394.073783] __sys_trace_return+0x0/0x4 [ 394.077612] Memory state around the buggy address: [ 394.082404] ffff800062e3b180: 00 00 00 00 00 00 00 00 fc fc fc fc fc fc fc fc [ 394.089623] ffff800062e3b200: 00 00 00 00 00 00 00 00 fc fc fc fc fc fc fc fc [ 394.096842] >ffff800062e3b280: 01 fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc [ 394.104056] ^ [ 394.107283] ffff800062e3b300: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc [ 394.114502] ffff800062e3b380: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc [ 394.121718] ================================================================== Signed-off-by: Yang Yang Signed-off-by: Richard Weinberger --- fs/jffs2/compr_rtime.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/jffs2/compr_rtime.c b/fs/jffs2/compr_rtime.c index 406d9cc84ba8..79e771ab624f 100644 --- a/fs/jffs2/compr_rtime.c +++ b/fs/jffs2/compr_rtime.c @@ -37,6 +37,9 @@ static int jffs2_rtime_compress(unsigned char *data_in, int outpos = 0; int pos=0; + if (*dstlen <= 3) + return -1; + memset(positions,0,sizeof(positions)); while (pos < (*sourcelen) && outpos <= (*dstlen)-2) { -- cgit v1.2.3 From e06aa2e94f0532d04bad7713eb7c6a32ab9ba674 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 12 Feb 2021 14:02:54 -0700 Subject: io-wq: clear out worker ->fs and ->files By default, kernel threads have init_fs and init_files assigned. In the past, this has triggered security problems, as commands that don't ask for (and hence don't get assigned) fs/files from the originating task can then attempt path resolution etc with access to parts of the system they should not be able to. Rather than add checks in the fs code for misuse, just set these to NULL. If we do attempt to use them, then the resulting code will oops rather than provide access to something that it should not permit. Suggested-by: Linus Torvalds Signed-off-by: Jens Axboe --- fs/io-wq.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 63ef195b1acb..c36bbcd823ce 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -64,9 +64,7 @@ struct io_worker { #endif const struct cred *cur_creds; const struct cred *saved_creds; - struct files_struct *restore_files; struct nsproxy *restore_nsproxy; - struct fs_struct *restore_fs; }; #if BITS_PER_LONG == 64 @@ -156,19 +154,19 @@ static bool __io_worker_unuse(struct io_wqe *wqe, struct io_worker *worker) worker->cur_creds = worker->saved_creds = NULL; } - if (current->files != worker->restore_files) { + if (current->files) { __acquire(&wqe->lock); raw_spin_unlock_irq(&wqe->lock); dropped_lock = true; task_lock(current); - current->files = worker->restore_files; + current->files = NULL; current->nsproxy = worker->restore_nsproxy; task_unlock(current); } - if (current->fs != worker->restore_fs) - current->fs = worker->restore_fs; + if (current->fs) + current->fs = NULL; /* * If we have an active mm, we need to drop the wq lock before unusing @@ -329,11 +327,11 @@ static void io_worker_start(struct io_wqe *wqe, struct io_worker *worker) allow_kernel_signal(SIGINT); current->flags |= PF_IO_WORKER; + current->fs = NULL; + current->files = NULL; worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); - worker->restore_files = current->files; worker->restore_nsproxy = current->nsproxy; - worker->restore_fs = current->fs; io_wqe_inc_running(wqe, worker); } -- cgit v1.2.3 From 938a184265d75ea474f1c6fe1da96a5196163789 Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Fri, 12 Feb 2021 14:09:54 -0800 Subject: f2fs: give a warning only for readonly partition Let's allow mounting readonly partition. We're able to recovery later once we have it as read-write back. Signed-off-by: Jaegeuk Kim --- fs/f2fs/super.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 4aa533cb4340..30d5abef4361 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -3933,12 +3933,10 @@ try_onemore: * previous checkpoint was not done by clean system shutdown. */ if (f2fs_hw_is_readonly(sbi)) { - if (!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) { - err = -EROFS; + if (!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) f2fs_err(sbi, "Need to recover fsync data, but write access unavailable"); - goto free_meta; - } - f2fs_info(sbi, "write access unavailable, skipping recovery"); + else + f2fs_info(sbi, "write access unavailable, skipping recovery"); goto reset_checkpoint; } -- cgit v1.2.3 From 68e68ee6e359318c40891f614612616d219066d0 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 13 Feb 2021 09:00:02 -0700 Subject: io_uring: allow task match to be passed to io_req_cache_free() No changes in this patch, just allows a caller to pass in a targeted task that we must match for freeing requests in the cache. Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 2e8cb739c835..9cd7b03a6f34 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8681,12 +8681,13 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) idr_destroy(&ctx->io_buffer_idr); } -static void io_req_cache_free(struct list_head *list) +static void io_req_cache_free(struct list_head *list, struct task_struct *tsk) { - while (!list_empty(list)) { - struct io_kiocb *req; + struct io_kiocb *req, *nxt; - req = list_first_entry(list, struct io_kiocb, compl.list); + list_for_each_entry_safe(req, nxt, list, compl.list) { + if (tsk && req->task != tsk) + continue; list_del(&req->compl.list); kmem_cache_free(req_cachep, req); } @@ -8742,8 +8743,8 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) free_uid(ctx->user); put_cred(ctx->creds); kfree(ctx->cancel_hash); - io_req_cache_free(&ctx->submit_state.comp.free_list); - io_req_cache_free(&ctx->submit_state.comp.locked_free_list); + io_req_cache_free(&ctx->submit_state.comp.free_list, NULL); + io_req_cache_free(&ctx->submit_state.comp.locked_free_list, NULL); kfree(ctx); } -- cgit v1.2.3 From 9a4fdbd8ee0d8aca0cb5692446e5ca583b230cd7 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 13 Feb 2021 09:09:44 -0700 Subject: io_uring: add helper to free all request caches We have three different ones, put it in a helper for easy calling. This is in preparation for doing it outside of ring freeing as well. With that in mind, also ensure that we do the proper locking for safe calling from a context where the ring it still live. Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 9cd7b03a6f34..1895fc132252 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8693,10 +8693,27 @@ static void io_req_cache_free(struct list_head *list, struct task_struct *tsk) } } -static void io_ring_ctx_free(struct io_ring_ctx *ctx) +static void io_req_caches_free(struct io_ring_ctx *ctx, struct task_struct *tsk) { struct io_submit_state *submit_state = &ctx->submit_state; + mutex_lock(&ctx->uring_lock); + + if (submit_state->free_reqs) + kmem_cache_free_bulk(req_cachep, submit_state->free_reqs, + submit_state->reqs); + + io_req_cache_free(&submit_state->comp.free_list, NULL); + + spin_lock_irq(&ctx->completion_lock); + io_req_cache_free(&submit_state->comp.locked_free_list, NULL); + spin_unlock_irq(&ctx->completion_lock); + + mutex_unlock(&ctx->uring_lock); +} + +static void io_ring_ctx_free(struct io_ring_ctx *ctx) +{ /* * Some may use context even when all refs and requests have been put, * and they are free to do so while still holding uring_lock, see @@ -8715,10 +8732,6 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) ctx->mm_account = NULL; } - if (submit_state->free_reqs) - kmem_cache_free_bulk(req_cachep, submit_state->free_reqs, - submit_state->reqs); - #ifdef CONFIG_BLK_CGROUP if (ctx->sqo_blkcg_css) css_put(ctx->sqo_blkcg_css); @@ -8742,9 +8755,8 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) percpu_ref_exit(&ctx->refs); free_uid(ctx->user); put_cred(ctx->creds); + io_req_caches_free(ctx, NULL); kfree(ctx->cancel_hash); - io_req_cache_free(&ctx->submit_state.comp.free_list, NULL); - io_req_cache_free(&ctx->submit_state.comp.locked_free_list, NULL); kfree(ctx); } -- cgit v1.2.3 From 41be53e94fb04cc69fdf2f524c2a05d8069e047b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 13 Feb 2021 09:11:04 -0700 Subject: io_uring: kill cached requests from exiting task closing the ring Be nice and prune these upfront, in case the ring is being shared and one of the tasks is going away. This is a bit more important now that we account the allocations. Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 1895fc132252..a9d094f7060f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9232,8 +9232,10 @@ static int io_uring_flush(struct file *file, void *data) struct io_uring_task *tctx = current->io_uring; struct io_ring_ctx *ctx = file->private_data; - if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) + if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) { io_uring_cancel_task_requests(ctx, NULL); + io_req_caches_free(ctx, current); + } if (!tctx) return 0; -- cgit v1.2.3 From 96c0a6a72d181a330db6dc9848ff2e6584b1aa5b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 10 Feb 2021 21:51:02 +0100 Subject: s390,alpha: switch to 64-bit ino_t s390 and alpha are the only 64 bit architectures with a 32-bit ino_t. Since this is quite unusual this causes bugs from time to time. See e.g. commit ebce3eb2f7ef ("ceph: fix inode number handling on arches with 32-bit ino_t") for an example. This (obviously) also prevents s390 and alpha to use 64-bit ino_t for tmpfs. See commit b85a7a8bb573 ("tmpfs: disallow CONFIG_TMPFS_INODE64 on s390"). Therefore switch both s390 and alpha to 64-bit ino_t. This should only have an effect on the ustat system call. To prevent ABI breakage define struct ustat compatible to the old layout and change sys_ustat() accordingly. Acked-by: Linus Torvalds Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/Kconfig | 4 ++++ arch/alpha/Kconfig | 1 + arch/s390/Kconfig | 1 + fs/statfs.c | 5 ++++- include/linux/types.h | 8 ++++++-- 5 files changed, 16 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/arch/Kconfig b/arch/Kconfig index 24862d15f3a3..383c98e86a70 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -327,6 +327,10 @@ config ARCH_32BIT_OFF_T still support 32-bit off_t. This option is enabled for all such architectures explicitly. +# Selected by 64 bit architectures which have a 32 bit f_tinode in struct ustat +config ARCH_32BIT_USTAT_F_TINODE + bool + config HAVE_ASM_MODVERSIONS bool help diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index 1f51437d5765..96ce6565890e 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -2,6 +2,7 @@ config ALPHA bool default y + select ARCH_32BIT_USTAT_F_TINODE select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO select ARCH_NO_PREEMPT diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 5de9f409e4d0..b68e6952866d 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -58,6 +58,7 @@ config S390 # Note: keep this list sorted alphabetically # imply IMA_SECURE_AND_OR_TRUSTED_BOOT + select ARCH_32BIT_USTAT_F_TINODE select ARCH_BINFMT_ELF_STATE select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_WX diff --git a/fs/statfs.c b/fs/statfs.c index 68cb07788750..0ba34c135593 100644 --- a/fs/statfs.c +++ b/fs/statfs.c @@ -255,7 +255,10 @@ SYSCALL_DEFINE2(ustat, unsigned, dev, struct ustat __user *, ubuf) memset(&tmp,0,sizeof(struct ustat)); tmp.f_tfree = sbuf.f_bfree; - tmp.f_tinode = sbuf.f_ffree; + if (IS_ENABLED(CONFIG_ARCH_32BIT_USTAT_F_TINODE)) + tmp.f_tinode = min_t(u64, sbuf.f_ffree, UINT_MAX); + else + tmp.f_tinode = sbuf.f_ffree; return copy_to_user(ubuf, &tmp, sizeof(struct ustat)) ? -EFAULT : 0; } diff --git a/include/linux/types.h b/include/linux/types.h index a147977602b5..ac825ad90e44 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -14,7 +14,7 @@ typedef u32 __kernel_dev_t; typedef __kernel_fd_set fd_set; typedef __kernel_dev_t dev_t; -typedef __kernel_ino_t ino_t; +typedef __kernel_ulong_t ino_t; typedef __kernel_mode_t mode_t; typedef unsigned short umode_t; typedef u32 nlink_t; @@ -189,7 +189,11 @@ struct hlist_node { struct ustat { __kernel_daddr_t f_tfree; - __kernel_ino_t f_tinode; +#ifdef CONFIG_ARCH_32BIT_USTAT_F_TINODE + unsigned int f_tinode; +#else + unsigned long f_tinode; +#endif char f_fname[6]; char f_fpack[6]; }; -- cgit v1.2.3 From 42119dbe571eb419dae99b81dd20fa42f47464e1 Mon Sep 17 00:00:00 2001 From: Wang ShaoBo Date: Sun, 15 Nov 2020 16:23:43 +0800 Subject: ubifs: Fix error return code in alloc_wbufs() Fix to return PTR_ERR() error code from the error handling case instead fo 0 in function alloc_wbufs(), as done elsewhere in this function. Fixes: 6a98bc4614de ("ubifs: Add authentication nodes to journal") Signed-off-by: Wang ShaoBo Reviewed-by: Sascha Hauer Signed-off-by: Richard Weinberger --- fs/ubifs/super.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c index 138b9426c6c1..ddb2ca636c93 100644 --- a/fs/ubifs/super.c +++ b/fs/ubifs/super.c @@ -838,8 +838,10 @@ static int alloc_wbufs(struct ubifs_info *c) c->jheads[i].wbuf.jhead = i; c->jheads[i].grouped = 1; c->jheads[i].log_hash = ubifs_hash_get_desc(c); - if (IS_ERR(c->jheads[i].log_hash)) + if (IS_ERR(c->jheads[i].log_hash)) { + err = PTR_ERR(c->jheads[i].log_hash); goto out; + } } /* -- cgit v1.2.3 From bd5ae9288d6451bd346a1b4a59d4fe7e62ba29b7 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Wed, 3 Feb 2021 11:42:13 -0500 Subject: nfsd: register pernet ops last, unregister first These pernet operations may depend on stuff set up or torn down in the module init/exit functions. And they may be called at any time in between. So it makes more sense for them to be the last to be registered in the init function, and the first to be unregistered in the exit function. In particular, without this, the drc slab is being destroyed before all the per-net drcs are shut down, resulting in an "Objects remaining in nfsd_drc on __kmem_cache_shutdown()" warning in exit_nfsd. Reported-by: Zhi Li Fixes: 3ba75830ce17 "nfsd4: drc containerization" Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfsctl.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index f6d5d783f4a4..0759e589ab52 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -1522,12 +1522,9 @@ static int __init init_nfsd(void) int retval; printk(KERN_INFO "Installing knfsd (copyright (C) 1996 okir@monad.swb.de).\n"); - retval = register_pernet_subsys(&nfsd_net_ops); - if (retval < 0) - return retval; retval = register_cld_notifier(); if (retval) - goto out_unregister_pernet; + return retval; retval = nfsd4_init_slabs(); if (retval) goto out_unregister_notifier; @@ -1544,9 +1541,14 @@ static int __init init_nfsd(void) goto out_free_lockd; retval = register_filesystem(&nfsd_fs_type); if (retval) + goto out_free_exports; + retval = register_pernet_subsys(&nfsd_net_ops); + if (retval < 0) goto out_free_all; return 0; out_free_all: + unregister_pernet_subsys(&nfsd_net_ops); +out_free_exports: remove_proc_entry("fs/nfs/exports", NULL); remove_proc_entry("fs/nfs", NULL); out_free_lockd: @@ -1559,13 +1561,12 @@ out_free_slabs: nfsd4_free_slabs(); out_unregister_notifier: unregister_cld_notifier(); -out_unregister_pernet: - unregister_pernet_subsys(&nfsd_net_ops); return retval; } static void __exit exit_nfsd(void) { + unregister_pernet_subsys(&nfsd_net_ops); nfsd_drc_slab_free(); remove_proc_entry("fs/nfs/exports", NULL); remove_proc_entry("fs/nfs", NULL); @@ -1575,7 +1576,6 @@ static void __exit exit_nfsd(void) nfsd4_exit_pnfs(); unregister_filesystem(&nfsd_fs_type); unregister_cld_notifier(); - unregister_pernet_subsys(&nfsd_net_ops); } MODULE_AUTHOR("Olaf Kirch "); -- cgit v1.2.3 From 6dffa4c22000595343fd676fd146a1318aab4073 Mon Sep 17 00:00:00 2001 From: Steve French Date: Tue, 2 Feb 2021 00:03:58 -0600 Subject: smb3: negotiate current dialect (SMB3.1.1) when version 3 or greater requested SMB3.1.1 is the newest, and preferred dialect, and is included in the requested dialect list by default (ie if no vers= is specified on mount) but it should also be requested if SMB3 or later is requested (vers=3 instead of a specific dialect: vers=2.1, vers=3.02 or vers=3.0). Currently specifying "vers=3" only requests smb3.0 and smb3.02 but this patch fixes it to also request smb3.1.1 dialect, as it is the newest and most secure dialect and is a "version 3 or later" dialect (the intent of "vers=3"). Signed-off-by: Steve French Suggested-by: Pavel Shilovsky Reviewed-by: Shyam Prasad N Reviewed-by: Pavel Shilovsky Signed-off-by: Steve French --- fs/cifs/fs_context.c | 2 +- fs/cifs/smb2pdu.c | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 12a5da0230b5..7d04f2255624 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -397,7 +397,7 @@ cifs_parse_smb_version(char *value, struct smb3_fs_context *ctx, bool is_smb3) ctx->vals = &smb3any_values; break; case Smb_default: - ctx->ops = &smb30_operations; /* currently identical with 3.0 */ + ctx->ops = &smb30_operations; ctx->vals = &smbdefault_values; break; default: diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index 794fc3b68b4f..e1391bd92768 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -814,8 +814,9 @@ SMB2_negotiate(const unsigned int xid, struct cifs_ses *ses) SMB3ANY_VERSION_STRING) == 0) { req->Dialects[0] = cpu_to_le16(SMB30_PROT_ID); req->Dialects[1] = cpu_to_le16(SMB302_PROT_ID); - req->DialectCount = cpu_to_le16(2); - total_len += 4; + req->Dialects[2] = cpu_to_le16(SMB311_PROT_ID); + req->DialectCount = cpu_to_le16(3); + total_len += 6; } else if (strcmp(server->vals->version_string, SMBDEFAULT_VERSION_STRING) == 0) { req->Dialects[0] = cpu_to_le16(SMB21_PROT_ID); @@ -848,6 +849,8 @@ SMB2_negotiate(const unsigned int xid, struct cifs_ses *ses) memcpy(req->ClientGUID, server->client_guid, SMB2_CLIENT_GUID_SIZE); if ((server->vals->protocol_id == SMB311_PROT_ID) || + (strcmp(server->vals->version_string, + SMB3ANY_VERSION_STRING) == 0) || (strcmp(server->vals->version_string, SMBDEFAULT_VERSION_STRING) == 0)) assemble_neg_contexts(req, server, &total_len); @@ -883,6 +886,10 @@ SMB2_negotiate(const unsigned int xid, struct cifs_ses *ses) cifs_server_dbg(VFS, "SMB2.1 dialect returned but not requested\n"); return -EIO; + } else if (rsp->DialectRevision == cpu_to_le16(SMB311_PROT_ID)) { + /* ops set to 3.0 by default for default so update */ + server->ops = &smb311_operations; + server->vals = &smb311_values; } } else if (strcmp(server->vals->version_string, SMBDEFAULT_VERSION_STRING) == 0) { @@ -1042,10 +1049,11 @@ int smb3_validate_negotiate(const unsigned int xid, struct cifs_tcon *tcon) SMB3ANY_VERSION_STRING) == 0) { pneg_inbuf->Dialects[0] = cpu_to_le16(SMB30_PROT_ID); pneg_inbuf->Dialects[1] = cpu_to_le16(SMB302_PROT_ID); - pneg_inbuf->DialectCount = cpu_to_le16(2); - /* structure is big enough for 3 dialects, sending only 2 */ + pneg_inbuf->Dialects[2] = cpu_to_le16(SMB311_PROT_ID); + pneg_inbuf->DialectCount = cpu_to_le16(3); + /* SMB 2.1 not included so subtract one dialect from len */ inbuflen = sizeof(*pneg_inbuf) - - (2 * sizeof(pneg_inbuf->Dialects[0])); + (sizeof(pneg_inbuf->Dialects[0])); } else if (strcmp(server->vals->version_string, SMBDEFAULT_VERSION_STRING) == 0) { pneg_inbuf->Dialects[0] = cpu_to_le16(SMB21_PROT_ID); @@ -1053,7 +1061,7 @@ int smb3_validate_negotiate(const unsigned int xid, struct cifs_tcon *tcon) pneg_inbuf->Dialects[2] = cpu_to_le16(SMB302_PROT_ID); pneg_inbuf->Dialects[3] = cpu_to_le16(SMB311_PROT_ID); pneg_inbuf->DialectCount = cpu_to_le16(4); - /* structure is big enough for 3 dialects */ + /* structure is big enough for 4 dialects */ inbuflen = sizeof(*pneg_inbuf); } else { /* otherwise specific dialect was requested */ -- cgit v1.2.3 From 2347961b11d4079deace3c81dceed460c08a8fc1 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 28 Jan 2020 14:25:39 +0100 Subject: binfmt_misc: pass binfmt_misc flags to the interpreter It can be useful to the interpreter to know which flags are in use. For instance, knowing if the preserve-argv[0] is in use would allow to skip the pathname argument. This patch uses an unused auxiliary vector, AT_FLAGS, to add a flag to inform interpreter if the preserve-argv[0] is enabled. Note by Helge Deller: The real-world user of this patch is qemu-user, which needs to know if it has to preserve the argv[0]. See Debian bug #970460. Signed-off-by: Laurent Vivier Reviewed-by: YunQiang Su URL: http://bugs.debian.org/970460 Signed-off-by: Helge Deller --- fs/binfmt_elf.c | 5 ++++- fs/binfmt_elf_fdpic.c | 5 ++++- fs/binfmt_misc.c | 4 +++- include/linux/binfmts.h | 4 ++++ include/uapi/linux/binfmts.h | 4 ++++ 5 files changed, 19 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 950bc177238a..11fb68e68f65 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -186,6 +186,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, unsigned char k_rand_bytes[16]; int items; elf_addr_t *elf_info; + elf_addr_t flags = 0; int ei_index; const struct cred *cred = current_cred(); struct vm_area_struct *vma; @@ -260,7 +261,9 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); NEW_AUX_ENT(AT_PHNUM, exec->e_phnum); NEW_AUX_ENT(AT_BASE, interp_load_addr); - NEW_AUX_ENT(AT_FLAGS, 0); + if (bprm->interp_flags & BINPRM_FLAGS_PRESERVE_ARGV0) + flags |= AT_FLAGS_PRESERVE_ARGV0; + NEW_AUX_ENT(AT_FLAGS, flags); NEW_AUX_ENT(AT_ENTRY, e_entry); NEW_AUX_ENT(AT_UID, from_kuid_munged(cred->user_ns, cred->uid)); NEW_AUX_ENT(AT_EUID, from_kuid_munged(cred->user_ns, cred->euid)); diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index be4062b8ba75..7a78c91bc1c1 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -506,6 +506,7 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm, char __user *u_platform, *u_base_platform, *p; int loop; int nr; /* reset for each csp adjustment */ + unsigned long flags = 0; #ifdef CONFIG_MMU /* In some cases (e.g. Hyper-Threading), we want to avoid L1 evictions @@ -648,7 +649,9 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm, NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); NEW_AUX_ENT(AT_PHNUM, exec_params->hdr.e_phnum); NEW_AUX_ENT(AT_BASE, interp_params->elfhdr_addr); - NEW_AUX_ENT(AT_FLAGS, 0); + if (bprm->interp_flags & BINPRM_FLAGS_PRESERVE_ARGV0) + flags |= AT_FLAGS_PRESERVE_ARGV0; + NEW_AUX_ENT(AT_FLAGS, flags); NEW_AUX_ENT(AT_ENTRY, exec_params->entry_addr); NEW_AUX_ENT(AT_UID, (elf_addr_t) from_kuid_munged(cred->user_ns, cred->uid)); NEW_AUX_ENT(AT_EUID, (elf_addr_t) from_kuid_munged(cred->user_ns, cred->euid)); diff --git a/fs/binfmt_misc.c b/fs/binfmt_misc.c index 3880a82da1dc..c457334de43f 100644 --- a/fs/binfmt_misc.c +++ b/fs/binfmt_misc.c @@ -153,7 +153,9 @@ static int load_misc_binary(struct linux_binprm *bprm) if (bprm->interp_flags & BINPRM_FLAGS_PATH_INACCESSIBLE) goto ret; - if (!(fmt->flags & MISC_FMT_PRESERVE_ARGV0)) { + if (fmt->flags & MISC_FMT_PRESERVE_ARGV0) { + bprm->interp_flags |= BINPRM_FLAGS_PRESERVE_ARGV0; + } else { retval = remove_arg_zero(bprm); if (retval) goto ret; diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index 0571701ab1c5..0abd93efc181 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -73,6 +73,10 @@ struct linux_binprm { #define BINPRM_FLAGS_PATH_INACCESSIBLE_BIT 2 #define BINPRM_FLAGS_PATH_INACCESSIBLE (1 << BINPRM_FLAGS_PATH_INACCESSIBLE_BIT) +/* preserve argv0 for the interpreter */ +#define BINPRM_FLAGS_PRESERVE_ARGV0_BIT 3 +#define BINPRM_FLAGS_PRESERVE_ARGV0 (1 << BINPRM_FLAGS_PRESERVE_ARGV0_BIT) + /* Function parameter for binfmt->coredump */ struct coredump_params { const kernel_siginfo_t *siginfo; diff --git a/include/uapi/linux/binfmts.h b/include/uapi/linux/binfmts.h index 689025d9c185..c6f9450efc12 100644 --- a/include/uapi/linux/binfmts.h +++ b/include/uapi/linux/binfmts.h @@ -18,4 +18,8 @@ struct pt_regs; /* sizeof(linux_binprm->buf) */ #define BINPRM_BUF_SIZE 256 +/* preserve argv0 for the interpreter */ +#define AT_FLAGS_PRESERVE_ARGV0_BIT 0 +#define AT_FLAGS_PRESERVE_ARGV0 (1 << AT_FLAGS_PRESERVE_ARGV0_BIT) + #endif /* _UAPI_LINUX_BINFMTS_H */ -- cgit v1.2.3 From 0d4370cfe36b7f1719123b621a4ec4d9c7a25f89 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 14 Feb 2021 13:21:43 -0700 Subject: proc: don't allow async path resolution of /proc/thread-self components If this is attempted by an io-wq kthread, then return -EOPNOTSUPP as we don't currently support that. Once we can get task_pid_ptr() doing the right thing, then this can go away again. Use PF_IO_WORKER for this to speciically target the io_uring workers. Modify the /proc/self/ check to use PF_IO_WORKER as well. Cc: stable@vger.kernel.org Fixes: 8d4c3e76e3be ("proc: don't allow async path resolution of /proc/self components") Reported-by: Eric W. Biederman Signed-off-by: Jens Axboe --- fs/proc/self.c | 2 +- fs/proc/thread_self.c | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/proc/self.c b/fs/proc/self.c index cc71ce3466dc..a4012154e109 100644 --- a/fs/proc/self.c +++ b/fs/proc/self.c @@ -20,7 +20,7 @@ static const char *proc_self_get_link(struct dentry *dentry, * Not currently supported. Once we can inherit all of struct pid, * we can allow this. */ - if (current->flags & PF_KTHREAD) + if (current->flags & PF_IO_WORKER) return ERR_PTR(-EOPNOTSUPP); if (!tgid) diff --git a/fs/proc/thread_self.c b/fs/proc/thread_self.c index a553273fbd41..d56681d86d28 100644 --- a/fs/proc/thread_self.c +++ b/fs/proc/thread_self.c @@ -17,6 +17,13 @@ static const char *proc_thread_self_get_link(struct dentry *dentry, pid_t pid = task_pid_nr_ns(current, ns); char *name; + /* + * Not currently supported. Once we can inherit all of struct pid, + * we can allow this. + */ + if (current->flags & PF_IO_WORKER) + return ERR_PTR(-EOPNOTSUPP); + if (!pid) return ERR_PTR(-ENOENT); name = kmalloc(10 + 6 + 10 + 1, dentry ? GFP_KERNEL : GFP_ATOMIC); -- cgit v1.2.3 From 62ab1aadcccd037a7ced4ed99b4d46d2b4190183 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Wed, 27 Jan 2021 05:21:15 +0900 Subject: zonefs: add tracepoints for file operations Add tracepoints for file I/O operations to aid in debugging of I/O errors with zonefs. The added tracepoints are in: - zonefs_zone_mgmt() for tracing zone management operations - zonefs_iomap_begin() for tracing regular file I/O - zonefs_file_dio_append() for tracing zone-append operations Signed-off-by: Johannes Thumshirn Signed-off-by: Damien Le Moal --- fs/zonefs/Makefile | 2 ++ fs/zonefs/super.c | 7 ++++ fs/zonefs/trace.h | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 fs/zonefs/trace.h (limited to 'fs') diff --git a/fs/zonefs/Makefile b/fs/zonefs/Makefile index 75a380aa1ae1..33c1a4f1132e 100644 --- a/fs/zonefs/Makefile +++ b/fs/zonefs/Makefile @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +ccflags-y += -I$(src) + obj-$(CONFIG_ZONEFS_FS) += zonefs.o zonefs-y := super.o diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index bec47f2d074b..96f0cb0c29aa 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -24,6 +24,9 @@ #include "zonefs.h" +#define CREATE_TRACE_POINTS +#include "trace.h" + static inline int zonefs_zone_mgmt(struct inode *inode, enum req_opf op) { @@ -32,6 +35,7 @@ static inline int zonefs_zone_mgmt(struct inode *inode, lockdep_assert_held(&zi->i_truncate_mutex); + trace_zonefs_zone_mgmt(inode, op); ret = blkdev_zone_mgmt(inode->i_sb->s_bdev, op, zi->i_zsector, zi->i_zone_size >> SECTOR_SHIFT, GFP_NOFS); if (ret) { @@ -100,6 +104,8 @@ static int zonefs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, iomap->bdev = inode->i_sb->s_bdev; iomap->addr = (zi->i_zsector << SECTOR_SHIFT) + iomap->offset; + trace_zonefs_iomap_begin(inode, iomap); + return 0; } @@ -703,6 +709,7 @@ static ssize_t zonefs_file_dio_append(struct kiocb *iocb, struct iov_iter *from) ret = submit_bio_wait(bio); zonefs_file_write_dio_end_io(iocb, size, ret, 0); + trace_zonefs_file_dio_append(inode, size, ret); out_release: bio_release_pages(bio, false); diff --git a/fs/zonefs/trace.h b/fs/zonefs/trace.h new file mode 100644 index 000000000000..f369d7d50303 --- /dev/null +++ b/fs/zonefs/trace.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * zonefs filesystem driver tracepoints. + * + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM zonefs + +#if !defined(_TRACE_ZONEFS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_ZONEFS_H + +#include +#include +#include + +#include "zonefs.h" + +#define show_dev(dev) MAJOR(dev), MINOR(dev) + +TRACE_EVENT(zonefs_zone_mgmt, + TP_PROTO(struct inode *inode, enum req_opf op), + TP_ARGS(inode, op), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(int, op) + __field(sector_t, sector) + __field(sector_t, nr_sectors) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->op = op; + __entry->sector = ZONEFS_I(inode)->i_zsector; + __entry->nr_sectors = + ZONEFS_I(inode)->i_zone_size >> SECTOR_SHIFT; + ), + TP_printk("bdev=(%d,%d), ino=%lu op=%s, sector=%llu, nr_sectors=%llu", + show_dev(__entry->dev), (unsigned long)__entry->ino, + blk_op_str(__entry->op), __entry->sector, + __entry->nr_sectors + ) +); + +TRACE_EVENT(zonefs_file_dio_append, + TP_PROTO(struct inode *inode, ssize_t size, ssize_t ret), + TP_ARGS(inode, size, ret), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(sector_t, sector) + __field(ssize_t, size) + __field(loff_t, wpoffset) + __field(ssize_t, ret) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->sector = ZONEFS_I(inode)->i_zsector; + __entry->size = size; + __entry->wpoffset = ZONEFS_I(inode)->i_wpoffset; + __entry->ret = ret; + ), + TP_printk("bdev=(%d, %d), ino=%lu, sector=%llu, size=%zu, wpoffset=%llu, ret=%zu", + show_dev(__entry->dev), (unsigned long)__entry->ino, + __entry->sector, __entry->size, __entry->wpoffset, + __entry->ret + ) +); + +TRACE_EVENT(zonefs_iomap_begin, + TP_PROTO(struct inode *inode, struct iomap *iomap), + TP_ARGS(inode, iomap), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(u64, addr) + __field(loff_t, offset) + __field(u64, length) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->addr = iomap->addr; + __entry->offset = iomap->offset; + __entry->length = iomap->length; + ), + TP_printk("bdev=(%d,%d), ino=%lu, addr=%llu, offset=%llu, length=%llu", + show_dev(__entry->dev), (unsigned long)__entry->ino, + __entry->addr, __entry->offset, __entry->length + ) +); + +#endif /* _TRACE_ZONEFS_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +/* This part must be outside protection */ +#include -- cgit v1.2.3 From bfe3911a91047557eb0e620f95a370aee6a248c7 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 5 Feb 2021 22:00:12 +0000 Subject: kcmp: Support selection of SYS_kcmp without CHECKPOINT_RESTORE Userspace has discovered the functionality offered by SYS_kcmp and has started to depend upon it. In particular, Mesa uses SYS_kcmp for os_same_file_description() in order to identify when two fd (e.g. device or dmabuf) point to the same struct file. Since they depend on it for core functionality, lift SYS_kcmp out of the non-default CONFIG_CHECKPOINT_RESTORE into the selectable syscall category. Rasmus Villemoes also pointed out that systemd uses SYS_kcmp to deduplicate the per-service file descriptor store. Note that some distributions such as Ubuntu are already enabling CHECKPOINT_RESTORE in their configs and so, by extension, SYS_kcmp. References: https://gitlab.freedesktop.org/drm/intel/-/issues/3046 Signed-off-by: Chris Wilson Cc: Kees Cook Cc: Andy Lutomirski Cc: Will Drewry Cc: Andrew Morton Cc: Dave Airlie Cc: Daniel Vetter Cc: Lucas Stach Cc: Rasmus Villemoes Cc: Cyrill Gorcunov Cc: stable@vger.kernel.org Acked-by: Daniel Vetter # DRM depends on kcmp Acked-by: Rasmus Villemoes # systemd uses kcmp Reviewed-by: Cyrill Gorcunov Reviewed-by: Kees Cook Acked-by: Thomas Zimmermann Signed-off-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20210205220012.1983-1-chris@chris-wilson.co.uk --- drivers/gpu/drm/Kconfig | 3 +++ fs/eventpoll.c | 4 ++-- include/linux/eventpoll.h | 2 +- init/Kconfig | 11 +++++++++++ kernel/Makefile | 2 +- tools/testing/selftests/seccomp/seccomp_bpf.c | 2 +- 6 files changed, 19 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 0973f408d75f..af6c6d214d91 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -15,6 +15,9 @@ menuconfig DRM select I2C_ALGOBIT select DMA_SHARED_BUFFER select SYNC_FILE +# gallium uses SYS_kcmp for os_same_file_description() to de-duplicate +# device and dmabuf fd. Let's make sure that is available for our userspace. + select KCMP help Kernel-level support for the Direct Rendering Infrastructure (DRI) introduced in XFree86 4.0. If you say Y here, you need to select diff --git a/fs/eventpoll.c b/fs/eventpoll.c index a829af074eb5..3196474cbe24 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -979,7 +979,7 @@ static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd) return epir; } -#ifdef CONFIG_CHECKPOINT_RESTORE +#ifdef CONFIG_KCMP static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff) { struct rb_node *rbp; @@ -1021,7 +1021,7 @@ struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd, return file_raw; } -#endif /* CONFIG_CHECKPOINT_RESTORE */ +#endif /* CONFIG_KCMP */ /** * Adds a new entry to the tail of the list in a lockless way, i.e. diff --git a/include/linux/eventpoll.h b/include/linux/eventpoll.h index 0350393465d4..593322c946e6 100644 --- a/include/linux/eventpoll.h +++ b/include/linux/eventpoll.h @@ -18,7 +18,7 @@ struct file; #ifdef CONFIG_EPOLL -#ifdef CONFIG_CHECKPOINT_RESTORE +#ifdef CONFIG_KCMP struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd, unsigned long toff); #endif diff --git a/init/Kconfig b/init/Kconfig index 29ad68325028..b7d3c6a12196 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1193,6 +1193,7 @@ endif # NAMESPACES config CHECKPOINT_RESTORE bool "Checkpoint/restore support" select PROC_CHILDREN + select KCMP default n help Enables additional kernel features in a sake of checkpoint/restore. @@ -1736,6 +1737,16 @@ config ARCH_HAS_MEMBARRIER_CALLBACKS config ARCH_HAS_MEMBARRIER_SYNC_CORE bool +config KCMP + bool "Enable kcmp() system call" if EXPERT + help + Enable the kernel resource comparison system call. It provides + user-space with the ability to compare two processes to see if they + share a common resource, such as a file descriptor or even virtual + memory space. + + If unsure, say N. + config RSEQ bool "Enable rseq() system call" if EXPERT default y diff --git a/kernel/Makefile b/kernel/Makefile index aa7368c7eabf..320f1f3941b7 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -51,7 +51,7 @@ obj-y += livepatch/ obj-y += dma/ obj-y += entry/ -obj-$(CONFIG_CHECKPOINT_RESTORE) += kcmp.o +obj-$(CONFIG_KCMP) += kcmp.o obj-$(CONFIG_FREEZER) += freezer.o obj-$(CONFIG_PROFILING) += profile.o obj-$(CONFIG_STACKTRACE) += stacktrace.o diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index 26c72f2b61b1..1b6c7d33c4ff 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -315,7 +315,7 @@ TEST(kcmp) ret = __filecmp(getpid(), getpid(), 1, 1); EXPECT_EQ(ret, 0); if (ret != 0 && errno == ENOSYS) - SKIP(return, "Kernel does not support kcmp() (missing CONFIG_CHECKPOINT_RESTORE?)"); + SKIP(return, "Kernel does not support kcmp() (missing CONFIG_KCMP?)"); } TEST(mode_strict_support) -- cgit v1.2.3 From 64f36da5625f7f9853b86750eaa89d499d16a2e9 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Thu, 10 Dec 2020 13:35:46 -0500 Subject: ceph: fix flush_snap logic after putting caps A primary reason for skipping ceph_check_caps after putting the references was to avoid the locking in ceph_check_caps during a reconnect. __ceph_put_cap_refs can still call ceph_flush_snaps in that case though, and that takes many of the same inconvenient locks. Fix the logic in __ceph_put_cap_refs to skip flushing snaps when the skip_checking_caps flag is set. Fixes: e64f44a88465 ("ceph: skip checking caps when session reconnecting and releasing reqs") Signed-off-by: Jeff Layton Reviewed-by: Ilya Dryomov Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 255a512f1277..638d18c198ea 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -3093,10 +3093,12 @@ static void __ceph_put_cap_refs(struct ceph_inode_info *ci, int had, dout("put_cap_refs %p had %s%s%s\n", inode, ceph_cap_string(had), last ? " last" : "", put ? " put" : ""); - if (last && !skip_checking_caps) - ceph_check_caps(ci, 0, NULL); - else if (flushsnaps) - ceph_flush_snaps(ci, NULL); + if (!skip_checking_caps) { + if (last) + ceph_check_caps(ci, 0, NULL); + else if (flushsnaps) + ceph_flush_snaps(ci, NULL); + } if (wake) wake_up_all(&ci->i_cap_wq); while (put-- > 0) -- cgit v1.2.3 From 64f28c627a27abb053561275bf94fbcc78e66198 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Fri, 9 Oct 2020 14:24:34 -0400 Subject: ceph: clean up inode work queueing Add a generic function for taking an inode reference, setting the I_WORK bit and queueing i_work. Turn the ceph_queue_* functions into static inline wrappers that pass in the right bit. Signed-off-by: Jeff Layton Reviewed-by: Ilya Dryomov Signed-off-by: Ilya Dryomov --- fs/ceph/inode.c | 55 ++++++------------------------------------------------- fs/ceph/super.h | 21 ++++++++++++++++++--- 2 files changed, 24 insertions(+), 52 deletions(-) (limited to 'fs') diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index adc8fc3c5d85..e64ccd93350f 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -1816,60 +1816,17 @@ void ceph_async_iput(struct inode *inode) } } -/* - * Write back inode data in a worker thread. (This can't be done - * in the message handler context.) - */ -void ceph_queue_writeback(struct inode *inode) -{ - struct ceph_inode_info *ci = ceph_inode(inode); - set_bit(CEPH_I_WORK_WRITEBACK, &ci->i_work_mask); - - ihold(inode); - if (queue_work(ceph_inode_to_client(inode)->inode_wq, - &ci->i_work)) { - dout("ceph_queue_writeback %p\n", inode); - } else { - dout("ceph_queue_writeback %p already queued, mask=%lx\n", - inode, ci->i_work_mask); - iput(inode); - } -} - -/* - * queue an async invalidation - */ -void ceph_queue_invalidate(struct inode *inode) -{ - struct ceph_inode_info *ci = ceph_inode(inode); - set_bit(CEPH_I_WORK_INVALIDATE_PAGES, &ci->i_work_mask); - - ihold(inode); - if (queue_work(ceph_inode_to_client(inode)->inode_wq, - &ceph_inode(inode)->i_work)) { - dout("ceph_queue_invalidate %p\n", inode); - } else { - dout("ceph_queue_invalidate %p already queued, mask=%lx\n", - inode, ci->i_work_mask); - iput(inode); - } -} - -/* - * Queue an async vmtruncate. If we fail to queue work, we will handle - * the truncation the next time we call __ceph_do_pending_vmtruncate. - */ -void ceph_queue_vmtruncate(struct inode *inode) +void ceph_queue_inode_work(struct inode *inode, int work_bit) { + struct ceph_fs_client *fsc = ceph_inode_to_client(inode); struct ceph_inode_info *ci = ceph_inode(inode); - set_bit(CEPH_I_WORK_VMTRUNCATE, &ci->i_work_mask); + set_bit(work_bit, &ci->i_work_mask); ihold(inode); - if (queue_work(ceph_inode_to_client(inode)->inode_wq, - &ci->i_work)) { - dout("ceph_queue_vmtruncate %p\n", inode); + if (queue_work(fsc->inode_wq, &ci->i_work)) { + dout("queue_inode_work %p, mask=%lx\n", inode, ci->i_work_mask); } else { - dout("ceph_queue_vmtruncate %p already queued, mask=%lx\n", + dout("queue_inode_work %p already queued, mask=%lx\n", inode, ci->i_work_mask); iput(inode); } diff --git a/fs/ceph/super.h b/fs/ceph/super.h index b62d8fee3b86..59153ee201c0 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -962,11 +962,26 @@ extern int ceph_inode_holds_cap(struct inode *inode, int mask); extern bool ceph_inode_set_size(struct inode *inode, loff_t size); extern void __ceph_do_pending_vmtruncate(struct inode *inode); -extern void ceph_queue_vmtruncate(struct inode *inode); -extern void ceph_queue_invalidate(struct inode *inode); -extern void ceph_queue_writeback(struct inode *inode); + extern void ceph_async_iput(struct inode *inode); +void ceph_queue_inode_work(struct inode *inode, int work_bit); + +static inline void ceph_queue_vmtruncate(struct inode *inode) +{ + ceph_queue_inode_work(inode, CEPH_I_WORK_VMTRUNCATE); +} + +static inline void ceph_queue_invalidate(struct inode *inode) +{ + ceph_queue_inode_work(inode, CEPH_I_WORK_INVALIDATE_PAGES); +} + +static inline void ceph_queue_writeback(struct inode *inode) +{ + ceph_queue_inode_work(inode, CEPH_I_WORK_WRITEBACK); +} + extern int __ceph_do_getattr(struct inode *inode, struct page *locked_page, int mask, bool force); static inline int ceph_do_getattr(struct inode *inode, int mask, bool force) -- cgit v1.2.3 From a8810cdc007f816e0e2448879ebd84152ce8c907 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Thu, 10 Dec 2020 14:39:26 -0500 Subject: ceph: allow queueing cap/snap handling after putting cap references Testing with the fscache overhaul has triggered some lockdep warnings about circular lock dependencies involving page_mkwrite and the mmap_lock. It'd be better to do the "real work" without the mmap lock being held. Change the skip_checking_caps parameter in __ceph_put_cap_refs to an enum, and use that to determine whether to queue check_caps, do it synchronously or not at all. Change ceph_page_mkwrite to do a ceph_put_cap_refs_async(). Signed-off-by: Jeff Layton Reviewed-by: Ilya Dryomov Signed-off-by: Ilya Dryomov --- fs/ceph/addr.c | 2 +- fs/ceph/caps.c | 29 +++++++++++++++++++++++++---- fs/ceph/inode.c | 6 ++++++ fs/ceph/super.h | 19 ++++++++++++++++--- 4 files changed, 48 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 950552944436..26e66436f005 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1662,7 +1662,7 @@ static vm_fault_t ceph_page_mkwrite(struct vm_fault *vmf) dout("page_mkwrite %p %llu~%zd dropping cap refs on %s ret %x\n", inode, off, len, ceph_cap_string(got), ret); - ceph_put_cap_refs(ci, got); + ceph_put_cap_refs_async(ci, got); out_free: ceph_restore_sigs(&oldset); sb_end_pagefault(inode->i_sb); diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 638d18c198ea..abbf48fc6230 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -3027,6 +3027,12 @@ static int ceph_try_drop_cap_snap(struct ceph_inode_info *ci, return 0; } +enum put_cap_refs_mode { + PUT_CAP_REFS_SYNC = 0, + PUT_CAP_REFS_NO_CHECK, + PUT_CAP_REFS_ASYNC, +}; + /* * Release cap refs. * @@ -3037,7 +3043,7 @@ static int ceph_try_drop_cap_snap(struct ceph_inode_info *ci, * cap_snap, and wake up any waiters. */ static void __ceph_put_cap_refs(struct ceph_inode_info *ci, int had, - bool skip_checking_caps) + enum put_cap_refs_mode mode) { struct inode *inode = &ci->vfs_inode; int last = 0, put = 0, flushsnaps = 0, wake = 0; @@ -3093,11 +3099,21 @@ static void __ceph_put_cap_refs(struct ceph_inode_info *ci, int had, dout("put_cap_refs %p had %s%s%s\n", inode, ceph_cap_string(had), last ? " last" : "", put ? " put" : ""); - if (!skip_checking_caps) { + switch (mode) { + case PUT_CAP_REFS_SYNC: if (last) ceph_check_caps(ci, 0, NULL); else if (flushsnaps) ceph_flush_snaps(ci, NULL); + break; + case PUT_CAP_REFS_ASYNC: + if (last) + ceph_queue_check_caps(inode); + else if (flushsnaps) + ceph_queue_flush_snaps(inode); + break; + default: + break; } if (wake) wake_up_all(&ci->i_cap_wq); @@ -3107,12 +3123,17 @@ static void __ceph_put_cap_refs(struct ceph_inode_info *ci, int had, void ceph_put_cap_refs(struct ceph_inode_info *ci, int had) { - __ceph_put_cap_refs(ci, had, false); + __ceph_put_cap_refs(ci, had, PUT_CAP_REFS_SYNC); +} + +void ceph_put_cap_refs_async(struct ceph_inode_info *ci, int had) +{ + __ceph_put_cap_refs(ci, had, PUT_CAP_REFS_ASYNC); } void ceph_put_cap_refs_no_check_caps(struct ceph_inode_info *ci, int had) { - __ceph_put_cap_refs(ci, had, true); + __ceph_put_cap_refs(ci, had, PUT_CAP_REFS_NO_CHECK); } /* diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index e64ccd93350f..5d20a620e96c 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -1965,6 +1965,12 @@ static void ceph_inode_work(struct work_struct *work) if (test_and_clear_bit(CEPH_I_WORK_VMTRUNCATE, &ci->i_work_mask)) __ceph_do_pending_vmtruncate(inode); + if (test_and_clear_bit(CEPH_I_WORK_CHECK_CAPS, &ci->i_work_mask)) + ceph_check_caps(ci, 0, NULL); + + if (test_and_clear_bit(CEPH_I_WORK_FLUSH_SNAPS, &ci->i_work_mask)) + ceph_flush_snaps(ci, NULL); + iput(inode); } diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 59153ee201c0..13b02887b085 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -562,9 +562,11 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, /* * Masks of ceph inode work. */ -#define CEPH_I_WORK_WRITEBACK 0 /* writeback */ -#define CEPH_I_WORK_INVALIDATE_PAGES 1 /* invalidate pages */ -#define CEPH_I_WORK_VMTRUNCATE 2 /* vmtruncate */ +#define CEPH_I_WORK_WRITEBACK 0 +#define CEPH_I_WORK_INVALIDATE_PAGES 1 +#define CEPH_I_WORK_VMTRUNCATE 2 +#define CEPH_I_WORK_CHECK_CAPS 3 +#define CEPH_I_WORK_FLUSH_SNAPS 4 /* * We set the ERROR_WRITE bit when we start seeing write errors on an inode @@ -982,6 +984,16 @@ static inline void ceph_queue_writeback(struct inode *inode) ceph_queue_inode_work(inode, CEPH_I_WORK_WRITEBACK); } +static inline void ceph_queue_check_caps(struct inode *inode) +{ + ceph_queue_inode_work(inode, CEPH_I_WORK_CHECK_CAPS); +} + +static inline void ceph_queue_flush_snaps(struct inode *inode) +{ + ceph_queue_inode_work(inode, CEPH_I_WORK_FLUSH_SNAPS); +} + extern int __ceph_do_getattr(struct inode *inode, struct page *locked_page, int mask, bool force); static inline int ceph_do_getattr(struct inode *inode, int mask, bool force) @@ -1120,6 +1132,7 @@ extern void ceph_take_cap_refs(struct ceph_inode_info *ci, int caps, bool snap_rwsem_locked); extern void ceph_get_cap_refs(struct ceph_inode_info *ci, int caps); extern void ceph_put_cap_refs(struct ceph_inode_info *ci, int had); +extern void ceph_put_cap_refs_async(struct ceph_inode_info *ci, int had); extern void ceph_put_cap_refs_no_check_caps(struct ceph_inode_info *ci, int had); extern void ceph_put_wrbuffer_cap_refs(struct ceph_inode_info *ci, int nr, -- cgit v1.2.3 From 558b4510f622a3d96cf9d95050a04e7793d343c7 Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Tue, 2 Feb 2021 14:54:53 +0800 Subject: ceph: defer flushing the capsnap if the Fb is used If the Fb cap is used it means the current inode is flushing the dirty data to OSD, just defer flushing the capsnap. URL: https://tracker.ceph.com/issues/48640 Signed-off-by: Xiubo Li Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- fs/ceph/caps.c | 33 ++++++++++++++++++++------------- fs/ceph/snap.c | 10 ++++++++++ 2 files changed, 30 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index abbf48fc6230..570731c4d019 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -3047,6 +3047,7 @@ static void __ceph_put_cap_refs(struct ceph_inode_info *ci, int had, { struct inode *inode = &ci->vfs_inode; int last = 0, put = 0, flushsnaps = 0, wake = 0; + bool check_flushsnaps = false; spin_lock(&ci->i_ceph_lock); if (had & CEPH_CAP_PIN) @@ -3063,26 +3064,17 @@ static void __ceph_put_cap_refs(struct ceph_inode_info *ci, int had, if (had & CEPH_CAP_FILE_BUFFER) { if (--ci->i_wb_ref == 0) { last++; + /* put the ref held by ceph_take_cap_refs() */ put++; + check_flushsnaps = true; } dout("put_cap_refs %p wb %d -> %d (?)\n", inode, ci->i_wb_ref+1, ci->i_wb_ref); } - if (had & CEPH_CAP_FILE_WR) + if (had & CEPH_CAP_FILE_WR) { if (--ci->i_wr_ref == 0) { last++; - if (__ceph_have_pending_cap_snap(ci)) { - struct ceph_cap_snap *capsnap = - list_last_entry(&ci->i_cap_snaps, - struct ceph_cap_snap, - ci_item); - capsnap->writing = 0; - if (ceph_try_drop_cap_snap(ci, capsnap)) - put++; - else if (__ceph_finish_cap_snap(ci, capsnap)) - flushsnaps = 1; - wake = 1; - } + check_flushsnaps = true; if (ci->i_wrbuffer_ref_head == 0 && ci->i_dirty_caps == 0 && ci->i_flushing_caps == 0) { @@ -3094,6 +3086,21 @@ static void __ceph_put_cap_refs(struct ceph_inode_info *ci, int had, if (!__ceph_is_any_real_caps(ci) && ci->i_snap_realm) drop_inode_snap_realm(ci); } + } + if (check_flushsnaps && __ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + + capsnap->writing = 0; + if (ceph_try_drop_cap_snap(ci, capsnap)) + /* put the ref held by ceph_queue_cap_snap() */ + put++; + else if (__ceph_finish_cap_snap(ci, capsnap)) + flushsnaps = 1; + wake = 1; + } spin_unlock(&ci->i_ceph_lock); dout("put_cap_refs %p had %s%s%s\n", inode, ceph_cap_string(had), diff --git a/fs/ceph/snap.c b/fs/ceph/snap.c index b611f829cb61..0728b01d4d43 100644 --- a/fs/ceph/snap.c +++ b/fs/ceph/snap.c @@ -623,6 +623,16 @@ int __ceph_finish_cap_snap(struct ceph_inode_info *ci, return 0; } + /* Fb cap still in use, delay it */ + if (ci->i_wb_ref) { + dout("finish_cap_snap %p cap_snap %p snapc %p %llu %s s=%llu " + "used WRBUFFER, delaying\n", inode, capsnap, + capsnap->context, capsnap->context->seq, + ceph_cap_string(capsnap->dirty), capsnap->size); + capsnap->writing = 1; + return 0; + } + ci->i_ceph_flags |= CEPH_I_FLUSH_SNAPS; dout("finish_cap_snap %p cap_snap %p snapc %p %llu %s s=%llu\n", inode, capsnap, capsnap->context, -- cgit v1.2.3 From 0b81e80c813f92520667c872d499a2dba8377be6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 16 Feb 2021 10:33:53 -0700 Subject: io_uring: tctx->task_lock should be IRQ safe We add task_work from any context, hence we need to ensure that we can tolerate it being from IRQ context as well. Fixes: 7cbf1722d5fc ("io_uring: provide FIFO ordering for task_work") Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index a9d094f7060f..58dd10481106 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2186,10 +2186,10 @@ static bool __tctx_task_work(struct io_uring_task *tctx) if (wq_list_empty(&tctx->task_list)) return false; - spin_lock(&tctx->task_lock); + spin_lock_irq(&tctx->task_lock); list = tctx->task_list; INIT_WQ_LIST(&tctx->task_list); - spin_unlock(&tctx->task_lock); + spin_unlock_irq(&tctx->task_lock); node = list.first; while (node) { @@ -2236,13 +2236,14 @@ static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req, { struct io_uring_task *tctx = tsk->io_uring; struct io_wq_work_node *node, *prev; + unsigned long flags; int ret; WARN_ON_ONCE(!tctx); - spin_lock(&tctx->task_lock); + spin_lock_irqsave(&tctx->task_lock, flags); wq_list_add_tail(&req->io_task_work.node, &tctx->task_list); - spin_unlock(&tctx->task_lock); + spin_unlock_irqrestore(&tctx->task_lock, flags); /* task_work already pending, we're done */ if (test_bit(0, &tctx->task_state) || @@ -2257,7 +2258,7 @@ static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req, * in the list, it got run and we're fine. */ ret = 0; - spin_lock(&tctx->task_lock); + spin_lock_irqsave(&tctx->task_lock, flags); wq_list_for_each(node, prev, &tctx->task_list) { if (&req->io_task_work.node == node) { wq_list_del(&tctx->task_list, node, prev); @@ -2265,7 +2266,7 @@ static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req, break; } } - spin_unlock(&tctx->task_lock); + spin_unlock_irqrestore(&tctx->task_lock, flags); clear_bit(0, &tctx->task_state); return ret; } -- cgit v1.2.3 From 201023c5b294d68bd370b9d81761ddfdb6cbcc86 Mon Sep 17 00:00:00 2001 From: Steve French Date: Mon, 15 Feb 2021 11:03:45 -0600 Subject: cifs: fix trivial typo Typo: exiting --> existing Signed-off-by: Steve French --- fs/cifs/connect.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 4bb9decbbf27..61418a1c7817 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -2911,7 +2911,7 @@ static int mount_setup_tlink(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses, #ifdef CONFIG_CIFS_DFS_UPCALL /* * cifs_build_path_to_root returns full path to root when we do not have an - * exiting connection (tcon) + * existing connection (tcon) */ static char * build_unc_path_to_root(const struct smb3_fs_context *ctx, -- cgit v1.2.3 From ed7bcdb374d20fab9e9dc36853a6735c047ad1b1 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 12 Feb 2021 16:49:48 -0500 Subject: NFS: Add support for eager writes Support eager writing to the server, meaning that we write the data to cache on the server, and wait for that to complete. This ensures that we see ENOSPC errors immediately. Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/file.c | 19 +++++++++++++++++-- fs/nfs/write.c | 17 ++++++++++++----- include/linux/nfs_fs_sb.h | 2 ++ 3 files changed, 31 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/nfs/file.c b/fs/nfs/file.c index 03fd1dcc96bd..16ad5050e046 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -606,8 +606,8 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); - unsigned long written = 0; - ssize_t result; + unsigned int mntflags = NFS_SERVER(inode)->flags; + ssize_t result, written; errseq_t since; int error; @@ -648,6 +648,21 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from) written = result; iocb->ki_pos += written; + + if (mntflags & NFS_MOUNT_WRITE_EAGER) { + result = filemap_fdatawrite_range(file->f_mapping, + iocb->ki_pos - written, + iocb->ki_pos - 1); + if (result < 0) + goto out; + } + if (mntflags & NFS_MOUNT_WRITE_WAIT) { + result = filemap_fdatawait_range(file->f_mapping, + iocb->ki_pos - written, + iocb->ki_pos - 1); + if (result < 0) + goto out; + } result = generic_write_sync(iocb, written); if (result < 0) goto out; diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 6193350356a8..82bdcb982186 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -712,16 +712,23 @@ int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc) { struct inode *inode = mapping->host; struct nfs_pageio_descriptor pgio; - struct nfs_io_completion *ioc; + struct nfs_io_completion *ioc = NULL; + unsigned int mntflags = NFS_SERVER(inode)->flags; + int priority = 0; int err; nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES); - ioc = nfs_io_completion_alloc(GFP_KERNEL); - if (ioc) - nfs_io_completion_init(ioc, nfs_io_completion_commit, inode); + if (!(mntflags & NFS_MOUNT_WRITE_EAGER) || wbc->for_kupdate || + wbc->for_background || wbc->for_sync || wbc->for_reclaim) { + ioc = nfs_io_completion_alloc(GFP_KERNEL); + if (ioc) + nfs_io_completion_init(ioc, nfs_io_completion_commit, + inode); + priority = wb_priority(wbc); + } - nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false, + nfs_pageio_init_write(&pgio, inode, priority, false, &nfs_async_write_completion_ops); pgio.pg_io_completion = ioc; err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio); diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 962e8313f007..6f76b32a0238 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -153,6 +153,8 @@ struct nfs_server { #define NFS_MOUNT_LOCAL_FCNTL 0x200000 #define NFS_MOUNT_SOFTERR 0x400000 #define NFS_MOUNT_SOFTREVAL 0x800000 +#define NFS_MOUNT_WRITE_EAGER 0x01000000 +#define NFS_MOUNT_WRITE_WAIT 0x02000000 unsigned int caps; /* server capabilities */ unsigned int rsize; /* read size */ -- cgit v1.2.3 From a0492339fc70f1f7aa98f0cab55b78b0be124711 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 12 Feb 2021 16:49:49 -0500 Subject: NFS: Add mount options supporting eager writes Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/fs_context.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'fs') diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index 06894bcdea2d..b6be02aa79f0 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -82,6 +82,7 @@ enum nfs_param { Opt_v, Opt_vers, Opt_wsize, + Opt_write, }; enum { @@ -113,6 +114,19 @@ static const struct constant_table nfs_param_enums_lookupcache[] = { {} }; +enum { + Opt_write_lazy, + Opt_write_eager, + Opt_write_wait, +}; + +static const struct constant_table nfs_param_enums_write[] = { + { "lazy", Opt_write_lazy }, + { "eager", Opt_write_eager }, + { "wait", Opt_write_wait }, + {} +}; + static const struct fs_parameter_spec nfs_fs_parameters[] = { fsparam_flag_no("ac", Opt_ac), fsparam_u32 ("acdirmax", Opt_acdirmax), @@ -171,6 +185,7 @@ static const struct fs_parameter_spec nfs_fs_parameters[] = { fsparam_flag ("v4.1", Opt_v), fsparam_flag ("v4.2", Opt_v), fsparam_string("vers", Opt_vers), + fsparam_enum ("write", Opt_write, nfs_param_enums_write), fsparam_u32 ("wsize", Opt_wsize), {} }; @@ -770,6 +785,24 @@ static int nfs_fs_context_parse_param(struct fs_context *fc, goto out_invalid_value; } break; + case Opt_write: + switch (result.uint_32) { + case Opt_write_lazy: + ctx->flags &= + ~(NFS_MOUNT_WRITE_EAGER | NFS_MOUNT_WRITE_WAIT); + break; + case Opt_write_eager: + ctx->flags |= NFS_MOUNT_WRITE_EAGER; + ctx->flags &= ~NFS_MOUNT_WRITE_WAIT; + break; + case Opt_write_wait: + ctx->flags |= + NFS_MOUNT_WRITE_EAGER | NFS_MOUNT_WRITE_WAIT; + break; + default: + goto out_invalid_value; + } + break; /* * Special options -- cgit v1.2.3 From 6c17260ca4aeb17d11461647c6b7eefcc2602acc Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 12 Feb 2021 16:41:19 -0500 Subject: NFS: Set the stable writes flag when initialising the super block We need to wait for outstanding writes on the page to complete before we can update it. Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/fs_context.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index b6be02aa79f0..971a9251c1d9 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -1512,6 +1512,8 @@ static int nfs_init_fs_context(struct fs_context *fc) ctx->selected_flavor = RPC_AUTH_MAXFLAVOR; ctx->minorversion = 0; ctx->need_mount = true; + + fc->s_iflags |= SB_I_STABLE_WRITES; } fc->fs_private = ctx; fc->ops = &nfs_fs_context_ops; -- cgit v1.2.3 From 0f56db831456cb4bf85a15c7a900b7138d89b6eb Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Wed, 3 Feb 2021 22:49:52 -0800 Subject: cifs: New optype for session operations. We used to share the CIFS_NEG_OP flag between negotiate and session authentication. There was an assumption in the code that CIFS_NEG_OP is used by negotiate only. So introcuded CIFS_SESS_OP and used it for session setup optypes. Signed-off-by: Shyam Prasad N Reviewed-by: Pavel Shilovsky Signed-off-by: Steve French --- fs/cifs/cifsglob.h | 4 +++- fs/cifs/smb2ops.c | 4 +++- fs/cifs/smb2pdu.c | 2 +- fs/cifs/transport.c | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index 50fcb65920e8..3152601a608b 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -1704,7 +1704,9 @@ static inline bool is_retryable_error(int error) #define CIFS_ECHO_OP 0x080 /* echo request */ #define CIFS_OBREAK_OP 0x0100 /* oplock break request */ #define CIFS_NEG_OP 0x0200 /* negotiate request */ -#define CIFS_OP_MASK 0x0380 /* mask request type */ +/* Lower bitmask values are reserved by others below. */ +#define CIFS_SESS_OP 0x2000 /* session setup request */ +#define CIFS_OP_MASK 0x2380 /* mask request type */ #define CIFS_HAS_CREDITS 0x0400 /* already has credits */ #define CIFS_TRANSFORM_REQ 0x0800 /* transform request before sending */ diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index f19274857292..84d1f265aa1d 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -84,7 +84,9 @@ smb2_add_credits(struct TCP_Server_Info *server, pr_warn_once("server overflowed SMB3 credits\n"); } server->in_flight--; - if (server->in_flight == 0 && (optype & CIFS_OP_MASK) != CIFS_NEG_OP) + if (server->in_flight == 0 && + ((optype & CIFS_OP_MASK) != CIFS_NEG_OP) && + ((optype & CIFS_OP_MASK) != CIFS_SESS_OP)) rc = change_conf(server); /* * Sometimes server returns 0 credits on oplock break ack - we need to diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index e1391bd92768..4bbb6126b14d 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -1261,7 +1261,7 @@ SMB2_sess_sendreceive(struct SMB2_sess_data *sess_data) cifs_ses_server(sess_data->ses), &rqst, &sess_data->buf0_type, - CIFS_LOG_ERROR | CIFS_NEG_OP, &rsp_iov); + CIFS_LOG_ERROR | CIFS_SESS_OP, &rsp_iov); cifs_small_buf_release(sess_data->iov[0].iov_base); memcpy(&sess_data->iov[0], &rsp_iov, sizeof(struct kvec)); diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c index 4a2b836eb017..41223a9ee086 100644 --- a/fs/cifs/transport.c +++ b/fs/cifs/transport.c @@ -1171,7 +1171,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, /* * Compounding is never used during session establish. */ - if ((ses->status == CifsNew) || (optype & CIFS_NEG_OP)) + if ((ses->status == CifsNew) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) smb311_update_preauth_hash(ses, rqst[0].rq_iov, rqst[0].rq_nvec); @@ -1236,7 +1236,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, /* * Compounding is never used during session establish. */ - if ((ses->status == CifsNew) || (optype & CIFS_NEG_OP)) { + if ((ses->status == CifsNew) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) { struct kvec iov = { .iov_base = resp_iov[0].iov_base, .iov_len = resp_iov[0].iov_len -- cgit v1.2.3 From 7de0394801da4f759684c4a33cf62f12da6e447d Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Wed, 3 Feb 2021 22:58:38 -0800 Subject: cifs: Fix in error types returned for out-of-credit situations. For failure by timeout waiting for credits, changed the error returned to the app with EBUSY, instead of ENOTSUPP. This is done because this situation is possible even in non-buggy cases. i.e. overloaded server can return 0 credits until done with outstanding requests. And this feels like a better error to return to the app. For cases of zero credits found even when there are no requests in flight, replaced ENOTSUPP with EDEADLK, since we're avoiding deadlock here by returning error. Signed-off-by: Shyam Prasad N Reviewed-by: Pavel Shilovsky Signed-off-by: Steve French --- fs/cifs/transport.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c index 41223a9ee086..39e87705840d 100644 --- a/fs/cifs/transport.c +++ b/fs/cifs/transport.c @@ -567,7 +567,7 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, server->hostname, num_credits, 0); cifs_server_dbg(VFS, "wait timed out after %d ms\n", timeout); - return -ENOTSUPP; + return -EBUSY; } if (rc == -ERESTARTSYS) return -ERESTARTSYS; @@ -609,7 +609,7 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, 0); cifs_server_dbg(VFS, "wait timed out after %d ms\n", timeout); - return -ENOTSUPP; + return -EBUSY; } if (rc == -ERESTARTSYS) return -ERESTARTSYS; @@ -687,7 +687,7 @@ wait_for_compound_request(struct TCP_Server_Info *server, int num, server->hostname, scredits, sin_flight); cifs_dbg(FYI, "%s: %d requests in flight, needed %d total=%d\n", __func__, sin_flight, num, scredits); - return -ENOTSUPP; + return -EDEADLK; } } spin_unlock(&server->req_lock); -- cgit v1.2.3 From 6d82c27ae5d048ba9219cccdf832f8406e507d5f Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Wed, 3 Feb 2021 23:20:46 -0800 Subject: cifs: Identify a connection by a conn_id. Introduced a new field conn_id in TCP_Server_Info structure. This is a non-persistent unique identifier maintained by the client for a connection to a file server. For this, a global counter named tcpSesNextId is maintained. On allocating a new TCP_Server_Info, this counter is incremented and assigned. Changed the dynamic tracepoints related to reconnects and crediting to be more informative (with conn_id printed). Debugging a crediting issue helped me understand the important things to print here. Always call dynamic tracepoints outside the scope of spinlocks. To do this, copy out the credits and in_flight fields of the server struct before dropping the lock. Signed-off-by: Shyam Prasad N Reviewed-by: Pavel Shilovsky Signed-off-by: Steve French --- fs/cifs/cifsfs.c | 1 + fs/cifs/cifsglob.h | 2 ++ fs/cifs/connect.c | 11 ++++++---- fs/cifs/smb2ops.c | 63 +++++++++++++++++++++++++++++++++++++++-------------- fs/cifs/trace.h | 36 +++++++++++++++++++++--------- fs/cifs/transport.c | 53 ++++++++++++++++++++++++++++++++------------ 6 files changed, 122 insertions(+), 44 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index ab883e84e116..6f33ff3f625f 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -1525,6 +1525,7 @@ init_cifs(void) */ atomic_set(&sesInfoAllocCount, 0); atomic_set(&tconInfoAllocCount, 0); + atomic_set(&tcpSesNextId, 0); atomic_set(&tcpSesAllocCount, 0); atomic_set(&tcpSesReconnectCount, 0); atomic_set(&tconInfoReconnectCount, 0); diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index 3152601a608b..0aa2c3c871c9 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -577,6 +577,7 @@ inc_rfc1001_len(void *buf, int count) struct TCP_Server_Info { struct list_head tcp_ses_list; struct list_head smb_ses_list; + __u64 conn_id; /* connection identifier (useful for debugging) */ int srv_count; /* reference counter */ /* 15 character server name + 0x20 16th byte indicating type = srv */ char server_RFC1001_name[RFC1001_NAME_LEN_WITH_NULL]; @@ -1846,6 +1847,7 @@ GLOBAL_EXTERN spinlock_t GlobalMid_Lock; /* protects above & list operations */ */ GLOBAL_EXTERN atomic_t sesInfoAllocCount; GLOBAL_EXTERN atomic_t tconInfoAllocCount; +GLOBAL_EXTERN atomic_t tcpSesNextId; GLOBAL_EXTERN atomic_t tcpSesAllocCount; GLOBAL_EXTERN atomic_t tcpSesReconnectCount; GLOBAL_EXTERN atomic_t tconInfoReconnectCount; diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 61418a1c7817..b3102a86fd81 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -242,7 +242,7 @@ cifs_reconnect(struct TCP_Server_Info *server) server->max_read = 0; cifs_dbg(FYI, "Mark tcp session as need reconnect\n"); - trace_smb3_reconnect(server->CurrentMid, server->hostname); + trace_smb3_reconnect(server->CurrentMid, server->conn_id, server->hostname); /* before reconnecting the tcp session, mark the smb session (uid) and the tid bad so they are not used until reconnected */ @@ -846,7 +846,7 @@ static void smb2_add_credits_from_hdr(char *buffer, struct TCP_Server_Info *server) { struct smb2_sync_hdr *shdr = (struct smb2_sync_hdr *)buffer; - int scredits = server->credits; + int scredits, in_flight; /* * SMB1 does not use credits. @@ -857,12 +857,14 @@ smb2_add_credits_from_hdr(char *buffer, struct TCP_Server_Info *server) if (shdr->CreditRequest) { spin_lock(&server->req_lock); server->credits += le16_to_cpu(shdr->CreditRequest); + scredits = server->credits; + in_flight = server->in_flight; spin_unlock(&server->req_lock); wake_up(&server->request_q); trace_smb3_add_credits(server->CurrentMid, - server->hostname, scredits, - le16_to_cpu(shdr->CreditRequest)); + server->conn_id, server->hostname, scredits, + le16_to_cpu(shdr->CreditRequest), in_flight); cifs_server_dbg(FYI, "%s: added %u credits total=%d\n", __func__, le16_to_cpu(shdr->CreditRequest), scredits); @@ -1317,6 +1319,7 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx) goto out_err_crypto_release; } + tcp_ses->conn_id = atomic_inc_return(&tcpSesNextId); tcp_ses->noblockcnt = ctx->rootfs; tcp_ses->noblocksnd = ctx->noblocksnd || ctx->rootfs; tcp_ses->noautotune = ctx->noautotune; diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index 84d1f265aa1d..fe171ccbe8e3 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -63,17 +63,19 @@ smb2_add_credits(struct TCP_Server_Info *server, const struct cifs_credits *credits, const int optype) { int *val, rc = -1; + int scredits, in_flight; unsigned int add = credits->value; unsigned int instance = credits->instance; bool reconnect_detected = false; + bool reconnect_with_invalid_credits = false; spin_lock(&server->req_lock); val = server->ops->get_credits_field(server, optype); /* eg found case where write overlapping reconnect messed up credits */ if (((optype & CIFS_OP_MASK) == CIFS_NEG_OP) && (*val != 0)) - trace_smb3_reconnect_with_invalid_credits(server->CurrentMid, - server->hostname, *val, add); + reconnect_with_invalid_credits = true; + if ((instance == 0) || (instance == server->reconnect_instance)) *val += add; else @@ -99,14 +101,26 @@ smb2_add_credits(struct TCP_Server_Info *server, server->oplock_credits++; } } + scredits = *val; + in_flight = server->in_flight; spin_unlock(&server->req_lock); wake_up(&server->request_q); if (reconnect_detected) { + trace_smb3_reconnect_detected(server->CurrentMid, + server->conn_id, server->hostname, scredits, add, in_flight); + cifs_dbg(FYI, "trying to put %d credits from the old server instance %d\n", add, instance); } + if (reconnect_with_invalid_credits) { + trace_smb3_reconnect_with_invalid_credits(server->CurrentMid, + server->conn_id, server->hostname, scredits, add, in_flight); + cifs_dbg(FYI, "Negotiate operation when server credits is non-zero. Optype: %d, server credits: %d, credits added: %d\n", + optype, scredits, add); + } + if (server->tcpStatus == CifsNeedReconnect || server->tcpStatus == CifsExiting) return; @@ -125,23 +139,30 @@ smb2_add_credits(struct TCP_Server_Info *server, cifs_dbg(FYI, "disabling oplocks\n"); break; default: - trace_smb3_add_credits(server->CurrentMid, - server->hostname, rc, add); - cifs_dbg(FYI, "%s: added %u credits total=%d\n", __func__, add, rc); + /* change_conf rebalanced credits for different types */ + break; } + + trace_smb3_add_credits(server->CurrentMid, + server->conn_id, server->hostname, scredits, add, in_flight); + cifs_dbg(FYI, "%s: added %u credits total=%d\n", __func__, add, scredits); } static void smb2_set_credits(struct TCP_Server_Info *server, const int val) { + int scredits, in_flight; + spin_lock(&server->req_lock); server->credits = val; if (val == 1) server->reconnect_instance++; + scredits = server->credits; + in_flight = server->in_flight; spin_unlock(&server->req_lock); trace_smb3_set_credits(server->CurrentMid, - server->hostname, val, val); + server->conn_id, server->hostname, scredits, val, in_flight); cifs_dbg(FYI, "%s: set %u credits\n", __func__, val); /* don't log while holding the lock */ @@ -173,7 +194,7 @@ smb2_wait_mtu_credits(struct TCP_Server_Info *server, unsigned int size, unsigned int *num, struct cifs_credits *credits) { int rc = 0; - unsigned int scredits; + unsigned int scredits, in_flight; spin_lock(&server->req_lock); while (1) { @@ -210,17 +231,18 @@ smb2_wait_mtu_credits(struct TCP_Server_Info *server, unsigned int size, DIV_ROUND_UP(*num, SMB2_MAX_BUFFER_SIZE); credits->instance = server->reconnect_instance; server->credits -= credits->value; - scredits = server->credits; server->in_flight++; if (server->in_flight > server->max_in_flight) server->max_in_flight = server->in_flight; break; } } + scredits = server->credits; + in_flight = server->in_flight; spin_unlock(&server->req_lock); trace_smb3_add_credits(server->CurrentMid, - server->hostname, scredits, -(credits->value)); + server->conn_id, server->hostname, scredits, -(credits->value), in_flight); cifs_dbg(FYI, "%s: removed %u credits total=%d\n", __func__, credits->value, scredits); @@ -233,14 +255,14 @@ smb2_adjust_credits(struct TCP_Server_Info *server, const unsigned int payload_size) { int new_val = DIV_ROUND_UP(payload_size, SMB2_MAX_BUFFER_SIZE); - int scredits; + int scredits, in_flight; if (!credits->value || credits->value == new_val) return 0; if (credits->value < new_val) { trace_smb3_too_many_credits(server->CurrentMid, - server->hostname, 0, credits->value - new_val); + server->conn_id, server->hostname, 0, credits->value - new_val, 0); cifs_server_dbg(VFS, "request has less credits (%d) than required (%d)", credits->value, new_val); @@ -250,9 +272,13 @@ smb2_adjust_credits(struct TCP_Server_Info *server, spin_lock(&server->req_lock); if (server->reconnect_instance != credits->instance) { + scredits = server->credits; + in_flight = server->in_flight; spin_unlock(&server->req_lock); + trace_smb3_reconnect_detected(server->CurrentMid, - server->hostname, 0, 0); + server->conn_id, server->hostname, scredits, + credits->value - new_val, in_flight); cifs_server_dbg(VFS, "trying to return %d credits to old session\n", credits->value - new_val); return -EAGAIN; @@ -260,15 +286,18 @@ smb2_adjust_credits(struct TCP_Server_Info *server, server->credits += credits->value - new_val; scredits = server->credits; + in_flight = server->in_flight; spin_unlock(&server->req_lock); wake_up(&server->request_q); - credits->value = new_val; trace_smb3_add_credits(server->CurrentMid, - server->hostname, scredits, credits->value - new_val); + server->conn_id, server->hostname, scredits, + credits->value - new_val, in_flight); cifs_dbg(FYI, "%s: adjust added %u credits total=%d\n", __func__, credits->value - new_val, scredits); + credits->value = new_val; + return 0; } @@ -2371,7 +2400,7 @@ static bool smb2_is_status_pending(char *buf, struct TCP_Server_Info *server) { struct smb2_sync_hdr *shdr = (struct smb2_sync_hdr *)buf; - int scredits; + int scredits, in_flight; if (shdr->Status != STATUS_PENDING) return false; @@ -2380,11 +2409,13 @@ smb2_is_status_pending(char *buf, struct TCP_Server_Info *server) spin_lock(&server->req_lock); server->credits += le16_to_cpu(shdr->CreditRequest); scredits = server->credits; + in_flight = server->in_flight; spin_unlock(&server->req_lock); wake_up(&server->request_q); trace_smb3_add_credits(server->CurrentMid, - server->hostname, scredits, le16_to_cpu(shdr->CreditRequest)); + server->conn_id, server->hostname, scredits, + le16_to_cpu(shdr->CreditRequest), in_flight); cifs_dbg(FYI, "%s: status pending add %u credits total=%d\n", __func__, le16_to_cpu(shdr->CreditRequest), scredits); } diff --git a/fs/cifs/trace.h b/fs/cifs/trace.h index c3d1a584f251..d6df908dccad 100644 --- a/fs/cifs/trace.h +++ b/fs/cifs/trace.h @@ -851,17 +851,21 @@ DEFINE_SMB3_LEASE_ERR_EVENT(lease_err); DECLARE_EVENT_CLASS(smb3_reconnect_class, TP_PROTO(__u64 currmid, + __u64 conn_id, char *hostname), - TP_ARGS(currmid, hostname), + TP_ARGS(currmid, conn_id, hostname), TP_STRUCT__entry( __field(__u64, currmid) + __field(__u64, conn_id) __field(char *, hostname) ), TP_fast_assign( __entry->currmid = currmid; + __entry->conn_id = conn_id; __entry->hostname = hostname; ), - TP_printk("server=%s current_mid=0x%llx", + TP_printk("conn_id=0x%llx server=%s current_mid=%llu", + __entry->conn_id, __entry->hostname, __entry->currmid) ) @@ -869,44 +873,56 @@ DECLARE_EVENT_CLASS(smb3_reconnect_class, #define DEFINE_SMB3_RECONNECT_EVENT(name) \ DEFINE_EVENT(smb3_reconnect_class, smb3_##name, \ TP_PROTO(__u64 currmid, \ - char *hostname), \ - TP_ARGS(currmid, hostname)) + __u64 conn_id, \ + char *hostname), \ + TP_ARGS(currmid, conn_id, hostname)) DEFINE_SMB3_RECONNECT_EVENT(reconnect); DEFINE_SMB3_RECONNECT_EVENT(partial_send_reconnect); DECLARE_EVENT_CLASS(smb3_credit_class, TP_PROTO(__u64 currmid, + __u64 conn_id, char *hostname, int credits, - int credits_to_add), - TP_ARGS(currmid, hostname, credits, credits_to_add), + int credits_to_add, + int in_flight), + TP_ARGS(currmid, conn_id, hostname, credits, credits_to_add, in_flight), TP_STRUCT__entry( __field(__u64, currmid) + __field(__u64, conn_id) __field(char *, hostname) __field(int, credits) __field(int, credits_to_add) + __field(int, in_flight) ), TP_fast_assign( __entry->currmid = currmid; + __entry->conn_id = conn_id; __entry->hostname = hostname; __entry->credits = credits; __entry->credits_to_add = credits_to_add; + __entry->in_flight = in_flight; ), - TP_printk("server=%s current_mid=0x%llx credits=%d credits_to_add=%d", + TP_printk("conn_id=0x%llx server=%s current_mid=%llu " + "credits=%d credit_change=%d in_flight=%d", + __entry->conn_id, __entry->hostname, __entry->currmid, __entry->credits, - __entry->credits_to_add) + __entry->credits_to_add, + __entry->in_flight) ) #define DEFINE_SMB3_CREDIT_EVENT(name) \ DEFINE_EVENT(smb3_credit_class, smb3_##name, \ TP_PROTO(__u64 currmid, \ + __u64 conn_id, \ char *hostname, \ int credits, \ - int credits_to_add), \ - TP_ARGS(currmid, hostname, credits, credits_to_add)) + int credits_to_add, \ + int in_flight), \ + TP_ARGS(currmid, conn_id, hostname, credits, credits_to_add, in_flight)) DEFINE_SMB3_CREDIT_EVENT(reconnect_with_invalid_credits); DEFINE_SMB3_CREDIT_EVENT(reconnect_detected); diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c index 39e87705840d..e90a1d1380b0 100644 --- a/fs/cifs/transport.c +++ b/fs/cifs/transport.c @@ -445,7 +445,7 @@ unmask: */ server->tcpStatus = CifsNeedReconnect; trace_smb3_partial_send_reconnect(server->CurrentMid, - server->hostname); + server->conn_id, server->hostname); } smbd_done: if (rc < 0 && rc != -EINTR) @@ -527,7 +527,7 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, int *credits; int optype; long int t; - int scredits = server->credits; + int scredits, in_flight; if (timeout < 0) t = MAX_JIFFY_OFFSET; @@ -551,22 +551,38 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, server->max_in_flight = server->in_flight; *credits -= 1; *instance = server->reconnect_instance; + scredits = *credits; + in_flight = server->in_flight; spin_unlock(&server->req_lock); + + trace_smb3_add_credits(server->CurrentMid, + server->conn_id, server->hostname, scredits, -1, in_flight); + cifs_dbg(FYI, "%s: remove %u credits total=%d\n", + __func__, 1, scredits); + return 0; } while (1) { if (*credits < num_credits) { + scredits = *credits; spin_unlock(&server->req_lock); + cifs_num_waiters_inc(server); rc = wait_event_killable_timeout(server->request_q, has_credits(server, credits, num_credits), t); cifs_num_waiters_dec(server); if (!rc) { + spin_lock(&server->req_lock); + scredits = *credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + trace_smb3_credit_timeout(server->CurrentMid, - server->hostname, num_credits, 0); + server->conn_id, server->hostname, scredits, + num_credits, in_flight); cifs_server_dbg(VFS, "wait timed out after %d ms\n", - timeout); + timeout); return -EBUSY; } if (rc == -ERESTARTSYS) @@ -595,6 +611,7 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, server->in_flight > 2 * MAX_COMPOUND && *credits <= MAX_COMPOUND) { spin_unlock(&server->req_lock); + cifs_num_waiters_inc(server); rc = wait_event_killable_timeout( server->request_q, @@ -603,12 +620,17 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, t); cifs_num_waiters_dec(server); if (!rc) { + spin_lock(&server->req_lock); + scredits = *credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + trace_smb3_credit_timeout( - server->CurrentMid, - server->hostname, num_credits, - 0); + server->CurrentMid, + server->conn_id, server->hostname, + scredits, num_credits, in_flight); cifs_server_dbg(VFS, "wait timed out after %d ms\n", - timeout); + timeout); return -EBUSY; } if (rc == -ERESTARTSYS) @@ -625,16 +647,18 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, /* update # of requests on the wire to server */ if ((flags & CIFS_TIMEOUT_MASK) != CIFS_BLOCKING_OP) { *credits -= num_credits; - scredits = *credits; server->in_flight += num_credits; if (server->in_flight > server->max_in_flight) server->max_in_flight = server->in_flight; *instance = server->reconnect_instance; } + scredits = *credits; + in_flight = server->in_flight; spin_unlock(&server->req_lock); trace_smb3_add_credits(server->CurrentMid, - server->hostname, scredits, -(num_credits)); + server->conn_id, server->hostname, scredits, + -(num_credits), in_flight); cifs_dbg(FYI, "%s: remove %u credits total=%d\n", __func__, num_credits, scredits); break; @@ -656,13 +680,13 @@ wait_for_compound_request(struct TCP_Server_Info *server, int num, const int flags, unsigned int *instance) { int *credits; - int scredits, sin_flight; + int scredits, in_flight; credits = server->ops->get_credits_field(server, flags & CIFS_OP_MASK); spin_lock(&server->req_lock); scredits = *credits; - sin_flight = server->in_flight; + in_flight = server->in_flight; if (*credits < num) { /* @@ -684,9 +708,10 @@ wait_for_compound_request(struct TCP_Server_Info *server, int num, if (server->in_flight == 0) { spin_unlock(&server->req_lock); trace_smb3_insufficient_credits(server->CurrentMid, - server->hostname, scredits, sin_flight); + server->conn_id, server->hostname, scredits, + num, in_flight); cifs_dbg(FYI, "%s: %d requests in flight, needed %d total=%d\n", - __func__, sin_flight, num, scredits); + __func__, in_flight, num, scredits); return -EDEADLK; } } -- cgit v1.2.3 From 03e9bb1a0b403c29d5e8679be68addc230472390 Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Wed, 3 Feb 2021 23:27:52 -0800 Subject: cifs: Reformat DebugData and index connections by conn_id. Reformat the output of /proc/fs/cifs/DebugData to print the conn_id for each connection. Also reordered and numbered the data into a more reader-friendly format. This is what the new format looks like: $ cat /proc/fs/cifs/DebugData Display Internal CIFS Data Structures for Debugging --------------------------------------------------- CIFS Version 2.30 Features: DFS,FSCACHE,STATS,DEBUG,ALLOW_INSECURE_LEGACY,WEAK_PW_HASH,CIFS_POSIX,UPCALL(SPNEGO),XATTR,ACL CIFSMaxBufSize: 16384 Active VFS Requests: 0 Servers: 1) ConnectionId: 0x1 Number of credits: 371 Dialect 0x300 TCP status: 1 Instance: 1 Local Users To Server: 1 SecMode: 0x1 Req On Wire: 0 In Send: 0 In MaxReq Wait: 0 Sessions: 1) Name: 10.10.10.10 Uses: 1 Capability: 0x300077 Session Status: 1 Security type: RawNTLMSSP SessionId: 0x785560000019 User: 1000 Cred User: 0 Shares: 0) IPC: \\10.10.10.10\IPC$ Mounts: 1 DevInfo: 0x0 Attributes: 0x0 PathComponentMax: 0 Status: 1 type: 0 Serial Number: 0x0 Share Capabilities: None Share Flags: 0x30 tid: 0x1 Maximal Access: 0x11f01ff 1) \\10.10.10.10\shyam_test2 Mounts: 1 DevInfo: 0x20020 Attributes: 0xc706ff PathComponentMax: 255 Status: 1 type: DISK Serial Number: 0xd4723975 Share Capabilities: None Aligned, Partition Aligned, Share Flags: 0x0 tid: 0x5 Optimal sector size: 0x1000 Maximal Access: 0x1f01ff MIDs: Server interfaces: 3 1) Speed: 10000000000 bps Capabilities: rss IPv4: 10.10.10.1 2) Speed: 10000000000 bps Capabilities: rss IPv6: fe80:0000:0000:0000:18b4:0000:0000:0000 3) Speed: 1000000000 bps Capabilities: rss IPv4: 10.10.10.10 [CONNECTED] Signed-off-by: Shyam Prasad N Reviewed-by: Pavel Shilovsky Reviewed-by: Aurelien Aptel Signed-off-by: Steve French --- fs/cifs/cifs_debug.c | 117 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 49 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c index b231dcf1d1f9..370cc88a3d02 100644 --- a/fs/cifs/cifs_debug.c +++ b/fs/cifs/cifs_debug.c @@ -133,11 +133,12 @@ cifs_dump_channel(struct seq_file *m, int i, struct cifs_chan *chan) { struct TCP_Server_Info *server = chan->server; - seq_printf(m, "\t\tChannel %d Number of credits: %d Dialect 0x%x " - "TCP status: %d Instance: %d Local Users To Server: %d " - "SecMode: 0x%x Req On Wire: %d In Send: %d " - "In MaxReq Wait: %d\n", - i+1, + seq_printf(m, "\n\n\t\tChannel: %d ConnectionId: 0x%llx" + "\n\t\tNumber of credits: %d Dialect 0x%x" + "\n\t\tTCP status: %d Instance: %d" + "\n\t\tLocal Users To Server: %d SecMode: 0x%x Req On Wire: %d" + "\n\t\tIn Send: %d In MaxReq Wait: %d", + i+1, server->conn_id, server->credits, server->dialect, server->tcpStatus, @@ -227,7 +228,7 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) struct TCP_Server_Info *server; struct cifs_ses *ses; struct cifs_tcon *tcon; - int i, j; + int c, i, j; seq_puts(m, "Display Internal CIFS Data Structures for Debugging\n" @@ -275,14 +276,23 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) seq_putc(m, '\n'); seq_printf(m, "CIFSMaxBufSize: %d\n", CIFSMaxBufSize); seq_printf(m, "Active VFS Requests: %d\n", GlobalTotalActiveXid); - seq_printf(m, "Servers:"); - i = 0; + seq_printf(m, "\nServers: "); + + c = 0; spin_lock(&cifs_tcp_ses_lock); list_for_each(tmp1, &cifs_tcp_ses_list) { server = list_entry(tmp1, struct TCP_Server_Info, tcp_ses_list); + /* channel info will be printed as a part of sessions below */ + if (server->is_channel) + continue; + + c++; + seq_printf(m, "\n%d) ConnectionId: 0x%llx ", + c, server->conn_id); + #ifdef CONFIG_CIFS_SMB_DIRECT if (!server->rdma) goto skip_rdma; @@ -362,46 +372,48 @@ skip_rdma: if (server->posix_ext_supported) seq_printf(m, " posix"); - i++; + if (server->rdma) + seq_printf(m, "\nRDMA "); + seq_printf(m, "\nTCP status: %d Instance: %d" + "\nLocal Users To Server: %d SecMode: 0x%x Req On Wire: %d", + server->tcpStatus, + server->reconnect_instance, + server->srv_count, + server->sec_mode, in_flight(server)); + + seq_printf(m, "\nIn Send: %d In MaxReq Wait: %d", + atomic_read(&server->in_send), + atomic_read(&server->num_waiters)); + + seq_printf(m, "\n\n\tSessions: "); + i = 0; list_for_each(tmp2, &server->smb_ses_list) { ses = list_entry(tmp2, struct cifs_ses, smb_ses_list); + i++; if ((ses->serverDomain == NULL) || (ses->serverOS == NULL) || (ses->serverNOS == NULL)) { - seq_printf(m, "\n%d) Name: %s Uses: %d Capability: 0x%x\tSession Status: %d ", + seq_printf(m, "\n\t%d) Name: %s Uses: %d Capability: 0x%x\tSession Status: %d ", i, ses->serverName, ses->ses_count, ses->capabilities, ses->status); if (ses->session_flags & SMB2_SESSION_FLAG_IS_GUEST) - seq_printf(m, "Guest\t"); + seq_printf(m, "Guest "); else if (ses->session_flags & SMB2_SESSION_FLAG_IS_NULL) - seq_printf(m, "Anonymous\t"); + seq_printf(m, "Anonymous "); } else { seq_printf(m, - "\n%d) Name: %s Domain: %s Uses: %d OS:" - " %s\n\tNOS: %s\tCapability: 0x%x\n\tSMB" - " session status: %d ", + "\n\t%d) Name: %s Domain: %s Uses: %d OS: %s " + "\n\tNOS: %s\tCapability: 0x%x" + "\n\tSMB session status: %d ", i, ses->serverName, ses->serverDomain, ses->ses_count, ses->serverOS, ses->serverNOS, ses->capabilities, ses->status); } - seq_printf(m,"Security type: %s\n", + seq_printf(m, "\n\tSecurity type: %s ", get_security_type_str(server->ops->select_sectype(server, ses->sectype))); - if (server->rdma) - seq_printf(m, "RDMA\n\t"); - seq_printf(m, "TCP status: %d Instance: %d\n\tLocal Users To " - "Server: %d SecMode: 0x%x Req On Wire: %d", - server->tcpStatus, - server->reconnect_instance, - server->srv_count, - server->sec_mode, in_flight(server)); - - seq_printf(m, " In Send: %d In MaxReq Wait: %d", - atomic_read(&server->in_send), - atomic_read(&server->num_waiters)); - /* dump session id helpful for use with network trace */ seq_printf(m, " SessionId: 0x%llx", ses->Suid); if (ses->session_flags & SMB2_SESSION_FLAG_ENCRYPT_DATA) @@ -414,13 +426,13 @@ skip_rdma: from_kuid(&init_user_ns, ses->cred_uid)); if (ses->chan_count > 1) { - seq_printf(m, "\n\n\tExtra Channels: %zu\n", + seq_printf(m, "\n\n\tExtra Channels: %zu ", ses->chan_count-1); for (j = 1; j < ses->chan_count; j++) cifs_dump_channel(m, j, &ses->chans[j]); } - seq_puts(m, "\n\n\tShares:"); + seq_puts(m, "\n\n\tShares: "); j = 0; seq_printf(m, "\n\t%d) IPC: ", j); @@ -437,38 +449,45 @@ skip_rdma: cifs_debug_tcon(m, tcon); } - seq_puts(m, "\n\tMIDs:\n"); - - spin_lock(&GlobalMid_Lock); - list_for_each(tmp3, &server->pending_mid_q) { - mid_entry = list_entry(tmp3, struct mid_q_entry, - qhead); - seq_printf(m, "\tState: %d com: %d pid:" - " %d cbdata: %p mid %llu\n", - mid_entry->mid_state, - le16_to_cpu(mid_entry->command), - mid_entry->pid, - mid_entry->callback_data, - mid_entry->mid); - } - spin_unlock(&GlobalMid_Lock); - spin_lock(&ses->iface_lock); if (ses->iface_count) - seq_printf(m, "\n\tServer interfaces: %zu\n", + seq_printf(m, "\n\n\tServer interfaces: %zu", ses->iface_count); for (j = 0; j < ses->iface_count; j++) { struct cifs_server_iface *iface; iface = &ses->iface_list[j]; - seq_printf(m, "\t%d)", j); + seq_printf(m, "\n\t%d)", j+1); cifs_dump_iface(m, iface); if (is_ses_using_iface(ses, iface)) seq_puts(m, "\t\t[CONNECTED]\n"); } + if (j == 0) + seq_printf(m, "\n\t[NONE]"); spin_unlock(&ses->iface_lock); } + if (i == 0) + seq_printf(m, "\n\t\t[NONE]"); + + seq_puts(m, "\n\n\tMIDs: "); + spin_lock(&GlobalMid_Lock); + list_for_each(tmp3, &server->pending_mid_q) { + mid_entry = list_entry(tmp3, struct mid_q_entry, + qhead); + seq_printf(m, "\n\tState: %d com: %d pid:" + " %d cbdata: %p mid %llu\n", + mid_entry->mid_state, + le16_to_cpu(mid_entry->command), + mid_entry->pid, + mid_entry->callback_data, + mid_entry->mid); + } + spin_unlock(&GlobalMid_Lock); + seq_printf(m, "\n--\n"); } + if (c == 0) + seq_printf(m, "\n\t[NONE]"); + spin_unlock(&cifs_tcp_ses_lock); seq_putc(m, '\n'); -- cgit v1.2.3 From f38e998fbbb5da6a097ecd4b2700ba95eabab0c9 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Mon, 5 Oct 2020 19:39:18 +0200 Subject: gfs2: Also reflect single-block allocations in rgd->rd_extfail_pt Pass a non-NULL minext to gfs2_rbm_find even for single-block allocations. In gfs2_rbm_find, also set rgd->rd_extfail_pt when a single-block allocation fails in a resource group: there is no reason for treating that case differently. In gfs2_reservation_check_and_update, only check how many free blocks we have if more than one block is requested; we already know there's at least one free block. In addition, when allocating N blocks fails in gfs2_rbm_find, we need to set rd_extfail_pt to N - 1 rather than N: rd_extfail_pt defines the biggest allocation that might still succeed. Finally, reset rd_extfail_pt when updating the resource group statistics in update_rgrp_lvb, as we already do in gfs2_rgrp_bh_get. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/rgrp.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index f878d82750f7..ad14a920f321 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -1278,6 +1278,8 @@ static int update_rgrp_lvb(struct gfs2_rgrpd *rgd) rgd->rd_free = be32_to_cpu(rgd->rd_rgl->rl_free); rgrp_set_bitmap_flags(rgd); rgd->rd_free_clone = rgd->rd_free; + /* max out the rgrp allocation failure point */ + rgd->rd_extfail_pt = rgd->rd_free; rgd->rd_dinodes = be32_to_cpu(rgd->rd_rgl->rl_dinodes); rgd->rd_igeneration = be64_to_cpu(rgd->rd_rgl->rl_igeneration); return 0; @@ -1676,7 +1678,7 @@ static int gfs2_reservation_check_and_update(struct gfs2_rbm *rbm, * If we have a minimum extent length, then skip over any extent * which is less than the min extent length in size. */ - if (minext) { + if (minext > 1) { extlen = gfs2_free_extlen(rbm, minext); if (extlen <= maxext->len) goto fail; @@ -1711,7 +1713,7 @@ fail: * gfs2_rbm_find - Look for blocks of a particular state * @rbm: Value/result starting position and final position * @state: The state which we want to find - * @minext: Pointer to the requested extent length (NULL for a single block) + * @minext: Pointer to the requested extent length * This is updated to be the actual reservation size. * @ip: If set, check for reservations * @nowrap: Stop looking at the end of the rgrp, rather than wrapping @@ -1767,8 +1769,7 @@ static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext, if (ip == NULL) return 0; - ret = gfs2_reservation_check_and_update(rbm, ip, - minext ? *minext : 0, + ret = gfs2_reservation_check_and_update(rbm, ip, *minext, &maxext); if (ret == 0) return 0; @@ -1800,7 +1801,7 @@ next_iter: break; } - if (minext == NULL || state != GFS2_BLKST_FREE) + if (state != GFS2_BLKST_FREE) return -ENOSPC; /* If the extent was too small, and it's smaller than the smallest @@ -1808,7 +1809,7 @@ next_iter: useless to search this rgrp again for this amount or more. */ if (wrapped && (scan_from_start || rbm->bii > last_bii) && *minext < rbm->rgd->rd_extfail_pt) - rbm->rgd->rd_extfail_pt = *minext; + rbm->rgd->rd_extfail_pt = *minext - 1; /* If the maximum extent we found is big enough to fulfill the minimum requirements, use it anyway. */ @@ -2382,14 +2383,15 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, struct buffer_head *dibh; struct gfs2_rbm rbm = { .rgd = ip->i_res.rs_rgd, }; u64 block; /* block, within the file system scope */ + u32 minext = 1; int error; gfs2_set_alloc_start(&rbm, ip, dinode); - error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, NULL, ip, false); + error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, ip, false); if (error == -ENOSPC) { gfs2_set_alloc_start(&rbm, ip, dinode); - error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, NULL, NULL, false); + error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, NULL, false); } /* Since all blocks are reserved in advance, this shouldn't happen */ -- cgit v1.2.3 From 3d39fcd16d885ebb0d9a2e345accb0e5ae13fed9 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Fri, 9 Oct 2020 19:30:16 +0200 Subject: gfs2: Only pass reservation down to gfs2_rbm_find Only pass the current reservation down to gfs2_rbm_find rather than the entire inode; we don't need any of the other information. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/rgrp.c | 26 +++++++++++++------------- fs/gfs2/rgrp.h | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index ad14a920f321..deb553624b3c 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -79,7 +79,7 @@ static const char valid_change[16] = { }; static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext, - const struct gfs2_inode *ip, bool nowrap); + struct gfs2_blkreserv *rs, bool nowrap); /** @@ -1590,7 +1590,7 @@ static void rg_mblk_search(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip, if (WARN_ON(gfs2_rbm_from_block(&rbm, goal))) return; - ret = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &extlen, ip, true); + ret = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &extlen, &ip->i_res, true); if (ret == 0) { rs->rs_start = gfs2_rbm_to_block(&rbm); rs->rs_free = extlen; @@ -1606,7 +1606,7 @@ static void rg_mblk_search(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip, * @rgd: The resource group * @block: The starting block * @length: The required length - * @ip: Ignore any reservations for this inode + * @ignore_rs: Reservation to ignore * * If the block does not appear in any reservation, then return the * block number unchanged. If it does appear in the reservation, then @@ -1616,7 +1616,7 @@ static void rg_mblk_search(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip, static u64 gfs2_next_unreserved_block(struct gfs2_rgrpd *rgd, u64 block, u32 length, - const struct gfs2_inode *ip) + struct gfs2_blkreserv *ignore_rs) { struct gfs2_blkreserv *rs; struct rb_node *n; @@ -1636,7 +1636,7 @@ static u64 gfs2_next_unreserved_block(struct gfs2_rgrpd *rgd, u64 block, } if (n) { - while ((rs_cmp(block, length, rs) == 0) && (&ip->i_res != rs)) { + while (rs_cmp(block, length, rs) == 0 && rs != ignore_rs) { block = rs->rs_start + rs->rs_free; n = n->rb_right; if (n == NULL) @@ -1652,7 +1652,7 @@ static u64 gfs2_next_unreserved_block(struct gfs2_rgrpd *rgd, u64 block, /** * gfs2_reservation_check_and_update - Check for reservations during block alloc * @rbm: The current position in the resource group - * @ip: The inode for which we are searching for blocks + * @rs: Our own reservation * @minext: The minimum extent length * @maxext: A pointer to the maximum extent structure * @@ -1666,7 +1666,7 @@ static u64 gfs2_next_unreserved_block(struct gfs2_rgrpd *rgd, u64 block, */ static int gfs2_reservation_check_and_update(struct gfs2_rbm *rbm, - const struct gfs2_inode *ip, + struct gfs2_blkreserv *rs, u32 minext, struct gfs2_extent *maxext) { @@ -1688,7 +1688,7 @@ static int gfs2_reservation_check_and_update(struct gfs2_rbm *rbm, * Check the extent which has been found against the reservations * and skip if parts of it are already reserved */ - nblock = gfs2_next_unreserved_block(rbm->rgd, block, extlen, ip); + nblock = gfs2_next_unreserved_block(rbm->rgd, block, extlen, rs); if (nblock == block) { if (!minext || extlen >= minext) return 0; @@ -1715,7 +1715,7 @@ fail: * @state: The state which we want to find * @minext: Pointer to the requested extent length * This is updated to be the actual reservation size. - * @ip: If set, check for reservations + * @rs: Our own reservation (NULL to skip checking for reservations) * @nowrap: Stop looking at the end of the rgrp, rather than wrapping * around until we've reached the starting point. * @@ -1729,7 +1729,7 @@ fail: */ static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext, - const struct gfs2_inode *ip, bool nowrap) + struct gfs2_blkreserv *rs, bool nowrap) { bool scan_from_start = rbm->bii == 0 && rbm->offset == 0; struct buffer_head *bh; @@ -1766,10 +1766,10 @@ static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext, goto next_bitmap; } rbm->offset = offset; - if (ip == NULL) + if (!rs) return 0; - ret = gfs2_reservation_check_and_update(rbm, ip, *minext, + ret = gfs2_reservation_check_and_update(rbm, rs, *minext, &maxext); if (ret == 0) return 0; @@ -2387,7 +2387,7 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, int error; gfs2_set_alloc_start(&rbm, ip, dinode); - error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, ip, false); + error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, &ip->i_res, false); if (error == -ENOSPC) { gfs2_set_alloc_start(&rbm, ip, dinode); diff --git a/fs/gfs2/rgrp.h b/fs/gfs2/rgrp.h index 9a587ada51ed..be1b2034f5ee 100644 --- a/fs/gfs2/rgrp.h +++ b/fs/gfs2/rgrp.h @@ -77,7 +77,7 @@ extern int gfs2_fitrim(struct file *filp, void __user *argp); /* This is how to tell if a reservation is in the rgrp tree: */ static inline bool gfs2_rs_active(const struct gfs2_blkreserv *rs) { - return rs && !RB_EMPTY_NODE(&rs->rs_node); + return !RB_EMPTY_NODE(&rs->rs_node); } static inline int rgrp_contains_block(struct gfs2_rgrpd *rgd, u64 block) -- cgit v1.2.3 From b2598965dc84bbcf8dd54accc80f60820e5d4965 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Mon, 12 Oct 2020 17:23:50 +0200 Subject: gfs2: Don't search for unreserved space twice If gfs2_inplace_reserve has chosen a resource group but it couldn't make a reservation there, there are too many other reservations in that resource group. In that case, don't even try to respect existing reservations in gfs2_alloc_blocks. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/rgrp.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index deb553624b3c..95ff4a3c89de 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -2384,11 +2384,12 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, struct gfs2_rbm rbm = { .rgd = ip->i_res.rs_rgd, }; u64 block; /* block, within the file system scope */ u32 minext = 1; - int error; - - gfs2_set_alloc_start(&rbm, ip, dinode); - error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, &ip->i_res, false); + int error = -ENOSPC; + if (gfs2_rs_active(&ip->i_res)) { + gfs2_set_alloc_start(&rbm, ip, dinode); + error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, &ip->i_res, false); + } if (error == -ENOSPC) { gfs2_set_alloc_start(&rbm, ip, dinode); error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, NULL, false); -- cgit v1.2.3 From 0ec9b9ea4f83303bfd8f052a3d8b2bd179b002e1 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Wed, 21 Oct 2020 16:37:54 +0200 Subject: gfs2: Check for active reservation in gfs2_release In gfs2_release, check if the inode has an active reservation to avoid unnecessary lock taking. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index b39b339feddc..177c4d74ca30 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -716,10 +716,10 @@ static int gfs2_release(struct inode *inode, struct file *file) kfree(file->private_data); file->private_data = NULL; - if (file->f_mode & FMODE_WRITE) { + if (gfs2_rs_active(&ip->i_res)) gfs2_rs_delete(ip, &inode->i_writecount); + if (file->f_mode & FMODE_WRITE) gfs2_qa_put(ip); - } return 0; } -- cgit v1.2.3 From 07974d2a2a98bc3ce15f3411ebe5175c0af8f1c3 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Thu, 22 Oct 2020 20:34:29 +0200 Subject: gfs2: Rename rs_{free -> requested} and rd_{reserved -> requested} We keep track of what we've so far been referring to as reservations in rd_rstree: the nodes in that tree indicate where in a resource group we'd like to allocate the next couple of blocks for a particular inode. Local processes take those as hints, but they may still "steal" blocks from those extents, so when actually allocating a block, we must double check in the bitmap whether that block is actually still free. Likewise, other cluster nodes may "steal" such blocks as well. One of the following patches introduces resource group glock sharing, i.e., sharing of an exclusively locked resource group glock among local processes to speed up allocations. To make that work, we'll need to keep track of how many blocks we've actually reserved for each inode, so we end up with two different kinds of reservations. Distinguish these two kinds by referring to blocks which are reserved but may still be "stolen" as "requested". This rename also makes it more obvious that rs_requested and rd_requested are strongly related. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/incore.h | 6 +++--- fs/gfs2/rgrp.c | 42 +++++++++++++++++++++--------------------- fs/gfs2/trace_gfs2.h | 18 +++++++++--------- 3 files changed, 33 insertions(+), 33 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index cca806ff611c..0640d0c70a75 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -106,7 +106,7 @@ struct gfs2_rgrpd { u32 rd_data; /* num of data blocks in rgrp */ u32 rd_bitbytes; /* number of bytes in data bitmaps */ u32 rd_free; - u32 rd_reserved; /* number of blocks reserved */ + u32 rd_requested; /* number of blocks in rd_rstree */ u32 rd_free_clone; u32 rd_dinodes; u64 rd_igeneration; @@ -290,8 +290,8 @@ struct gfs2_qadata { /* quota allocation data */ struct gfs2_blkreserv { struct rb_node rs_node; /* node within rd_rstree */ struct gfs2_rgrpd *rs_rgd; - u64 rs_start; /* start of reservation */ - u32 rs_free; /* how many blocks are still free */ + u64 rs_start; + u32 rs_requested; }; /* diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index 95ff4a3c89de..bc8d1ab9e07f 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -203,7 +203,7 @@ static inline u64 gfs2_bit_search(const __le64 *ptr, u64 mask, u8 state) */ static inline int rs_cmp(u64 start, u32 len, struct gfs2_blkreserv *rs) { - if (start >= rs->rs_start + rs->rs_free) + if (start >= rs->rs_start + rs->rs_requested) return 1; if (rs->rs_start >= start + len) return -1; @@ -625,7 +625,7 @@ static void dump_rs(struct seq_file *seq, const struct gfs2_blkreserv *rs, fs_id_buf, (unsigned long long)ip->i_no_addr, (unsigned long long)rs->rs_start, - rs->rs_free); + rs->rs_requested); } /** @@ -645,17 +645,17 @@ static void __rs_deltree(struct gfs2_blkreserv *rs) rb_erase(&rs->rs_node, &rgd->rd_rstree); RB_CLEAR_NODE(&rs->rs_node); - if (rs->rs_free) { - /* return reserved blocks to the rgrp */ - BUG_ON(rs->rs_rgd->rd_reserved < rs->rs_free); - rs->rs_rgd->rd_reserved -= rs->rs_free; + if (rs->rs_requested) { + /* return requested blocks to the rgrp */ + BUG_ON(rs->rs_rgd->rd_requested < rs->rs_requested); + rs->rs_rgd->rd_requested -= rs->rs_requested; /* The rgrp extent failure point is likely not to increase; it will only do so if the freed blocks are somehow contiguous with a span of free blocks that follows. Still, it will force the number to be recalculated later. */ - rgd->rd_extfail_pt += rs->rs_free; - rs->rs_free = 0; + rgd->rd_extfail_pt += rs->rs_requested; + rs->rs_requested = 0; } } @@ -672,7 +672,7 @@ void gfs2_rs_deltree(struct gfs2_blkreserv *rs) if (rgd) { spin_lock(&rgd->rd_rsspin); __rs_deltree(rs); - BUG_ON(rs->rs_free); + BUG_ON(rs->rs_requested); spin_unlock(&rgd->rd_rsspin); } } @@ -1504,7 +1504,7 @@ static void rs_insert(struct gfs2_inode *ip) rb_entry(*newn, struct gfs2_blkreserv, rs_node); parent = *newn; - rc = rs_cmp(rs->rs_start, rs->rs_free, cur); + rc = rs_cmp(rs->rs_start, rs->rs_requested, cur); if (rc > 0) newn = &((*newn)->rb_right); else if (rc < 0) @@ -1520,7 +1520,7 @@ static void rs_insert(struct gfs2_inode *ip) rb_insert_color(&rs->rs_node, &rgd->rd_rstree); /* Do our rgrp accounting for the reservation */ - rgd->rd_reserved += rs->rs_free; /* blocks reserved */ + rgd->rd_requested += rs->rs_requested; /* blocks requested */ spin_unlock(&rgd->rd_rsspin); trace_gfs2_rs(rs, TRACE_RS_INSERT); } @@ -1541,9 +1541,9 @@ static inline u32 rgd_free(struct gfs2_rgrpd *rgd, struct gfs2_blkreserv *rs) { u32 tot_reserved, tot_free; - if (WARN_ON_ONCE(rgd->rd_reserved < rs->rs_free)) + if (WARN_ON_ONCE(rgd->rd_requested < rs->rs_requested)) return 0; - tot_reserved = rgd->rd_reserved - rs->rs_free; + tot_reserved = rgd->rd_requested - rs->rs_requested; if (rgd->rd_free_clone < tot_reserved) tot_reserved = 0; @@ -1578,7 +1578,7 @@ static void rg_mblk_search(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip, extlen = max_t(u32, atomic_read(&ip->i_sizehint), ap->target); extlen = clamp(extlen, (u32)RGRP_RSRV_MINBLKS, free_blocks); } - if ((rgd->rd_free_clone < rgd->rd_reserved) || (free_blocks < extlen)) + if ((rgd->rd_free_clone < rgd->rd_requested) || (free_blocks < extlen)) return; /* Find bitmap block that contains bits for goal block */ @@ -1593,7 +1593,7 @@ static void rg_mblk_search(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip, ret = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &extlen, &ip->i_res, true); if (ret == 0) { rs->rs_start = gfs2_rbm_to_block(&rbm); - rs->rs_free = extlen; + rs->rs_requested = extlen; rs_insert(ip); } else { if (goal == rgd->rd_last_alloc + rgd->rd_data0) @@ -1637,7 +1637,7 @@ static u64 gfs2_next_unreserved_block(struct gfs2_rgrpd *rgd, u64 block, if (n) { while (rs_cmp(block, length, rs) == 0 && rs != ignore_rs) { - block = rs->rs_start + rs->rs_free; + block = rs->rs_start + rs->rs_requested; n = n->rb_right; if (n == NULL) break; @@ -2263,7 +2263,7 @@ void gfs2_rgrp_dump(struct seq_file *seq, struct gfs2_rgrpd *rgd, fs_id_buf, (unsigned long long)rgd->rd_addr, rgd->rd_flags, rgd->rd_free, rgd->rd_free_clone, rgd->rd_dinodes, - rgd->rd_reserved, rgd->rd_extfail_pt); + rgd->rd_requested, rgd->rd_extfail_pt); if (rgd->rd_sbd->sd_args.ar_rgrplvb) { struct gfs2_rgrp_lvb *rgl = rgd->rd_rgl; @@ -2318,12 +2318,12 @@ static void gfs2_adjust_reservation(struct gfs2_inode *ip, unsigned int rlen; rs->rs_start += len; - rlen = min(rs->rs_free, len); - rs->rs_free -= rlen; - rgd->rd_reserved -= rlen; + rlen = min(rs->rs_requested, len); + rs->rs_requested -= rlen; + rgd->rd_requested -= rlen; trace_gfs2_rs(rs, TRACE_RS_CLAIM); if (rs->rs_start < rgd->rd_data0 + rgd->rd_data && - rs->rs_free) + rs->rs_requested) goto out; /* We used up our block reservation, so we should reserve more blocks next time. */ diff --git a/fs/gfs2/trace_gfs2.h b/fs/gfs2/trace_gfs2.h index e461a691e5e8..d24bdcdd42e5 100644 --- a/fs/gfs2/trace_gfs2.h +++ b/fs/gfs2/trace_gfs2.h @@ -560,7 +560,7 @@ TRACE_EVENT(gfs2_block_alloc, __field( u8, block_state ) __field( u64, rd_addr ) __field( u32, rd_free_clone ) - __field( u32, rd_reserved ) + __field( u32, rd_requested ) ), TP_fast_assign( @@ -571,7 +571,7 @@ TRACE_EVENT(gfs2_block_alloc, __entry->block_state = block_state; __entry->rd_addr = rgd->rd_addr; __entry->rd_free_clone = rgd->rd_free_clone; - __entry->rd_reserved = rgd->rd_reserved; + __entry->rd_requested = rgd->rd_requested; ), TP_printk("%u,%u bmap %llu alloc %llu/%lu %s rg:%llu rf:%u rr:%lu", @@ -581,7 +581,7 @@ TRACE_EVENT(gfs2_block_alloc, (unsigned long)__entry->len, block_state_name(__entry->block_state), (unsigned long long)__entry->rd_addr, - __entry->rd_free_clone, (unsigned long)__entry->rd_reserved) + __entry->rd_free_clone, (unsigned long)__entry->rd_requested) ); /* Keep track of multi-block reservations as they are allocated/freed */ @@ -595,10 +595,10 @@ TRACE_EVENT(gfs2_rs, __field( dev_t, dev ) __field( u64, rd_addr ) __field( u32, rd_free_clone ) - __field( u32, rd_reserved ) + __field( u32, rd_requested ) __field( u64, inum ) __field( u64, start ) - __field( u32, free ) + __field( u32, requested ) __field( u8, func ) ), @@ -606,11 +606,11 @@ TRACE_EVENT(gfs2_rs, __entry->dev = rs->rs_rgd->rd_sbd->sd_vfs->s_dev; __entry->rd_addr = rs->rs_rgd->rd_addr; __entry->rd_free_clone = rs->rs_rgd->rd_free_clone; - __entry->rd_reserved = rs->rs_rgd->rd_reserved; + __entry->rd_requested = rs->rs_rgd->rd_requested; __entry->inum = container_of(rs, struct gfs2_inode, i_res)->i_no_addr; __entry->start = rs->rs_start; - __entry->free = rs->rs_free; + __entry->requested = rs->rs_requested; __entry->func = func; ), @@ -620,8 +620,8 @@ TRACE_EVENT(gfs2_rs, (unsigned long long)__entry->start, (unsigned long long)__entry->rd_addr, (unsigned long)__entry->rd_free_clone, - (unsigned long)__entry->rd_reserved, - rs_func_name(__entry->func), (unsigned long)__entry->free) + (unsigned long)__entry->rd_requested, + rs_func_name(__entry->func), (unsigned long)__entry->requested) ); #endif /* _TRACE_GFS2_H */ -- cgit v1.2.3 From 725d0e9d464d567cd9290e29879d8bffc92013f8 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Tue, 2 Oct 2018 14:59:54 +0100 Subject: gfs2: Add per-reservation reserved block accounting Add a rs_reserved field to struct gfs2_blkreserv to keep track of the number of blocks reserved by this particular reservation, and a rd_reserved field to struct gfs2_rgrpd to keep track of the total number of reserved blocks in the resource group. Those blocks are exclusively reserved, as opposed to the rs_requested / rd_requested blocks which are tracked in the reservation tree (rd_rstree) and which can be stolen if necessary. When making a reservation with gfs2_inplace_reserve, rs_reserved is set to somewhere between ap->min_target and ap->target depending on the number of free blocks in the resource group. When allocating blocks with gfs2_alloc_blocks, rs_reserved is decremented accordingly. Eventually, any reserved but not consumed blocks are returned to the resource group by gfs2_inplace_release. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/file.c | 4 +-- fs/gfs2/incore.h | 2 ++ fs/gfs2/lops.c | 1 + fs/gfs2/rgrp.c | 80 +++++++++++++++++++++++++++++++++++++++------------- fs/gfs2/trace_gfs2.h | 23 +++++++++++---- 5 files changed, 82 insertions(+), 28 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index 177c4d74ca30..294087516ce0 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -1115,8 +1115,8 @@ static long __gfs2_fallocate(struct file *file, int mode, loff_t offset, loff_t goto out_qunlock; /* check if the selected rgrp limits our max_blks further */ - if (ap.allowed && ap.allowed < max_blks) - max_blks = ap.allowed; + if (ip->i_res.rs_reserved < max_blks) + max_blks = ip->i_res.rs_reserved; /* Almost done. Calculate bytes that can be written using * max_blks. We also recompute max_bytes, data_blocks and diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index 0640d0c70a75..2679ba54798c 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -107,6 +107,7 @@ struct gfs2_rgrpd { u32 rd_bitbytes; /* number of bytes in data bitmaps */ u32 rd_free; u32 rd_requested; /* number of blocks in rd_rstree */ + u32 rd_reserved; /* number of reserved blocks */ u32 rd_free_clone; u32 rd_dinodes; u64 rd_igeneration; @@ -292,6 +293,7 @@ struct gfs2_blkreserv { struct gfs2_rgrpd *rs_rgd; u64 rs_start; u32 rs_requested; + u32 rs_reserved; /* number of reserved blocks */ }; /* diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index 3922b26264f5..802bc15f9f11 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c @@ -84,6 +84,7 @@ static void maybe_release_space(struct gfs2_bufdata *bd) bd->bd_bh->b_data + bi->bi_offset, bi->bi_bytes); clear_bit(GBF_FULL, &bi->bi_flags); rgd->rd_free_clone = rgd->rd_free; + BUG_ON(rgd->rd_free_clone < rgd->rd_reserved); rgd->rd_extfail_pt = rgd->rd_free; } diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index bc8d1ab9e07f..f1df5e75364a 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -1229,6 +1229,7 @@ static int gfs2_rgrp_bh_get(struct gfs2_rgrpd *rgd) rgrp_set_bitmap_flags(rgd); rgd->rd_flags |= (GFS2_RDF_UPTODATE | GFS2_RDF_CHECK); rgd->rd_free_clone = rgd->rd_free; + BUG_ON(rgd->rd_reserved); /* max out the rgrp allocation failure point */ rgd->rd_extfail_pt = rgd->rd_free; } @@ -1278,6 +1279,7 @@ static int update_rgrp_lvb(struct gfs2_rgrpd *rgd) rgd->rd_free = be32_to_cpu(rgd->rd_rgl->rl_free); rgrp_set_bitmap_flags(rgd); rgd->rd_free_clone = rgd->rd_free; + BUG_ON(rgd->rd_reserved); /* max out the rgrp allocation failure point */ rgd->rd_extfail_pt = rgd->rd_free; rgd->rd_dinodes = be32_to_cpu(rgd->rd_rgl->rl_dinodes); @@ -1568,17 +1570,26 @@ static void rg_mblk_search(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip, u64 goal; struct gfs2_blkreserv *rs = &ip->i_res; u32 extlen; - u32 free_blocks = rgd_free(rgd, rs); + u32 free_blocks, blocks_available; int ret; struct inode *inode = &ip->i_inode; + spin_lock(&rgd->rd_rsspin); + free_blocks = rgd_free(rgd, rs); + if (rgd->rd_free_clone < rgd->rd_requested) + free_blocks = 0; + blocks_available = rgd->rd_free_clone - rgd->rd_reserved; + if (rgd == rs->rs_rgd) + blocks_available += rs->rs_reserved; + spin_unlock(&rgd->rd_rsspin); + if (S_ISDIR(inode->i_mode)) extlen = 1; else { extlen = max_t(u32, atomic_read(&ip->i_sizehint), ap->target); extlen = clamp(extlen, (u32)RGRP_RSRV_MINBLKS, free_blocks); } - if ((rgd->rd_free_clone < rgd->rd_requested) || (free_blocks < extlen)) + if (free_blocks < extlen || blocks_available < extlen) return; /* Find bitmap block that contains bits for goal block */ @@ -2027,8 +2038,7 @@ static inline int fast_to_acquire(struct gfs2_rgrpd *rgd) * We try our best to find an rgrp that has at least ap->target blocks * available. After a couple of passes (loops == 2), the prospects of finding * such an rgrp diminish. At this stage, we return the first rgrp that has - * at least ap->min_target blocks available. Either way, we set ap->allowed to - * the number of blocks available in the chosen rgrp. + * at least ap->min_target blocks available. * * Returns: 0 on success, * -ENOMEM if a suitable rgrp can't be found @@ -2044,7 +2054,9 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) u64 last_unlinked = NO_BLOCK; u32 target = ap->target; int loops = 0; - u32 free_blocks, skip = 0; + u32 free_blocks, blocks_available, skip = 0; + + BUG_ON(rs->rs_reserved); if (sdp->sd_args.ar_rgrplvb) flags |= GL_SKIP; @@ -2065,6 +2077,8 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) return -EBADSLT; while (loops < 3) { + struct gfs2_rgrpd *rgd; + rg_locked = 1; if (!gfs2_glock_is_locked_by_me(rs->rs_rgd->rd_gl)) { @@ -2115,11 +2129,20 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) goto check_rgrp; /* If rgrp has enough free space, use it */ - free_blocks = rgd_free(rs->rs_rgd, rs); - if (free_blocks >= target) { - ap->allowed = free_blocks; - return 0; + rgd = rs->rs_rgd; + spin_lock(&rgd->rd_rsspin); + free_blocks = rgd_free(rgd, rs); + blocks_available = rgd->rd_free_clone - rgd->rd_reserved; + if (free_blocks < target || blocks_available < target) { + spin_unlock(&rgd->rd_rsspin); + goto check_rgrp; } + rs->rs_reserved = ap->target; + if (rs->rs_reserved > blocks_available) + rs->rs_reserved = blocks_available; + rgd->rd_reserved += rs->rs_reserved; + spin_unlock(&rgd->rd_rsspin); + return 0; check_rgrp: /* Check for unlinked inodes which can be reclaimed */ if (rs->rs_rgd->rd_flags & GFS2_RDF_CHECK) @@ -2172,6 +2195,17 @@ next_rgrp: void gfs2_inplace_release(struct gfs2_inode *ip) { + struct gfs2_blkreserv *rs = &ip->i_res; + + if (rs->rs_reserved) { + struct gfs2_rgrpd *rgd = rs->rs_rgd; + + spin_lock(&rgd->rd_rsspin); + BUG_ON(rgd->rd_reserved < rs->rs_reserved); + rgd->rd_reserved -= rs->rs_reserved; + spin_unlock(&rgd->rd_rsspin); + rs->rs_reserved = 0; + } if (gfs2_holder_initialized(&ip->i_rgd_gh)) gfs2_glock_dq_uninit(&ip->i_rgd_gh); } @@ -2259,11 +2293,11 @@ void gfs2_rgrp_dump(struct seq_file *seq, struct gfs2_rgrpd *rgd, struct gfs2_blkreserv *trs; const struct rb_node *n; - gfs2_print_dbg(seq, "%s R: n:%llu f:%02x b:%u/%u i:%u r:%u e:%u\n", + gfs2_print_dbg(seq, "%s R: n:%llu f:%02x b:%u/%u i:%u q:%u r:%u e:%u\n", fs_id_buf, (unsigned long long)rgd->rd_addr, rgd->rd_flags, rgd->rd_free, rgd->rd_free_clone, rgd->rd_dinodes, - rgd->rd_requested, rgd->rd_extfail_pt); + rgd->rd_requested, rgd->rd_reserved, rgd->rd_extfail_pt); if (rgd->rd_sbd->sd_args.ar_rgrplvb) { struct gfs2_rgrp_lvb *rgl = rgd->rd_rgl; @@ -2310,7 +2344,8 @@ static void gfs2_adjust_reservation(struct gfs2_inode *ip, struct gfs2_blkreserv *rs = &ip->i_res; struct gfs2_rgrpd *rgd = rbm->rgd; - spin_lock(&rgd->rd_rsspin); + BUG_ON(rs->rs_reserved < len); + rs->rs_reserved -= len; if (gfs2_rs_active(rs)) { u64 start = gfs2_rbm_to_block(rbm); @@ -2324,15 +2359,13 @@ static void gfs2_adjust_reservation(struct gfs2_inode *ip, trace_gfs2_rs(rs, TRACE_RS_CLAIM); if (rs->rs_start < rgd->rd_data0 + rgd->rd_data && rs->rs_requested) - goto out; + return; /* We used up our block reservation, so we should reserve more blocks next time. */ atomic_add(RGRP_RSRV_ADDBLKS, &ip->i_sizehint); } __rs_deltree(rs); } -out: - spin_unlock(&rgd->rd_rsspin); } /** @@ -2386,6 +2419,8 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, u32 minext = 1; int error = -ENOSPC; + BUG_ON(ip->i_res.rs_reserved < *nblocks); + if (gfs2_rs_active(&ip->i_res)) { gfs2_set_alloc_start(&rbm, ip, dinode); error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, &ip->i_res, false); @@ -2407,8 +2442,6 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, gfs2_alloc_extent(&rbm, dinode, nblocks); block = gfs2_rbm_to_block(&rbm); rbm.rgd->rd_last_alloc = block - rbm.rgd->rd_data0; - if (gfs2_rs_active(&ip->i_res)) - gfs2_adjust_reservation(ip, &rbm, *nblocks); if (!dinode) { ip->i_goal = block + *nblocks - 1; error = gfs2_meta_inode_buffer(ip, &dibh); @@ -2421,12 +2454,20 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, brelse(dibh); } } - if (rbm.rgd->rd_free < *nblocks) { + spin_lock(&rbm.rgd->rd_rsspin); + gfs2_adjust_reservation(ip, &rbm, *nblocks); + if (rbm.rgd->rd_free < *nblocks || rbm.rgd->rd_reserved < *nblocks) { fs_warn(sdp, "nblocks=%u\n", *nblocks); + spin_unlock(&rbm.rgd->rd_rsspin); goto rgrp_error; } - + BUG_ON(rbm.rgd->rd_reserved < *nblocks); + BUG_ON(rbm.rgd->rd_free_clone < *nblocks); + BUG_ON(rbm.rgd->rd_free < *nblocks); + rbm.rgd->rd_reserved -= *nblocks; + rbm.rgd->rd_free_clone -= *nblocks; rbm.rgd->rd_free -= *nblocks; + spin_unlock(&rbm.rgd->rd_rsspin); if (dinode) { rbm.rgd->rd_dinodes++; *generation = rbm.rgd->rd_igeneration++; @@ -2443,7 +2484,6 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, gfs2_quota_change(ip, *nblocks, ip->i_inode.i_uid, ip->i_inode.i_gid); - rbm.rgd->rd_free_clone -= *nblocks; trace_gfs2_block_alloc(ip, rbm.rgd, block, *nblocks, dinode ? GFS2_BLKST_DINODE : GFS2_BLKST_USED); *bn = block; diff --git a/fs/gfs2/trace_gfs2.h b/fs/gfs2/trace_gfs2.h index d24bdcdd42e5..bd6c8e9e49db 100644 --- a/fs/gfs2/trace_gfs2.h +++ b/fs/gfs2/trace_gfs2.h @@ -561,6 +561,7 @@ TRACE_EVENT(gfs2_block_alloc, __field( u64, rd_addr ) __field( u32, rd_free_clone ) __field( u32, rd_requested ) + __field( u32, rd_reserved ) ), TP_fast_assign( @@ -572,16 +573,19 @@ TRACE_EVENT(gfs2_block_alloc, __entry->rd_addr = rgd->rd_addr; __entry->rd_free_clone = rgd->rd_free_clone; __entry->rd_requested = rgd->rd_requested; + __entry->rd_reserved = rgd->rd_reserved; ), - TP_printk("%u,%u bmap %llu alloc %llu/%lu %s rg:%llu rf:%u rr:%lu", + TP_printk("%u,%u bmap %llu alloc %llu/%lu %s rg:%llu rf:%u rq:%u rr:%u", MAJOR(__entry->dev), MINOR(__entry->dev), (unsigned long long)__entry->inum, (unsigned long long)__entry->start, (unsigned long)__entry->len, block_state_name(__entry->block_state), (unsigned long long)__entry->rd_addr, - __entry->rd_free_clone, (unsigned long)__entry->rd_requested) + __entry->rd_free_clone, + __entry->rd_requested, + __entry->rd_reserved) ); /* Keep track of multi-block reservations as they are allocated/freed */ @@ -596,9 +600,11 @@ TRACE_EVENT(gfs2_rs, __field( u64, rd_addr ) __field( u32, rd_free_clone ) __field( u32, rd_requested ) + __field( u32, rd_reserved ) __field( u64, inum ) __field( u64, start ) __field( u32, requested ) + __field( u32, reserved ) __field( u8, func ) ), @@ -607,21 +613,26 @@ TRACE_EVENT(gfs2_rs, __entry->rd_addr = rs->rs_rgd->rd_addr; __entry->rd_free_clone = rs->rs_rgd->rd_free_clone; __entry->rd_requested = rs->rs_rgd->rd_requested; + __entry->rd_reserved = rs->rs_rgd->rd_reserved; __entry->inum = container_of(rs, struct gfs2_inode, i_res)->i_no_addr; __entry->start = rs->rs_start; __entry->requested = rs->rs_requested; + __entry->reserved = rs->rs_reserved; __entry->func = func; ), - TP_printk("%u,%u bmap %llu resrv %llu rg:%llu rf:%lu rr:%lu %s f:%lu", + TP_printk("%u,%u bmap %llu resrv %llu rg:%llu rf:%u rq:%u rr:%u %s q:%u r:%u", MAJOR(__entry->dev), MINOR(__entry->dev), (unsigned long long)__entry->inum, (unsigned long long)__entry->start, (unsigned long long)__entry->rd_addr, - (unsigned long)__entry->rd_free_clone, - (unsigned long)__entry->rd_requested, - rs_func_name(__entry->func), (unsigned long)__entry->requested) + __entry->rd_free_clone, + __entry->rd_requested, + __entry->rd_reserved, + rs_func_name(__entry->func), + __entry->requested, + __entry->reserved) ); #endif /* _TRACE_GFS2_H */ -- cgit v1.2.3 From 9e514605c77451745ea9fca5a26fc3153893686a Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Mon, 8 Feb 2021 20:44:26 +0100 Subject: gfs2: Add local resource group locking Prepare for treating resource group glocks as exclusive among nodes but shared among all tasks running on a node: introduce another layer of node-specific locking that the local tasks can use to coordinate their accesses. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/incore.h | 2 ++ fs/gfs2/lops.c | 6 +++++- fs/gfs2/rgrp.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++------ fs/gfs2/rgrp.h | 4 ++++ 4 files changed, 59 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index 2679ba54798c..92f5a920ce61 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -20,6 +20,7 @@ #include #include #include +#include #define DIO_WAIT 0x00000010 #define DIO_METADATA 0x00000020 @@ -123,6 +124,7 @@ struct gfs2_rgrpd { #define GFS2_RDF_PREFERRED 0x80000000 /* This rgrp is preferred */ #define GFS2_RDF_MASK 0xf0000000 /* mask for internal flags */ spinlock_t rd_rsspin; /* protects reservation related vars */ + struct mutex rd_mutex; struct rb_root rd_rstree; /* multi-block reservation tree */ }; diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index 802bc15f9f11..dffa6cc8b5b8 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c @@ -76,8 +76,9 @@ static void maybe_release_space(struct gfs2_bufdata *bd) unsigned int index = bd->bd_bh->b_blocknr - gl->gl_name.ln_number; struct gfs2_bitmap *bi = rgd->rd_bits + index; + rgrp_lock_local(rgd); if (bi->bi_clone == NULL) - return; + goto out; if (sdp->sd_args.ar_discard) gfs2_rgrp_send_discards(sdp, rgd->rd_data0, bd->bd_bh, bi, 1, NULL); memcpy(bi->bi_clone + bi->bi_offset, @@ -86,6 +87,9 @@ static void maybe_release_space(struct gfs2_bufdata *bd) rgd->rd_free_clone = rgd->rd_free; BUG_ON(rgd->rd_free_clone < rgd->rd_reserved); rgd->rd_extfail_pt = rgd->rd_free; + +out: + rgrp_unlock_local(rgd); } /** diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index f1df5e75364a..f45b787307d0 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -920,6 +920,7 @@ static int read_rindex_entry(struct gfs2_inode *ip) rgd->rd_data = be32_to_cpu(buf.ri_data); rgd->rd_bitbytes = be32_to_cpu(buf.ri_bitbytes); spin_lock_init(&rgd->rd_rsspin); + mutex_init(&rgd->rd_mutex); error = compute_bitstructs(rgd); if (error) @@ -1449,9 +1450,11 @@ int gfs2_fitrim(struct file *filp, void __user *argp) /* Trim each bitmap in the rgrp */ for (x = 0; x < rgd->rd_length; x++) { struct gfs2_bitmap *bi = rgd->rd_bits + x; + rgrp_lock_local(rgd); ret = gfs2_rgrp_send_discards(sdp, rgd->rd_data0, NULL, bi, minlen, &amt); + rgrp_unlock_local(rgd); if (ret) { gfs2_glock_dq_uninit(&gh); goto out; @@ -1463,9 +1466,11 @@ int gfs2_fitrim(struct file *filp, void __user *argp) ret = gfs2_trans_begin(sdp, RES_RG_HDR, 0); if (ret == 0) { bh = rgd->rd_bits[0].bi_bh; + rgrp_lock_local(rgd); rgd->rd_flags |= GFS2_RGF_TRIMMED; gfs2_trans_add_meta(rgd->rd_gl, bh); gfs2_rgrp_out(rgd, bh->b_data); + rgrp_unlock_local(rgd); gfs2_trans_end(sdp); } } @@ -2050,7 +2055,8 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); struct gfs2_rgrpd *begin = NULL; struct gfs2_blkreserv *rs = &ip->i_res; - int error = 0, rg_locked, flags = 0; + int error = 0, flags = 0; + bool rg_locked; u64 last_unlinked = NO_BLOCK; u32 target = ap->target; int loops = 0; @@ -2079,10 +2085,10 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) while (loops < 3) { struct gfs2_rgrpd *rgd; - rg_locked = 1; - - if (!gfs2_glock_is_locked_by_me(rs->rs_rgd->rd_gl)) { - rg_locked = 0; + rg_locked = gfs2_glock_is_locked_by_me(rs->rs_rgd->rd_gl); + if (rg_locked) { + rgrp_lock_local(rs->rs_rgd); + } else { if (skip && skip--) goto next_rgrp; if (!gfs2_rs_active(rs)) { @@ -2099,12 +2105,14 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) &ip->i_rgd_gh); if (unlikely(error)) return error; + rgrp_lock_local(rs->rs_rgd); if (!gfs2_rs_active(rs) && (loops < 2) && gfs2_rgrp_congested(rs->rs_rgd, loops)) goto skip_rgrp; if (sdp->sd_args.ar_rgrplvb) { error = update_rgrp_lvb(rs->rs_rgd); if (unlikely(error)) { + rgrp_unlock_local(rs->rs_rgd); gfs2_glock_dq_uninit(&ip->i_rgd_gh); return error; } @@ -2142,6 +2150,7 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) rs->rs_reserved = blocks_available; rgd->rd_reserved += rs->rs_reserved; spin_unlock(&rgd->rd_rsspin); + rgrp_unlock_local(rs->rs_rgd); return 0; check_rgrp: /* Check for unlinked inodes which can be reclaimed */ @@ -2149,6 +2158,8 @@ check_rgrp: try_rgrp_unlink(rs->rs_rgd, &last_unlinked, ip->i_no_addr); skip_rgrp: + rgrp_unlock_local(rs->rs_rgd); + /* Drop reservation, if we couldn't use reserved rgrp */ if (gfs2_rs_active(rs)) gfs2_rs_deltree(rs); @@ -2293,6 +2304,7 @@ void gfs2_rgrp_dump(struct seq_file *seq, struct gfs2_rgrpd *rgd, struct gfs2_blkreserv *trs; const struct rb_node *n; + spin_lock(&rgd->rd_rsspin); gfs2_print_dbg(seq, "%s R: n:%llu f:%02x b:%u/%u i:%u q:%u r:%u e:%u\n", fs_id_buf, (unsigned long long)rgd->rd_addr, rgd->rd_flags, @@ -2306,7 +2318,6 @@ void gfs2_rgrp_dump(struct seq_file *seq, struct gfs2_rgrpd *rgd, be32_to_cpu(rgl->rl_free), be32_to_cpu(rgl->rl_dinodes)); } - spin_lock(&rgd->rd_rsspin); for (n = rb_first(&rgd->rd_rstree); n; n = rb_next(&trs->rs_node)) { trs = rb_entry(n, struct gfs2_blkreserv, rs_node); dump_rs(seq, trs, fs_id_buf); @@ -2421,6 +2432,7 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, BUG_ON(ip->i_res.rs_reserved < *nblocks); + rgrp_lock_local(rbm.rgd); if (gfs2_rs_active(&ip->i_res)) { gfs2_set_alloc_start(&rbm, ip, dinode); error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, &ip->i_res, false); @@ -2477,6 +2489,7 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, gfs2_trans_add_meta(rbm.rgd->rd_gl, rbm.rgd->rd_bits[0].bi_bh); gfs2_rgrp_out(rbm.rgd, rbm.rgd->rd_bits[0].bi_bh->b_data); + rgrp_unlock_local(rbm.rgd); gfs2_statfs_change(sdp, 0, -(s64)*nblocks, dinode ? 1 : 0); if (dinode) @@ -2490,6 +2503,7 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, return 0; rgrp_error: + rgrp_unlock_local(rbm.rgd); gfs2_rgrp_error(rbm.rgd); return -EIO; } @@ -2509,12 +2523,14 @@ void __gfs2_free_blocks(struct gfs2_inode *ip, struct gfs2_rgrpd *rgd, { struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + rgrp_lock_local(rgd); rgblk_free(sdp, rgd, bstart, blen, GFS2_BLKST_FREE); trace_gfs2_block_alloc(ip, rgd, bstart, blen, GFS2_BLKST_FREE); rgd->rd_free += blen; rgd->rd_flags &= ~GFS2_RGF_TRIMMED; gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh); gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data); + rgrp_unlock_local(rgd); /* Directories keep their data in the metadata address space */ if (meta || ip->i_depth || gfs2_is_jdata(ip)) @@ -2550,17 +2566,20 @@ void gfs2_unlink_di(struct inode *inode) rgd = gfs2_blk2rgrpd(sdp, blkno, true); if (!rgd) return; + rgrp_lock_local(rgd); rgblk_free(sdp, rgd, blkno, 1, GFS2_BLKST_UNLINKED); trace_gfs2_block_alloc(ip, rgd, blkno, 1, GFS2_BLKST_UNLINKED); gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh); gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data); be32_add_cpu(&rgd->rd_rgl->rl_unlinked, 1); + rgrp_unlock_local(rgd); } void gfs2_free_di(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip) { struct gfs2_sbd *sdp = rgd->rd_sbd; + rgrp_lock_local(rgd); rgblk_free(sdp, rgd, ip->i_no_addr, 1, GFS2_BLKST_FREE); if (!rgd->rd_dinodes) gfs2_consist_rgrpd(rgd); @@ -2569,6 +2588,7 @@ void gfs2_free_di(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip) gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh); gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data); + rgrp_unlock_local(rgd); be32_add_cpu(&rgd->rd_rgl->rl_unlinked, -1); gfs2_statfs_change(sdp, 0, +1, -1); @@ -2583,6 +2603,10 @@ void gfs2_free_di(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip) * @no_addr: The block number to check * @type: The block type we are looking for * + * The inode glock of @no_addr must be held. The @type to check for is either + * GFS2_BLKST_DINODE or GFS2_BLKST_UNLINKED; checking for type GFS2_BLKST_FREE + * or GFS2_BLKST_USED would make no sense. + * * Returns: 0 if the block type matches the expected type * -ESTALE if it doesn't match * or -ve errno if something went wrong while checking @@ -2606,6 +2630,13 @@ int gfs2_check_blk_type(struct gfs2_sbd *sdp, u64 no_addr, unsigned int type) rbm.rgd = rgd; error = gfs2_rbm_from_block(&rbm, no_addr); if (!WARN_ON_ONCE(error)) { + /* + * No need to take the local resource group lock here; the + * inode glock of @no_addr provides the necessary + * synchronization in case the block is an inode. (In case + * the block is not an inode, the block type will not match + * the @type we are looking for.) + */ if (gfs2_testbit(&rbm, false) != type) error = -ESTALE; } @@ -2730,3 +2761,14 @@ void gfs2_rlist_free(struct gfs2_rgrp_list *rlist) } } +void rgrp_lock_local(struct gfs2_rgrpd *rgd) +{ + BUG_ON(!gfs2_glock_is_held_excl(rgd->rd_gl) && + !test_bit(SDF_NORECOVERY, &rgd->rd_sbd->sd_flags)); + mutex_lock(&rgd->rd_mutex); +} + +void rgrp_unlock_local(struct gfs2_rgrpd *rgd) +{ + mutex_unlock(&rgd->rd_mutex); +} diff --git a/fs/gfs2/rgrp.h b/fs/gfs2/rgrp.h index be1b2034f5ee..a6855fd796e0 100644 --- a/fs/gfs2/rgrp.h +++ b/fs/gfs2/rgrp.h @@ -88,4 +88,8 @@ static inline int rgrp_contains_block(struct gfs2_rgrpd *rgd, u64 block) } extern void check_and_update_goal(struct gfs2_inode *ip); + +extern void rgrp_lock_local(struct gfs2_rgrpd *rgd); +extern void rgrp_unlock_local(struct gfs2_rgrpd *rgd); + #endif /* __RGRP_DOT_H__ */ -- cgit v1.2.3 From 06e908cd9ead1f215cc30132aac5ce132a352fbe Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Wed, 18 Apr 2018 13:58:19 -0700 Subject: gfs2: Allow node-wide exclusive glock sharing Introduce a new LM_FLAG_NODE_SCOPE glock holder flag: when taking a glock in LM_ST_EXCLUSIVE (EX) mode and with the LM_FLAG_NODE_SCOPE flag set, the exclusive lock is shared among all local processes who are holding the glock in EX mode and have the LM_FLAG_NODE_SCOPE flag set. From the point of view of other nodes, the lock is still held exclusively. A future patch will start using this flag to improve performance with rgrp sharing. Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/glock.c | 22 +++++++++++++++++++--- fs/gfs2/glock.h | 6 ++++++ 2 files changed, 25 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index d87a5bc3607b..9567520d79f7 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -313,9 +313,23 @@ void gfs2_glock_put(struct gfs2_glock *gl) static inline int may_grant(const struct gfs2_glock *gl, const struct gfs2_holder *gh) { const struct gfs2_holder *gh_head = list_first_entry(&gl->gl_holders, const struct gfs2_holder, gh_list); - if ((gh->gh_state == LM_ST_EXCLUSIVE || - gh_head->gh_state == LM_ST_EXCLUSIVE) && gh != gh_head) - return 0; + + if (gh != gh_head) { + /** + * Here we make a special exception to grant holders who agree + * to share the EX lock with other holders who also have the + * bit set. If the original holder has the LM_FLAG_NODE_SCOPE bit + * is set, we grant more holders with the bit set. + */ + if (gh_head->gh_state == LM_ST_EXCLUSIVE && + (gh_head->gh_flags & LM_FLAG_NODE_SCOPE) && + gh->gh_state == LM_ST_EXCLUSIVE && + (gh->gh_flags & LM_FLAG_NODE_SCOPE)) + return 1; + if ((gh->gh_state == LM_ST_EXCLUSIVE || + gh_head->gh_state == LM_ST_EXCLUSIVE)) + return 0; + } if (gl->gl_state == gh->gh_state) return 1; if (gh->gh_flags & GL_EXACT) @@ -2030,6 +2044,8 @@ static const char *hflags2str(char *buf, u16 flags, unsigned long iflags) *p++ = 'A'; if (flags & LM_FLAG_PRIORITY) *p++ = 'p'; + if (flags & LM_FLAG_NODE_SCOPE) + *p++ = 'n'; if (flags & GL_ASYNC) *p++ = 'a'; if (flags & GL_EXACT) diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h index 53813364517b..31a8f2f649b5 100644 --- a/fs/gfs2/glock.h +++ b/fs/gfs2/glock.h @@ -75,6 +75,11 @@ enum { * request and directly join the other shared lock. A shared lock request * without the priority flag might be forced to wait until the deferred * requested had acquired and released the lock. + * + * LM_FLAG_NODE_SCOPE + * This holder agrees to share the lock within this node. In other words, + * the glock is held in EX mode according to DLM, but local holders on the + * same node can share it. */ #define LM_FLAG_TRY 0x0001 @@ -82,6 +87,7 @@ enum { #define LM_FLAG_NOEXP 0x0004 #define LM_FLAG_ANY 0x0008 #define LM_FLAG_PRIORITY 0x0010 +#define LM_FLAG_NODE_SCOPE 0x0020 #define GL_ASYNC 0x0040 #define GL_EXACT 0x0080 #define GL_SKIP 0x0100 -- cgit v1.2.3 From 4fc7ec31c3c44031e8a8872bb8432cf4f6cb6ddd Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Tue, 24 Apr 2018 10:35:02 -0700 Subject: gfs2: Use resource group glock sharing This patch takes advantage of the new glock holder sharing feature for resource groups. We have already introduced local resource group locking in a previous patch, so competing accesses of local processes are already under control. Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/bmap.c | 2 +- fs/gfs2/inode.c | 6 +++--- fs/gfs2/rgrp.c | 10 +++++----- fs/gfs2/super.c | 3 ++- fs/gfs2/xattr.c | 6 ++++-- 5 files changed, 15 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index e2ee3703f7df..cf6ccdd00587 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1538,7 +1538,7 @@ more_rgrps: goto out; } ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, - 0, rd_gh); + LM_FLAG_NODE_SCOPE, rd_gh); if (ret) goto out; diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index c1b77e8d6b1c..4d3d8624a95b 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -1145,7 +1145,7 @@ static int gfs2_unlink(struct inode *dir, struct dentry *dentry) if (!rgd) goto out_inodes; - gfs2_holder_init(rgd->rd_gl, LM_ST_EXCLUSIVE, 0, ghs + 2); + gfs2_holder_init(rgd->rd_gl, LM_ST_EXCLUSIVE, LM_FLAG_NODE_SCOPE, ghs + 2); error = gfs2_glock_nq(ghs); /* parent */ @@ -1450,8 +1450,8 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, error = -ENOENT; goto out_gunlock; } - error = gfs2_glock_nq_init(nrgd->rd_gl, LM_ST_EXCLUSIVE, 0, - &rd_gh); + error = gfs2_glock_nq_init(nrgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &rd_gh); if (error) goto out_gunlock; } diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index f45b787307d0..89c37a845e64 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -1442,7 +1442,8 @@ int gfs2_fitrim(struct file *filp, void __user *argp) while (1) { - ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, 0, &gh); + ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &gh); if (ret) goto out; @@ -2055,7 +2056,7 @@ int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); struct gfs2_rgrpd *begin = NULL; struct gfs2_blkreserv *rs = &ip->i_res; - int error = 0, flags = 0; + int error = 0, flags = LM_FLAG_NODE_SCOPE; bool rg_locked; u64 last_unlinked = NO_BLOCK; u32 target = ap->target; @@ -2736,9 +2737,8 @@ void gfs2_rlist_alloc(struct gfs2_rgrp_list *rlist) sizeof(struct gfs2_holder), GFP_NOFS | __GFP_NOFAIL); for (x = 0; x < rlist->rl_rgrps; x++) - gfs2_holder_init(rlist->rl_rgd[x]->rd_gl, - LM_ST_EXCLUSIVE, 0, - &rlist->rl_ghs[x]); + gfs2_holder_init(rlist->rl_rgd[x]->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &rlist->rl_ghs[x]); } /** diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index 2f56acc41c04..e138a2d15c77 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -1229,7 +1229,8 @@ static int gfs2_dinode_dealloc(struct gfs2_inode *ip) goto out_qs; } - error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, 0, &gh); + error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &gh); if (error) goto out_qs; diff --git a/fs/gfs2/xattr.c b/fs/gfs2/xattr.c index 9d7667bc4292..d332bf31f0ae 100644 --- a/fs/gfs2/xattr.c +++ b/fs/gfs2/xattr.c @@ -259,7 +259,8 @@ static int ea_dealloc_unstuffed(struct gfs2_inode *ip, struct buffer_head *bh, return -EIO; } - error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, 0, &rg_gh); + error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &rg_gh); if (error) return error; @@ -1385,7 +1386,8 @@ static int ea_dealloc_block(struct gfs2_inode *ip) return -EIO; } - error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, 0, &gh); + error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &gh); if (error) return error; -- cgit v1.2.3 From 7ae017c7322e2b12472033e65a48aa25cde2fb22 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Wed, 17 Feb 2021 10:12:33 -0500 Subject: NFS: Support the '-owrite=' option in /proc/self/mounts and mountinfo Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/super.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'fs') diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 4034102010f0..bd22c9338600 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -511,6 +511,13 @@ static void nfs_show_mount_options(struct seq_file *m, struct nfs_server *nfss, seq_puts(m, ",local_lock=flock"); else seq_puts(m, ",local_lock=posix"); + + if (nfss->flags & NFS_MOUNT_WRITE_EAGER) { + if (nfss->flags & NFS_MOUNT_WRITE_WAIT) + seq_puts(m, ",write=wait"); + else + seq_puts(m, ",write=eager"); + } } /* -- cgit v1.2.3 From fe1cdd558619546f76643878e7aa521c32d52131 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 17 Feb 2021 21:02:36 +0000 Subject: io_uring: fix read memory leak Don't forget to free iovec read inline completion and bunch of other cases that do "goto done" before setting up an async context. Fixes: 5ea5dd45844d ("io_uring: inline io_read()'s iovec freeing") Reported-by: Jens Axboe Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 58dd10481106..4352bcea3d9d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3602,10 +3602,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) ret = io_iter_do_read(req, iter); if (ret == -EIOCBQUEUED) { - /* it's faster to check here then delegate to kfree */ - if (iovec) - kfree(iovec); - return 0; + goto out_free; } else if (ret == -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) @@ -3626,6 +3623,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) if (ret2) return ret2; + iovec = NULL; rw = req->async_data; /* now use our persistent iterator, if we aren't already */ iter = &rw->iter; @@ -3652,6 +3650,10 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) } while (ret > 0 && ret < io_size); done: kiocb_done(kiocb, ret, issue_flags); +out_free: + /* it's faster to check here then delegate to kfree */ + if (iovec) + kfree(iovec); return 0; } -- cgit v1.2.3 From 059c01039c0185dbee7ed080f1f2bd22cb1e4dab Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki Date: Wed, 17 Feb 2021 18:58:11 +0900 Subject: zonefs: Fix file size of zones in full condition Per ZBC/ZAC/ZNS specifications, write pointers may not have valid values when zones are in full condition. However, when zonefs mounts a zoned block device, zonefs refers write pointers to set file size even when the zones are in full condition. This results in wrong file size. To fix this, refer maximum file size in place of write pointers for zones in full condition. Signed-off-by: Shin'ichiro Kawasaki Fixes: 8dcc1a9d90c1 ("fs: New zonefs file system") Cc: # 5.6+ Signed-off-by: Damien Le Moal --- fs/zonefs/super.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 96f0cb0c29aa..2e2a234d387b 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -256,6 +256,9 @@ static loff_t zonefs_check_zone_condition(struct inode *inode, } inode->i_mode &= ~0222; return i_size_read(inode); + case BLK_ZONE_COND_FULL: + /* The write pointer of full zones is invalid. */ + return zi->i_max_size; default: if (zi->i_ztype == ZONEFS_ZTYPE_CNV) return zi->i_max_size; -- cgit v1.2.3 From bc6de804d36b3709d54fa22bd128cbac91c11526 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 18 Feb 2021 11:08:17 +0100 Subject: debugfs: be more robust at handling improper input in debugfs_lookup() debugfs_lookup() doesn't like it if it is passed an illegal name pointer, or if the filesystem isn't even initialized yet. If either of these happen, it will crash the system, so fix it up by properly testing for valid input and that we are up and running before trying to find a file in the filesystem. Cc: "Rafael J. Wysocki" Cc: stable Reported-by: Michael Walle Tested-by: Michael Walle Tested-by: Marc Zyngier Link: https://lore.kernel.org/r/20210218100818.3622317-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- fs/debugfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/debugfs/inode.c b/fs/debugfs/inode.c index 2fcf66473436..bbeb563cbe78 100644 --- a/fs/debugfs/inode.c +++ b/fs/debugfs/inode.c @@ -297,7 +297,7 @@ struct dentry *debugfs_lookup(const char *name, struct dentry *parent) { struct dentry *dentry; - if (IS_ERR(parent)) + if (!debugfs_initialized() || IS_ERR_OR_NULL(name) || IS_ERR(parent)) return NULL; if (!parent) -- cgit v1.2.3 From 56348560d495d2501e87db559a61de717cd3ab02 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 18 Feb 2021 11:08:18 +0100 Subject: debugfs: do not attempt to create a new file before the filesystem is initalized Some subsystems want to add debugfs files at early boot, way before debugfs is initialized. This seems to work somehow as the vfs layer will not allow it to happen, but let's be explicit and test to ensure we are properly up and running before allowing files to be created. Cc: "Rafael J. Wysocki" Cc: stable Reported-by: Michael Walle Reported-by: Marc Zyngier Link: https://lore.kernel.org/r/20210218100818.3622317-2-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- fs/debugfs/inode.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/debugfs/inode.c b/fs/debugfs/inode.c index bbeb563cbe78..86c7f0489620 100644 --- a/fs/debugfs/inode.c +++ b/fs/debugfs/inode.c @@ -318,6 +318,9 @@ static struct dentry *start_creating(const char *name, struct dentry *parent) if (!(debugfs_allow & DEBUGFS_ALLOW_API)) return ERR_PTR(-EPERM); + if (!debugfs_initialized()) + return ERR_PTR(-ENOENT); + pr_debug("creating file '%s'\n", name); if (IS_ERR(parent)) -- cgit v1.2.3 From 46c4e16a8625f7afdd8eee1ac8c3b3e592cba974 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:37 +0000 Subject: io_uring: kill fictitious submit iteration index @i and @submitted are very much coupled together, and there is no need to keep them both. Remove @i, it doesn't change generated binary but helps to keep a single source of truth. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 4352bcea3d9d..32a6c89e69b1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6884,7 +6884,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) { struct io_submit_link link; - int i, submitted = 0; + int submitted = 0; /* if we have a backlog and couldn't flush it all, return BUSY */ if (test_bit(0, &ctx->sq_check_overflow)) { @@ -6904,7 +6904,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) io_submit_state_start(&ctx->submit_state, nr); link.head = NULL; - for (i = 0; i < nr; i++) { + while (submitted < nr) { const struct io_uring_sqe *sqe; struct io_kiocb *req; int err; -- cgit v1.2.3 From 1155c76a248364dd182bde90fea6f5682a6a766f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:38 +0000 Subject: io_uring: keep io_*_prep() naming consistent Follow io_*_prep() naming pattern, there are only fsync and sfr that don't do that. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 32a6c89e69b1..adb5cd4b760d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4008,7 +4008,7 @@ static int io_nop(struct io_kiocb *req, unsigned int issue_flags) return 0; } -static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_ring_ctx *ctx = req->ctx; @@ -4595,7 +4595,7 @@ err: return 0; } -static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_ring_ctx *ctx = req->ctx; @@ -6081,9 +6081,9 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) case IORING_OP_POLL_REMOVE: return io_poll_remove_prep(req, sqe); case IORING_OP_FSYNC: - return io_prep_fsync(req, sqe); + return io_fsync_prep(req, sqe); case IORING_OP_SYNC_FILE_RANGE: - return io_prep_sfr(req, sqe); + return io_sfr_prep(req, sqe); case IORING_OP_SENDMSG: case IORING_OP_SEND: return io_sendmsg_prep(req, sqe); -- cgit v1.2.3 From 441960f3b9b8ee6aeea847e3e67093e0840e7059 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:39 +0000 Subject: io_uring: don't duplicate ->file check in sfr IORING_OP_SYNC_FILE_RANGE is marked as .needs_file, so the common path will take care of assigning and validating req->file, no need to duplicate it in io_sfr_prep(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index adb5cd4b760d..db6680bb02d3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4599,9 +4599,6 @@ static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_ring_ctx *ctx = req->ctx; - if (!req->file) - return -EBADF; - if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index)) -- cgit v1.2.3 From b16fed66bc7dca1a5dfd0af8991e9f58b5ef8d5f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:40 +0000 Subject: io_uring: move io_init_req()'s definition A preparation patch, symbol to symbol move io_init_req() + io_check_restriction() a bit up. The submission path is pretty settled down, so don't worry about backports and move the functions instead of relying on forward declarations in the future. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 214 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 107 insertions(+), 107 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index db6680bb02d3..1563853caac5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -104,6 +104,10 @@ #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST) +#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ + IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ + IOSQE_BUFFER_SELECT) + struct io_uring { u32 head ____cacheline_aligned_in_smp; u32 tail ____cacheline_aligned_in_smp; @@ -6639,6 +6643,109 @@ static inline void io_queue_link_head(struct io_kiocb *req) io_queue_sqe(req, NULL); } +/* + * Check SQE restrictions (opcode and flags). + * + * Returns 'true' if SQE is allowed, 'false' otherwise. + */ +static inline bool io_check_restriction(struct io_ring_ctx *ctx, + struct io_kiocb *req, + unsigned int sqe_flags) +{ + if (!ctx->restricted) + return true; + + if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) + return false; + + if ((sqe_flags & ctx->restrictions.sqe_flags_required) != + ctx->restrictions.sqe_flags_required) + return false; + + if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed | + ctx->restrictions.sqe_flags_required)) + return false; + + return true; +} + +static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, + const struct io_uring_sqe *sqe) +{ + struct io_submit_state *state; + unsigned int sqe_flags; + int id, ret = 0; + + req->opcode = READ_ONCE(sqe->opcode); + /* same numerical values with corresponding REQ_F_*, safe to copy */ + req->flags = sqe_flags = READ_ONCE(sqe->flags); + req->user_data = READ_ONCE(sqe->user_data); + req->async_data = NULL; + req->file = NULL; + req->ctx = ctx; + req->link = NULL; + req->fixed_rsrc_refs = NULL; + /* one is dropped after submission, the other at completion */ + refcount_set(&req->refs, 2); + req->task = current; + req->result = 0; + + /* enforce forwards compatibility on users */ + if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) + return -EINVAL; + + if (unlikely(req->opcode >= IORING_OP_LAST)) + return -EINVAL; + + if (unlikely(io_sq_thread_acquire_mm_files(ctx, req))) + return -EFAULT; + + if (unlikely(!io_check_restriction(ctx, req, sqe_flags))) + return -EACCES; + + if ((sqe_flags & IOSQE_BUFFER_SELECT) && + !io_op_defs[req->opcode].buffer_select) + return -EOPNOTSUPP; + + id = READ_ONCE(sqe->personality); + if (id) { + struct io_identity *iod; + + iod = idr_find(&ctx->personality_idr, id); + if (unlikely(!iod)) + return -EINVAL; + refcount_inc(&iod->count); + + __io_req_init_async(req); + get_cred(iod->creds); + req->work.identity = iod; + req->work.flags |= IO_WQ_WORK_CREDS; + } + + state = &ctx->submit_state; + + /* + * Plug now if we have more than 1 IO left after this, and the target + * is potentially a read/write to block based storage. + */ + if (!state->plug_started && state->ios_left > 1 && + io_op_defs[req->opcode].plug) { + blk_start_plug(&state->plug); + state->plug_started = true; + } + + if (io_op_defs[req->opcode].needs_file) { + bool fixed = req->flags & REQ_F_FIXED_FILE; + + req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed); + if (unlikely(!req->file)) + ret = -EBADF; + } + + state->ios_left--; + return ret; +} + struct io_submit_link { struct io_kiocb *head; struct io_kiocb *last; @@ -6771,113 +6878,6 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) return NULL; } -/* - * Check SQE restrictions (opcode and flags). - * - * Returns 'true' if SQE is allowed, 'false' otherwise. - */ -static inline bool io_check_restriction(struct io_ring_ctx *ctx, - struct io_kiocb *req, - unsigned int sqe_flags) -{ - if (!ctx->restricted) - return true; - - if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) - return false; - - if ((sqe_flags & ctx->restrictions.sqe_flags_required) != - ctx->restrictions.sqe_flags_required) - return false; - - if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed | - ctx->restrictions.sqe_flags_required)) - return false; - - return true; -} - -#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ - IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ - IOSQE_BUFFER_SELECT) - -static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, - const struct io_uring_sqe *sqe) -{ - struct io_submit_state *state; - unsigned int sqe_flags; - int id, ret = 0; - - req->opcode = READ_ONCE(sqe->opcode); - /* same numerical values with corresponding REQ_F_*, safe to copy */ - req->flags = sqe_flags = READ_ONCE(sqe->flags); - req->user_data = READ_ONCE(sqe->user_data); - req->async_data = NULL; - req->file = NULL; - req->ctx = ctx; - req->link = NULL; - req->fixed_rsrc_refs = NULL; - /* one is dropped after submission, the other at completion */ - refcount_set(&req->refs, 2); - req->task = current; - req->result = 0; - - /* enforce forwards compatibility on users */ - if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) - return -EINVAL; - - if (unlikely(req->opcode >= IORING_OP_LAST)) - return -EINVAL; - - if (unlikely(io_sq_thread_acquire_mm_files(ctx, req))) - return -EFAULT; - - if (unlikely(!io_check_restriction(ctx, req, sqe_flags))) - return -EACCES; - - if ((sqe_flags & IOSQE_BUFFER_SELECT) && - !io_op_defs[req->opcode].buffer_select) - return -EOPNOTSUPP; - - id = READ_ONCE(sqe->personality); - if (id) { - struct io_identity *iod; - - iod = idr_find(&ctx->personality_idr, id); - if (unlikely(!iod)) - return -EINVAL; - refcount_inc(&iod->count); - - __io_req_init_async(req); - get_cred(iod->creds); - req->work.identity = iod; - req->work.flags |= IO_WQ_WORK_CREDS; - } - - state = &ctx->submit_state; - - /* - * Plug now if we have more than 1 IO left after this, and the target - * is potentially a read/write to block based storage. - */ - if (!state->plug_started && state->ios_left > 1 && - io_op_defs[req->opcode].plug) { - blk_start_plug(&state->plug); - state->plug_started = true; - } - - if (io_op_defs[req->opcode].needs_file) { - bool fixed = req->flags & REQ_F_FIXED_FILE; - - req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed); - if (unlikely(!req->file)) - ret = -EBADF; - } - - state->ios_left--; - return ret; -} - static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) { struct io_submit_link link; -- cgit v1.2.3 From a6b8cadcea86da0fe92de5c2e6e82824cb6fb57c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:41 +0000 Subject: io_uring: move io_init_req() into io_submit_sqe() Behaves identically, just move io_init_req() call into the beginning of io_submit_sqes(). That looks better unloads io_submit_sqes(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 1563853caac5..5c9b3b9ff92f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6751,12 +6751,23 @@ struct io_submit_link { struct io_kiocb *last; }; -static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, +static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, + const struct io_uring_sqe *sqe, struct io_submit_link *link) { - struct io_ring_ctx *ctx = req->ctx; int ret; + ret = io_init_req(ctx, req, sqe); + if (unlikely(ret)) { +fail_req: + io_put_req(req); + io_req_complete(req, ret); + return ret; + } + + trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, + true, ctx->flags & IORING_SETUP_SQPOLL); + /* * If we already have a head request, queue this one for async * submittal once the head completes. If we don't have a head but @@ -6782,7 +6793,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, if (unlikely(ret)) { /* fail even hard links since we don't submit */ head->flags |= REQ_F_FAIL_LINK; - return ret; + goto fail_req; } trace_io_uring_link(ctx, req, head); link->last->link = req; @@ -6904,7 +6915,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) while (submitted < nr) { const struct io_uring_sqe *sqe; struct io_kiocb *req; - int err; req = io_alloc_req(ctx); if (unlikely(!req)) { @@ -6919,20 +6929,8 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) } /* will complete beyond this point, count as submitted */ submitted++; - - err = io_init_req(ctx, req, sqe); - if (unlikely(err)) { -fail_req: - io_put_req(req); - io_req_complete(req, err); + if (io_submit_sqe(ctx, req, sqe, &link)) break; - } - - trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, - true, ctx->flags & IORING_SETUP_SQPOLL); - err = io_submit_sqe(req, sqe, &link); - if (err) - goto fail_req; } if (unlikely(submitted != nr)) { -- cgit v1.2.3 From a1ab7b35db8f262cd74edff62b47b4d90f84f997 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:42 +0000 Subject: io_uring: move req link into submit_state Move struct io_submit_link into submit_state, which is a part of a submission state and so belongs to it. It saves us from explicitly passing it, and init/deinit is now nicely hidden in io_submit_state_[start,end]. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5c9b3b9ff92f..fe2379179b00 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -283,8 +283,14 @@ struct io_comp_state { struct list_head locked_free_list; }; +struct io_submit_link { + struct io_kiocb *head; + struct io_kiocb *last; +}; + struct io_submit_state { struct blk_plug plug; + struct io_submit_link link; /* * io_kiocb alloc cache @@ -6746,15 +6752,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, return ret; } -struct io_submit_link { - struct io_kiocb *head; - struct io_kiocb *last; -}; - static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, - const struct io_uring_sqe *sqe, - struct io_submit_link *link) + const struct io_uring_sqe *sqe) { + struct io_submit_link *link = &ctx->submit_state.link; int ret; ret = io_init_req(ctx, req, sqe); @@ -6829,6 +6830,8 @@ fail_req: static void io_submit_state_end(struct io_submit_state *state, struct io_ring_ctx *ctx) { + if (state->link.head) + io_queue_link_head(state->link.head); if (state->comp.nr) io_submit_flush_completions(&state->comp, ctx); if (state->plug_started) @@ -6844,6 +6847,8 @@ static void io_submit_state_start(struct io_submit_state *state, { state->plug_started = false; state->ios_left = max_ios; + /* set only head, no need to init link_last in advance */ + state->link.head = NULL; } static void io_commit_sqring(struct io_ring_ctx *ctx) @@ -6891,7 +6896,6 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) { - struct io_submit_link link; int submitted = 0; /* if we have a backlog and couldn't flush it all, return BUSY */ @@ -6908,9 +6912,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) percpu_counter_add(¤t->io_uring->inflight, nr); refcount_add(nr, ¤t->usage); - io_submit_state_start(&ctx->submit_state, nr); - link.head = NULL; while (submitted < nr) { const struct io_uring_sqe *sqe; @@ -6929,7 +6931,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) } /* will complete beyond this point, count as submitted */ submitted++; - if (io_submit_sqe(ctx, req, sqe, &link)) + if (io_submit_sqe(ctx, req, sqe)) break; } @@ -6942,10 +6944,8 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) percpu_counter_sub(&tctx->inflight, unused); put_task_struct_many(current, unused); } - if (link.head) - io_queue_link_head(link.head); - io_submit_state_end(&ctx->submit_state, ctx); + io_submit_state_end(&ctx->submit_state, ctx); /* Commit SQ ring head once we've consumed and submitted all SQEs */ io_commit_sqring(ctx); -- cgit v1.2.3 From cf109604265156bb22c45e0c2aa62f53a697a3f4 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:43 +0000 Subject: io_uring: don't submit link on error If we get an error in io_init_req() for a request that would have been linked, we break the submission but still issue a partially composed link, that's nasty, fail it instead. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index fe2379179b00..62688866357c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6763,6 +6763,9 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, fail_req: io_put_req(req); io_req_complete(req, ret); + /* fail even hard links since we don't submit */ + if (link->head) + link->head->flags |= REQ_F_FAIL_LINK; return ret; } @@ -6791,11 +6794,8 @@ fail_req: ctx->drain_next = 1; } ret = io_req_defer_prep(req, sqe); - if (unlikely(ret)) { - /* fail even hard links since we don't submit */ - head->flags |= REQ_F_FAIL_LINK; + if (unlikely(ret)) goto fail_req; - } trace_io_uring_link(ctx, req, head); link->last->link = req; link->last = req; -- cgit v1.2.3 From 93642ef8843445f72a1e6b0c68914746c7aa5b9c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:44 +0000 Subject: io_uring: split sqe-prep and async setup There are two kinds of opcode-specific preparations we do. The first is just initialising req with what is always needed for an opcode and reading all non-generic SQE fields. And the second is copying some of the stuff like iovec preparing to punt a request to somewhere async, e.g. to io-wq or for draining. For requests that have tried an inline execution but still needing to be punted, the second prep type is done by the opcode handler itself. Currently, we don't explicitly split those preparation steps, but combining both of them into io_*_prep(), altering the behaviour by allocating ->async_data. That's pretty messy and hard to follow and also gets in the way of some optimisations. Split the steps, leave the first type as where it is now, and put the second into a new io_req_prep_async() helper. It may make us to do opcode switch twice, but it's worth it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 120 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 70 insertions(+), 50 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 62688866357c..987cfd8db213 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3472,19 +3472,9 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw) static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - ssize_t ret; - - ret = io_prep_rw(req, sqe); - if (ret) - return ret; - if (unlikely(!(req->file->f_mode & FMODE_READ))) return -EBADF; - - /* either don't need iovec imported or already have it */ - if (!req->async_data) - return 0; - return io_rw_prep_async(req, READ); + return io_prep_rw(req, sqe); } /* @@ -3669,19 +3659,9 @@ out_free: static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - ssize_t ret; - - ret = io_prep_rw(req, sqe); - if (ret) - return ret; - if (unlikely(!(req->file->f_mode & FMODE_WRITE))) return -EBADF; - - /* either don't need iovec imported or already have it */ - if (!req->async_data) - return 0; - return io_rw_prep_async(req, WRITE); + return io_prep_rw(req, sqe); } static int io_write(struct io_kiocb *req, unsigned int issue_flags) @@ -4668,11 +4648,21 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req, req->sr_msg.msg_flags, &iomsg->free_iov); } +static int io_sendmsg_prep_async(struct io_kiocb *req) +{ + int ret; + + if (!io_op_defs[req->opcode].needs_async_data) + return 0; + ret = io_sendmsg_copy_hdr(req, req->async_data); + if (!ret) + req->flags |= REQ_F_NEED_CLEANUP; + return ret; +} + static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_async_msghdr *async_msg = req->async_data; struct io_sr_msg *sr = &req->sr_msg; - int ret; if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; @@ -4685,13 +4675,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (req->ctx->compat) sr->msg_flags |= MSG_CMSG_COMPAT; #endif - - if (!async_msg || !io_op_defs[req->opcode].needs_async_data) - return 0; - ret = io_sendmsg_copy_hdr(req, async_msg); - if (!ret) - req->flags |= REQ_F_NEED_CLEANUP; - return ret; + return 0; } static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) @@ -4885,13 +4869,22 @@ static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req) return io_put_kbuf(req, req->sr_msg.kbuf); } -static int io_recvmsg_prep(struct io_kiocb *req, - const struct io_uring_sqe *sqe) +static int io_recvmsg_prep_async(struct io_kiocb *req) { - struct io_async_msghdr *async_msg = req->async_data; - struct io_sr_msg *sr = &req->sr_msg; int ret; + if (!io_op_defs[req->opcode].needs_async_data) + return 0; + ret = io_recvmsg_copy_hdr(req, req->async_data); + if (!ret) + req->flags |= REQ_F_NEED_CLEANUP; + return ret; +} + +static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +{ + struct io_sr_msg *sr = &req->sr_msg; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; @@ -4904,13 +4897,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, if (req->ctx->compat) sr->msg_flags |= MSG_CMSG_COMPAT; #endif - - if (!async_msg || !io_op_defs[req->opcode].needs_async_data) - return 0; - ret = io_recvmsg_copy_hdr(req, async_msg); - if (!ret) - req->flags |= REQ_F_NEED_CLEANUP; - return ret; + return 0; } static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) @@ -5063,10 +5050,17 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) return 0; } +static int io_connect_prep_async(struct io_kiocb *req) +{ + struct io_async_connect *io = req->async_data; + struct io_connect *conn = &req->connect; + + return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); +} + static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_connect *conn = &req->connect; - struct io_async_connect *io = req->async_data; if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; @@ -5075,12 +5069,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); conn->addr_len = READ_ONCE(sqe->addr2); - - if (!io) - return 0; - - return move_addr_to_kernel(conn->addr, conn->addr_len, - &io->address); + return 0; } static int io_connect(struct io_kiocb *req, unsigned int issue_flags) @@ -6148,14 +6137,45 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return-EINVAL; } +static int io_req_prep_async(struct io_kiocb *req) +{ + switch (req->opcode) { + case IORING_OP_READV: + case IORING_OP_READ_FIXED: + case IORING_OP_READ: + return io_rw_prep_async(req, READ); + case IORING_OP_WRITEV: + case IORING_OP_WRITE_FIXED: + case IORING_OP_WRITE: + return io_rw_prep_async(req, WRITE); + case IORING_OP_SENDMSG: + case IORING_OP_SEND: + return io_sendmsg_prep_async(req); + case IORING_OP_RECVMSG: + case IORING_OP_RECV: + return io_recvmsg_prep_async(req); + case IORING_OP_CONNECT: + return io_connect_prep_async(req); + } + return 0; +} + static int io_req_defer_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { + int ret; + if (!sqe) return 0; if (io_alloc_async_data(req)) return -EAGAIN; - return io_req_prep(req, sqe); + ret = io_req_prep(req, sqe); + if (ret) + return ret; + if (req->async_data) + return io_req_prep_async(req); + return 0; + } static u32 io_get_sequence(struct io_kiocb *req) -- cgit v1.2.3 From be7053b7d028dc891857ca3e23b401a901257789 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:45 +0000 Subject: io_uring: do io_*_prep() early in io_submit_sqe() Now as preparations are split from async setup, we can do the first one pretty early not spilling it across multiple call sites. And after it's done SQE is not needed anymore and we can save on passing it deeply into the submission stack. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 59 ++++++++++++++++++++++++----------------------------------- 1 file changed, 24 insertions(+), 35 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 987cfd8db213..7d54b0abbb82 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6160,22 +6160,16 @@ static int io_req_prep_async(struct io_kiocb *req) return 0; } -static int io_req_defer_prep(struct io_kiocb *req, - const struct io_uring_sqe *sqe) +static int io_req_defer_prep(struct io_kiocb *req) { - int ret; - - if (!sqe) + if (!io_op_defs[req->opcode].needs_async_data) return 0; - if (io_alloc_async_data(req)) - return -EAGAIN; - ret = io_req_prep(req, sqe); - if (ret) - return ret; + /* some opcodes init it during the inital prep */ if (req->async_data) - return io_req_prep_async(req); - return 0; - + return 0; + if (__io_alloc_async_data(req)) + return -EAGAIN; + return io_req_prep_async(req); } static u32 io_get_sequence(struct io_kiocb *req) @@ -6191,7 +6185,7 @@ static u32 io_get_sequence(struct io_kiocb *req) return total_submitted - nr_reqs; } -static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static int io_req_defer(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; struct io_defer_entry *de; @@ -6208,11 +6202,9 @@ static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) return 0; - if (!req->async_data) { - ret = io_req_defer_prep(req, sqe); - if (ret) - return ret; - } + ret = io_req_defer_prep(req); + if (ret) + return ret; io_prep_async_link(req); de = kmalloc(sizeof(*de), GFP_KERNEL); if (!de) @@ -6631,11 +6623,11 @@ static void __io_queue_sqe(struct io_kiocb *req) io_queue_linked_timeout(linked_timeout); } -static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static void io_queue_sqe(struct io_kiocb *req) { int ret; - ret = io_req_defer(req, sqe); + ret = io_req_defer(req); if (ret) { if (ret != -EIOCBQUEUED) { fail_req: @@ -6644,18 +6636,11 @@ fail_req: io_req_complete(req, ret); } } else if (req->flags & REQ_F_FORCE_ASYNC) { - if (!req->async_data) { - ret = io_req_defer_prep(req, sqe); - if (unlikely(ret)) - goto fail_req; - } + ret = io_req_defer_prep(req); + if (unlikely(ret)) + goto fail_req; io_queue_async_work(req); } else { - if (sqe) { - ret = io_req_prep(req, sqe); - if (unlikely(ret)) - goto fail_req; - } __io_queue_sqe(req); } } @@ -6666,7 +6651,7 @@ static inline void io_queue_link_head(struct io_kiocb *req) io_put_req(req); io_req_complete(req, -ECANCELED); } else - io_queue_sqe(req, NULL); + io_queue_sqe(req); } /* @@ -6788,7 +6773,11 @@ fail_req: link->head->flags |= REQ_F_FAIL_LINK; return ret; } + ret = io_req_prep(req, sqe); + if (unlikely(ret)) + goto fail_req; + /* don't need @sqe from now on */ trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, true, ctx->flags & IORING_SETUP_SQPOLL); @@ -6813,7 +6802,7 @@ fail_req: head->flags |= REQ_F_IO_DRAIN; ctx->drain_next = 1; } - ret = io_req_defer_prep(req, sqe); + ret = io_req_defer_prep(req); if (unlikely(ret)) goto fail_req; trace_io_uring_link(ctx, req, head); @@ -6831,13 +6820,13 @@ fail_req: ctx->drain_next = 0; } if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { - ret = io_req_defer_prep(req, sqe); + ret = io_req_defer_prep(req); if (unlikely(ret)) req->flags |= REQ_F_FAIL_LINK; link->head = req; link->last = req; } else { - io_queue_sqe(req, sqe); + io_queue_sqe(req); } } -- cgit v1.2.3 From 1ee43ba8d267b5e6729c45b8756263f69c2978cc Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:46 +0000 Subject: io_uring: don't do async setup for links' heads Now, as we can do async setup without holding an SQE, we can skip doing io_req_defer_prep() for link heads, it will be tried to be executed inline and follows all the rules of the non-linked requests. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 7d54b0abbb82..45f78fd25ce2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6820,9 +6820,6 @@ fail_req: ctx->drain_next = 0; } if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { - ret = io_req_defer_prep(req); - if (unlikely(ret)) - req->flags |= REQ_F_FAIL_LINK; link->head = req; link->last = req; } else { -- cgit v1.2.3 From de59bc104c24f2e8637464a9e3ebbd8fd4c0f115 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 18:29:47 +0000 Subject: io_uring: fail links more in io_submit_sqe() Instead of marking a link with REQ_F_FAIL_LINK on an error and delaying its failing to the caller, do it eagerly right when after getting an error in io_submit_sqe(). This renders FAIL_LINK checks in io_queue_link_head() useless and we can skip it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 45f78fd25ce2..2fdfe5fa00b0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6645,15 +6645,6 @@ fail_req: } } -static inline void io_queue_link_head(struct io_kiocb *req) -{ - if (unlikely(req->flags & REQ_F_FAIL_LINK)) { - io_put_req(req); - io_req_complete(req, -ECANCELED); - } else - io_queue_sqe(req); -} - /* * Check SQE restrictions (opcode and flags). * @@ -6768,9 +6759,13 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, fail_req: io_put_req(req); io_req_complete(req, ret); - /* fail even hard links since we don't submit */ - if (link->head) + if (link->head) { + /* fail even hard links since we don't submit */ link->head->flags |= REQ_F_FAIL_LINK; + io_put_req(link->head); + io_req_complete(link->head, -ECANCELED); + link->head = NULL; + } return ret; } ret = io_req_prep(req, sqe); @@ -6811,7 +6806,7 @@ fail_req: /* last request of a link, enqueue the link */ if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { - io_queue_link_head(head); + io_queue_sqe(head); link->head = NULL; } } else { @@ -6837,7 +6832,7 @@ static void io_submit_state_end(struct io_submit_state *state, struct io_ring_ctx *ctx) { if (state->link.head) - io_queue_link_head(state->link.head); + io_queue_sqe(state->link.head); if (state->comp.nr) io_submit_flush_completions(&state->comp, ctx); if (state->plug_started) -- cgit v1.2.3 From 19d8e9149c27b689c6224f5c84b96a159342195a Mon Sep 17 00:00:00 2001 From: Jiri Bohac Date: Thu, 18 Feb 2021 12:15:47 +0100 Subject: pstore: Fix typo in compression option name Both pstore_compress() and decompress_record() use a mistyped config option name ("PSTORE_COMPRESSION" instead of "PSTORE_COMPRESS"). As a result compression and decompression of pstore records was always disabled. Use the correct config option name. Signed-off-by: Jiri Bohac Fixes: fd49e03280e5 ("pstore: Fix linking when crypto API disabled") Acked-by: Matteo Croce Signed-off-by: Kees Cook Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20210218111547.johvp5klpv3xrpnn@dwarf.suse.cz --- fs/pstore/platform.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c index 32f64abc277c..d963ae7902f9 100644 --- a/fs/pstore/platform.c +++ b/fs/pstore/platform.c @@ -269,7 +269,7 @@ static int pstore_compress(const void *in, void *out, { int ret; - if (!IS_ENABLED(CONFIG_PSTORE_COMPRESSION)) + if (!IS_ENABLED(CONFIG_PSTORE_COMPRESS)) return -EINVAL; ret = crypto_comp_compress(tfm, in, inlen, out, &outlen); @@ -671,7 +671,7 @@ static void decompress_record(struct pstore_record *record) int unzipped_len; char *unzipped, *workspace; - if (!IS_ENABLED(CONFIG_PSTORE_COMPRESSION) || !record->compressed) + if (!IS_ENABLED(CONFIG_PSTORE_COMPRESS) || !record->compressed) return; /* Only PSTORE_TYPE_DMESG support compression. */ -- cgit v1.2.3 From 792bb6eb862333658bf1bd2260133f0507e2da8d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 22:32:51 +0000 Subject: io_uring: don't take uring_lock during iowq cancel [ 97.866748] a.out/2890 is trying to acquire lock: [ 97.867829] ffff8881046763e8 (&ctx->uring_lock){+.+.}-{3:3}, at: io_wq_submit_work+0x155/0x240 [ 97.869735] [ 97.869735] but task is already holding lock: [ 97.871033] ffff88810dfe0be8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_enter+0x3f0/0x5b0 [ 97.873074] [ 97.873074] other info that might help us debug this: [ 97.874520] Possible unsafe locking scenario: [ 97.874520] [ 97.875845] CPU0 [ 97.876440] ---- [ 97.877048] lock(&ctx->uring_lock); [ 97.877961] lock(&ctx->uring_lock); [ 97.878881] [ 97.878881] *** DEADLOCK *** [ 97.878881] [ 97.880341] May be due to missing lock nesting notation [ 97.880341] [ 97.881952] 1 lock held by a.out/2890: [ 97.882873] #0: ffff88810dfe0be8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_enter+0x3f0/0x5b0 [ 97.885108] [ 97.885108] stack backtrace: [ 97.890457] Call Trace: [ 97.891121] dump_stack+0xac/0xe3 [ 97.891972] __lock_acquire+0xab6/0x13a0 [ 97.892940] lock_acquire+0x2c3/0x390 [ 97.894894] __mutex_lock+0xae/0x9f0 [ 97.901101] io_wq_submit_work+0x155/0x240 [ 97.902112] io_wq_cancel_cb+0x162/0x490 [ 97.904126] io_async_find_and_cancel+0x3b/0x140 [ 97.905247] io_issue_sqe+0x86d/0x13e0 [ 97.909122] __io_queue_sqe+0x10b/0x550 [ 97.913971] io_queue_sqe+0x235/0x470 [ 97.914894] io_submit_sqes+0xcce/0xf10 [ 97.917872] __x64_sys_io_uring_enter+0x3fb/0x5b0 [ 97.921424] do_syscall_64+0x2d/0x40 [ 97.922329] entry_SYSCALL_64_after_hwframe+0x44/0xa9 While holding uring_lock, e.g. from inline execution, async cancel request may attempt cancellations through io_wq_submit_work, which may try to grab a lock. Delay it to task_work, so we do it from a clean context and don't have to worry about locking. Cc: # 5.5+ Fixes: c07e6719511e ("io_uring: hold uring_lock while completing failed polled io in io_wq_submit_work()") Reported-by: Abaci Reported-by: Hao Xu Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 2fdfe5fa00b0..8dab07f42b34 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2337,7 +2337,9 @@ static void io_req_task_cancel(struct callback_head *cb) struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); struct io_ring_ctx *ctx = req->ctx; + mutex_lock(&ctx->uring_lock); __io_req_task_cancel(req, -ECANCELED); + mutex_unlock(&ctx->uring_lock); percpu_ref_put(&ctx->refs); } @@ -6426,8 +6428,13 @@ static void io_wq_submit_work(struct io_wq_work *work) if (timeout) io_queue_linked_timeout(timeout); - if (work->flags & IO_WQ_WORK_CANCEL) - ret = -ECANCELED; + if (work->flags & IO_WQ_WORK_CANCEL) { + /* io-wq is going to take down one */ + refcount_inc(&req->refs); + percpu_ref_get(&req->ctx->refs); + io_req_task_work_add_fallback(req, io_req_task_cancel); + return; + } if (!ret) { do { -- cgit v1.2.3 From af982da9a612295a91f367469f8945c916a20dfd Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Thu, 18 Feb 2021 17:28:12 +0800 Subject: cifs: Fix inconsistent IS_ERR and PTR_ERR Fix inconsistent IS_ERR and PTR_ERR in cifs_find_swn_reg(). The proper pointer to be passed as argument to PTR_ERR() is share_name. This bug was detected with the help of Coccinelle. Fixes: bf80e5d4259a ("cifs: Send witness register and unregister commands to userspace daemon") Signed-off-by: YueHaibing Reviewed-by: Samuel Cabrero Signed-off-by: Steve French --- fs/cifs/cifs_swn.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/cifs/cifs_swn.c b/fs/cifs/cifs_swn.c index d35f599aa00e..f2d730fffccb 100644 --- a/fs/cifs/cifs_swn.c +++ b/fs/cifs/cifs_swn.c @@ -272,7 +272,7 @@ static struct cifs_swn_reg *cifs_find_swn_reg(struct cifs_tcon *tcon) if (IS_ERR(share_name)) { int ret; - ret = PTR_ERR(net_name); + ret = PTR_ERR(share_name); cifs_dbg(VFS, "%s: failed to extract share name from target '%s': %d\n", __func__, tcon->treeName, ret); kfree(net_name); -- cgit v1.2.3 From eacd9aa8cedeb412842c7b339adbaa0477fdd5ad Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 15 Feb 2021 12:03:23 -0500 Subject: fix handling of nd->depth on LOOKUP_CACHED failures in try_to_unlazy* After switching to non-RCU mode, we want nd->depth to match the number of entries in nd->stack[] that need eventual path_put(). legitimize_links() takes care of that on failures; unfortunately, failure exits added for LOOKUP_CACHED do not. We could add the logics for that into those failure exits, both in try_to_unlazy() and in try_to_unlazy_next(), but since both checks are immediately followed by legitimize_links() and there's no calls of legitimize_links() other than those two... It's easier to move the check (and required handling of nd->depth on failure) into legitimize_links() itself. [caught by Jens: ... and since we are zeroing ->depth here, we need to do drop_links() first] Fixes: 6c6ec2b0a3e0 "fs: add support for LOOKUP_CACHED" Tested-by: Jens Axboe Signed-off-by: Al Viro --- fs/namei.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/namei.c b/fs/namei.c index 4cae88733a5c..de74ad2bc6e2 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -630,6 +630,11 @@ static inline bool legitimize_path(struct nameidata *nd, static bool legitimize_links(struct nameidata *nd) { int i; + if (unlikely(nd->flags & LOOKUP_CACHED)) { + drop_links(nd); + nd->depth = 0; + return false; + } for (i = 0; i < nd->depth; i++) { struct saved *last = nd->stack + i; if (unlikely(!legitimize_path(nd, &last->link, last->seq))) { @@ -686,8 +691,6 @@ static bool try_to_unlazy(struct nameidata *nd) BUG_ON(!(nd->flags & LOOKUP_RCU)); nd->flags &= ~LOOKUP_RCU; - if (nd->flags & LOOKUP_CACHED) - goto out1; if (unlikely(!legitimize_links(nd))) goto out1; if (unlikely(!legitimize_path(nd, &nd->path, nd->seq))) @@ -724,8 +727,6 @@ static bool try_to_unlazy_next(struct nameidata *nd, struct dentry *dentry, unsi BUG_ON(!(nd->flags & LOOKUP_RCU)); nd->flags &= ~LOOKUP_RCU; - if (nd->flags & LOOKUP_CACHED) - goto out2; if (unlikely(!legitimize_links(nd))) goto out2; if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq))) -- cgit v1.2.3 From a3df769899c0bdc224c94d1d8cc9cbb3f3a72553 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Feb 2021 22:32:52 +0000 Subject: io_uring: fail io-wq submission from a task_work In case of failure io_wq_submit_work() needs to post an CQE and so potentially take uring_lock. The safest way to deal with it is to do that from under task_work where we can safely take the lock. Also, as io_iopoll_check() holds the lock tight and releases it reluctantly, it will play nicer in the furuter with notifying an iopolling task about new such pending failed requests. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 48 ++++++++++++++++++------------------------------ 1 file changed, 18 insertions(+), 30 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 8dab07f42b34..582306b1dfd1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2338,7 +2338,7 @@ static void io_req_task_cancel(struct callback_head *cb) struct io_ring_ctx *ctx = req->ctx; mutex_lock(&ctx->uring_lock); - __io_req_task_cancel(req, -ECANCELED); + __io_req_task_cancel(req, req->result); mutex_unlock(&ctx->uring_lock); percpu_ref_put(&ctx->refs); } @@ -2371,11 +2371,22 @@ static void io_req_task_queue(struct io_kiocb *req) req->task_work.func = io_req_task_submit; ret = io_req_task_work_add(req); if (unlikely(ret)) { + req->result = -ECANCELED; percpu_ref_get(&req->ctx->refs); io_req_task_work_add_fallback(req, io_req_task_cancel); } } +static void io_req_task_queue_fail(struct io_kiocb *req, int ret) +{ + percpu_ref_get(&req->ctx->refs); + req->result = ret; + req->task_work.func = io_req_task_cancel; + + if (unlikely(io_req_task_work_add(req))) + io_req_task_work_add_fallback(req, io_req_task_cancel); +} + static inline void io_queue_next(struct io_kiocb *req) { struct io_kiocb *nxt = io_req_find_next(req); @@ -6428,13 +6439,8 @@ static void io_wq_submit_work(struct io_wq_work *work) if (timeout) io_queue_linked_timeout(timeout); - if (work->flags & IO_WQ_WORK_CANCEL) { - /* io-wq is going to take down one */ - refcount_inc(&req->refs); - percpu_ref_get(&req->ctx->refs); - io_req_task_work_add_fallback(req, io_req_task_cancel); - return; - } + if (work->flags & IO_WQ_WORK_CANCEL) + ret = -ECANCELED; if (!ret) { do { @@ -6450,29 +6456,11 @@ static void io_wq_submit_work(struct io_wq_work *work) } while (1); } + /* avoid locking problems by failing it from a clean context */ if (ret) { - struct io_ring_ctx *lock_ctx = NULL; - - if (req->ctx->flags & IORING_SETUP_IOPOLL) - lock_ctx = req->ctx; - - /* - * io_iopoll_complete() does not hold completion_lock to - * complete polled io, so here for polled io, we can not call - * io_req_complete() directly, otherwise there maybe concurrent - * access to cqring, defer_list, etc, which is not safe. Given - * that io_iopoll_complete() is always called under uring_lock, - * so here for polled io, we also get uring_lock to complete - * it. - */ - if (lock_ctx) - mutex_lock(&lock_ctx->uring_lock); - - req_set_fail_links(req); - io_req_complete(req, ret); - - if (lock_ctx) - mutex_unlock(&lock_ctx->uring_lock); + /* io-wq is going to take one down */ + refcount_inc(&req->refs); + io_req_task_queue_fail(req, ret); } } -- cgit v1.2.3 From 8bad28d8a305b0e5ae444c8c3051e8744f5a4296 Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Fri, 19 Feb 2021 17:19:36 +0800 Subject: io_uring: don't hold uring_lock when calling io_run_task_work* Abaci reported the below issue: [ 141.400455] hrtimer: interrupt took 205853 ns [ 189.869316] process 'usr/local/ilogtail/ilogtail_0.16.26' started with executable stack [ 250.188042] [ 250.188327] ============================================ [ 250.189015] WARNING: possible recursive locking detected [ 250.189732] 5.11.0-rc4 #1 Not tainted [ 250.190267] -------------------------------------------- [ 250.190917] a.out/7363 is trying to acquire lock: [ 250.191506] ffff888114dbcbe8 (&ctx->uring_lock){+.+.}-{3:3}, at: __io_req_task_submit+0x29/0xa0 [ 250.192599] [ 250.192599] but task is already holding lock: [ 250.193309] ffff888114dbfbe8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_register+0xad/0x210 [ 250.194426] [ 250.194426] other info that might help us debug this: [ 250.195238] Possible unsafe locking scenario: [ 250.195238] [ 250.196019] CPU0 [ 250.196411] ---- [ 250.196803] lock(&ctx->uring_lock); [ 250.197420] lock(&ctx->uring_lock); [ 250.197966] [ 250.197966] *** DEADLOCK *** [ 250.197966] [ 250.198837] May be due to missing lock nesting notation [ 250.198837] [ 250.199780] 1 lock held by a.out/7363: [ 250.200373] #0: ffff888114dbfbe8 (&ctx->uring_lock){+.+.}-{3:3}, at: __x64_sys_io_uring_register+0xad/0x210 [ 250.201645] [ 250.201645] stack backtrace: [ 250.202298] CPU: 0 PID: 7363 Comm: a.out Not tainted 5.11.0-rc4 #1 [ 250.203144] Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011 [ 250.203887] Call Trace: [ 250.204302] dump_stack+0xac/0xe3 [ 250.204804] __lock_acquire+0xab6/0x13a0 [ 250.205392] lock_acquire+0x2c3/0x390 [ 250.205928] ? __io_req_task_submit+0x29/0xa0 [ 250.206541] __mutex_lock+0xae/0x9f0 [ 250.207071] ? __io_req_task_submit+0x29/0xa0 [ 250.207745] ? 0xffffffffa0006083 [ 250.208248] ? __io_req_task_submit+0x29/0xa0 [ 250.208845] ? __io_req_task_submit+0x29/0xa0 [ 250.209452] ? __io_req_task_submit+0x5/0xa0 [ 250.210083] __io_req_task_submit+0x29/0xa0 [ 250.210687] io_async_task_func+0x23d/0x4c0 [ 250.211278] task_work_run+0x89/0xd0 [ 250.211884] io_run_task_work_sig+0x50/0xc0 [ 250.212464] io_sqe_files_unregister+0xb2/0x1f0 [ 250.213109] __io_uring_register+0x115a/0x1750 [ 250.213718] ? __x64_sys_io_uring_register+0xad/0x210 [ 250.214395] ? __fget_files+0x15a/0x260 [ 250.214956] __x64_sys_io_uring_register+0xbe/0x210 [ 250.215620] ? trace_hardirqs_on+0x46/0x110 [ 250.216205] do_syscall_64+0x2d/0x40 [ 250.216731] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 250.217455] RIP: 0033:0x7f0fa17e5239 [ 250.218034] Code: 01 00 48 81 c4 80 00 00 00 e9 f1 fe ff ff 0f 1f 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 3d 01 f0 ff ff 73 01 c3 48 8b 0d 27 ec 2c 00 f7 d8 64 89 01 48 [ 250.220343] RSP: 002b:00007f0fa1eeac48 EFLAGS: 00000246 ORIG_RAX: 00000000000001ab [ 250.221360] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f0fa17e5239 [ 250.222272] RDX: 0000000000000000 RSI: 0000000000000003 RDI: 0000000000000008 [ 250.223185] RBP: 00007f0fa1eeae20 R08: 0000000000000000 R09: 0000000000000000 [ 250.224091] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000 [ 250.224999] R13: 0000000000021000 R14: 0000000000000000 R15: 00007f0fa1eeb700 This is caused by calling io_run_task_work_sig() to do work under uring_lock while the caller io_sqe_files_unregister() already held uring_lock. To fix this issue, briefly drop uring_lock when calling io_run_task_work_sig(), and there are two things to concern: - hold uring_lock in io_ring_ctx_free() around io_sqe_files_unregister() this is for consistency of lock/unlock. - add new fixed rsrc ref node before dropping uring_lock it's not safe to do io_uring_enter-->percpu_ref_get() with a dying one. - check if rsrc_data->refs is dying to avoid parallel io_sqe_files_unregister Reported-by: Abaci Fixes: 1ffc54220c44 ("io_uring: fix io_sqe_files_unregister() hangs") Suggested-by: Pavel Begunkov Signed-off-by: Hao Xu [axboe: fixes from Pavel folded in] Signed-off-by: Jens Axboe --- fs/io_uring.c | 61 ++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 582306b1dfd1..7956c6751a67 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -236,6 +236,7 @@ struct fixed_rsrc_data { struct fixed_rsrc_ref_node *node; struct percpu_ref refs; struct completion done; + bool quiesce; }; struct io_buffer { @@ -7316,38 +7317,57 @@ static void io_sqe_rsrc_set_node(struct io_ring_ctx *ctx, percpu_ref_get(&rsrc_data->refs); } -static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, - struct io_ring_ctx *ctx, - struct fixed_rsrc_ref_node *backup_node) +static void io_sqe_rsrc_kill_node(struct io_ring_ctx *ctx, struct fixed_rsrc_data *data) { - struct fixed_rsrc_ref_node *ref_node; - int ret; + struct fixed_rsrc_ref_node *ref_node = NULL; io_rsrc_ref_lock(ctx); ref_node = data->node; io_rsrc_ref_unlock(ctx); if (ref_node) percpu_ref_kill(&ref_node->refs); +} + +static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, + struct io_ring_ctx *ctx, + struct fixed_rsrc_ref_node *backup_node) +{ + int ret; - percpu_ref_kill(&data->refs); + if (data->quiesce) + return -ENXIO; - /* wait for all refs nodes to complete */ - flush_delayed_work(&ctx->rsrc_put_work); + data->quiesce = true; do { + io_sqe_rsrc_kill_node(ctx, data); + percpu_ref_kill(&data->refs); + flush_delayed_work(&ctx->rsrc_put_work); + ret = wait_for_completion_interruptible(&data->done); if (!ret) break; + + percpu_ref_resurrect(&data->refs); + io_sqe_rsrc_set_node(ctx, data, backup_node); + backup_node = NULL; + reinit_completion(&data->done); + mutex_unlock(&ctx->uring_lock); ret = io_run_task_work_sig(); - if (ret < 0) { - percpu_ref_resurrect(&data->refs); - reinit_completion(&data->done); - io_sqe_rsrc_set_node(ctx, data, backup_node); - return ret; - } + mutex_lock(&ctx->uring_lock); + + if (ret < 0) + break; + backup_node = alloc_fixed_rsrc_ref_node(ctx); + ret = -ENOMEM; + if (!backup_node) + break; + init_fixed_file_ref_node(ctx, backup_node); } while (1); + data->quiesce = false; - destroy_fixed_rsrc_ref_node(backup_node); - return 0; + if (backup_node) + destroy_fixed_rsrc_ref_node(backup_node); + return ret; } static struct fixed_rsrc_data *alloc_fixed_rsrc_data(struct io_ring_ctx *ctx) @@ -7382,7 +7402,12 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) unsigned nr_tables, i; int ret; - if (!data) + /* + * percpu_ref_is_dying() is to stop parallel files unregister + * Since we possibly drop uring lock later in this function to + * run task work. + */ + if (!data || percpu_ref_is_dying(&data->refs)) return -ENXIO; backup_node = alloc_fixed_rsrc_ref_node(ctx); if (!backup_node) @@ -8731,7 +8756,9 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) css_put(ctx->sqo_blkcg_css); #endif + mutex_lock(&ctx->uring_lock); io_sqe_files_unregister(ctx); + mutex_unlock(&ctx->uring_lock); io_eventfd_unregister(ctx); io_destroy_buffers(ctx); idr_destroy(&ctx->personality_idr); -- cgit v1.2.3 From 99a10081647168022745859bb2f1c28b2f70dc83 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 19 Feb 2021 09:35:19 -0700 Subject: io_uring: make the !CONFIG_NET helpers a bit more robust With the prep and prep async split, we now have potentially 3 helpers that need to be defined for !CONFIG_NET. Add some helpers to do just that. Fixes the following compile error on !CONFIG_NET: fs/io_uring.c:6171:10: error: implicit declaration of function 'io_sendmsg_prep_async'; did you mean 'io_req_prep_async'? [-Werror=implicit-function-declaration] return io_sendmsg_prep_async(req); ^~~~~~~~~~~~~~~~~~~~~ io_req_prep_async Fixes: 93642ef88434 ("io_uring: split sqe-prep and async setup") Reported-by: Naresh Kamboju Signed-off-by: Jens Axboe --- fs/io_uring.c | 76 ++++++++++++++++++++--------------------------------------- 1 file changed, 26 insertions(+), 50 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 7956c6751a67..cef80106b305 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5128,56 +5128,32 @@ out: return 0; } #else /* !CONFIG_NET */ -static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -{ - return -EOPNOTSUPP; -} - -static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} - -static int io_send(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} - -static int io_recvmsg_prep(struct io_kiocb *req, - const struct io_uring_sqe *sqe) -{ - return -EOPNOTSUPP; -} - -static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} - -static int io_recv(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} - -static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -{ - return -EOPNOTSUPP; -} - -static int io_accept(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} - -static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) -{ - return -EOPNOTSUPP; -} - -static int io_connect(struct io_kiocb *req, unsigned int issue_flags) -{ - return -EOPNOTSUPP; -} +#define IO_NETOP_FN(op) \ +static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \ +{ \ + return -EOPNOTSUPP; \ +} + +#define IO_NETOP_PREP(op) \ +IO_NETOP_FN(op) \ +static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \ +{ \ + return -EOPNOTSUPP; \ +} \ + +#define IO_NETOP_PREP_ASYNC(op) \ +IO_NETOP_PREP(op) \ +static int io_##op##_prep_async(struct io_kiocb *req) \ +{ \ + return -EOPNOTSUPP; \ +} + +IO_NETOP_PREP_ASYNC(sendmsg); +IO_NETOP_PREP_ASYNC(recvmsg); +IO_NETOP_PREP_ASYNC(connect); +IO_NETOP_PREP(accept); +IO_NETOP_FN(send); +IO_NETOP_FN(recv); #endif /* CONFIG_NET */ struct io_poll_table { -- cgit v1.2.3 From e6cb007c45dedada0a847eaa486c49509d63b1e8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 20 Feb 2021 18:03:47 +0000 Subject: io_uring: zero ref_node after killing it After a rsrc/files reference node's refs are killed, it must never be used. And that's how it works, it either assigns a new node or kills the whole data table. Let's explicitly NULL it, that shouldn't be necessary, but if something would go wrong I'd rather catch a NULL dereference to using a dangling pointer. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index cef80106b305..5215d32c4f8c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7299,6 +7299,7 @@ static void io_sqe_rsrc_kill_node(struct io_ring_ctx *ctx, struct fixed_rsrc_dat io_rsrc_ref_lock(ctx); ref_node = data->node; + data->node = NULL; io_rsrc_ref_unlock(ctx); if (ref_node) percpu_ref_kill(&ref_node->refs); -- cgit v1.2.3 From f2303b1f8244d88ffca28d3be6166ce4835cc27a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 20 Feb 2021 18:03:49 +0000 Subject: io_uring: keep generic rsrc infra generic io_rsrc_ref_quiesce() is a generic resource function, though now it was wired to allocate and initialise ref nodes with file-specific callbacks/etc. Keep it sane by passing in as a parameters everything we need for initialisations, otherwise it will hurt us badly one day. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5215d32c4f8c..c98b673f0bb1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1037,8 +1037,7 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node); static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( struct io_ring_ctx *ctx); -static void init_fixed_file_ref_node(struct io_ring_ctx *ctx, - struct fixed_rsrc_ref_node *ref_node); +static void io_ring_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc); static bool io_rw_reissue(struct io_kiocb *req); static void io_cqring_fill_event(struct io_kiocb *req, long res); @@ -7307,8 +7306,10 @@ static void io_sqe_rsrc_kill_node(struct io_ring_ctx *ctx, struct fixed_rsrc_dat static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, struct io_ring_ctx *ctx, - struct fixed_rsrc_ref_node *backup_node) + void (*rsrc_put)(struct io_ring_ctx *ctx, + struct io_rsrc_put *prsrc)) { + struct fixed_rsrc_ref_node *backup_node; int ret; if (data->quiesce) @@ -7316,6 +7317,13 @@ static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, data->quiesce = true; do { + ret = -ENOMEM; + backup_node = alloc_fixed_rsrc_ref_node(ctx); + if (!backup_node) + break; + backup_node->rsrc_data = data; + backup_node->rsrc_put = rsrc_put; + io_sqe_rsrc_kill_node(ctx, data); percpu_ref_kill(&data->refs); flush_delayed_work(&ctx->rsrc_put_work); @@ -7331,15 +7339,7 @@ static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, mutex_unlock(&ctx->uring_lock); ret = io_run_task_work_sig(); mutex_lock(&ctx->uring_lock); - - if (ret < 0) - break; - backup_node = alloc_fixed_rsrc_ref_node(ctx); - ret = -ENOMEM; - if (!backup_node) - break; - init_fixed_file_ref_node(ctx, backup_node); - } while (1); + } while (ret >= 0); data->quiesce = false; if (backup_node) @@ -7375,7 +7375,6 @@ static void free_fixed_rsrc_data(struct fixed_rsrc_data *data) static int io_sqe_files_unregister(struct io_ring_ctx *ctx) { struct fixed_rsrc_data *data = ctx->file_data; - struct fixed_rsrc_ref_node *backup_node; unsigned nr_tables, i; int ret; @@ -7386,12 +7385,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) */ if (!data || percpu_ref_is_dying(&data->refs)) return -ENXIO; - backup_node = alloc_fixed_rsrc_ref_node(ctx); - if (!backup_node) - return -ENOMEM; - init_fixed_file_ref_node(ctx, backup_node); - - ret = io_rsrc_ref_quiesce(data, ctx, backup_node); + ret = io_rsrc_ref_quiesce(data, ctx, io_ring_file_put); if (ret) return ret; -- cgit v1.2.3 From 88f171ab7798a1ed0b9e39867ee16f307466e870 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 20 Feb 2021 18:03:50 +0000 Subject: io_uring: wait potential ->release() on resurrect There is a short window where percpu_refs are already turned zero, but we try to do resurrect(). Play nicer and wait for ->release() to happen in this case and proceed as everything is ok. One downside for ctx refs is that we can ignore signal_pending() on a rare occasion, but someone else should check for it later if needed. Cc: # 5.5+ Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index c98b673f0bb1..5cc02226bb38 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1104,6 +1104,21 @@ static inline void io_set_resource_node(struct io_kiocb *req) } } +static bool io_refs_resurrect(struct percpu_ref *ref, struct completion *compl) +{ + if (!percpu_ref_tryget(ref)) { + /* already at zero, wait for ->release() */ + if (!try_wait_for_completion(compl)) + synchronize_rcu(); + return false; + } + + percpu_ref_resurrect(ref); + reinit_completion(compl); + percpu_ref_put(ref); + return true; +} + static bool io_match_task(struct io_kiocb *head, struct task_struct *task, struct files_struct *files) @@ -7329,13 +7344,11 @@ static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, flush_delayed_work(&ctx->rsrc_put_work); ret = wait_for_completion_interruptible(&data->done); - if (!ret) + if (!ret || !io_refs_resurrect(&data->refs, &data->done)) break; - percpu_ref_resurrect(&data->refs); io_sqe_rsrc_set_node(ctx, data, backup_node); backup_node = NULL; - reinit_completion(&data->done); mutex_unlock(&ctx->uring_lock); ret = io_run_task_work_sig(); mutex_lock(&ctx->uring_lock); @@ -10070,10 +10083,8 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, mutex_lock(&ctx->uring_lock); - if (ret) { - percpu_ref_resurrect(&ctx->refs); - goto out_quiesce; - } + if (ret && io_refs_resurrect(&ctx->refs, &ctx->ref_comp)) + return ret; } if (ctx->restricted) { @@ -10165,7 +10176,6 @@ out: if (io_register_op_must_quiesce(opcode)) { /* bring the ctx back to life */ percpu_ref_reinit(&ctx->refs); -out_quiesce: reinit_completion(&ctx->ref_comp); } return ret; -- cgit v1.2.3 From ebf4a5db690a47e71056381ead8a134de7202694 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 20 Feb 2021 01:39:53 +0000 Subject: io_uring: fix leaving invalid req->flags sqe->flags are subset of req flags, so incorrectly copied may span into in-kernel flags and wreck havoc, e.g. by setting REQ_F_INFLIGHT. Fixes: 5be9ad1e4287e ("io_uring: optimise io_init_req() flags setting") Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5cc02226bb38..1501f20fde84 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6679,8 +6679,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->result = 0; /* enforce forwards compatibility on users */ - if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) + if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) { + req->flags = 0; return -EINVAL; + } if (unlikely(req->opcode >= IORING_OP_LAST)) return -EINVAL; -- cgit v1.2.3 From b6c23dd5a483174f386e4c2e1711d9532e090c00 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 20 Feb 2021 15:17:18 +0000 Subject: io_uring: run task_work on io_uring_register() Do run task_work before io_uring_register(), that might make a first quiesce round much nicer. We generally do that for any syscall invocation to avoid spurious -EINTR/-ERESTARTSYS, for task_work that we generate. This patch brings io_uring_register() inline with the two other io_uring syscalls. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 1501f20fde84..3ecc3c08bf12 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -10200,6 +10200,8 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, ctx = f.file->private_data; + io_run_task_work(); + mutex_lock(&ctx->uring_lock); ret = __io_uring_register(ctx, opcode, arg, nr_args); mutex_unlock(&ctx->uring_lock); -- cgit v1.2.3 From 7c25c0d16ef3c37e49c593ac92f69fa3884d4bb9 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 16 Feb 2021 07:17:00 -0700 Subject: io_uring: remove the need for relying on an io-wq fallback worker We hit this case when the task is exiting, and we need somewhere to do background cleanup of requests. Instead of relying on the io-wq task manager to do this work for us, just stuff it somewhere where we can safely run it ourselves directly. Signed-off-by: Jens Axboe --- fs/io-wq.c | 12 ------------ fs/io-wq.h | 2 -- fs/io_uring.c | 37 ++++++++++++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index c36bbcd823ce..800b299f9772 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -775,9 +774,6 @@ static int io_wq_manager(void *data) complete(&wq->done); while (!kthread_should_stop()) { - if (current->task_works) - task_work_run(); - for_each_node(node) { struct io_wqe *wqe = wq->wqes[node]; bool fork_worker[2] = { false, false }; @@ -800,9 +796,6 @@ static int io_wq_manager(void *data) schedule_timeout(HZ); } - if (current->task_works) - task_work_run(); - out: if (refcount_dec_and_test(&wq->refs)) { complete(&wq->done); @@ -1160,11 +1153,6 @@ void io_wq_destroy(struct io_wq *wq) __io_wq_destroy(wq); } -struct task_struct *io_wq_get_task(struct io_wq *wq) -{ - return wq->manager; -} - static bool io_wq_worker_affinity(struct io_worker *worker, void *data) { struct task_struct *task = worker->task; diff --git a/fs/io-wq.h b/fs/io-wq.h index 096f1021018e..a1610702f222 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -124,8 +124,6 @@ typedef bool (work_cancel_fn)(struct io_wq_work *, void *); enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, void *data, bool cancel_all); -struct task_struct *io_wq_get_task(struct io_wq *wq); - #if defined(CONFIG_IO_WQ) extern void io_wq_worker_sleeping(struct task_struct *); extern void io_wq_worker_running(struct task_struct *); diff --git a/fs/io_uring.c b/fs/io_uring.c index ace7494e7404..e3eb37304e24 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -456,6 +456,9 @@ struct io_ring_ctx { struct io_restriction restrictions; + /* exit task_work */ + struct callback_head *exit_task_work; + /* Keep this last, we don't need it for the fast path */ struct work_struct exit_work; }; @@ -2328,11 +2331,14 @@ static int io_req_task_work_add(struct io_kiocb *req) static void io_req_task_work_add_fallback(struct io_kiocb *req, task_work_func_t cb) { - struct task_struct *tsk = io_wq_get_task(req->ctx->io_wq); + struct io_ring_ctx *ctx = req->ctx; + struct callback_head *head; init_task_work(&req->task_work, cb); - task_work_add(tsk, &req->task_work, TWA_NONE); - wake_up_process(tsk); + do { + head = READ_ONCE(ctx->exit_task_work); + req->task_work.next = head; + } while (cmpxchg(&ctx->exit_task_work, head, &req->task_work) != head); } static void __io_req_task_cancel(struct io_kiocb *req, int error) @@ -8835,6 +8841,28 @@ static int io_remove_personalities(int id, void *p, void *data) return 0; } +static void io_run_ctx_fallback(struct io_ring_ctx *ctx) +{ + struct callback_head *work, *head, *next; + + do { + do { + head = NULL; + work = READ_ONCE(ctx->exit_task_work); + } while (cmpxchg(&ctx->exit_task_work, work, head) != work); + + if (!work) + break; + + do { + next = work->next; + work->func(work); + work = next; + cond_resched(); + } while (work); + } while (1); +} + static void io_ring_exit_work(struct work_struct *work) { struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, @@ -8848,6 +8876,7 @@ static void io_ring_exit_work(struct work_struct *work) */ do { io_uring_try_cancel_requests(ctx, NULL, NULL); + io_run_ctx_fallback(ctx); } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)); io_ring_ctx_free(ctx); } @@ -9243,6 +9272,8 @@ static int io_uring_flush(struct file *file, void *data) io_req_caches_free(ctx, current); } + io_run_ctx_fallback(ctx); + if (!tctx) return 0; -- cgit v1.2.3 From 1cbd9c2bcf02a3be91e14c7206d4b6c0346540ed Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 16 Feb 2021 10:06:21 -0700 Subject: io-wq: don't create any IO workers upfront When the manager thread starts up, it creates a worker per node for the given context. Just let these get created dynamically, like we do for adding further workers. Signed-off-by: Jens Axboe --- fs/io-wq.c | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 800b299f9772..e9e218274c76 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -759,18 +759,7 @@ static int io_wq_manager(void *data) struct io_wq *wq = data; int node; - /* create fixed workers */ refcount_set(&wq->refs, 1); - for_each_node(node) { - if (!node_online(node)) - continue; - if (create_io_worker(wq, wq->wqes[node], IO_WQ_ACCT_BOUND)) - continue; - set_bit(IO_WQ_BIT_ERROR, &wq->state); - set_bit(IO_WQ_BIT_EXIT, &wq->state); - goto out; - } - complete(&wq->done); while (!kthread_should_stop()) { @@ -796,7 +785,6 @@ static int io_wq_manager(void *data) schedule_timeout(HZ); } -out: if (refcount_dec_and_test(&wq->refs)) { complete(&wq->done); return 0; -- cgit v1.2.3 From d25e3a3de0d6fb2f660dbc7d643b2c632beb1743 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 16 Feb 2021 11:41:41 -0700 Subject: io_uring: disable io-wq attaching Moving towards making the io_wq per ring per task, so we can't really share it between rings. Which is fine, since we've now dropped some of that fat from it. Retain compatibility with how attaching works, so that any attempt to attach to an fd that doesn't exist, or isn't an io_uring fd, will fail like it did before. Signed-off-by: Jens Axboe --- fs/io_uring.c | 55 ++++++++++++++++++++++--------------------------------- 1 file changed, 22 insertions(+), 33 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index e3eb37304e24..d6c2ff6124fd 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8105,12 +8105,9 @@ static struct io_wq_work *io_free_work(struct io_wq_work *work) return req ? &req->work : NULL; } -static int io_init_wq_offload(struct io_ring_ctx *ctx, - struct io_uring_params *p) +static int io_init_wq_offload(struct io_ring_ctx *ctx) { struct io_wq_data data; - struct fd f; - struct io_ring_ctx *ctx_attach; unsigned int concurrency; int ret = 0; @@ -8118,37 +8115,15 @@ static int io_init_wq_offload(struct io_ring_ctx *ctx, data.free_work = io_free_work; data.do_work = io_wq_submit_work; - if (!(p->flags & IORING_SETUP_ATTACH_WQ)) { - /* Do QD, or 4 * CPUS, whatever is smallest */ - concurrency = min(ctx->sq_entries, 4 * num_online_cpus()); - - ctx->io_wq = io_wq_create(concurrency, &data); - if (IS_ERR(ctx->io_wq)) { - ret = PTR_ERR(ctx->io_wq); - ctx->io_wq = NULL; - } - return ret; - } - - f = fdget(p->wq_fd); - if (!f.file) - return -EBADF; - - if (f.file->f_op != &io_uring_fops) { - ret = -EINVAL; - goto out_fput; - } + /* Do QD, or 4 * CPUS, whatever is smallest */ + concurrency = min(ctx->sq_entries, 4 * num_online_cpus()); - ctx_attach = f.file->private_data; - /* @io_wq is protected by holding the fd */ - if (!io_wq_get(ctx_attach->io_wq, &data)) { - ret = -EINVAL; - goto out_fput; + ctx->io_wq = io_wq_create(concurrency, &data); + if (IS_ERR(ctx->io_wq)) { + ret = PTR_ERR(ctx->io_wq); + ctx->io_wq = NULL; } - ctx->io_wq = ctx_attach->io_wq; -out_fput: - fdput(f); return ret; } @@ -8200,6 +8175,20 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, { int ret; + /* Retain compatibility with failing for an invalid attach attempt */ + if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) == + IORING_SETUP_ATTACH_WQ) { + struct fd f; + + f = fdget(p->wq_fd); + if (!f.file) + return -ENXIO; + if (f.file->f_op != &io_uring_fops) { + fdput(f); + return -EINVAL; + } + fdput(f); + } if (ctx->flags & IORING_SETUP_SQPOLL) { struct io_sq_data *sqd; @@ -8257,7 +8246,7 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, } done: - ret = io_init_wq_offload(ctx, p); + ret = io_init_wq_offload(ctx); if (ret) goto err; -- cgit v1.2.3 From 3b094e727dd5b24b4b259a8617b375dd20c16347 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 16 Feb 2021 15:42:24 -0700 Subject: io-wq: get rid of wq->use_refs We don't support attach anymore, so doesn't make sense to carry the use_refs reference count. Get rid of it. Signed-off-by: Jens Axboe --- fs/io-wq.c | 19 +------------------ fs/io-wq.h | 1 - 2 files changed, 1 insertion(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index e9e218274c76..0c47febfed9b 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -122,8 +122,6 @@ struct io_wq { struct completion done; struct hlist_node cpuhp_node; - - refcount_t use_refs; }; static enum cpuhp_state io_wq_online; @@ -1086,7 +1084,6 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) ret = -ENOMEM; goto err; } - refcount_set(&wq->use_refs, 1); reinit_completion(&wq->done); return wq; } @@ -1104,15 +1101,7 @@ err_wq: return ERR_PTR(ret); } -bool io_wq_get(struct io_wq *wq, struct io_wq_data *data) -{ - if (data->free_work != wq->free_work || data->do_work != wq->do_work) - return false; - - return refcount_inc_not_zero(&wq->use_refs); -} - -static void __io_wq_destroy(struct io_wq *wq) +void io_wq_destroy(struct io_wq *wq) { int node; @@ -1135,12 +1124,6 @@ static void __io_wq_destroy(struct io_wq *wq) kfree(wq); } -void io_wq_destroy(struct io_wq *wq) -{ - if (refcount_dec_and_test(&wq->use_refs)) - __io_wq_destroy(wq); -} - static bool io_wq_worker_affinity(struct io_worker *worker, void *data) { struct task_struct *task = worker->task; diff --git a/fs/io-wq.h b/fs/io-wq.h index a1610702f222..d2cf284b4641 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -108,7 +108,6 @@ struct io_wq_data { }; struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); -bool io_wq_get(struct io_wq *wq, struct io_wq_data *data); void io_wq_destroy(struct io_wq *wq); void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); -- cgit v1.2.3 From 5aa75ed5b93f086c455a3c67239b0471ff5a1526 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 16 Feb 2021 12:56:50 -0700 Subject: io_uring: tie async worker side to the task context Move it outside of the io_ring_ctx, and tie it to the io_uring task context. Signed-off-by: Jens Axboe --- fs/io_uring.c | 84 ++++++++++++++++++++---------------------------- include/linux/io_uring.h | 1 + 2 files changed, 35 insertions(+), 50 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index d6c2ff6124fd..31402a19fca6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -366,9 +366,6 @@ struct io_ring_ctx { struct io_rings *rings; - /* IO offload */ - struct io_wq *io_wq; - /* * For SQPOLL usage - we hold a reference to the parent task, so we * have access to the ->files @@ -1634,10 +1631,11 @@ static struct io_kiocb *__io_queue_async_work(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; struct io_kiocb *link = io_prep_linked_timeout(req); + struct io_uring_task *tctx = req->task->io_uring; trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req, &req->work, req->flags); - io_wq_enqueue(ctx->io_wq, &req->work); + io_wq_enqueue(tctx->io_wq, &req->work); return link; } @@ -5960,12 +5958,15 @@ static bool io_cancel_cb(struct io_wq_work *work, void *data) return req->user_data == (unsigned long) data; } -static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr) +static int io_async_cancel_one(struct io_uring_task *tctx, void *sqe_addr) { enum io_wq_cancel cancel_ret; int ret = 0; - cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr, false); + if (!tctx->io_wq) + return -ENOENT; + + cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, sqe_addr, false); switch (cancel_ret) { case IO_WQ_CANCEL_OK: ret = 0; @@ -5988,7 +5989,8 @@ static void io_async_find_and_cancel(struct io_ring_ctx *ctx, unsigned long flags; int ret; - ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr); + ret = io_async_cancel_one(req->task->io_uring, + (void *) (unsigned long) sqe_addr); if (ret != -ENOENT) { spin_lock_irqsave(&ctx->completion_lock, flags); goto done; @@ -7537,16 +7539,6 @@ static void io_sq_thread_stop(struct io_ring_ctx *ctx) } } -static void io_finish_async(struct io_ring_ctx *ctx) -{ - io_sq_thread_stop(ctx); - - if (ctx->io_wq) { - io_wq_destroy(ctx->io_wq); - ctx->io_wq = NULL; - } -} - #if defined(CONFIG_UNIX) /* * Ensure the UNIX gc is aware of our file set, so we are certain that @@ -8105,11 +8097,10 @@ static struct io_wq_work *io_free_work(struct io_wq_work *work) return req ? &req->work : NULL; } -static int io_init_wq_offload(struct io_ring_ctx *ctx) +static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx) { struct io_wq_data data; unsigned int concurrency; - int ret = 0; data.user = ctx->user; data.free_work = io_free_work; @@ -8118,16 +8109,11 @@ static int io_init_wq_offload(struct io_ring_ctx *ctx) /* Do QD, or 4 * CPUS, whatever is smallest */ concurrency = min(ctx->sq_entries, 4 * num_online_cpus()); - ctx->io_wq = io_wq_create(concurrency, &data); - if (IS_ERR(ctx->io_wq)) { - ret = PTR_ERR(ctx->io_wq); - ctx->io_wq = NULL; - } - - return ret; + return io_wq_create(concurrency, &data); } -static int io_uring_alloc_task_context(struct task_struct *task) +static int io_uring_alloc_task_context(struct task_struct *task, + struct io_ring_ctx *ctx) { struct io_uring_task *tctx; int ret; @@ -8142,6 +8128,14 @@ static int io_uring_alloc_task_context(struct task_struct *task) return ret; } + tctx->io_wq = io_init_wq_offload(ctx); + if (IS_ERR(tctx->io_wq)) { + ret = PTR_ERR(tctx->io_wq); + percpu_counter_destroy(&tctx->inflight); + kfree(tctx); + return ret; + } + xa_init(&tctx->xa); init_waitqueue_head(&tctx->wait); tctx->last = NULL; @@ -8214,7 +8208,7 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, ctx->sq_thread_idle = HZ; if (sqd->thread) - goto done; + return 0; if (p->flags & IORING_SETUP_SQ_AFF) { int cpu = p->sq_thread_cpu; @@ -8236,7 +8230,7 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, sqd->thread = NULL; goto err; } - ret = io_uring_alloc_task_context(sqd->thread); + ret = io_uring_alloc_task_context(sqd->thread, ctx); if (ret) goto err; } else if (p->flags & IORING_SETUP_SQ_AFF) { @@ -8245,14 +8239,9 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, goto err; } -done: - ret = io_init_wq_offload(ctx); - if (ret) - goto err; - return 0; err: - io_finish_async(ctx); + io_sq_thread_stop(ctx); return ret; } @@ -8727,7 +8716,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) mutex_lock(&ctx->uring_lock); mutex_unlock(&ctx->uring_lock); - io_finish_async(ctx); + io_sq_thread_stop(ctx); io_sqe_buffers_unregister(ctx); if (ctx->sqo_task) { @@ -8870,13 +8859,6 @@ static void io_ring_exit_work(struct work_struct *work) io_ring_ctx_free(ctx); } -static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) -{ - struct io_kiocb *req = container_of(work, struct io_kiocb, work); - - return req->ctx == data; -} - static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) { mutex_lock(&ctx->uring_lock); @@ -8895,9 +8877,6 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) io_kill_timeouts(ctx, NULL, NULL); io_poll_remove_all(ctx, NULL, NULL); - if (ctx->io_wq) - io_wq_cancel_cb(ctx->io_wq, io_cancel_ctx_cb, ctx, true); - /* if we failed setting up the ctx, we might not have any rings */ io_iopoll_try_reap_events(ctx); @@ -8976,13 +8955,14 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct files_struct *files) { struct io_task_cancel cancel = { .task = task, .files = files, }; + struct io_uring_task *tctx = current->io_uring; while (1) { enum io_wq_cancel cret; bool ret = false; - if (ctx->io_wq) { - cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, + if (tctx && tctx->io_wq) { + cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb, &cancel, true); ret |= (cret != IO_WQ_CANCEL_NOTFOUND); } @@ -9094,7 +9074,7 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) int ret; if (unlikely(!tctx)) { - ret = io_uring_alloc_task_context(current); + ret = io_uring_alloc_task_context(current, ctx); if (unlikely(ret)) return ret; tctx = current->io_uring; @@ -9164,8 +9144,12 @@ void __io_uring_files_cancel(struct files_struct *files) io_uring_cancel_task_requests(file->private_data, files); atomic_dec(&tctx->in_idle); - if (files) + if (files) { io_uring_remove_task_files(tctx); + } else if (tctx->io_wq && current->flags & PF_EXITING) { + io_wq_destroy(tctx->io_wq); + tctx->io_wq = NULL; + } } static s64 tctx_inflight(struct io_uring_task *tctx) diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 2eb6d19de336..0e95398998b6 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -36,6 +36,7 @@ struct io_uring_task { struct xarray xa; struct wait_queue_head wait; struct file *last; + void *io_wq; struct percpu_counter inflight; struct io_identity __identity; struct io_identity *identity; -- cgit v1.2.3 From 958234d5ec9321445500dc5e69dfefb405b3d82c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 17 Feb 2021 09:00:57 -0700 Subject: io-wq: don't pass 'wqe' needlessly around Just grab it from the worker itself, which we're already passing in. Signed-off-by: Jens Axboe --- fs/io-wq.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 0c47febfed9b..ec7f1106b659 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -201,9 +201,10 @@ static inline struct io_wqe_acct *io_work_get_acct(struct io_wqe *wqe, return &wqe->acct[IO_WQ_ACCT_BOUND]; } -static inline struct io_wqe_acct *io_wqe_get_acct(struct io_wqe *wqe, - struct io_worker *worker) +static inline struct io_wqe_acct *io_wqe_get_acct(struct io_worker *worker) { + struct io_wqe *wqe = worker->wqe; + if (worker->flags & IO_WORKER_F_BOUND) return &wqe->acct[IO_WQ_ACCT_BOUND]; @@ -213,7 +214,7 @@ static inline struct io_wqe_acct *io_wqe_get_acct(struct io_wqe *wqe, static void io_worker_exit(struct io_worker *worker) { struct io_wqe *wqe = worker->wqe; - struct io_wqe_acct *acct = io_wqe_get_acct(wqe, worker); + struct io_wqe_acct *acct = io_wqe_get_acct(worker); /* * If we're not at zero, someone else is holding a brief reference @@ -303,23 +304,24 @@ static void io_wqe_wake_worker(struct io_wqe *wqe, struct io_wqe_acct *acct) wake_up_process(wqe->wq->manager); } -static void io_wqe_inc_running(struct io_wqe *wqe, struct io_worker *worker) +static void io_wqe_inc_running(struct io_worker *worker) { - struct io_wqe_acct *acct = io_wqe_get_acct(wqe, worker); + struct io_wqe_acct *acct = io_wqe_get_acct(worker); atomic_inc(&acct->nr_running); } -static void io_wqe_dec_running(struct io_wqe *wqe, struct io_worker *worker) +static void io_wqe_dec_running(struct io_worker *worker) __must_hold(wqe->lock) { - struct io_wqe_acct *acct = io_wqe_get_acct(wqe, worker); + struct io_wqe_acct *acct = io_wqe_get_acct(worker); + struct io_wqe *wqe = worker->wqe; if (atomic_dec_and_test(&acct->nr_running) && io_wqe_run_queue(wqe)) io_wqe_wake_worker(wqe, acct); } -static void io_worker_start(struct io_wqe *wqe, struct io_worker *worker) +static void io_worker_start(struct io_worker *worker) { allow_kernel_signal(SIGINT); @@ -329,7 +331,7 @@ static void io_worker_start(struct io_wqe *wqe, struct io_worker *worker) worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); worker->restore_nsproxy = current->nsproxy; - io_wqe_inc_running(wqe, worker); + io_wqe_inc_running(worker); } /* @@ -354,7 +356,7 @@ static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker, worker_bound = (worker->flags & IO_WORKER_F_BOUND) != 0; work_bound = (work->flags & IO_WQ_WORK_UNBOUND) == 0; if (worker_bound != work_bound) { - io_wqe_dec_running(wqe, worker); + io_wqe_dec_running(worker); if (work_bound) { worker->flags |= IO_WORKER_F_BOUND; wqe->acct[IO_WQ_ACCT_UNBOUND].nr_workers--; @@ -366,7 +368,7 @@ static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker, wqe->acct[IO_WQ_ACCT_BOUND].nr_workers--; atomic_inc(&wqe->wq->user->processes); } - io_wqe_inc_running(wqe, worker); + io_wqe_inc_running(worker); } } @@ -589,7 +591,7 @@ static int io_wqe_worker(void *data) struct io_wqe *wqe = worker->wqe; struct io_wq *wq = wqe->wq; - io_worker_start(wqe, worker); + io_worker_start(worker); while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { set_current_state(TASK_INTERRUPTIBLE); @@ -634,14 +636,13 @@ loop: void io_wq_worker_running(struct task_struct *tsk) { struct io_worker *worker = kthread_data(tsk); - struct io_wqe *wqe = worker->wqe; if (!(worker->flags & IO_WORKER_F_UP)) return; if (worker->flags & IO_WORKER_F_RUNNING) return; worker->flags |= IO_WORKER_F_RUNNING; - io_wqe_inc_running(wqe, worker); + io_wqe_inc_running(worker); } /* @@ -662,7 +663,7 @@ void io_wq_worker_sleeping(struct task_struct *tsk) worker->flags &= ~IO_WORKER_F_RUNNING; raw_spin_lock_irq(&wqe->lock); - io_wqe_dec_running(wqe, worker); + io_wqe_dec_running(worker); raw_spin_unlock_irq(&wqe->lock); } -- cgit v1.2.3 From 3bfe6106693b6b4ba175ad1f929c4660b8f59ca8 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 16 Feb 2021 14:15:30 -0700 Subject: io-wq: fork worker threads from original task Instead of using regular kthread kernel threads, create kernel threads that are like a real thread that the task would create. This ensures that we get all the context that we need, without having to carry that state around. This greatly reduces the code complexity, and the risk of missing state for a given request type. With the move away from kthread, we can also dump everything related to assigned state to the new threads. Signed-off-by: Jens Axboe --- fs/io-wq.c | 301 +++++++++++++++++--------------------------------- fs/io-wq.h | 3 +- fs/io_uring.c | 7 ++ include/linux/sched.h | 3 + 4 files changed, 116 insertions(+), 198 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index ec7f1106b659..b53f569b5b4e 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -13,12 +13,9 @@ #include #include #include -#include #include -#include -#include -#include #include +#include #include "../kernel/sched/sched.h" #include "io-wq.h" @@ -57,13 +54,6 @@ struct io_worker { spinlock_t lock; struct rcu_head rcu; - struct mm_struct *mm; -#ifdef CONFIG_BLK_CGROUP - struct cgroup_subsys_state *blkcg_css; -#endif - const struct cred *cur_creds; - const struct cred *saved_creds; - struct nsproxy *restore_nsproxy; }; #if BITS_PER_LONG == 64 @@ -122,6 +112,8 @@ struct io_wq { struct completion done; struct hlist_node cpuhp_node; + + pid_t task_pid; }; static enum cpuhp_state io_wq_online; @@ -137,61 +129,6 @@ static void io_worker_release(struct io_worker *worker) wake_up_process(worker->task); } -/* - * Note: drops the wqe->lock if returning true! The caller must re-acquire - * the lock in that case. Some callers need to restart handling if this - * happens, so we can't just re-acquire the lock on behalf of the caller. - */ -static bool __io_worker_unuse(struct io_wqe *wqe, struct io_worker *worker) -{ - bool dropped_lock = false; - - if (worker->saved_creds) { - revert_creds(worker->saved_creds); - worker->cur_creds = worker->saved_creds = NULL; - } - - if (current->files) { - __acquire(&wqe->lock); - raw_spin_unlock_irq(&wqe->lock); - dropped_lock = true; - - task_lock(current); - current->files = NULL; - current->nsproxy = worker->restore_nsproxy; - task_unlock(current); - } - - if (current->fs) - current->fs = NULL; - - /* - * If we have an active mm, we need to drop the wq lock before unusing - * it. If we do, return true and let the caller retry the idle loop. - */ - if (worker->mm) { - if (!dropped_lock) { - __acquire(&wqe->lock); - raw_spin_unlock_irq(&wqe->lock); - dropped_lock = true; - } - __set_current_state(TASK_RUNNING); - kthread_unuse_mm(worker->mm); - mmput(worker->mm); - worker->mm = NULL; - } - -#ifdef CONFIG_BLK_CGROUP - if (worker->blkcg_css) { - kthread_associate_blkcg(NULL); - worker->blkcg_css = NULL; - } -#endif - if (current->signal->rlim[RLIMIT_FSIZE].rlim_cur != RLIM_INFINITY) - current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY; - return dropped_lock; -} - static inline struct io_wqe_acct *io_work_get_acct(struct io_wqe *wqe, struct io_wq_work *work) { @@ -237,10 +174,6 @@ static void io_worker_exit(struct io_worker *worker) raw_spin_lock_irq(&wqe->lock); hlist_nulls_del_rcu(&worker->nulls_node); list_del_rcu(&worker->all_list); - if (__io_worker_unuse(wqe, worker)) { - __release(&wqe->lock); - raw_spin_lock_irq(&wqe->lock); - } acct->nr_workers--; raw_spin_unlock_irq(&wqe->lock); @@ -323,14 +256,7 @@ static void io_wqe_dec_running(struct io_worker *worker) static void io_worker_start(struct io_worker *worker) { - allow_kernel_signal(SIGINT); - - current->flags |= PF_IO_WORKER; - current->fs = NULL; - current->files = NULL; - worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); - worker->restore_nsproxy = current->nsproxy; io_wqe_inc_running(worker); } @@ -387,7 +313,7 @@ static bool __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); } - return __io_worker_unuse(wqe, worker); + return false; } static inline unsigned int io_get_work_hash(struct io_wq_work *work) @@ -426,96 +352,23 @@ static struct io_wq_work *io_get_next_work(struct io_wqe *wqe) return NULL; } -static void io_wq_switch_mm(struct io_worker *worker, struct io_wq_work *work) +static void io_flush_signals(void) { - if (worker->mm) { - kthread_unuse_mm(worker->mm); - mmput(worker->mm); - worker->mm = NULL; + if (unlikely(test_tsk_thread_flag(current, TIF_NOTIFY_SIGNAL))) { + if (current->task_works) + task_work_run(); + clear_tsk_thread_flag(current, TIF_NOTIFY_SIGNAL); } - - if (mmget_not_zero(work->identity->mm)) { - kthread_use_mm(work->identity->mm); - worker->mm = work->identity->mm; - return; - } - - /* failed grabbing mm, ensure work gets cancelled */ - work->flags |= IO_WQ_WORK_CANCEL; -} - -static inline void io_wq_switch_blkcg(struct io_worker *worker, - struct io_wq_work *work) -{ -#ifdef CONFIG_BLK_CGROUP - if (!(work->flags & IO_WQ_WORK_BLKCG)) - return; - if (work->identity->blkcg_css != worker->blkcg_css) { - kthread_associate_blkcg(work->identity->blkcg_css); - worker->blkcg_css = work->identity->blkcg_css; - } -#endif -} - -static void io_wq_switch_creds(struct io_worker *worker, - struct io_wq_work *work) -{ - const struct cred *old_creds = override_creds(work->identity->creds); - - worker->cur_creds = work->identity->creds; - if (worker->saved_creds) - put_cred(old_creds); /* creds set by previous switch */ - else - worker->saved_creds = old_creds; -} - -static void io_impersonate_work(struct io_worker *worker, - struct io_wq_work *work) -{ - if ((work->flags & IO_WQ_WORK_FILES) && - current->files != work->identity->files) { - task_lock(current); - current->files = work->identity->files; - current->nsproxy = work->identity->nsproxy; - task_unlock(current); - if (!work->identity->files) { - /* failed grabbing files, ensure work gets cancelled */ - work->flags |= IO_WQ_WORK_CANCEL; - } - } - if ((work->flags & IO_WQ_WORK_FS) && current->fs != work->identity->fs) - current->fs = work->identity->fs; - if ((work->flags & IO_WQ_WORK_MM) && work->identity->mm != worker->mm) - io_wq_switch_mm(worker, work); - if ((work->flags & IO_WQ_WORK_CREDS) && - worker->cur_creds != work->identity->creds) - io_wq_switch_creds(worker, work); - if (work->flags & IO_WQ_WORK_FSIZE) - current->signal->rlim[RLIMIT_FSIZE].rlim_cur = work->identity->fsize; - else if (current->signal->rlim[RLIMIT_FSIZE].rlim_cur != RLIM_INFINITY) - current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY; - io_wq_switch_blkcg(worker, work); -#ifdef CONFIG_AUDIT - current->loginuid = work->identity->loginuid; - current->sessionid = work->identity->sessionid; -#endif } static void io_assign_current_work(struct io_worker *worker, struct io_wq_work *work) { if (work) { - /* flush pending signals before assigning new work */ - if (signal_pending(current)) - flush_signals(current); + io_flush_signals(); cond_resched(); } -#ifdef CONFIG_AUDIT - current->loginuid = KUIDT_INIT(AUDIT_UID_UNSET); - current->sessionid = AUDIT_SID_UNSET; -#endif - spin_lock_irq(&worker->lock); worker->cur_work = work; spin_unlock_irq(&worker->lock); @@ -556,7 +409,6 @@ get_next: unsigned int hash = io_get_work_hash(work); next_hashed = wq_next_work(work); - io_impersonate_work(worker, work); wq->do_work(work); io_assign_current_work(worker, NULL); @@ -608,10 +460,11 @@ loop: goto loop; } raw_spin_unlock_irq(&wqe->lock); - if (signal_pending(current)) - flush_signals(current); + io_flush_signals(); if (schedule_timeout(WORKER_IDLE_TIMEOUT)) continue; + if (fatal_signal_pending(current)) + break; /* timed out, exit unless we're the fixed worker */ if (test_bit(IO_WQ_BIT_EXIT, &wq->state) || !(worker->flags & IO_WORKER_F_FIXED)) @@ -635,8 +488,10 @@ loop: */ void io_wq_worker_running(struct task_struct *tsk) { - struct io_worker *worker = kthread_data(tsk); + struct io_worker *worker = tsk->pf_io_worker; + if (!worker) + return; if (!(worker->flags & IO_WORKER_F_UP)) return; if (worker->flags & IO_WORKER_F_RUNNING) @@ -652,9 +507,10 @@ void io_wq_worker_running(struct task_struct *tsk) */ void io_wq_worker_sleeping(struct task_struct *tsk) { - struct io_worker *worker = kthread_data(tsk); - struct io_wqe *wqe = worker->wqe; + struct io_worker *worker = tsk->pf_io_worker; + if (!worker) + return; if (!(worker->flags & IO_WORKER_F_UP)) return; if (!(worker->flags & IO_WORKER_F_RUNNING)) @@ -662,32 +518,27 @@ void io_wq_worker_sleeping(struct task_struct *tsk) worker->flags &= ~IO_WORKER_F_RUNNING; - raw_spin_lock_irq(&wqe->lock); + raw_spin_lock_irq(&worker->wqe->lock); io_wqe_dec_running(worker); - raw_spin_unlock_irq(&wqe->lock); + raw_spin_unlock_irq(&worker->wqe->lock); } -static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) +static int task_thread(void *data, int index) { + struct io_worker *worker = data; + struct io_wqe *wqe = worker->wqe; struct io_wqe_acct *acct = &wqe->acct[index]; - struct io_worker *worker; + struct io_wq *wq = wqe->wq; + char buf[TASK_COMM_LEN]; - worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, wqe->node); - if (!worker) - return false; + sprintf(buf, "iou-wrk-%d", wq->task_pid); + set_task_comm(current, buf); - refcount_set(&worker->ref, 1); - worker->nulls_node.pprev = NULL; - worker->wqe = wqe; - spin_lock_init(&worker->lock); + current->pf_io_worker = worker; + worker->task = current; - worker->task = kthread_create_on_node(io_wqe_worker, worker, wqe->node, - "io_wqe_worker-%d/%d", index, wqe->node); - if (IS_ERR(worker->task)) { - kfree(worker); - return false; - } - kthread_bind_mask(worker->task, cpumask_of_node(wqe->node)); + set_cpus_allowed_ptr(current, cpumask_of_node(wqe->node)); + current->flags |= PF_NO_SETAFFINITY; raw_spin_lock_irq(&wqe->lock); hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); @@ -703,8 +554,58 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) if (index == IO_WQ_ACCT_UNBOUND) atomic_inc(&wq->user->processes); + io_wqe_worker(data); + do_exit(0); +} + +static int task_thread_bound(void *data) +{ + return task_thread(data, IO_WQ_ACCT_BOUND); +} + +static int task_thread_unbound(void *data) +{ + return task_thread(data, IO_WQ_ACCT_UNBOUND); +} + +static pid_t fork_thread(int (*fn)(void *), void *arg) +{ + unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD| + CLONE_IO|SIGCHLD; + struct kernel_clone_args args = { + .flags = ((lower_32_bits(flags) | CLONE_VM | + CLONE_UNTRACED) & ~CSIGNAL), + .exit_signal = (lower_32_bits(flags) & CSIGNAL), + .stack = (unsigned long)fn, + .stack_size = (unsigned long)arg, + }; + + return kernel_clone(&args); +} + +static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) +{ + struct io_worker *worker; + pid_t pid; + + worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, wqe->node); + if (!worker) + return false; + + refcount_set(&worker->ref, 1); + worker->nulls_node.pprev = NULL; + worker->wqe = wqe; + spin_lock_init(&worker->lock); + + if (index == IO_WQ_ACCT_BOUND) + pid = fork_thread(task_thread_bound, worker); + else + pid = fork_thread(task_thread_unbound, worker); + if (pid < 0) { + kfree(worker); + return false; + } refcount_inc(&wq->refs); - wake_up_process(worker->task); return true; } @@ -756,12 +657,17 @@ static bool io_wq_worker_wake(struct io_worker *worker, void *data) static int io_wq_manager(void *data) { struct io_wq *wq = data; + char buf[TASK_COMM_LEN]; int node; - refcount_set(&wq->refs, 1); + sprintf(buf, "iou-mgr-%d", wq->task_pid); + set_task_comm(current, buf); + current->flags |= PF_IO_WORKER; + wq->manager = current; + complete(&wq->done); - while (!kthread_should_stop()) { + while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { for_each_node(node) { struct io_wqe *wqe = wq->wqes[node]; bool fork_worker[2] = { false, false }; @@ -782,11 +688,13 @@ static int io_wq_manager(void *data) } set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(HZ); + if (fatal_signal_pending(current)) + set_bit(IO_WQ_BIT_EXIT, &wq->state); } if (refcount_dec_and_test(&wq->refs)) { complete(&wq->done); - return 0; + do_exit(0); } /* if ERROR is set and we get here, we have workers to wake */ if (test_bit(IO_WQ_BIT_ERROR, &wq->state)) { @@ -795,7 +703,7 @@ static int io_wq_manager(void *data) io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); rcu_read_unlock(); } - return 0; + do_exit(0); } static bool io_wq_can_queue(struct io_wqe *wqe, struct io_wqe_acct *acct, @@ -919,7 +827,7 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data) spin_lock_irqsave(&worker->lock, flags); if (worker->cur_work && match->fn(worker->cur_work, match->data)) { - send_sig(SIGINT, worker->task, 1); + set_notify_signal(worker->task); match->nr_running++; } spin_unlock_irqrestore(&worker->lock, flags); @@ -1075,22 +983,21 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) INIT_LIST_HEAD(&wqe->all_list); } + wq->task_pid = current->pid; init_completion(&wq->done); + refcount_set(&wq->refs, 1); - wq->manager = kthread_create(io_wq_manager, wq, "io_wq_manager"); - if (!IS_ERR(wq->manager)) { - wake_up_process(wq->manager); + current->flags |= PF_IO_WORKER; + ret = fork_thread(io_wq_manager, wq); + current->flags &= ~PF_IO_WORKER; + if (ret >= 0) { wait_for_completion(&wq->done); - if (test_bit(IO_WQ_BIT_ERROR, &wq->state)) { - ret = -ENOMEM; - goto err; - } reinit_completion(&wq->done); return wq; } - ret = PTR_ERR(wq->manager); - complete(&wq->done); + if (refcount_dec_and_test(&wq->refs)) + complete(&wq->done); err: cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); for_each_node(node) @@ -1110,7 +1017,7 @@ void io_wq_destroy(struct io_wq *wq) set_bit(IO_WQ_BIT_EXIT, &wq->state); if (wq->manager) - kthread_stop(wq->manager); + wake_up_process(wq->manager); rcu_read_lock(); for_each_node(node) diff --git a/fs/io-wq.h b/fs/io-wq.h index d2cf284b4641..83d56adabd16 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -137,6 +137,7 @@ static inline void io_wq_worker_running(struct task_struct *tsk) static inline bool io_wq_current_is_worker(void) { - return in_task() && (current->flags & PF_IO_WORKER); + return in_task() && (current->flags & PF_IO_WORKER) && + current->pf_io_worker; } #endif diff --git a/fs/io_uring.c b/fs/io_uring.c index 31402a19fca6..9d22ec9d9406 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1633,6 +1633,9 @@ static struct io_kiocb *__io_queue_async_work(struct io_kiocb *req) struct io_kiocb *link = io_prep_linked_timeout(req); struct io_uring_task *tctx = req->task->io_uring; + BUG_ON(!tctx); + BUG_ON(!tctx->io_wq); + trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req, &req->work, req->flags); io_wq_enqueue(tctx->io_wq, &req->work); @@ -9240,6 +9243,10 @@ static int io_uring_flush(struct file *file, void *data) struct io_uring_task *tctx = current->io_uring; struct io_ring_ctx *ctx = file->private_data; + /* Ignore helper thread files exit */ + if (current->flags & PF_IO_WORKER) + return 0; + if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) { io_uring_cancel_task_requests(ctx, NULL); io_req_caches_free(ctx, current); diff --git a/include/linux/sched.h b/include/linux/sched.h index 26f499810dfa..ef00bb22164c 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -895,6 +895,9 @@ struct task_struct { /* CLONE_CHILD_CLEARTID: */ int __user *clear_child_tid; + /* PF_IO_WORKER */ + void *pf_io_worker; + u64 utime; u64 stime; #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME -- cgit v1.2.3 From c6d77d92b7e53b24e8e74a58e6ef2056385cc780 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 15 Feb 2021 13:26:34 -0700 Subject: io-wq: worker idling always returns false Remove the bool return, and the checking for it in the caller. Signed-off-by: Jens Axboe --- fs/io-wq.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index b53f569b5b4e..41042119bf0f 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -305,15 +305,13 @@ static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker, * retry the loop in that case (we changed task state), we don't regrab * the lock if we return success. */ -static bool __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) +static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) __must_hold(wqe->lock) { if (!(worker->flags & IO_WORKER_F_FREE)) { worker->flags |= IO_WORKER_F_FREE; hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); } - - return false; } static inline unsigned int io_get_work_hash(struct io_wq_work *work) @@ -454,11 +452,7 @@ loop: io_worker_handle_work(worker); goto loop; } - /* drops the lock on success, retry */ - if (__io_worker_idle(wqe, worker)) { - __release(&wqe->lock); - goto loop; - } + __io_worker_idle(wqe, worker); raw_spin_unlock_irq(&wqe->lock); io_flush_signals(); if (schedule_timeout(WORKER_IDLE_TIMEOUT)) -- cgit v1.2.3 From 44526bedc2ff8fcd58552e3c5bae928524b6f13c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 15 Feb 2021 13:32:18 -0700 Subject: io_uring: remove any grabbing of context The async workers are siblings of the task itself, so by definition we have all the state that we need. Remove any of the state grabbing that we have, and requests flagging what they need. Signed-off-by: Jens Axboe --- fs/io-wq.h | 7 -- fs/io_uring.c | 236 ++++------------------------------------------------------ 2 files changed, 14 insertions(+), 229 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.h b/fs/io-wq.h index 83d56adabd16..bbe05dd54716 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -11,13 +11,6 @@ enum { IO_WQ_WORK_UNBOUND = 4, IO_WQ_WORK_CONCURRENT = 16, - IO_WQ_WORK_FILES = 32, - IO_WQ_WORK_FS = 64, - IO_WQ_WORK_MM = 128, - IO_WQ_WORK_CREDS = 256, - IO_WQ_WORK_BLKCG = 512, - IO_WQ_WORK_FSIZE = 1024, - IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */ }; diff --git a/fs/io_uring.c b/fs/io_uring.c index 9d22ec9d9406..6e88295758b5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -838,7 +838,6 @@ struct io_op_def { unsigned plug : 1; /* size of async data needed, if any */ unsigned short async_size; - unsigned work_flags; }; static const struct io_op_def io_op_defs[] = { @@ -851,7 +850,6 @@ static const struct io_op_def io_op_defs[] = { .needs_async_data = 1, .plug = 1, .async_size = sizeof(struct io_async_rw), - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, }, [IORING_OP_WRITEV] = { .needs_file = 1, @@ -861,12 +859,9 @@ static const struct io_op_def io_op_defs[] = { .needs_async_data = 1, .plug = 1, .async_size = sizeof(struct io_async_rw), - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | - IO_WQ_WORK_FSIZE, }, [IORING_OP_FSYNC] = { .needs_file = 1, - .work_flags = IO_WQ_WORK_BLKCG, }, [IORING_OP_READ_FIXED] = { .needs_file = 1, @@ -874,7 +869,6 @@ static const struct io_op_def io_op_defs[] = { .pollin = 1, .plug = 1, .async_size = sizeof(struct io_async_rw), - .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_MM, }, [IORING_OP_WRITE_FIXED] = { .needs_file = 1, @@ -883,8 +877,6 @@ static const struct io_op_def io_op_defs[] = { .pollout = 1, .plug = 1, .async_size = sizeof(struct io_async_rw), - .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE | - IO_WQ_WORK_MM, }, [IORING_OP_POLL_ADD] = { .needs_file = 1, @@ -893,7 +885,6 @@ static const struct io_op_def io_op_defs[] = { [IORING_OP_POLL_REMOVE] = {}, [IORING_OP_SYNC_FILE_RANGE] = { .needs_file = 1, - .work_flags = IO_WQ_WORK_BLKCG, }, [IORING_OP_SENDMSG] = { .needs_file = 1, @@ -901,8 +892,6 @@ static const struct io_op_def io_op_defs[] = { .pollout = 1, .needs_async_data = 1, .async_size = sizeof(struct io_async_msghdr), - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | - IO_WQ_WORK_FS, }, [IORING_OP_RECVMSG] = { .needs_file = 1, @@ -911,29 +900,23 @@ static const struct io_op_def io_op_defs[] = { .buffer_select = 1, .needs_async_data = 1, .async_size = sizeof(struct io_async_msghdr), - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | - IO_WQ_WORK_FS, }, [IORING_OP_TIMEOUT] = { .needs_async_data = 1, .async_size = sizeof(struct io_timeout_data), - .work_flags = IO_WQ_WORK_MM, }, [IORING_OP_TIMEOUT_REMOVE] = { /* used by timeout updates' prep() */ - .work_flags = IO_WQ_WORK_MM, }, [IORING_OP_ACCEPT] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollin = 1, - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FILES, }, [IORING_OP_ASYNC_CANCEL] = {}, [IORING_OP_LINK_TIMEOUT] = { .needs_async_data = 1, .async_size = sizeof(struct io_timeout_data), - .work_flags = IO_WQ_WORK_MM, }, [IORING_OP_CONNECT] = { .needs_file = 1, @@ -941,26 +924,14 @@ static const struct io_op_def io_op_defs[] = { .pollout = 1, .needs_async_data = 1, .async_size = sizeof(struct io_async_connect), - .work_flags = IO_WQ_WORK_MM, }, [IORING_OP_FALLOCATE] = { .needs_file = 1, - .work_flags = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE, - }, - [IORING_OP_OPENAT] = { - .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG | - IO_WQ_WORK_FS | IO_WQ_WORK_MM, - }, - [IORING_OP_CLOSE] = { - .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_BLKCG, - }, - [IORING_OP_FILES_UPDATE] = { - .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_MM, - }, - [IORING_OP_STATX] = { - .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_MM | - IO_WQ_WORK_FS | IO_WQ_WORK_BLKCG, }, + [IORING_OP_OPENAT] = {}, + [IORING_OP_CLOSE] = {}, + [IORING_OP_FILES_UPDATE] = {}, + [IORING_OP_STATX] = {}, [IORING_OP_READ] = { .needs_file = 1, .unbound_nonreg_file = 1, @@ -968,7 +939,6 @@ static const struct io_op_def io_op_defs[] = { .buffer_select = 1, .plug = 1, .async_size = sizeof(struct io_async_rw), - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, }, [IORING_OP_WRITE] = { .needs_file = 1, @@ -976,42 +946,31 @@ static const struct io_op_def io_op_defs[] = { .pollout = 1, .plug = 1, .async_size = sizeof(struct io_async_rw), - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | - IO_WQ_WORK_FSIZE, }, [IORING_OP_FADVISE] = { .needs_file = 1, - .work_flags = IO_WQ_WORK_BLKCG, - }, - [IORING_OP_MADVISE] = { - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, }, + [IORING_OP_MADVISE] = {}, [IORING_OP_SEND] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollout = 1, - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, }, [IORING_OP_RECV] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollin = 1, .buffer_select = 1, - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG, }, [IORING_OP_OPENAT2] = { - .work_flags = IO_WQ_WORK_FILES | IO_WQ_WORK_FS | - IO_WQ_WORK_BLKCG | IO_WQ_WORK_MM, }, [IORING_OP_EPOLL_CTL] = { .unbound_nonreg_file = 1, - .work_flags = IO_WQ_WORK_FILES, }, [IORING_OP_SPLICE] = { .needs_file = 1, .hash_reg_file = 1, .unbound_nonreg_file = 1, - .work_flags = IO_WQ_WORK_BLKCG, }, [IORING_OP_PROVIDE_BUFFERS] = {}, [IORING_OP_REMOVE_BUFFERS] = {}, @@ -1023,14 +982,8 @@ static const struct io_op_def io_op_defs[] = { [IORING_OP_SHUTDOWN] = { .needs_file = 1, }, - [IORING_OP_RENAMEAT] = { - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FILES | - IO_WQ_WORK_FS | IO_WQ_WORK_BLKCG, - }, - [IORING_OP_UNLINKAT] = { - .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FILES | - IO_WQ_WORK_FS | IO_WQ_WORK_BLKCG, - }, + [IORING_OP_RENAMEAT] = {}, + [IORING_OP_UNLINKAT] = {}, }; static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, @@ -1141,8 +1094,7 @@ static bool io_match_task(struct io_kiocb *head, continue; if (req->file && req->file->f_op == &io_uring_fops) return true; - if ((req->work.flags & IO_WQ_WORK_FILES) && - req->work.identity->files == files) + if (req->work.identity->files == files) return true; } return false; @@ -1219,20 +1171,15 @@ static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx) static int __io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, struct io_kiocb *req) { - const struct io_op_def *def = &io_op_defs[req->opcode]; int ret; - if (def->work_flags & IO_WQ_WORK_MM) { - ret = __io_sq_thread_acquire_mm(ctx); - if (unlikely(ret)) - return ret; - } + ret = __io_sq_thread_acquire_mm(ctx); + if (unlikely(ret)) + return ret; - if (def->needs_file || (def->work_flags & IO_WQ_WORK_FILES)) { - ret = __io_sq_thread_acquire_files(ctx); - if (unlikely(ret)) - return ret; - } + ret = __io_sq_thread_acquire_files(ctx); + if (unlikely(ret)) + return ret; return 0; } @@ -1416,28 +1363,6 @@ static void io_req_clean_work(struct io_kiocb *req) if (!(req->flags & REQ_F_WORK_INITIALIZED)) return; - if (req->work.flags & IO_WQ_WORK_MM) - mmdrop(req->work.identity->mm); -#ifdef CONFIG_BLK_CGROUP - if (req->work.flags & IO_WQ_WORK_BLKCG) - css_put(req->work.identity->blkcg_css); -#endif - if (req->work.flags & IO_WQ_WORK_CREDS) - put_cred(req->work.identity->creds); - if (req->work.flags & IO_WQ_WORK_FS) { - struct fs_struct *fs = req->work.identity->fs; - - spin_lock(&req->work.identity->fs->lock); - if (--fs->users) - fs = NULL; - spin_unlock(&req->work.identity->fs->lock); - if (fs) - free_fs_struct(fs); - } - if (req->work.flags & IO_WQ_WORK_FILES) { - put_files_struct(req->work.identity->files); - put_nsproxy(req->work.identity->nsproxy); - } if (req->flags & REQ_F_INFLIGHT) { struct io_ring_ctx *ctx = req->ctx; struct io_uring_task *tctx = req->task->io_uring; @@ -1452,56 +1377,9 @@ static void io_req_clean_work(struct io_kiocb *req) } req->flags &= ~REQ_F_WORK_INITIALIZED; - req->work.flags &= ~(IO_WQ_WORK_MM | IO_WQ_WORK_BLKCG | IO_WQ_WORK_FS | - IO_WQ_WORK_CREDS | IO_WQ_WORK_FILES); io_put_identity(req->task->io_uring, req); } -/* - * Create a private copy of io_identity, since some fields don't match - * the current context. - */ -static bool io_identity_cow(struct io_kiocb *req) -{ - struct io_uring_task *tctx = current->io_uring; - const struct cred *creds = NULL; - struct io_identity *id; - - if (req->work.flags & IO_WQ_WORK_CREDS) - creds = req->work.identity->creds; - - id = kmemdup(req->work.identity, sizeof(*id), GFP_KERNEL); - if (unlikely(!id)) { - req->work.flags |= IO_WQ_WORK_CANCEL; - return false; - } - - /* - * We can safely just re-init the creds we copied Either the field - * matches the current one, or we haven't grabbed it yet. The only - * exception is ->creds, through registered personalities, so handle - * that one separately. - */ - io_init_identity(id); - if (creds) - id->creds = creds; - - /* add one for this request */ - refcount_inc(&id->count); - - /* drop tctx and req identity references, if needed */ - if (tctx->identity != &tctx->__identity && - refcount_dec_and_test(&tctx->identity->count)) - kfree(tctx->identity); - if (req->work.identity != &tctx->__identity && - refcount_dec_and_test(&req->work.identity->count)) - kfree(req->work.identity); - - req->work.identity = id; - tctx->identity = id; - return true; -} - static void io_req_track_inflight(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; @@ -1516,79 +1394,6 @@ static void io_req_track_inflight(struct io_kiocb *req) } } -static bool io_grab_identity(struct io_kiocb *req) -{ - const struct io_op_def *def = &io_op_defs[req->opcode]; - struct io_identity *id = req->work.identity; - - if (def->work_flags & IO_WQ_WORK_FSIZE) { - if (id->fsize != rlimit(RLIMIT_FSIZE)) - return false; - req->work.flags |= IO_WQ_WORK_FSIZE; - } -#ifdef CONFIG_BLK_CGROUP - if (!(req->work.flags & IO_WQ_WORK_BLKCG) && - (def->work_flags & IO_WQ_WORK_BLKCG)) { - rcu_read_lock(); - if (id->blkcg_css != blkcg_css()) { - rcu_read_unlock(); - return false; - } - /* - * This should be rare, either the cgroup is dying or the task - * is moving cgroups. Just punt to root for the handful of ios. - */ - if (css_tryget_online(id->blkcg_css)) - req->work.flags |= IO_WQ_WORK_BLKCG; - rcu_read_unlock(); - } -#endif - if (!(req->work.flags & IO_WQ_WORK_CREDS)) { - if (id->creds != current_cred()) - return false; - get_cred(id->creds); - req->work.flags |= IO_WQ_WORK_CREDS; - } -#ifdef CONFIG_AUDIT - if (!uid_eq(current->loginuid, id->loginuid) || - current->sessionid != id->sessionid) - return false; -#endif - if (!(req->work.flags & IO_WQ_WORK_FS) && - (def->work_flags & IO_WQ_WORK_FS)) { - if (current->fs != id->fs) - return false; - spin_lock(&id->fs->lock); - if (!id->fs->in_exec) { - id->fs->users++; - req->work.flags |= IO_WQ_WORK_FS; - } else { - req->work.flags |= IO_WQ_WORK_CANCEL; - } - spin_unlock(¤t->fs->lock); - } - if (!(req->work.flags & IO_WQ_WORK_FILES) && - (def->work_flags & IO_WQ_WORK_FILES) && - !(req->flags & REQ_F_NO_FILE_TABLE)) { - if (id->files != current->files || - id->nsproxy != current->nsproxy) - return false; - atomic_inc(&id->files->count); - get_nsproxy(id->nsproxy); - req->work.flags |= IO_WQ_WORK_FILES; - io_req_track_inflight(req); - } - if (!(req->work.flags & IO_WQ_WORK_MM) && - (def->work_flags & IO_WQ_WORK_MM)) { - if (id->mm != current->mm) - return false; - mmgrab(id->mm); - req->work.flags |= IO_WQ_WORK_MM; - } - - return true; -} - static void io_prep_async_work(struct io_kiocb *req) { const struct io_op_def *def = &io_op_defs[req->opcode]; @@ -1606,17 +1411,6 @@ static void io_prep_async_work(struct io_kiocb *req) if (def->unbound_nonreg_file) req->work.flags |= IO_WQ_WORK_UNBOUND; } - - /* if we fail grabbing identity, we must COW, regrab, and retry */ - if (io_grab_identity(req)) - return; - - if (!io_identity_cow(req)) - return; - - /* can't fail at this point */ - if (!io_grab_identity(req)) - WARN_ON(1); } static void io_prep_async_link(struct io_kiocb *req) @@ -6583,7 +6377,6 @@ static void __io_queue_sqe(struct io_kiocb *req) int ret; if ((req->flags & REQ_F_WORK_INITIALIZED) && - (req->work.flags & IO_WQ_WORK_CREDS) && req->work.identity->creds != current_cred()) old_creds = override_creds(req->work.identity->creds); @@ -6725,7 +6518,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, __io_req_init_async(req); get_cred(iod->creds); req->work.identity = iod; - req->work.flags |= IO_WQ_WORK_CREDS; } state = &ctx->submit_state; -- cgit v1.2.3 From 4379bf8bd70b5de6bba7d53015b0c36c57a634ee Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 15 Feb 2021 13:40:22 -0700 Subject: io_uring: remove io_identity We are no longer grabbing state, so no need to maintain an IO identity that we COW if there are changes. Signed-off-by: Jens Axboe --- fs/io-wq.c | 26 ++++++++++++ fs/io-wq.h | 2 +- fs/io_uring.c | 104 ++++++++++++----------------------------------- include/linux/io_uring.h | 19 --------- 4 files changed, 52 insertions(+), 99 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 41042119bf0f..acc67ed3a52c 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -53,6 +53,9 @@ struct io_worker { struct io_wq_work *cur_work; spinlock_t lock; + const struct cred *cur_creds; + const struct cred *saved_creds; + struct rcu_head rcu; }; @@ -171,6 +174,11 @@ static void io_worker_exit(struct io_worker *worker) worker->flags = 0; preempt_enable(); + if (worker->saved_creds) { + revert_creds(worker->saved_creds); + worker->cur_creds = worker->saved_creds = NULL; + } + raw_spin_lock_irq(&wqe->lock); hlist_nulls_del_rcu(&worker->nulls_node); list_del_rcu(&worker->all_list); @@ -312,6 +320,10 @@ static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) worker->flags |= IO_WORKER_F_FREE; hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); } + if (worker->saved_creds) { + revert_creds(worker->saved_creds); + worker->cur_creds = worker->saved_creds = NULL; + } } static inline unsigned int io_get_work_hash(struct io_wq_work *work) @@ -359,6 +371,18 @@ static void io_flush_signals(void) } } +static void io_wq_switch_creds(struct io_worker *worker, + struct io_wq_work *work) +{ + const struct cred *old_creds = override_creds(work->creds); + + worker->cur_creds = work->creds; + if (worker->saved_creds) + put_cred(old_creds); /* creds set by previous switch */ + else + worker->saved_creds = old_creds; +} + static void io_assign_current_work(struct io_worker *worker, struct io_wq_work *work) { @@ -407,6 +431,8 @@ get_next: unsigned int hash = io_get_work_hash(work); next_hashed = wq_next_work(work); + if (work->creds && worker->cur_creds != work->creds) + io_wq_switch_creds(worker, work); wq->do_work(work); io_assign_current_work(worker, NULL); diff --git a/fs/io-wq.h b/fs/io-wq.h index bbe05dd54716..584f0bd5a83d 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -78,7 +78,7 @@ static inline void wq_list_del(struct io_wq_work_list *list, struct io_wq_work { struct io_wq_work_node list; - struct io_identity *identity; + const struct cred *creds; unsigned flags; }; diff --git a/fs/io_uring.c b/fs/io_uring.c index 6e88295758b5..6d851033e48d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1094,7 +1094,7 @@ static bool io_match_task(struct io_kiocb *head, continue; if (req->file && req->file->f_op == &io_uring_fops) return true; - if (req->work.identity->files == files) + if (req->task->files == files) return true; } return false; @@ -1218,31 +1218,6 @@ static inline void req_set_fail_links(struct io_kiocb *req) req->flags |= REQ_F_FAIL_LINK; } -/* - * None of these are dereferenced, they are simply used to check if any of - * them have changed. If we're under current and check they are still the - * same, we're fine to grab references to them for actual out-of-line use. - */ -static void io_init_identity(struct io_identity *id) -{ - id->files = current->files; - id->mm = current->mm; -#ifdef CONFIG_BLK_CGROUP - rcu_read_lock(); - id->blkcg_css = blkcg_css(); - rcu_read_unlock(); -#endif - id->creds = current_cred(); - id->nsproxy = current->nsproxy; - id->fs = current->fs; - id->fsize = rlimit(RLIMIT_FSIZE); -#ifdef CONFIG_AUDIT - id->loginuid = current->loginuid; - id->sessionid = current->sessionid; -#endif - refcount_set(&id->count, 1); -} - static inline void __io_req_init_async(struct io_kiocb *req) { memset(&req->work, 0, sizeof(req->work)); @@ -1255,17 +1230,10 @@ static inline void __io_req_init_async(struct io_kiocb *req) */ static inline void io_req_init_async(struct io_kiocb *req) { - struct io_uring_task *tctx = current->io_uring; - if (req->flags & REQ_F_WORK_INITIALIZED) return; __io_req_init_async(req); - - /* Grab a ref if this isn't our static identity */ - req->work.identity = tctx->identity; - if (tctx->identity != &tctx->__identity) - refcount_inc(&req->work.identity->count); } static void io_ring_ctx_ref_free(struct percpu_ref *ref) @@ -1350,19 +1318,15 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) return false; } -static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req) -{ - if (req->work.identity == &tctx->__identity) - return; - if (refcount_dec_and_test(&req->work.identity->count)) - kfree(req->work.identity); -} - static void io_req_clean_work(struct io_kiocb *req) { if (!(req->flags & REQ_F_WORK_INITIALIZED)) return; + if (req->work.creds) { + put_cred(req->work.creds); + req->work.creds = NULL; + } if (req->flags & REQ_F_INFLIGHT) { struct io_ring_ctx *ctx = req->ctx; struct io_uring_task *tctx = req->task->io_uring; @@ -1377,7 +1341,6 @@ static void io_req_clean_work(struct io_kiocb *req) } req->flags &= ~REQ_F_WORK_INITIALIZED; - io_put_identity(req->task->io_uring, req); } static void io_req_track_inflight(struct io_kiocb *req) @@ -1411,6 +1374,8 @@ static void io_prep_async_work(struct io_kiocb *req) if (def->unbound_nonreg_file) req->work.flags |= IO_WQ_WORK_UNBOUND; } + if (!req->work.creds) + req->work.creds = get_current_cred(); } static void io_prep_async_link(struct io_kiocb *req) @@ -6376,9 +6341,9 @@ static void __io_queue_sqe(struct io_kiocb *req) const struct cred *old_creds = NULL; int ret; - if ((req->flags & REQ_F_WORK_INITIALIZED) && - req->work.identity->creds != current_cred()) - old_creds = override_creds(req->work.identity->creds); + if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds && + req->work.creds != current_cred()) + old_creds = override_creds(req->work.creds); ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); @@ -6508,16 +6473,11 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, id = READ_ONCE(sqe->personality); if (id) { - struct io_identity *iod; - - iod = idr_find(&ctx->personality_idr, id); - if (unlikely(!iod)) - return -EINVAL; - refcount_inc(&iod->count); - __io_req_init_async(req); - get_cred(iod->creds); - req->work.identity = iod; + req->work.creds = idr_find(&ctx->personality_idr, id); + if (unlikely(!req->work.creds)) + return -EINVAL; + get_cred(req->work.creds); } state = &ctx->submit_state; @@ -7936,8 +7896,6 @@ static int io_uring_alloc_task_context(struct task_struct *task, tctx->last = NULL; atomic_set(&tctx->in_idle, 0); tctx->sqpoll = false; - io_init_identity(&tctx->__identity); - tctx->identity = &tctx->__identity; task->io_uring = tctx; spin_lock_init(&tctx->task_lock); INIT_WQ_LIST(&tctx->task_list); @@ -7951,9 +7909,6 @@ void __io_uring_free(struct task_struct *tsk) struct io_uring_task *tctx = tsk->io_uring; WARN_ON_ONCE(!xa_empty(&tctx->xa)); - WARN_ON_ONCE(refcount_read(&tctx->identity->count) != 1); - if (tctx->identity != &tctx->__identity) - kfree(tctx->identity); percpu_counter_destroy(&tctx->inflight); kfree(tctx); tsk->io_uring = NULL; @@ -8593,13 +8548,11 @@ static int io_uring_fasync(int fd, struct file *file, int on) static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) { - struct io_identity *iod; + const struct cred *creds; - iod = idr_remove(&ctx->personality_idr, id); - if (iod) { - put_cred(iod->creds); - if (refcount_dec_and_test(&iod->count)) - kfree(iod); + creds = idr_remove(&ctx->personality_idr, id); + if (creds) { + put_cred(creds); return 0; } @@ -9300,8 +9253,7 @@ out_fput: #ifdef CONFIG_PROC_FS static int io_uring_show_cred(int id, void *p, void *data) { - struct io_identity *iod = p; - const struct cred *cred = iod->creds; + const struct cred *cred = p; struct seq_file *m = data; struct user_namespace *uns = seq_user_ns(m); struct group_info *gi; @@ -9732,21 +9684,15 @@ out: static int io_register_personality(struct io_ring_ctx *ctx) { - struct io_identity *id; + const struct cred *creds; int ret; - id = kmalloc(sizeof(*id), GFP_KERNEL); - if (unlikely(!id)) - return -ENOMEM; - - io_init_identity(id); - id->creds = get_current_cred(); + creds = get_current_cred(); - ret = idr_alloc_cyclic(&ctx->personality_idr, id, 1, USHRT_MAX, GFP_KERNEL); - if (ret < 0) { - put_cred(id->creds); - kfree(id); - } + ret = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1, + USHRT_MAX, GFP_KERNEL); + if (ret < 0) + put_cred(creds); return ret; } diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 0e95398998b6..c48fcbdc2ea8 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -5,23 +5,6 @@ #include #include -struct io_identity { - struct files_struct *files; - struct mm_struct *mm; -#ifdef CONFIG_BLK_CGROUP - struct cgroup_subsys_state *blkcg_css; -#endif - const struct cred *creds; - struct nsproxy *nsproxy; - struct fs_struct *fs; - unsigned long fsize; -#ifdef CONFIG_AUDIT - kuid_t loginuid; - unsigned int sessionid; -#endif - refcount_t count; -}; - struct io_wq_work_node { struct io_wq_work_node *next; }; @@ -38,8 +21,6 @@ struct io_uring_task { struct file *last; void *io_wq; struct percpu_counter inflight; - struct io_identity __identity; - struct io_identity *identity; atomic_t in_idle; bool sqpoll; -- cgit v1.2.3 From bf1daa4bfc77a60e58bed392e659c9ddd0174340 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 16 Feb 2021 18:00:55 -0700 Subject: io-wq: only remove worker from free_list, if it was there If the worker isn't on the free_list, don't attempt to delete it. Signed-off-by: Jens Axboe --- fs/io-wq.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index acc67ed3a52c..3a506f1c7838 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -155,6 +155,7 @@ static void io_worker_exit(struct io_worker *worker) { struct io_wqe *wqe = worker->wqe; struct io_wqe_acct *acct = io_wqe_get_acct(worker); + unsigned flags; /* * If we're not at zero, someone else is holding a brief reference @@ -167,9 +168,11 @@ static void io_worker_exit(struct io_worker *worker) preempt_disable(); current->flags &= ~PF_IO_WORKER; - if (worker->flags & IO_WORKER_F_RUNNING) + flags = worker->flags; + worker->flags = 0; + if (flags & IO_WORKER_F_RUNNING) atomic_dec(&acct->nr_running); - if (!(worker->flags & IO_WORKER_F_BOUND)) + if (!(flags & IO_WORKER_F_BOUND)) atomic_dec(&wqe->wq->user->processes); worker->flags = 0; preempt_enable(); @@ -180,7 +183,8 @@ static void io_worker_exit(struct io_worker *worker) } raw_spin_lock_irq(&wqe->lock); - hlist_nulls_del_rcu(&worker->nulls_node); + if (flags & IO_WORKER_F_FREE) + hlist_nulls_del_rcu(&worker->nulls_node); list_del_rcu(&worker->all_list); acct->nr_workers--; raw_spin_unlock_irq(&wqe->lock); -- cgit v1.2.3 From 843bbfd49f02caab7186910480a86378bb84e975 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 17 Feb 2021 21:05:41 -0700 Subject: io-wq: make io_wq_fork_thread() available to other users We want to use this in io_uring proper as well, for the SQPOLL thread. Rename it from fork_thread() to io_wq_fork_thread(), and make it available through the io-wq.h header. Signed-off-by: Jens Axboe --- fs/io-wq.c | 8 ++++---- fs/io-wq.h | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 3a506f1c7838..b0d09f60200b 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -592,7 +592,7 @@ static int task_thread_unbound(void *data) return task_thread(data, IO_WQ_ACCT_UNBOUND); } -static pid_t fork_thread(int (*fn)(void *), void *arg) +pid_t io_wq_fork_thread(int (*fn)(void *), void *arg) { unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD| CLONE_IO|SIGCHLD; @@ -622,9 +622,9 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) spin_lock_init(&worker->lock); if (index == IO_WQ_ACCT_BOUND) - pid = fork_thread(task_thread_bound, worker); + pid = io_wq_fork_thread(task_thread_bound, worker); else - pid = fork_thread(task_thread_unbound, worker); + pid = io_wq_fork_thread(task_thread_unbound, worker); if (pid < 0) { kfree(worker); return false; @@ -1012,7 +1012,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) refcount_set(&wq->refs, 1); current->flags |= PF_IO_WORKER; - ret = fork_thread(io_wq_manager, wq); + ret = io_wq_fork_thread(io_wq_manager, wq); current->flags &= ~PF_IO_WORKER; if (ret >= 0) { wait_for_completion(&wq->done); diff --git a/fs/io-wq.h b/fs/io-wq.h index 584f0bd5a83d..23f6cbd620f8 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -106,6 +106,8 @@ void io_wq_destroy(struct io_wq *wq); void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); void io_wq_hash_work(struct io_wq_work *work, void *val); +pid_t io_wq_fork_thread(int (*fn)(void *), void *arg); + static inline bool io_wq_is_hashed(struct io_wq_work *work) { return work->flags & IO_WQ_WORK_HASHED; -- cgit v1.2.3 From 78c276f5495aa53a8beebb627e5bf6a54f0af34f Mon Sep 17 00:00:00 2001 From: Namjae Jeon Date: Mon, 1 Feb 2021 09:23:37 +0900 Subject: exfat: fix shift-out-of-bounds in exfat_fill_super() syzbot reported a warning which could cause shift-out-of-bounds issue. Call Trace: __dump_stack lib/dump_stack.c:79 [inline] dump_stack+0x183/0x22e lib/dump_stack.c:120 ubsan_epilogue lib/ubsan.c:148 [inline] __ubsan_handle_shift_out_of_bounds+0x432/0x4d0 lib/ubsan.c:395 exfat_read_boot_sector fs/exfat/super.c:471 [inline] __exfat_fill_super fs/exfat/super.c:556 [inline] exfat_fill_super+0x2acb/0x2d00 fs/exfat/super.c:624 get_tree_bdev+0x406/0x630 fs/super.c:1291 vfs_get_tree+0x86/0x270 fs/super.c:1496 do_new_mount fs/namespace.c:2881 [inline] path_mount+0x1937/0x2c50 fs/namespace.c:3211 do_mount fs/namespace.c:3224 [inline] __do_sys_mount fs/namespace.c:3432 [inline] __se_sys_mount+0x2f9/0x3b0 fs/namespace.c:3409 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xa9 exfat specification describe sect_per_clus_bits field of boot sector could be at most 25 - sect_size_bits and at least 0. And sect_size_bits can also affect this calculation, It also needs validation. This patch add validation for sect_per_clus_bits and sect_size_bits field of boot sector. Fixes: 719c1e182916 ("exfat: add super block operations") Cc: stable@vger.kernel.org # v5.9+ Reported-by: syzbot+da4fe66aaadd3c2e2d1c@syzkaller.appspotmail.com Reviewed-by: Sungjong Seo Tested-by: Randy Dunlap Signed-off-by: Namjae Jeon --- fs/exfat/exfat_raw.h | 4 ++++ fs/exfat/super.c | 31 ++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/exfat/exfat_raw.h b/fs/exfat/exfat_raw.h index 6aec6288e1f2..7f39b1c6469c 100644 --- a/fs/exfat/exfat_raw.h +++ b/fs/exfat/exfat_raw.h @@ -77,6 +77,10 @@ #define EXFAT_FILE_NAME_LEN 15 +#define EXFAT_MIN_SECT_SIZE_BITS 9 +#define EXFAT_MAX_SECT_SIZE_BITS 12 +#define EXFAT_MAX_SECT_PER_CLUS_BITS(x) (25 - (x)->sect_size_bits) + /* EXFAT: Main and Backup Boot Sector (512 bytes) */ struct boot_sector { __u8 jmp_boot[BOOTSEC_JUMP_BOOT_LEN]; diff --git a/fs/exfat/super.c b/fs/exfat/super.c index 87be5bfc31eb..c6d8d2e53486 100644 --- a/fs/exfat/super.c +++ b/fs/exfat/super.c @@ -381,8 +381,7 @@ static int exfat_calibrate_blocksize(struct super_block *sb, int logical_sect) { struct exfat_sb_info *sbi = EXFAT_SB(sb); - if (!is_power_of_2(logical_sect) || - logical_sect < 512 || logical_sect > 4096) { + if (!is_power_of_2(logical_sect)) { exfat_err(sb, "bogus logical sector size %u", logical_sect); return -EIO; } @@ -451,6 +450,25 @@ static int exfat_read_boot_sector(struct super_block *sb) return -EINVAL; } + /* + * sect_size_bits could be at least 9 and at most 12. + */ + if (p_boot->sect_size_bits < EXFAT_MIN_SECT_SIZE_BITS || + p_boot->sect_size_bits > EXFAT_MAX_SECT_SIZE_BITS) { + exfat_err(sb, "bogus sector size bits : %u\n", + p_boot->sect_size_bits); + return -EINVAL; + } + + /* + * sect_per_clus_bits could be at least 0 and at most 25 - sect_size_bits. + */ + if (p_boot->sect_per_clus_bits > EXFAT_MAX_SECT_PER_CLUS_BITS(p_boot)) { + exfat_err(sb, "bogus sectors bits per cluster : %u\n", + p_boot->sect_per_clus_bits); + return -EINVAL; + } + sbi->sect_per_clus = 1 << p_boot->sect_per_clus_bits; sbi->sect_per_clus_bits = p_boot->sect_per_clus_bits; sbi->cluster_size_bits = p_boot->sect_per_clus_bits + @@ -477,16 +495,19 @@ static int exfat_read_boot_sector(struct super_block *sb) sbi->used_clusters = EXFAT_CLUSTERS_UNTRACKED; /* check consistencies */ - if (sbi->num_FAT_sectors << p_boot->sect_size_bits < - sbi->num_clusters * 4) { + if ((u64)sbi->num_FAT_sectors << p_boot->sect_size_bits < + (u64)sbi->num_clusters * 4) { exfat_err(sb, "bogus fat length"); return -EINVAL; } + if (sbi->data_start_sector < - sbi->FAT1_start_sector + sbi->num_FAT_sectors * p_boot->num_fats) { + (u64)sbi->FAT1_start_sector + + (u64)sbi->num_FAT_sectors * p_boot->num_fats) { exfat_err(sb, "bogus data start sector"); return -EINVAL; } + if (sbi->vol_flags & VOLUME_DIRTY) exfat_warn(sb, "Volume was not properly unmounted. Some data may be corrupt. Please run fsck."); if (sbi->vol_flags & MEDIA_FAILURE) -- cgit v1.2.3 From f728760aa923f1dd3a4818368dbdbd2c7d63b370 Mon Sep 17 00:00:00 2001 From: Hyeongseok Kim Date: Mon, 1 Feb 2021 10:02:46 +0900 Subject: exfat: improve performance of exfat_free_cluster when using dirsync mount option There are stressful update of cluster allocation bitmap when using dirsync mount option which is doing sync buffer on every cluster bit clearing. This could result in performance degradation when deleting big size file. Fix to update only when the bitmap buffer index is changed would make less disk access, improving performance especially for truncate operation. Testing with Samsung 256GB sdcard, mounted with dirsync option (mount -t exfat /dev/block/mmcblk0p1 /temp/mount -o dirsync) Remove 4GB file, blktrace result. [Before] : 39 secs. Total (blktrace): Reads Queued: 0, 0KiB Writes Queued: 32775, 16387KiB Read Dispatches: 0, 0KiB Write Dispatches: 32775, 16387KiB Reads Requeued: 0 Writes Requeued: 0 Reads Completed: 0, 0KiB Writes Completed: 32775, 16387KiB Read Merges: 0, 0KiB Write Merges: 0, 0KiB IO unplugs: 2 Timer unplugs: 0 [After] : 1 sec. Total (blktrace): Reads Queued: 0, 0KiB Writes Queued: 13, 6KiB Read Dispatches: 0, 0KiB Write Dispatches: 13, 6KiB Reads Requeued: 0 Writes Requeued: 0 Reads Completed: 0, 0KiB Writes Completed: 13, 6KiB Read Merges: 0, 0KiB Write Merges: 0, 0KiB IO unplugs: 1 Timer unplugs: 0 Signed-off-by: Hyeongseok Kim Acked-by: Sungjong Seo Signed-off-by: Namjae Jeon --- fs/exfat/balloc.c | 4 ++-- fs/exfat/exfat_fs.h | 2 +- fs/exfat/fatent.c | 43 +++++++++++++++++++++++++++++++++++++------ 3 files changed, 40 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/exfat/balloc.c b/fs/exfat/balloc.c index a987919686c0..761c79c3a4ba 100644 --- a/fs/exfat/balloc.c +++ b/fs/exfat/balloc.c @@ -166,7 +166,7 @@ int exfat_set_bitmap(struct inode *inode, unsigned int clu) * If the value of "clu" is 0, it means cluster 2 which is the first cluster of * the cluster heap. */ -void exfat_clear_bitmap(struct inode *inode, unsigned int clu) +void exfat_clear_bitmap(struct inode *inode, unsigned int clu, bool sync) { int i, b; unsigned int ent_idx; @@ -180,7 +180,7 @@ void exfat_clear_bitmap(struct inode *inode, unsigned int clu) b = BITMAP_OFFSET_BIT_IN_SECTOR(sb, ent_idx); clear_bit_le(b, sbi->vol_amap[i]->b_data); - exfat_update_bh(sbi->vol_amap[i], IS_DIRSYNC(inode)); + exfat_update_bh(sbi->vol_amap[i], sync); if (opts->discard) { int ret_discard; diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h index b8f0e829ecbd..764bc645241e 100644 --- a/fs/exfat/exfat_fs.h +++ b/fs/exfat/exfat_fs.h @@ -408,7 +408,7 @@ int exfat_count_num_clusters(struct super_block *sb, int exfat_load_bitmap(struct super_block *sb); void exfat_free_bitmap(struct exfat_sb_info *sbi); int exfat_set_bitmap(struct inode *inode, unsigned int clu); -void exfat_clear_bitmap(struct inode *inode, unsigned int clu); +void exfat_clear_bitmap(struct inode *inode, unsigned int clu, bool sync); unsigned int exfat_find_free_bitmap(struct super_block *sb, unsigned int clu); int exfat_count_used_clusters(struct super_block *sb, unsigned int *ret_count); diff --git a/fs/exfat/fatent.c b/fs/exfat/fatent.c index c3c9afee7418..7b2e8af17193 100644 --- a/fs/exfat/fatent.c +++ b/fs/exfat/fatent.c @@ -157,6 +157,7 @@ int exfat_free_cluster(struct inode *inode, struct exfat_chain *p_chain) unsigned int clu; struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); + int cur_cmap_i, next_cmap_i; /* invalid cluster number */ if (p_chain->dir == EXFAT_FREE_CLUSTER || @@ -176,21 +177,51 @@ int exfat_free_cluster(struct inode *inode, struct exfat_chain *p_chain) clu = p_chain->dir; + cur_cmap_i = next_cmap_i = + BITMAP_OFFSET_SECTOR_INDEX(sb, CLUSTER_TO_BITMAP_ENT(clu)); + if (p_chain->flags == ALLOC_NO_FAT_CHAIN) { + unsigned int last_cluster = p_chain->dir + p_chain->size - 1; do { - exfat_clear_bitmap(inode, clu); - clu++; + bool sync = false; + + if (clu < last_cluster) + next_cmap_i = + BITMAP_OFFSET_SECTOR_INDEX(sb, CLUSTER_TO_BITMAP_ENT(clu+1)); + /* flush bitmap only if index would be changed or for last cluster */ + if (clu == last_cluster || cur_cmap_i != next_cmap_i) { + sync = true; + cur_cmap_i = next_cmap_i; + } + + exfat_clear_bitmap(inode, clu, (sync && IS_DIRSYNC(inode))); + clu++; num_clusters++; } while (num_clusters < p_chain->size); } else { do { - exfat_clear_bitmap(inode, clu); - - if (exfat_get_next_cluster(sb, &clu)) - goto dec_used_clus; + bool sync = false; + unsigned int n_clu = clu; + int err = exfat_get_next_cluster(sb, &n_clu); + + if (err || n_clu == EXFAT_EOF_CLUSTER) + sync = true; + else + next_cmap_i = + BITMAP_OFFSET_SECTOR_INDEX(sb, CLUSTER_TO_BITMAP_ENT(n_clu)); + + if (cur_cmap_i != next_cmap_i) { + sync = true; + cur_cmap_i = next_cmap_i; + } + exfat_clear_bitmap(inode, clu, (sync && IS_DIRSYNC(inode))); + clu = n_clu; num_clusters++; + + if (err) + goto dec_used_clus; } while (clu != EXFAT_EOF_CLUSTER); } -- cgit v1.2.3 From 8e5c66c485a8af3f39a8b0358e9e09f002016d92 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 22 Feb 2021 11:45:55 +0000 Subject: io_uring: clear request count when freeing caches BUG: KASAN: double-free or invalid-free in io_req_caches_free.constprop.0+0x3ce/0x530 fs/io_uring.c:8709 Workqueue: events_unbound io_ring_exit_work Call Trace: [...] __cache_free mm/slab.c:3424 [inline] kmem_cache_free_bulk+0x4b/0x1b0 mm/slab.c:3744 io_req_caches_free.constprop.0+0x3ce/0x530 fs/io_uring.c:8709 io_ring_ctx_free fs/io_uring.c:8764 [inline] io_ring_exit_work+0x518/0x6b0 fs/io_uring.c:8846 process_one_work+0x98d/0x1600 kernel/workqueue.c:2275 worker_thread+0x64c/0x1120 kernel/workqueue.c:2421 kthread+0x3b1/0x4a0 kernel/kthread.c:292 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:294 Freed by task 11900: [...] kmem_cache_free_bulk+0x4b/0x1b0 mm/slab.c:3744 io_req_caches_free.constprop.0+0x3ce/0x530 fs/io_uring.c:8709 io_uring_flush+0x483/0x6e0 fs/io_uring.c:9237 filp_close+0xb4/0x170 fs/open.c:1286 close_files fs/file.c:403 [inline] put_files_struct fs/file.c:418 [inline] put_files_struct+0x1d0/0x350 fs/file.c:415 exit_files+0x7e/0xa0 fs/file.c:435 do_exit+0xc27/0x2ae0 kernel/exit.c:820 do_group_exit+0x125/0x310 kernel/exit.c:922 [...] io_req_caches_free() doesn't zero submit_state->free_reqs, so io_uring considers just freed requests to be good and sound and will reuse or double free them. Zero the counter. Reported-by: syzbot+30b4936dcdb3aafa4fb4@syzkaller.appspotmail.com Fixes: 41be53e94fb04 ("io_uring: kill cached requests from exiting task closing the ring") Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 3ecc3c08bf12..bf9ad810c621 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8704,9 +8704,11 @@ static void io_req_caches_free(struct io_ring_ctx *ctx, struct task_struct *tsk) mutex_lock(&ctx->uring_lock); - if (submit_state->free_reqs) + if (submit_state->free_reqs) { kmem_cache_free_bulk(req_cachep, submit_state->free_reqs, submit_state->reqs); + submit_state->free_reqs = 0; + } io_req_cache_free(&submit_state->comp.free_list, NULL); -- cgit v1.2.3 From d70cef0d46729808dc53f145372c02b145c92604 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 27 Jan 2021 22:15:03 -0800 Subject: btrfs: fix raid6 qstripe kmap When a qstripe is required an extra page is allocated and mapped. There were 3 problems: 1) There is no corresponding call of kunmap() for the qstripe page. 2) There is no reason to map the qstripe page more than once if the number of bits set in rbio->dbitmap is greater than one. 3) There is no reason to map the parity page and unmap it each time through the loop. The page memory can continue to be reused with a single mapping on each iteration by raid6_call.gen_syndrome() without remapping. So map the page for the duration of the loop. Similarly, improve the algorithm by mapping the parity page just 1 time. Fixes: 5a6ac9eacb49 ("Btrfs, raid56: support parity scrub on raid56") CC: stable@vger.kernel.org # 4.4.x: c17af96554a8: btrfs: raid56: simplify tracking of Q stripe presence CC: stable@vger.kernel.org # 4.4.x Signed-off-by: Ira Weiny Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 5394641541f7..abf17fae0912 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -2362,16 +2362,21 @@ static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, SetPageUptodate(p_page); if (has_qstripe) { + /* RAID6, allocate and map temp space for the Q stripe */ q_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); if (!q_page) { __free_page(p_page); goto cleanup; } SetPageUptodate(q_page); + pointers[rbio->real_stripes - 1] = kmap(q_page); } atomic_set(&rbio->error, 0); + /* Map the parity stripe just once */ + pointers[nr_data] = kmap(p_page); + for_each_set_bit(pagenr, rbio->dbitmap, rbio->stripe_npages) { struct page *p; void *parity; @@ -2381,16 +2386,8 @@ static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, pointers[stripe] = kmap(p); } - /* then add the parity stripe */ - pointers[stripe++] = kmap(p_page); - if (has_qstripe) { - /* - * raid6, add the qstripe and call the - * library function to fill in our p/q - */ - pointers[stripe++] = kmap(q_page); - + /* RAID6, call the library function to fill in our P/Q */ raid6_call.gen_syndrome(rbio->real_stripes, PAGE_SIZE, pointers); } else { @@ -2411,12 +2408,14 @@ static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, for (stripe = 0; stripe < nr_data; stripe++) kunmap(page_in_rbio(rbio, stripe, pagenr, 0)); - kunmap(p_page); } + kunmap(p_page); __free_page(p_page); - if (q_page) + if (q_page) { + kunmap(q_page); __free_page(q_page); + } writeback: /* -- cgit v1.2.3 From be6a13613fd35602ea9e65d6634cf7af79f0a93d Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 4 Feb 2021 15:03:23 +0800 Subject: btrfs: make btrfs_submit_compressed_read() subpage compatible For compressed read, we always submit page read using page size. This doesn't work well with subpage, as for subpage one page can contain several sectors. Such submission will read range out of what we want, and cause problems. Thankfully to make it subpage compatible, we only need to change how the last page of the compressed extent is read. Instead of always adding a full page to the compressed read bio, if we're at the last page, calculate the size using compressed length, so that we only add part of the range into the compressed read bio. Since we are here, also change the PAGE_SIZE used in lookup_extent_mapping() to sectorsize. This modification won't cause any functional change, as lookup_extent_mapping() can handle the case where the search range is larger than found extent range. Reviewed-by: Anand Jain Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/compression.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 6d203acfdeb3..5bad3a0b8a88 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -640,7 +640,7 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, read_lock(&em_tree->lock); em = lookup_extent_mapping(em_tree, page_offset(bio_first_page_all(bio)), - PAGE_SIZE); + fs_info->sectorsize); read_unlock(&em_tree->lock); if (!em) return BLK_STS_IOERR; @@ -698,19 +698,30 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, refcount_set(&cb->pending_bios, 1); for (pg_index = 0; pg_index < nr_pages; pg_index++) { + u32 pg_len = PAGE_SIZE; int submit = 0; + /* + * To handle subpage case, we need to make sure the bio only + * covers the range we need. + * + * If we're at the last page, truncate the length to only cover + * the remaining part. + */ + if (pg_index == nr_pages - 1) + pg_len = min_t(u32, PAGE_SIZE, + compressed_len - pg_index * PAGE_SIZE); + page = cb->compressed_pages[pg_index]; page->mapping = inode->i_mapping; page->index = em_start >> PAGE_SHIFT; if (comp_bio->bi_iter.bi_size) - submit = btrfs_bio_fits_in_stripe(page, PAGE_SIZE, + submit = btrfs_bio_fits_in_stripe(page, pg_len, comp_bio, 0); page->mapping = NULL; - if (submit || bio_add_page(comp_bio, page, PAGE_SIZE, 0) < - PAGE_SIZE) { + if (submit || bio_add_page(comp_bio, page, pg_len, 0) < pg_len) { unsigned int nr_sectors; ret = btrfs_bio_wq_end_io(fs_info, comp_bio, @@ -743,9 +754,9 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, comp_bio->bi_private = cb; comp_bio->bi_end_io = end_compressed_bio_read; - bio_add_page(comp_bio, page, PAGE_SIZE, 0); + bio_add_page(comp_bio, page, pg_len, 0); } - cur_disk_byte += PAGE_SIZE; + cur_disk_byte += pg_len; } ret = btrfs_bio_wq_end_io(fs_info, comp_bio, BTRFS_WQ_ENDIO_DATA); -- cgit v1.2.3 From 04d4ba4c90759844fb4ffa735214c1c41508d2f7 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 4 Feb 2021 15:03:24 +0800 Subject: btrfs: make check_compressed_csum() to be subpage compatible Currently check_compressed_csum() completely relies on sectorsize == PAGE_SIZE to do checksum verification for compressed extents. To make it subpage compatible, this patch will: - Do extra calculation for the csum range Since we have multiple sectors inside a page, we need to only hash the range we want, not the full page anymore. - Do sector-by-sector hash inside the page With this patch and previous conversion on btrfs_submit_compressed_read(), now we can read subpage compressed extents properly, and do proper csum verification. Reviewed-by: Anand Jain Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/compression.c | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 5bad3a0b8a88..375dee691a37 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -141,6 +141,7 @@ static int check_compressed_csum(struct btrfs_inode *inode, struct bio *bio, struct btrfs_fs_info *fs_info = inode->root->fs_info; SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); const u32 csum_size = fs_info->csum_size; + const u32 sectorsize = fs_info->sectorsize; struct page *page; unsigned long i; char *kaddr; @@ -154,22 +155,34 @@ static int check_compressed_csum(struct btrfs_inode *inode, struct bio *bio, shash->tfm = fs_info->csum_shash; for (i = 0; i < cb->nr_pages; i++) { + u32 pg_offset; + u32 bytes_left = PAGE_SIZE; page = cb->compressed_pages[i]; - kaddr = kmap_atomic(page); - crypto_shash_digest(shash, kaddr, PAGE_SIZE, csum); - kunmap_atomic(kaddr); - - if (memcmp(&csum, cb_sum, csum_size)) { - btrfs_print_data_csum_error(inode, disk_start, - csum, cb_sum, cb->mirror_num); - if (btrfs_io_bio(bio)->device) - btrfs_dev_stat_inc_and_print( - btrfs_io_bio(bio)->device, - BTRFS_DEV_STAT_CORRUPTION_ERRS); - return -EIO; + /* Determine the remaining bytes inside the page first */ + if (i == cb->nr_pages - 1) + bytes_left = cb->compressed_len - i * PAGE_SIZE; + + /* Hash through the page sector by sector */ + for (pg_offset = 0; pg_offset < bytes_left; + pg_offset += sectorsize) { + kaddr = kmap_atomic(page); + crypto_shash_digest(shash, kaddr + pg_offset, + sectorsize, csum); + kunmap_atomic(kaddr); + + if (memcmp(&csum, cb_sum, csum_size) != 0) { + btrfs_print_data_csum_error(inode, disk_start, + csum, cb_sum, cb->mirror_num); + if (btrfs_io_bio(bio)->device) + btrfs_dev_stat_inc_and_print( + btrfs_io_bio(bio)->device, + BTRFS_DEV_STAT_CORRUPTION_ERRS); + return -EIO; + } + cb_sum += csum_size; + disk_start += sectorsize; } - cb_sum += csum_size; } return 0; } -- cgit v1.2.3 From 3c17916510428dbccdf657de050c34e208347089 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 8 Feb 2021 10:26:54 +0200 Subject: btrfs: fix race between extent freeing/allocation when using bitmaps During allocation the allocator will try to allocate an extent using cluster policy. Once the current cluster is exhausted it will remove the entry under btrfs_free_cluster::lock and subsequently acquire btrfs_free_space_ctl::tree_lock to dispose of the already-deleted entry and adjust btrfs_free_space_ctl::total_bitmap. This poses a problem because there exists a race condition between removing the entry under one lock and doing the necessary accounting holding a different lock since extent freeing only uses the 2nd lock. This can result in the following situation: T1: T2: btrfs_alloc_from_cluster insert_into_bitmap if (entry->bytes == 0) if (block_group && !list_empty(&block_group->cluster_list)) { rb_erase(entry) spin_unlock(&cluster->lock); (total_bitmaps is still 4) spin_lock(&cluster->lock); root> spin_lock(&ctl->tree_lock); recalculate_thresholds To fix this ensure that once depleted, the cluster entry is deleted when both cluster lock and tree locks are held in the allocator (T1), this ensures that even if there is a race with a concurrent insert_into_bitmap call it will correctly find the entry in the cluster and add the new space to it. CC: # 4.4+ Reviewed-by: Josef Bacik Signed-off-by: Nikolay Borisov Signed-off-by: David Sterba --- fs/btrfs/free-space-cache.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 5400294bd271..abcf951e6b44 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -3125,8 +3125,6 @@ u64 btrfs_alloc_from_cluster(struct btrfs_block_group *block_group, entry->bytes -= bytes; } - if (entry->bytes == 0) - rb_erase(&entry->offset_index, &cluster->root); break; } out: @@ -3143,7 +3141,10 @@ out: ctl->free_space -= bytes; if (!entry->bitmap && !btrfs_free_space_trimmed(entry)) ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; + + spin_lock(&cluster->lock); if (entry->bytes == 0) { + rb_erase(&entry->offset_index, &cluster->root); ctl->free_extents--; if (entry->bitmap) { kmem_cache_free(btrfs_free_space_bitmap_cachep, @@ -3156,6 +3157,7 @@ out: kmem_cache_free(btrfs_free_space_cachep, entry); } + spin_unlock(&cluster->lock); spin_unlock(&ctl->tree_lock); return ret; -- cgit v1.2.3 From 20903032cd9f0260b99aeab92e6540f0350e4a23 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 5 Feb 2021 12:55:36 +0000 Subject: btrfs: avoid checking for RO block group twice during nocow writeback During the nocow writeback path, we currently iterate the rbtree of block groups twice: once for checking if the target block group is RO with the call to btrfs_extent_readonly()), and once again for getting a nocow reference on the block group with a call to btrfs_inc_nocow_writers(). Since btrfs_inc_nocow_writers() already returns false when the target block group is RO, remove the call to btrfs_extent_readonly(). Not only we avoid searching the blocks group rbtree twice, it also helps reduce contention on the lock that protects it (specially since it is a spin lock and not a read-write lock). That may make a noticeable difference on very large filesystems, with thousands of allocated block groups. Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/inode.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 535abf898225..d6f0e1ad3711 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -1674,9 +1674,6 @@ next_slot: */ btrfs_release_path(path); - /* If extent is RO, we must COW it */ - if (btrfs_extent_readonly(fs_info, disk_bytenr)) - goto out_check; ret = btrfs_cross_ref_exist(root, ino, found_key.offset - extent_offset, disk_bytenr, false); @@ -1723,6 +1720,7 @@ next_slot: WARN_ON_ONCE(freespace_inode); goto out_check; } + /* If the extent's block group is RO, we must COW */ if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) goto out_check; nocow = true; -- cgit v1.2.3 From 195a49eaf655eb914896c92cecd96bc863c9feb3 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 5 Feb 2021 12:55:37 +0000 Subject: btrfs: fix race between writes to swap files and scrub When we active a swap file, at btrfs_swap_activate(), we acquire the exclusive operation lock to prevent the physical location of the swap file extents to be changed by operations such as balance and device replace/resize/remove. We also call there can_nocow_extent() which, among other things, checks if the block group of a swap file extent is currently RO, and if it is we can not use the extent, since a write into it would result in COWing the extent. However we have no protection against a scrub operation running after we activate the swap file, which can result in the swap file extents to be COWed while the scrub is running and operating on the respective block group, because scrub turns a block group into RO before it processes it and then back again to RW mode after processing it. That means an attempt to write into a swap file extent while scrub is processing the respective block group, will result in COWing the extent, changing its physical location on disk. Fix this by making sure that block groups that have extents that are used by active swap files can not be turned into RO mode, therefore making it not possible for a scrub to turn them into RO mode. When a scrub finds a block group that can not be turned to RO due to the existence of extents used by swap files, it proceeds to the next block group and logs a warning message that mentions the block group was skipped due to active swap files - this is the same approach we currently use for balance. Fixes: ed46ff3d42378 ("Btrfs: support swap files") CC: stable@vger.kernel.org # 5.4+ Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 33 ++++++++++++++++++++++++++++++++- fs/btrfs/block-group.h | 9 +++++++++ fs/btrfs/ctree.h | 5 +++++ fs/btrfs/inode.c | 19 ++++++++++++++++++- fs/btrfs/scrub.c | 9 ++++++++- 5 files changed, 72 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 5064be59dac5..744b99ddc28c 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1162,6 +1162,11 @@ static int inc_block_group_ro(struct btrfs_block_group *cache, int force) spin_lock(&sinfo->lock); spin_lock(&cache->lock); + if (cache->swap_extents) { + ret = -ETXTBSY; + goto out; + } + if (cache->ro) { cache->ro++; ret = 0; @@ -2307,7 +2312,7 @@ again: } ret = inc_block_group_ro(cache, 0); - if (!do_chunk_alloc) + if (!do_chunk_alloc || ret == -ETXTBSY) goto unlock_out; if (!ret) goto out; @@ -2316,6 +2321,8 @@ again: if (ret < 0) goto out; ret = inc_block_group_ro(cache, 0); + if (ret == -ETXTBSY) + goto unlock_out; out: if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) { alloc_flags = btrfs_get_alloc_profile(fs_info, cache->flags); @@ -3406,6 +3413,7 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info) ASSERT(list_empty(&block_group->io_list)); ASSERT(list_empty(&block_group->bg_list)); ASSERT(refcount_read(&block_group->refs) == 1); + ASSERT(block_group->swap_extents == 0); btrfs_put_block_group(block_group); spin_lock(&info->block_group_cache_lock); @@ -3472,3 +3480,26 @@ void btrfs_unfreeze_block_group(struct btrfs_block_group *block_group) __btrfs_remove_free_space_cache(block_group->free_space_ctl); } } + +bool btrfs_inc_block_group_swap_extents(struct btrfs_block_group *bg) +{ + bool ret = true; + + spin_lock(&bg->lock); + if (bg->ro) + ret = false; + else + bg->swap_extents++; + spin_unlock(&bg->lock); + + return ret; +} + +void btrfs_dec_block_group_swap_extents(struct btrfs_block_group *bg, int amount) +{ + spin_lock(&bg->lock); + ASSERT(!bg->ro); + ASSERT(bg->swap_extents >= amount); + bg->swap_extents -= amount; + spin_unlock(&bg->lock); +} diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 29678426247d..3ecc3372a5ce 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -186,6 +186,12 @@ struct btrfs_block_group { /* Flag indicating this block group is placed on a sequential zone */ bool seq_zone; + /* + * Number of extents in this block group used for swap files. + * All accesses protected by the spinlock 'lock'. + */ + int swap_extents; + /* Record locked full stripes for RAID5/6 block group */ struct btrfs_full_stripe_locks_tree full_stripe_locks_root; @@ -312,4 +318,7 @@ static inline int btrfs_block_group_done(struct btrfs_block_group *cache) void btrfs_freeze_block_group(struct btrfs_block_group *cache); void btrfs_unfreeze_block_group(struct btrfs_block_group *cache); +bool btrfs_inc_block_group_swap_extents(struct btrfs_block_group *bg); +void btrfs_dec_block_group_swap_extents(struct btrfs_block_group *bg, int amount); + #endif /* BTRFS_BLOCK_GROUP_H */ diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 3bc00aed13b2..40ec3393d2a1 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -524,6 +524,11 @@ struct btrfs_swapfile_pin { * points to a struct btrfs_device. */ bool is_block_group; + /* + * Only used when 'is_block_group' is true and it is the number of + * extents used by a swapfile for this block group ('ptr' field). + */ + int bg_extent_count; }; bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index d6f0e1ad3711..30358a2e2bc0 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -10192,6 +10192,7 @@ static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr, sp->ptr = ptr; sp->inode = inode; sp->is_block_group = is_block_group; + sp->bg_extent_count = 1; spin_lock(&fs_info->swapfile_pins_lock); p = &fs_info->swapfile_pins.rb_node; @@ -10205,6 +10206,8 @@ static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr, (sp->ptr == entry->ptr && sp->inode > entry->inode)) { p = &(*p)->rb_right; } else { + if (is_block_group) + entry->bg_extent_count++; spin_unlock(&fs_info->swapfile_pins_lock); kfree(sp); return 1; @@ -10230,8 +10233,11 @@ static void btrfs_free_swapfile_pins(struct inode *inode) sp = rb_entry(node, struct btrfs_swapfile_pin, node); if (sp->inode == inode) { rb_erase(&sp->node, &fs_info->swapfile_pins); - if (sp->is_block_group) + if (sp->is_block_group) { + btrfs_dec_block_group_swap_extents(sp->ptr, + sp->bg_extent_count); btrfs_put_block_group(sp->ptr); + } kfree(sp); } node = next; @@ -10446,6 +10452,17 @@ static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, goto out; } + if (!btrfs_inc_block_group_swap_extents(bg)) { + btrfs_warn(fs_info, + "block group for swapfile at %llu is read-only%s", + bg->start, + atomic_read(&fs_info->scrubs_running) ? + " (scrub running)" : ""); + btrfs_put_block_group(bg); + ret = -EINVAL; + goto out; + } + ret = btrfs_add_swapfile_pin(inode, bg, true); if (ret) { btrfs_put_block_group(bg); diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 310fce00fcda..70a90ca2c8da 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -3767,6 +3767,13 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, * commit_transactions. */ ro_set = 0; + } else if (ret == -ETXTBSY) { + btrfs_warn(fs_info, + "skipping scrub of block group %llu due to active swapfile", + cache->start); + scrub_pause_off(fs_info); + ret = 0; + goto skip_unfreeze; } else { btrfs_warn(fs_info, "failed setting block group ro: %d", ret); @@ -3862,7 +3869,7 @@ done: } else { spin_unlock(&cache->lock); } - +skip_unfreeze: btrfs_unfreeze_block_group(cache); btrfs_put_block_group(cache); if (ret) -- cgit v1.2.3 From dd0734f2a866f9d619d4abf97c3d71bcdee40ea9 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 5 Feb 2021 12:55:38 +0000 Subject: btrfs: fix race between swap file activation and snapshot creation When creating a snapshot we check if the current number of swap files, in the root, is non-zero, and if it is, we error out and warn that we can not create the snapshot because there are active swap files. However this is racy because when a task started activation of a swap file, another task might have started already snapshot creation and might have seen the counter for the number of swap files as zero. This means that after the swap file is activated we may end up with a snapshot of the same root successfully created, and therefore when the first write to the swap file happens it has to fall back into COW mode, which should never happen for active swap files. Basically what can happen is: 1) Task A starts snapshot creation and enters ioctl.c:create_snapshot(). There it sees that root->nr_swapfiles has a value of 0 so it continues; 2) Task B enters btrfs_swap_activate(). It is not aware that another task started snapshot creation but it did not finish yet. It increments root->nr_swapfiles from 0 to 1; 3) Task B checks that the file meets all requirements to be an active swap file - it has NOCOW set, there are no snapshots for the inode's root at the moment, no file holes, no reflinked extents, etc; 4) Task B returns success and now the file is an active swap file; 5) Task A commits the transaction to create the snapshot and finishes. The swap file's extents are now shared between the original root and the snapshot; 6) A write into an extent of the swap file is attempted - there is a snapshot of the file's root, so we fall back to COW mode and therefore the physical location of the extent changes on disk. So fix this by taking the snapshot lock during swap file activation before locking the extent range, as that is the order in which we lock these during buffered writes. Fixes: ed46ff3d42378 ("Btrfs: support swap files") CC: stable@vger.kernel.org # 5.4+ Reviewed-by: Anand Jain Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/inode.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 30358a2e2bc0..4f2f1e932751 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -10298,7 +10298,8 @@ static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, sector_t *span) { struct inode *inode = file_inode(file); - struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_fs_info *fs_info = root->fs_info; struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; struct extent_state *cached_state = NULL; struct extent_map *em = NULL; @@ -10349,13 +10350,27 @@ static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, "cannot activate swapfile while exclusive operation is running"); return -EBUSY; } + + /* + * Prevent snapshot creation while we are activating the swap file. + * We do not want to race with snapshot creation. If snapshot creation + * already started before we bumped nr_swapfiles from 0 to 1 and + * completes before the first write into the swap file after it is + * activated, than that write would fallback to COW. + */ + if (!btrfs_drew_try_write_lock(&root->snapshot_lock)) { + btrfs_exclop_finish(fs_info); + btrfs_warn(fs_info, + "cannot activate swapfile because snapshot creation is in progress"); + return -EINVAL; + } /* * Snapshots can create extents which require COW even if NODATACOW is * set. We use this counter to prevent snapshots. We must increment it * before walking the extents because we don't want a concurrent * snapshot to run after we've already checked the extents. */ - atomic_inc(&BTRFS_I(inode)->root->nr_swapfiles); + atomic_inc(&root->nr_swapfiles); isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize); @@ -10501,6 +10516,8 @@ out: if (ret) btrfs_swap_deactivate(file); + btrfs_drew_write_unlock(&root->snapshot_lock); + btrfs_exclop_finish(fs_info); if (ret) -- cgit v1.2.3 From 1119a72e223f3073a604f8fccb3a470ccd8a4416 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Tue, 16 Feb 2021 15:43:22 -0500 Subject: btrfs: tree-checker: do not error out if extent ref hash doesn't match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tree checker checks the extent ref hash at read and write time to make sure we do not corrupt the file system. Generally extent references go inline, but if we have enough of them we need to make an item, which looks like key.objectid = key.type = key.offset = hash(tree, owner, offset) However if key.offset collide with an unrelated extent reference we'll simply key.offset++ until we get something that doesn't collide. Obviously this doesn't match at tree checker time, and thus we error while writing out the transaction. This is relatively easy to reproduce, simply do something like the following xfs_io -f -c "pwrite 0 1M" file offset=2 for i in {0..10000} do xfs_io -c "reflink file 0 ${offset}M 1M" file offset=$(( offset + 2 )) done xfs_io -c "reflink file 0 17999258914816 1M" file xfs_io -c "reflink file 0 35998517829632 1M" file xfs_io -c "reflink file 0 53752752058368 1M" file btrfs filesystem sync And the sync will error out because we'll abort the transaction. The magic values above are used because they generate hash collisions with the first file in the main subvol. The fix for this is to remove the hash value check from tree checker, as we have no idea which offset ours should belong to. Reported-by: Tuomas Lähdekorpi Fixes: 0785a9aacf9d ("btrfs: tree-checker: Add EXTENT_DATA_REF check") CC: stable@vger.kernel.org # 5.4+ Reviewed-by: Filipe Manana Signed-off-by: Josef Bacik Reviewed-by: David Sterba [ add comment] Signed-off-by: David Sterba --- fs/btrfs/tree-checker.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 582061c7b547..f4ade821307d 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -1453,22 +1453,14 @@ static int check_extent_data_ref(struct extent_buffer *leaf, return -EUCLEAN; } for (; ptr < end; ptr += sizeof(*dref)) { - u64 root_objectid; - u64 owner; u64 offset; - u64 hash; + /* + * We cannot check the extent_data_ref hash due to possible + * overflow from the leaf due to hash collisions. + */ dref = (struct btrfs_extent_data_ref *)ptr; - root_objectid = btrfs_extent_data_ref_root(leaf, dref); - owner = btrfs_extent_data_ref_objectid(leaf, dref); offset = btrfs_extent_data_ref_offset(leaf, dref); - hash = hash_extent_data_ref(root_objectid, owner, offset); - if (unlikely(hash != key->offset)) { - extent_err(leaf, slot, - "invalid extent data ref hash, item has 0x%016llx key has 0x%016llx", - hash, key->offset); - return -EUCLEAN; - } if (unlikely(!IS_ALIGNED(offset, leaf->fs_info->sectorsize))) { extent_err(leaf, slot, "invalid extent data backref offset, have %llu expect aligned to %u", -- cgit v1.2.3 From 3660d0bcdb82807d434da9d2e57d88b37331182d Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 16 Feb 2021 11:09:25 +0000 Subject: btrfs: fix stale data exposure after cloning a hole with NO_HOLES enabled When using the NO_HOLES feature, if we clone a file range that spans only a hole into a range that is at or beyond the current i_size of the destination file, we end up not setting the full sync runtime flag on the inode. As a result, if we then fsync the destination file and have a power failure, after log replay we can end up exposing stale data instead of having a hole for that range. The conditions for this to happen are the following: 1) We have a file with a size of, for example, 1280K; 2) There is a written (non-prealloc) extent for the file range from 1024K to 1280K with a length of 256K; 3) This particular file extent layout is durably persisted, so that the existing superblock persisted on disk points to a subvolume root where the file has that exact file extent layout and state; 4) The file is truncated to a smaller size, to an offset lower than the start offset of its last extent, for example to 800K. The truncate sets the full sync runtime flag on the inode; 6) Fsync the file to log it and clear the full sync runtime flag; 7) Clone a region that covers only a hole (implicit hole due to NO_HOLES) into the file with a destination offset that starts at or beyond the 256K file extent item we had - for example to offset 1024K; 8) Since the clone operation does not find extents in the source range, we end up in the if branch at the bottom of btrfs_clone() where we punch a hole for the file range starting at offset 1024K by calling btrfs_replace_file_extents(). There we end up not setting the full sync flag on the inode, because we don't know we are being called in a clone context (and not fallocate's punch hole operation), and neither do we create an extent map to represent a hole because the requested range is beyond eof; 9) A further fsync to the file will be a fast fsync, since the clone operation did not set the full sync flag, and therefore it relies on modified extent maps to correctly log the file layout. But since it does not find any extent map marking the range from 1024K (the previous eof) to the new eof, it does not log a file extent item for that range representing the hole; 10) After a power failure no hole for the range starting at 1024K is punched and we end up exposing stale data from the old 256K extent. Turning this into exact steps: $ mkfs.btrfs -f -O no-holes /dev/sdi $ mount /dev/sdi /mnt # Create our test file with 3 extents of 256K and a 256K hole at offset # 256K. The file has a size of 1280K. $ xfs_io -f -s \ -c "pwrite -S 0xab -b 256K 0 256K" \ -c "pwrite -S 0xcd -b 256K 512K 256K" \ -c "pwrite -S 0xef -b 256K 768K 256K" \ -c "pwrite -S 0x73 -b 256K 1024K 256K" \ /mnt/sdi/foobar # Make sure it's durably persisted. We want the last committed super # block to point to this particular file extent layout. sync # Now truncate our file to a smaller size, falling within a position of # the second extent. This sets the full sync runtime flag on the inode. # Then fsync the file to log it and clear the full sync flag from the # inode. The third extent is no longer part of the file and therefore # it is not logged. $ xfs_io -c "truncate 800K" -c "fsync" /mnt/foobar # Now do a clone operation that only clones the hole and sets back the # file size to match the size it had before the truncate operation # (1280K). $ xfs_io \ -c "reflink /mnt/foobar 256K 1024K 256K" \ -c "fsync" \ /mnt/foobar # File data before power failure: $ od -A d -t x1 /mnt/foobar 0000000 ab ab ab ab ab ab ab ab ab ab ab ab ab ab ab ab * 0262144 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 * 0524288 cd cd cd cd cd cd cd cd cd cd cd cd cd cd cd cd * 0786432 ef ef ef ef ef ef ef ef ef ef ef ef ef ef ef ef * 0819200 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 * 1310720 # Mount the fs again to replay the log tree. $ mount /dev/sdi /mnt # File data after power failure: $ od -A d -t x1 /mnt/foobar 0000000 ab ab ab ab ab ab ab ab ab ab ab ab ab ab ab ab * 0262144 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 * 0524288 cd cd cd cd cd cd cd cd cd cd cd cd cd cd cd cd * 0786432 ef ef ef ef ef ef ef ef ef ef ef ef ef ef ef ef * 0819200 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 * 1048576 73 73 73 73 73 73 73 73 73 73 73 73 73 73 73 73 * 1310720 The range from 1024K to 1280K should correspond to a hole but instead it points to stale data, to the 256K extent that should not exist after the truncate operation. The issue does not exists when not using NO_HOLES, because for that case we use file extent items to represent holes, these are found and copied during the loop that iterates over extents at btrfs_clone(), and that causes btrfs_replace_file_extents() to be called with a non-NULL extent_info argument and therefore set the full sync runtime flag on the inode. So fix this by making the code that deals with a trailing hole during cloning, at btrfs_clone(), to set the full sync flag on the inode, if the range starts at or beyond the current i_size. A test case for fstests will follow soon. Backporting notes: for kernel 5.4 the change goes to ioctl.c into btrfs_clone before the last call to btrfs_punch_hole_range. CC: stable@vger.kernel.org # 5.4+ Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/reflink.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index b24396cf2f99..5413578d2c32 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -553,6 +553,24 @@ process_slot: */ btrfs_release_path(path); + /* + * When using NO_HOLES and we are cloning a range that covers + * only a hole (no extents) into a range beyond the current + * i_size, punching a hole in the target range will not create + * an extent map defining a hole, because the range starts at or + * beyond current i_size. If the file previously had an i_size + * greater than the new i_size set by this clone operation, we + * need to make sure the next fsync is a full fsync, so that it + * detects and logs a hole covering a range from the current + * i_size to the new i_size. If the clone range covers extents, + * besides a hole, then we know the full sync flag was already + * set by previous calls to btrfs_replace_file_extents() that + * replaced file extent items. + */ + if (last_dest_end >= i_size_read(inode)) + set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, + &BTRFS_I(inode)->runtime_flags); + ret = btrfs_replace_file_extents(inode, path, last_dest_end, destoff + len - 1, NULL, &trans); if (ret) -- cgit v1.2.3 From 95c85fba1f64c3249c67f0078a29f8a125078189 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Mon, 25 Jan 2021 16:42:35 -0500 Subject: btrfs: avoid double put of block group when emptying cluster It's wrong calling btrfs_put_block_group in __btrfs_return_cluster_to_free_space if the block group passed is different than the block group the cluster represents. As this means the cluster doesn't have a reference to the passed block group. This results in double put and a use-after-free bug. Fix this by simply bailing if the block group we passed in does not match the block group on the cluster. Fixes: fa9c0d795f7b ("Btrfs: rework allocation clustering") CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Josef Bacik Reviewed-by: David Sterba [ update changelog ] Signed-off-by: David Sterba --- fs/btrfs/free-space-cache.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index abcf951e6b44..711a6a751ae9 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -2801,8 +2801,10 @@ static void __btrfs_return_cluster_to_free_space( struct rb_node *node; spin_lock(&cluster->lock); - if (cluster->block_group != block_group) - goto out; + if (cluster->block_group != block_group) { + spin_unlock(&cluster->lock); + return; + } cluster->block_group = NULL; cluster->window_start = 0; @@ -2840,8 +2842,6 @@ static void __btrfs_return_cluster_to_free_space( entry->offset, &entry->offset_index, bitmap); } cluster->root = RB_ROOT; - -out: spin_unlock(&cluster->lock); btrfs_put_block_group(block_group); } -- cgit v1.2.3 From 6e37d245994189ba757df7dc2950a44d31421ac6 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Wed, 17 Feb 2021 16:06:18 +0900 Subject: btrfs: zoned: fix deadlock on log sync Lockdep with fstests test case btrfs/041 detected a unsafe locking scenario when we allocate the log node on a zoned filesystem. btrfs/041 ============================================ WARNING: possible recursive locking detected 5.11.0-rc7+ #939 Not tainted -------------------------------------------- xfs_io/698 is trying to acquire lock: ffff88810cd673a0 (&root->log_mutex){+.+.}-{3:3}, at: btrfs_sync_log+0x3d1/0xee0 [btrfs] but task is already holding lock: ffff88810b0fc3a0 (&root->log_mutex){+.+.}-{3:3}, at: btrfs_sync_log+0x313/0xee0 [btrfs] other info that might help us debug this: Possible unsafe locking scenario: CPU0 ---- lock(&root->log_mutex); lock(&root->log_mutex); *** DEADLOCK *** May be due to missing lock nesting notation 2 locks held by xfs_io/698: #0: ffff88810cd66620 (sb_internal){.+.+}-{0:0}, at: btrfs_sync_file+0x2c3/0x570 [btrfs] #1: ffff88810b0fc3a0 (&root->log_mutex){+.+.}-{3:3}, at: btrfs_sync_log+0x313/0xee0 [btrfs] stack backtrace: CPU: 0 PID: 698 Comm: xfs_io Not tainted 5.11.0-rc7+ #939 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.13.0-0-gf21b5a4-rebuilt.opensuse.org 04/01/2014 Call Trace: dump_stack+0x77/0x97 __lock_acquire.cold+0xb9/0x32a lock_acquire+0xb5/0x400 ? btrfs_sync_log+0x3d1/0xee0 [btrfs] __mutex_lock+0x7b/0x8d0 ? btrfs_sync_log+0x3d1/0xee0 [btrfs] ? btrfs_sync_log+0x3d1/0xee0 [btrfs] ? find_first_extent_bit+0x9f/0x100 [btrfs] ? __mutex_unlock_slowpath+0x35/0x270 btrfs_sync_log+0x3d1/0xee0 [btrfs] btrfs_sync_file+0x3a8/0x570 [btrfs] __x64_sys_fsync+0x34/0x60 do_syscall_64+0x33/0x40 entry_SYSCALL_64_after_hwframe+0x44/0xa9 This happens, because we are taking the ->log_mutex albeit it has already been locked. Also while at it, fix the bogus unlock of the tree_log_mutex in the error handling. Fixes: 3ddebf27fcd3 ("btrfs: zoned: reorder log node allocation on zoned filesystem") Reviewed-by: Filipe Manana Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index d90695c1ab6c..2f1acc9aea9e 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3174,16 +3174,13 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, root_log_ctx.log_transid = log_root_tree->log_transid; if (btrfs_is_zoned(fs_info)) { - mutex_lock(&fs_info->tree_root->log_mutex); if (!log_root_tree->node) { ret = btrfs_alloc_log_tree_node(trans, log_root_tree); if (ret) { - mutex_unlock(&fs_info->tree_log_mutex); mutex_unlock(&log_root_tree->log_mutex); goto out; } } - mutex_unlock(&fs_info->tree_root->log_mutex); } /* -- cgit v1.2.3 From 71b219f4e50b12efffbc8107408e17904f9c47e6 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Fri, 29 Jan 2021 19:56:37 +0100 Subject: gfs2: Minor calc_reserved cleanup No functional change. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 6fd4ded1e357..0c28ddd0e410 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -546,17 +546,14 @@ static inline unsigned int log_distance(struct gfs2_sbd *sdp, unsigned int newer static unsigned int calc_reserved(struct gfs2_sbd *sdp) { unsigned int reserved = 0; - unsigned int mbuf; - unsigned int dbuf; + unsigned int blocks; struct gfs2_trans *tr = sdp->sd_log_tr; if (tr) { - mbuf = tr->tr_num_buf_new - tr->tr_num_buf_rm; - dbuf = tr->tr_num_databuf_new - tr->tr_num_databuf_rm; - reserved = mbuf + dbuf; - /* Account for header blocks */ - reserved += DIV_ROUND_UP(mbuf, buf_limit(sdp)); - reserved += DIV_ROUND_UP(dbuf, databuf_limit(sdp)); + blocks = tr->tr_num_buf_new - tr->tr_num_buf_rm; + reserved += blocks + DIV_ROUND_UP(blocks, buf_limit(sdp)); + blocks = tr->tr_num_databuf_new - tr->tr_num_databuf_rm; + reserved += blocks + DIV_ROUND_UP(blocks, databuf_limit(sdp)); } if (sdp->sd_log_committed_revoke > 0) -- cgit v1.2.3 From fe3e397668775e20ad0962459733158838b926af Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Thu, 10 Dec 2020 12:49:56 +0100 Subject: gfs2: Rework the log space allocation logic The current log space allocation logic is hard to understand or extend. The principle it that when the log is flushed, we may or may not have a transaction active that has space allocated in the log. To deal with that, we set aside a magical number of blocks to be used in case we don't have an active transaction. It isn't clear that the pool will always be big enough. In addition, we can't return unused log space at the end of a transaction, so the number of blocks allocated must exactly match the number of blocks used. Simplify this as follows: * When transactions are allocated or merged, always reserve enough blocks to flush the transaction (err on the safe side). * In gfs2_log_flush, return any allocated blocks that haven't been used. * Maintain a pool of spare blocks big enough to do one log flush, as before. * In gfs2_log_flush, when we have no active transaction, allocate a suitable number of blocks. For that, use the spare pool when called from logd, and leave the pool alone otherwise. This means that when the log is almost full, logd will still be able to do one more log flush, which will result in more log space becoming available. This will make the log space allocator code easier to work with in the future. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 162 +++++++++++++++++++++++++++++++++----------------------- fs/gfs2/log.h | 7 +++ fs/gfs2/trans.c | 4 +- 3 files changed, 104 insertions(+), 69 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 0c28ddd0e410..913150e60f52 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -447,15 +447,42 @@ void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks) } /** - * gfs2_log_reserve - Make a log reservation + * __gfs2_log_try_reserve - Try to make a log reservation + * @sdp: The GFS2 superblock + * @blks: The number of blocks to reserve + * @taboo_blks: The number of blocks to leave free + * + * Try to do the same as __gfs2_log_reserve(), but fail if no more log + * space is immediately available. + */ +static bool __gfs2_log_try_reserve(struct gfs2_sbd *sdp, unsigned int blks, + unsigned int taboo_blks) +{ + unsigned wanted = blks + taboo_blks; + unsigned int free_blocks; + + free_blocks = atomic_read(&sdp->sd_log_blks_free); + while (free_blocks >= wanted) { + if (atomic_try_cmpxchg(&sdp->sd_log_blks_free, &free_blocks, + free_blocks - blks)) { + trace_gfs2_log_blocks(sdp, -blks); + return true; + } + } + return false; +} + +/** + * __gfs2_log_reserve - Make a log reservation * @sdp: The GFS2 superblock * @blks: The number of blocks to reserve + * @taboo_blks: The number of blocks to leave free * - * Note that we never give out the last few blocks of the journal. Thats - * due to the fact that there is a small number of header blocks - * associated with each log flush. The exact number can't be known until - * flush time, so we ensure that we have just enough free blocks at all - * times to avoid running out during a log flush. + * @taboo_blks is set to 0 for logd, and to GFS2_LOG_FLUSH_MIN_BLOCKS + * for all other processes. This ensures that when the log is almost full, + * logd will still be able to call gfs2_log_flush one more time without + * blocking, which will advance the tail and make some more log space + * available. * * We no longer flush the log here, instead we wake up logd to do that * for us. To avoid the thundering herd and to ensure that we deal fairly @@ -464,19 +491,12 @@ void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks) * wake the next waiter on the list. */ -void gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks) +static void __gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks, + unsigned int taboo_blks) { - unsigned reserved_blks = 7 * (4096 / sdp->sd_vfs->s_blocksize); - unsigned wanted = blks + reserved_blks; + unsigned wanted = blks + taboo_blks; unsigned int free_blocks; - free_blocks = atomic_read(&sdp->sd_log_blks_free); - while (free_blocks >= wanted) { - if (atomic_try_cmpxchg(&sdp->sd_log_blks_free, &free_blocks, - free_blocks - blks)) - return; - } - atomic_add(blks, &sdp->sd_log_blks_needed); for (;;) { if (current != sdp->sd_logd_process) @@ -498,6 +518,19 @@ reserved: wake_up(&sdp->sd_log_waitq); } +/** + * gfs2_log_reserve - Make a log reservation + * @sdp: The GFS2 superblock + * @blks: The number of blocks to reserve + */ + +void gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks) +{ + if (__gfs2_log_try_reserve(sdp, blks, GFS2_LOG_FLUSH_MIN_BLOCKS)) + return; + __gfs2_log_reserve(sdp, blks, GFS2_LOG_FLUSH_MIN_BLOCKS); +} + /** * log_distance - Compute distance between two journal blocks * @sdp: The GFS2 superblock @@ -545,7 +578,7 @@ static inline unsigned int log_distance(struct gfs2_sbd *sdp, unsigned int newer */ static unsigned int calc_reserved(struct gfs2_sbd *sdp) { - unsigned int reserved = 0; + unsigned int reserved = GFS2_LOG_FLUSH_MIN_BLOCKS; unsigned int blocks; struct gfs2_trans *tr = sdp->sd_log_tr; @@ -557,10 +590,7 @@ static unsigned int calc_reserved(struct gfs2_sbd *sdp) } if (sdp->sd_log_committed_revoke > 0) - reserved += gfs2_struct2blk(sdp, sdp->sd_log_committed_revoke); - /* One for the overall header */ - if (reserved) - reserved++; + reserved += gfs2_struct2blk(sdp, sdp->sd_log_committed_revoke) - 1; return reserved; } @@ -708,29 +738,8 @@ void gfs2_flush_revokes(struct gfs2_sbd *sdp) max_revokes += roundup(sdp->sd_log_num_revoke - sdp->sd_ldptrs, sdp->sd_inptrs); max_revokes -= sdp->sd_log_num_revoke; - if (!sdp->sd_log_num_revoke) { - atomic_dec(&sdp->sd_log_blks_free); - /* If no blocks have been reserved, we need to also - * reserve a block for the header */ - if (!sdp->sd_log_blks_reserved) { - atomic_dec(&sdp->sd_log_blks_free); - trace_gfs2_log_blocks(sdp, -2); - } else { - trace_gfs2_log_blocks(sdp, -1); - } - } gfs2_ail1_empty(sdp, max_revokes); gfs2_log_unlock(sdp); - - if (!sdp->sd_log_num_revoke) { - atomic_inc(&sdp->sd_log_blks_free); - if (!sdp->sd_log_blks_reserved) { - atomic_inc(&sdp->sd_log_blks_free); - trace_gfs2_log_blocks(sdp, 2); - } else { - trace_gfs2_log_blocks(sdp, 1); - } - } } /** @@ -843,6 +852,7 @@ static void log_write_header(struct gfs2_sbd *sdp, u32 flags) gfs2_log_incr_head(sdp); log_flush_wait(sdp); log_pull_tail(sdp); + gfs2_log_update_head(sdp); } /** @@ -942,10 +952,14 @@ static void trans_drain(struct gfs2_trans *tr) void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) { struct gfs2_trans *tr = NULL; + unsigned int reserved_blocks = 0, used_blocks = 0; enum gfs2_freeze_state state = atomic_read(&sdp->sd_freeze_state); + unsigned int first_log_head; down_write(&sdp->sd_log_flush_lock); + trace_gfs2_log_flush(sdp, 1, flags); +repeat: /* * Do this check while holding the log_flush_lock to prevent new * buffers from being added to the ail via gfs2_pin() @@ -956,22 +970,41 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) /* Log might have been flushed while we waited for the flush lock */ if (gl && !test_bit(GLF_LFLUSH, &gl->gl_flags)) goto out; - trace_gfs2_log_flush(sdp, 1, flags); - if (flags & GFS2_LOG_HEAD_FLUSH_SHUTDOWN) - clear_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); + first_log_head = sdp->sd_log_head; + sdp->sd_log_flush_head = first_log_head; - sdp->sd_log_flush_head = sdp->sd_log_head; tr = sdp->sd_log_tr; - if (tr) { - sdp->sd_log_tr = NULL; - tr->tr_first = sdp->sd_log_flush_head; - if (unlikely (state == SFS_FROZEN)) - if (gfs2_assert_withdraw_delayed(sdp, - !tr->tr_num_buf_new && !tr->tr_num_databuf_new)) - goto out_withdraw; + if (tr || sdp->sd_log_num_revoke) { + if (reserved_blocks) + gfs2_log_release(sdp, reserved_blocks); + reserved_blocks = sdp->sd_log_blks_reserved; + if (tr) { + sdp->sd_log_tr = NULL; + tr->tr_first = first_log_head; + if (unlikely (state == SFS_FROZEN)) + if (gfs2_assert_withdraw_delayed(sdp, + !tr->tr_num_buf_new && !tr->tr_num_databuf_new)) + goto out_withdraw; + } + } else if (!reserved_blocks) { + unsigned int taboo_blocks = GFS2_LOG_FLUSH_MIN_BLOCKS; + + reserved_blocks = GFS2_LOG_FLUSH_MIN_BLOCKS; + if (current == sdp->sd_logd_process) + taboo_blocks = 0; + + if (!__gfs2_log_try_reserve(sdp, reserved_blocks, taboo_blocks)) { + up_write(&sdp->sd_log_flush_lock); + __gfs2_log_reserve(sdp, reserved_blocks, taboo_blocks); + down_write(&sdp->sd_log_flush_lock); + goto repeat; + } } + if (flags & GFS2_LOG_HEAD_FLUSH_SHUTDOWN) + clear_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); + if (unlikely(state == SFS_FROZEN)) if (gfs2_assert_withdraw_delayed(sdp, !sdp->sd_log_num_revoke)) goto out_withdraw; @@ -993,8 +1026,6 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) log_flush_wait(sdp); log_write_header(sdp, flags); } else if (sdp->sd_log_tail != sdp->sd_log_flush_tail && !sdp->sd_log_idle) { - atomic_dec(&sdp->sd_log_blks_free); /* Adjust for unreserved buffer */ - trace_gfs2_log_blocks(sdp, -1); log_write_header(sdp, flags); } if (gfs2_withdrawn(sdp)) @@ -1002,7 +1033,6 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) lops_after_commit(sdp, tr); gfs2_log_lock(sdp); - gfs2_log_update_head(sdp); sdp->sd_log_blks_reserved = 0; sdp->sd_log_committed_revoke = 0; @@ -1019,10 +1049,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) empty_ail1_list(sdp); if (gfs2_withdrawn(sdp)) goto out_withdraw; - atomic_dec(&sdp->sd_log_blks_free); /* Adjust for unreserved buffer */ - trace_gfs2_log_blocks(sdp, -1); log_write_header(sdp, flags); - gfs2_log_update_head(sdp); } if (flags & (GFS2_LOG_HEAD_FLUSH_SHUTDOWN | GFS2_LOG_HEAD_FLUSH_FREEZE)) @@ -1032,12 +1059,17 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) } out_end: - trace_gfs2_log_flush(sdp, 0, flags); + used_blocks = log_distance(sdp, sdp->sd_log_flush_head, first_log_head); + if (gfs2_assert_withdraw_delayed(sdp, used_blocks <= reserved_blocks)) + goto out; out: + if (used_blocks != reserved_blocks) + gfs2_log_release(sdp, reserved_blocks - used_blocks); up_write(&sdp->sd_log_flush_lock); gfs2_trans_free(sdp, tr); if (gfs2_withdrawing(sdp)) gfs2_withdraw(sdp); + trace_gfs2_log_flush(sdp, 0, flags); return; out_withdraw: @@ -1150,15 +1182,11 @@ static void gfs2_log_shutdown(struct gfs2_sbd *sdp) gfs2_assert_withdraw(sdp, !sdp->sd_log_num_revoke); gfs2_assert_withdraw(sdp, list_empty(&sdp->sd_ail1_list)); - sdp->sd_log_flush_head = sdp->sd_log_head; - log_write_header(sdp, GFS2_LOG_HEAD_UNMOUNT | GFS2_LFC_SHUTDOWN); + log_pull_tail(sdp); gfs2_assert_warn(sdp, sdp->sd_log_head == sdp->sd_log_tail); gfs2_assert_warn(sdp, list_empty(&sdp->sd_ail2_list)); - - gfs2_log_update_head(sdp); - sdp->sd_log_tail = sdp->sd_log_head; } static inline int gfs2_jrnl_flush_reqd(struct gfs2_sbd *sdp) @@ -1213,7 +1241,7 @@ int gfs2_logd(void *data) if (gfs2_jrnl_flush_reqd(sdp) || t == 0) { gfs2_ail1_empty(sdp, 0); gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | - GFS2_LFC_LOGD_JFLUSH_REQD); + GFS2_LFC_LOGD_JFLUSH_REQD); } if (gfs2_ail_flush_reqd(sdp)) { @@ -1221,7 +1249,7 @@ int gfs2_logd(void *data) gfs2_ail1_wait(sdp); gfs2_ail1_empty(sdp, 0); gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | - GFS2_LFC_LOGD_AIL_FLUSH_REQD); + GFS2_LFC_LOGD_AIL_FLUSH_REQD); } t = gfs2_tune_get(sdp, gt_logd_secs) * HZ; diff --git a/fs/gfs2/log.h b/fs/gfs2/log.h index b36a3539f352..e7f4a8d6be64 100644 --- a/fs/gfs2/log.h +++ b/fs/gfs2/log.h @@ -13,6 +13,13 @@ #include "incore.h" #include "inode.h" +/* + * The minimum amount of log space required for a log flush is one block for + * revokes and one block for the log header. Log flushes other than + * GFS2_LOG_HEAD_FLUSH_NORMAL may write one or two more log headers. + */ +#define GFS2_LOG_FLUSH_MIN_BLOCKS 4 + /** * gfs2_log_lock - acquire the right to mess with the log manager * @sdp: the filesystem diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index f73d6b8f3b53..231ca1a41a73 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -53,7 +53,7 @@ int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, tr->tr_ip = ip; tr->tr_blocks = blocks; tr->tr_revokes = revokes; - tr->tr_reserved = 1; + tr->tr_reserved = GFS2_LOG_FLUSH_MIN_BLOCKS; if (blocks) { /* * The reserved blocks are either used for data or metadata. @@ -63,7 +63,7 @@ int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, tr->tr_reserved += blocks + 1 + DIV_ROUND_UP(blocks - 1, databuf_limit(sdp)); } if (revokes) - tr->tr_reserved += gfs2_struct2blk(sdp, revokes); + tr->tr_reserved += gfs2_struct2blk(sdp, revokes) - 1; INIT_LIST_HEAD(&tr->tr_databuf); INIT_LIST_HEAD(&tr->tr_buf); INIT_LIST_HEAD(&tr->tr_list); -- cgit v1.2.3 From 2129b4288852cf872c42870c7f6e813ce0611199 Mon Sep 17 00:00:00 2001 From: Andreas Gruenbacher Date: Thu, 17 Dec 2020 16:14:30 +0100 Subject: gfs2: Per-revoke accounting in transactions In the log, revokes are stored as a revoke descriptor (struct gfs2_log_descriptor), followed by zero or more additional revoke blocks (struct gfs2_meta_header). On filesystems with a blocksize of 4k, the revoke descriptor contains up to 503 revokes, and the metadata blocks contain up to 509 revokes each. We've so far been reserving space for revokes in transactions in block granularity, so a lot more space than necessary was being allocated and then released again. This patch switches to assigning revokes to transactions individually instead. Initially, space for the revoke descriptor is reserved and handed out to transactions. When more revokes than that are reserved, additional revoke blocks are added. When the log is flushed, the space for the additional revoke blocks is released, but we keep the space for the revoke descriptor block allocated. Transactions may still reserve more revokes than they will actually need in the end, but now we won't overshoot the target as much, and by only returning the space for excess revokes at log flush time, we further reduce the amount of contention between processes. Signed-off-by: Andreas Gruenbacher --- fs/gfs2/glops.c | 8 +--- fs/gfs2/incore.h | 3 +- fs/gfs2/log.c | 114 ++++++++++++++++++++++++++++++++++++++++----------- fs/gfs2/log.h | 6 ++- fs/gfs2/lops.c | 1 - fs/gfs2/ops_fstype.c | 7 ++++ fs/gfs2/trans.c | 34 +++++++++++---- 7 files changed, 131 insertions(+), 42 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c index a067924341e3..8e32d569c8bf 100644 --- a/fs/gfs2/glops.c +++ b/fs/gfs2/glops.c @@ -136,19 +136,15 @@ void gfs2_ail_flush(struct gfs2_glock *gl, bool fsync) { struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; unsigned int revokes = atomic_read(&gl->gl_ail_count); - unsigned int max_revokes = (sdp->sd_sb.sb_bsize - sizeof(struct gfs2_log_descriptor)) / sizeof(u64); int ret; if (!revokes) return; - while (revokes > max_revokes) - max_revokes += (sdp->sd_sb.sb_bsize - sizeof(struct gfs2_meta_header)) / sizeof(u64); - - ret = gfs2_trans_begin(sdp, 0, max_revokes); + ret = gfs2_trans_begin(sdp, 0, revokes); if (ret) return; - __gfs2_ail_flush(gl, fsync, max_revokes); + __gfs2_ail_flush(gl, fsync, revokes); gfs2_trans_end(sdp); gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_AIL_FLUSH); diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h index 51656b053170..30f67c3c87b0 100644 --- a/fs/gfs2/incore.h +++ b/fs/gfs2/incore.h @@ -506,7 +506,6 @@ struct gfs2_trans { unsigned int tr_num_buf_rm; unsigned int tr_num_databuf_rm; unsigned int tr_num_revoke; - unsigned int tr_num_revoke_rm; struct list_head tr_list; struct list_head tr_databuf; @@ -821,7 +820,6 @@ struct gfs2_sbd { struct gfs2_trans *sd_log_tr; unsigned int sd_log_blks_reserved; - int sd_log_committed_revoke; atomic_t sd_log_pinned; unsigned int sd_log_num_revoke; @@ -834,6 +832,7 @@ struct gfs2_sbd { atomic_t sd_log_thresh2; atomic_t sd_log_blks_free; atomic_t sd_log_blks_needed; + atomic_t sd_log_revokes_available; wait_queue_head_t sd_log_waitq; wait_queue_head_t sd_logd_waitq; diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 913150e60f52..68e7afdd19a8 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -429,6 +429,32 @@ bool gfs2_log_is_empty(struct gfs2_sbd *sdp) { return atomic_read(&sdp->sd_log_blks_free) == sdp->sd_jdesc->jd_blocks; } +static bool __gfs2_log_try_reserve_revokes(struct gfs2_sbd *sdp, unsigned int revokes) +{ + unsigned int available; + + available = atomic_read(&sdp->sd_log_revokes_available); + while (available >= revokes) { + if (atomic_try_cmpxchg(&sdp->sd_log_revokes_available, + &available, available - revokes)) + return true; + } + return false; +} + +/** + * gfs2_log_release_revokes - Release a given number of revokes + * @sdp: The GFS2 superblock + * @revokes: The number of revokes to release + * + * sdp->sd_log_flush_lock must be held. + */ +void gfs2_log_release_revokes(struct gfs2_sbd *sdp, unsigned int revokes) +{ + if (revokes) + atomic_add(revokes, &sdp->sd_log_revokes_available); +} + /** * gfs2_log_release - Release a given number of log blocks * @sdp: The GFS2 superblock @@ -519,15 +545,59 @@ reserved: } /** - * gfs2_log_reserve - Make a log reservation + * gfs2_log_try_reserve - Try to make a log reservation * @sdp: The GFS2 superblock - * @blks: The number of blocks to reserve + * @tr: The transaction + * @extra_revokes: The number of additional revokes reserved (output) + * + * This is similar to gfs2_log_reserve, but sdp->sd_log_flush_lock must be + * held for correct revoke accounting. */ -void gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks) +bool gfs2_log_try_reserve(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + unsigned int *extra_revokes) { + unsigned int blks = tr->tr_reserved; + unsigned int revokes = tr->tr_revokes; + unsigned int revoke_blks = 0; + + *extra_revokes = 0; + if (revokes && !__gfs2_log_try_reserve_revokes(sdp, revokes)) { + revoke_blks = DIV_ROUND_UP(revokes, sdp->sd_inptrs); + *extra_revokes = revoke_blks * sdp->sd_inptrs - revokes; + blks += revoke_blks; + } + if (!blks) + return true; if (__gfs2_log_try_reserve(sdp, blks, GFS2_LOG_FLUSH_MIN_BLOCKS)) - return; + return true; + if (!revoke_blks) + gfs2_log_release_revokes(sdp, revokes); + return false; +} + +/** + * gfs2_log_reserve - Make a log reservation + * @sdp: The GFS2 superblock + * @tr: The transaction + * @extra_revokes: The number of additional revokes reserved (output) + * + * sdp->sd_log_flush_lock must not be held. + */ + +void gfs2_log_reserve(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + unsigned int *extra_revokes) +{ + unsigned int blks = tr->tr_reserved; + unsigned int revokes = tr->tr_revokes; + unsigned int revoke_blks = 0; + + *extra_revokes = 0; + if (revokes) { + revoke_blks = DIV_ROUND_UP(revokes, sdp->sd_inptrs); + *extra_revokes = revoke_blks * sdp->sd_inptrs - revokes; + blks += revoke_blks; + } __gfs2_log_reserve(sdp, blks, GFS2_LOG_FLUSH_MIN_BLOCKS); } @@ -588,9 +658,6 @@ static unsigned int calc_reserved(struct gfs2_sbd *sdp) blocks = tr->tr_num_databuf_new - tr->tr_num_databuf_rm; reserved += blocks + DIV_ROUND_UP(blocks, databuf_limit(sdp)); } - - if (sdp->sd_log_committed_revoke > 0) - reserved += gfs2_struct2blk(sdp, sdp->sd_log_committed_revoke) - 1; return reserved; } @@ -730,14 +797,9 @@ void gfs2_glock_remove_revoke(struct gfs2_glock *gl) void gfs2_flush_revokes(struct gfs2_sbd *sdp) { /* number of revokes we still have room for */ - unsigned int max_revokes; + unsigned int max_revokes = atomic_read(&sdp->sd_log_revokes_available); gfs2_log_lock(sdp); - max_revokes = sdp->sd_ldptrs; - if (sdp->sd_log_num_revoke > sdp->sd_ldptrs) - max_revokes += roundup(sdp->sd_log_num_revoke - sdp->sd_ldptrs, - sdp->sd_inptrs); - max_revokes -= sdp->sd_log_num_revoke; gfs2_ail1_empty(sdp, max_revokes); gfs2_log_unlock(sdp); } @@ -955,6 +1017,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) unsigned int reserved_blocks = 0, used_blocks = 0; enum gfs2_freeze_state state = atomic_read(&sdp->sd_freeze_state); unsigned int first_log_head; + unsigned int reserved_revokes = 0; down_write(&sdp->sd_log_flush_lock); trace_gfs2_log_flush(sdp, 1, flags); @@ -979,13 +1042,15 @@ repeat: if (reserved_blocks) gfs2_log_release(sdp, reserved_blocks); reserved_blocks = sdp->sd_log_blks_reserved; + reserved_revokes = sdp->sd_log_num_revoke; if (tr) { sdp->sd_log_tr = NULL; tr->tr_first = first_log_head; - if (unlikely (state == SFS_FROZEN)) + if (unlikely (state == SFS_FROZEN)) { if (gfs2_assert_withdraw_delayed(sdp, !tr->tr_num_buf_new && !tr->tr_num_databuf_new)) goto out_withdraw; + } } } else if (!reserved_blocks) { unsigned int taboo_blocks = GFS2_LOG_FLUSH_MIN_BLOCKS; @@ -1000,17 +1065,15 @@ repeat: down_write(&sdp->sd_log_flush_lock); goto repeat; } + BUG_ON(sdp->sd_log_num_revoke); } if (flags & GFS2_LOG_HEAD_FLUSH_SHUTDOWN) clear_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); if (unlikely(state == SFS_FROZEN)) - if (gfs2_assert_withdraw_delayed(sdp, !sdp->sd_log_num_revoke)) + if (gfs2_assert_withdraw_delayed(sdp, !reserved_revokes)) goto out_withdraw; - if (gfs2_assert_withdraw_delayed(sdp, - sdp->sd_log_num_revoke == sdp->sd_log_committed_revoke)) - goto out_withdraw; gfs2_ordered_write(sdp); if (gfs2_withdrawn(sdp)) @@ -1034,7 +1097,6 @@ repeat: gfs2_log_lock(sdp); sdp->sd_log_blks_reserved = 0; - sdp->sd_log_committed_revoke = 0; spin_lock(&sdp->sd_ail_lock); if (tr && !list_empty(&tr->tr_ail1_list)) { @@ -1060,11 +1122,16 @@ repeat: out_end: used_blocks = log_distance(sdp, sdp->sd_log_flush_head, first_log_head); - if (gfs2_assert_withdraw_delayed(sdp, used_blocks <= reserved_blocks)) - goto out; + reserved_revokes += atomic_read(&sdp->sd_log_revokes_available); + atomic_set(&sdp->sd_log_revokes_available, sdp->sd_ldptrs); + gfs2_assert_withdraw(sdp, reserved_revokes % sdp->sd_inptrs == sdp->sd_ldptrs); + if (reserved_revokes > sdp->sd_ldptrs) + reserved_blocks += (reserved_revokes - sdp->sd_ldptrs) / sdp->sd_inptrs; out: - if (used_blocks != reserved_blocks) + if (used_blocks != reserved_blocks) { + gfs2_assert_withdraw_delayed(sdp, used_blocks < reserved_blocks); gfs2_log_release(sdp, reserved_blocks - used_blocks); + } up_write(&sdp->sd_log_flush_lock); gfs2_trans_free(sdp, tr); if (gfs2_withdrawing(sdp)) @@ -1105,8 +1172,8 @@ static void gfs2_merge_trans(struct gfs2_sbd *sdp, struct gfs2_trans *new) old->tr_num_databuf_new += new->tr_num_databuf_new; old->tr_num_buf_rm += new->tr_num_buf_rm; old->tr_num_databuf_rm += new->tr_num_databuf_rm; + old->tr_revokes += new->tr_revokes; old->tr_num_revoke += new->tr_num_revoke; - old->tr_num_revoke_rm += new->tr_num_revoke_rm; list_splice_tail_init(&new->tr_databuf, &old->tr_databuf); list_splice_tail_init(&new->tr_buf, &old->tr_buf); @@ -1133,7 +1200,6 @@ static void log_refund(struct gfs2_sbd *sdp, struct gfs2_trans *tr) set_bit(TR_ATTACHED, &tr->tr_flags); } - sdp->sd_log_committed_revoke += tr->tr_num_revoke - tr->tr_num_revoke_rm; reserved = calc_reserved(sdp); maxres = sdp->sd_log_blks_reserved + tr->tr_reserved; gfs2_assert_withdraw(sdp, maxres >= reserved); diff --git a/fs/gfs2/log.h b/fs/gfs2/log.h index e7f4a8d6be64..eea58015710e 100644 --- a/fs/gfs2/log.h +++ b/fs/gfs2/log.h @@ -74,8 +74,12 @@ extern void gfs2_ordered_del_inode(struct gfs2_inode *ip); extern unsigned int gfs2_struct2blk(struct gfs2_sbd *sdp, unsigned int nstruct); extern void gfs2_remove_from_ail(struct gfs2_bufdata *bd); extern bool gfs2_log_is_empty(struct gfs2_sbd *sdp); +extern void gfs2_log_release_revokes(struct gfs2_sbd *sdp, unsigned int revokes); extern void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks); -extern void gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks); +extern bool gfs2_log_try_reserve(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + unsigned int *extra_revokes); +extern void gfs2_log_reserve(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + unsigned int *extra_revokes); extern void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, u64 seq, u32 tail, u32 lblock, u32 flags, int op_flags); diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index 8658ebbcb4a9..dfe8537cb88e 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c @@ -857,7 +857,6 @@ static void revoke_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr) sdp->sd_log_num_revoke--; if (offset + sizeof(u64) > sdp->sd_sb.sb_bsize) { - gfs2_log_write_page(sdp, page); page = mempool_alloc(gfs2_page_pool, GFP_NOIO); mh = page_address(page); diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index 986dc2ebebf0..316ca5fc99e8 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -315,6 +315,13 @@ static int gfs2_read_sb(struct gfs2_sbd *sdp, int silent) sizeof(struct gfs2_meta_header)) * GFS2_NBBY; /* not the rgrp bitmap, subsequent bitmaps only */ + /* + * We always keep at least one block reserved for revokes in + * transactions. This greatly simplifies allocating additional + * revoke blocks. + */ + atomic_set(&sdp->sd_log_revokes_available, sdp->sd_ldptrs); + /* Compute maximum reservation required to add a entry to a directory */ hash_blocks = DIV_ROUND_UP(sizeof(u64) * BIT(GFS2_DIR_MAX_DEPTH), diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index 231ca1a41a73..ab96cf0bf26b 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -31,16 +31,18 @@ static void gfs2_print_trans(struct gfs2_sbd *sdp, const struct gfs2_trans *tr) fs_warn(sdp, "blocks=%u revokes=%u reserved=%u touched=%u\n", tr->tr_blocks, tr->tr_revokes, tr->tr_reserved, test_bit(TR_TOUCHED, &tr->tr_flags)); - fs_warn(sdp, "Buf %u/%u Databuf %u/%u Revoke %u/%u\n", + fs_warn(sdp, "Buf %u/%u Databuf %u/%u Revoke %u\n", tr->tr_num_buf_new, tr->tr_num_buf_rm, tr->tr_num_databuf_new, tr->tr_num_databuf_rm, - tr->tr_num_revoke, tr->tr_num_revoke_rm); + tr->tr_num_revoke); } int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, unsigned int blocks, unsigned int revokes, unsigned long ip) { + unsigned int extra_revokes; + if (current->journal_info) { gfs2_print_trans(sdp, current->journal_info); BUG(); @@ -62,8 +64,6 @@ int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, */ tr->tr_reserved += blocks + 1 + DIV_ROUND_UP(blocks - 1, databuf_limit(sdp)); } - if (revokes) - tr->tr_reserved += gfs2_struct2blk(sdp, revokes) - 1; INIT_LIST_HEAD(&tr->tr_databuf); INIT_LIST_HEAD(&tr->tr_buf); INIT_LIST_HEAD(&tr->tr_list); @@ -75,10 +75,26 @@ int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, sb_start_intwrite(sdp->sd_vfs); - gfs2_log_reserve(sdp, tr->tr_reserved); + /* + * Try the reservations under sd_log_flush_lock to prevent log flushes + * from creating inconsistencies between the number of allocated and + * reserved revokes. If that fails, do a full-block allocation outside + * of the lock to avoid stalling log flushes. Then, allot the + * appropriate number of blocks to revokes, use as many revokes locally + * as needed, and "release" the surplus into the revokes pool. + */ down_read(&sdp->sd_log_flush_lock); + if (gfs2_log_try_reserve(sdp, tr, &extra_revokes)) + goto reserved; + up_read(&sdp->sd_log_flush_lock); + gfs2_log_reserve(sdp, tr, &extra_revokes); + down_read(&sdp->sd_log_flush_lock); + +reserved: + gfs2_log_release_revokes(sdp, extra_revokes); if (unlikely(!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags))) { + gfs2_log_release_revokes(sdp, tr->tr_revokes); up_read(&sdp->sd_log_flush_lock); gfs2_log_release(sdp, tr->tr_reserved); sb_end_intwrite(sdp->sd_vfs); @@ -113,14 +129,17 @@ void gfs2_trans_end(struct gfs2_sbd *sdp) current->journal_info = NULL; if (!test_bit(TR_TOUCHED, &tr->tr_flags)) { - gfs2_log_release(sdp, tr->tr_reserved); + gfs2_log_release_revokes(sdp, tr->tr_revokes); up_read(&sdp->sd_log_flush_lock); + gfs2_log_release(sdp, tr->tr_reserved); if (!test_bit(TR_ONSTACK, &tr->tr_flags)) gfs2_trans_free(sdp, tr); sb_end_intwrite(sdp->sd_vfs); return; } + gfs2_log_release_revokes(sdp, tr->tr_revokes - tr->tr_num_revoke); + nbuf = tr->tr_num_buf_new + tr->tr_num_databuf_new; nbuf -= tr->tr_num_buf_rm; nbuf -= tr->tr_num_databuf_rm; @@ -278,7 +297,6 @@ void gfs2_trans_add_revoke(struct gfs2_sbd *sdp, struct gfs2_bufdata *bd) void gfs2_trans_remove_revoke(struct gfs2_sbd *sdp, u64 blkno, unsigned int len) { struct gfs2_bufdata *bd, *tmp; - struct gfs2_trans *tr = current->journal_info; unsigned int n = len; gfs2_log_lock(sdp); @@ -290,7 +308,7 @@ void gfs2_trans_remove_revoke(struct gfs2_sbd *sdp, u64 blkno, unsigned int len) if (bd->bd_gl) gfs2_glock_remove_revoke(bd->bd_gl); kmem_cache_free(gfs2_bufdata_cachep, bd); - tr->tr_num_revoke_rm++; + gfs2_log_release_revokes(sdp, 1); if (--n == 0) break; } -- cgit v1.2.3 From b438fcf12815db794403652f0ceeb216650a6a04 Mon Sep 17 00:00:00 2001 From: Steve French Date: Sat, 20 Feb 2021 19:24:11 -0600 Subject: cifs: change confusing field serverName (to ip_addr) ses->serverName is not the server name, but the string form of the ip address of the server. Change the name to ip_addr to avoid confusion (and fix the array length to match maximum length of ipv6 address). Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/cifs_debug.c | 4 ++-- fs/cifs/cifsencrypt.c | 6 +++--- fs/cifs/cifsglob.h | 3 ++- fs/cifs/connect.c | 4 ++-- fs/cifs/file.c | 2 +- fs/cifs/sess.c | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c index 370cc88a3d02..1048180f4722 100644 --- a/fs/cifs/cifs_debug.c +++ b/fs/cifs/cifs_debug.c @@ -395,7 +395,7 @@ skip_rdma: (ses->serverOS == NULL) || (ses->serverNOS == NULL)) { seq_printf(m, "\n\t%d) Name: %s Uses: %d Capability: 0x%x\tSession Status: %d ", - i, ses->serverName, ses->ses_count, + i, ses->ip_addr, ses->ses_count, ses->capabilities, ses->status); if (ses->session_flags & SMB2_SESSION_FLAG_IS_GUEST) seq_printf(m, "Guest "); @@ -406,7 +406,7 @@ skip_rdma: "\n\t%d) Name: %s Domain: %s Uses: %d OS: %s " "\n\tNOS: %s\tCapability: 0x%x" "\n\tSMB session status: %d ", - i, ses->serverName, ses->serverDomain, + i, ses->ip_addr, ses->serverDomain, ses->ses_count, ses->serverOS, ses->serverNOS, ses->capabilities, ses->status); } diff --git a/fs/cifs/cifsencrypt.c b/fs/cifs/cifsencrypt.c index 51d53e4bdf6b..b8f1ff9a83f3 100644 --- a/fs/cifs/cifsencrypt.c +++ b/fs/cifs/cifsencrypt.c @@ -568,15 +568,15 @@ static int calc_ntlmv2_hash(struct cifs_ses *ses, char *ntlmv2_hash, return rc; } } else { - /* We use ses->serverName if no domain name available */ - len = strlen(ses->serverName); + /* We use ses->ip_addr if no domain name available */ + len = strlen(ses->ip_addr); server = kmalloc(2 + (len * 2), GFP_KERNEL); if (server == NULL) { rc = -ENOMEM; return rc; } - len = cifs_strtoUTF16((__le16 *)server, ses->serverName, len, + len = cifs_strtoUTF16((__le16 *)server, ses->ip_addr, len, nls_cp); rc = crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash, diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index 0aa2c3c871c9..fb904236f07f 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -902,7 +903,7 @@ struct cifs_ses { kuid_t linux_uid; /* overriding owner of files on the mount */ kuid_t cred_uid; /* owner of credentials */ unsigned int capabilities; - char serverName[SERVER_NAME_LEN_WITH_NULL]; + char ip_addr[INET6_ADDRSTRLEN + 1]; /* Max ipv6 (or v4) addr string len */ char *user_name; /* must not be null except during init of sess and after mount option parsing we fill it */ char *domainName; diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index b3102a86fd81..139e306305df 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -1841,9 +1841,9 @@ cifs_get_smb_ses(struct TCP_Server_Info *server, struct smb3_fs_context *ctx) /* new SMB session uses our server ref */ ses->server = server; if (server->dstaddr.ss_family == AF_INET6) - sprintf(ses->serverName, "%pI6", &addr6->sin6_addr); + sprintf(ses->ip_addr, "%pI6", &addr6->sin6_addr); else - sprintf(ses->serverName, "%pI4", &addr->sin_addr); + sprintf(ses->ip_addr, "%pI4", &addr->sin_addr); if (ctx->username) { ses->user_name = kstrdup(ctx->username, GFP_KERNEL); diff --git a/fs/cifs/file.c b/fs/cifs/file.c index 6d001905c8e5..26de4329d161 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c @@ -580,7 +580,7 @@ int cifs_open(struct inode *inode, struct file *file) } else if ((rc == -EINVAL) || (rc == -EOPNOTSUPP)) { if (tcon->ses->serverNOS) cifs_dbg(VFS, "server %s of type %s returned unexpected error on SMB posix open, disabling posix open support. Check if server update available.\n", - tcon->ses->serverName, + tcon->ses->ip_addr, tcon->ses->serverNOS); tcon->broken_posix_open = true; } else if ((rc != -EIO) && (rc != -EREMOTE) && diff --git a/fs/cifs/sess.c b/fs/cifs/sess.c index 213465718fa8..183a3a868d7b 100644 --- a/fs/cifs/sess.c +++ b/fs/cifs/sess.c @@ -218,7 +218,7 @@ cifs_ses_add_channel(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses, /* UNC and paths */ /* XXX: Use ses->server->hostname? */ - sprintf(unc, unc_fmt, ses->serverName); + sprintf(unc, unc_fmt, ses->ip_addr); ctx.UNC = unc; ctx.prepath = ""; -- cgit v1.2.3 From 40f077a02bf9d70719128d2a807e28a3503711eb Mon Sep 17 00:00:00 2001 From: Steve French Date: Sun, 21 Feb 2021 23:00:46 -0600 Subject: cifs: clarify hostname vs ip address in /proc/fs/cifs/DebugData /proc/fs/cifs/DebugData called the ip address for server sessions "Name" which is confusing since it is not a hostname. Change this field name to "Address" and for the list of servers add new field "Hostname" which is populated from the hostname used to connect to the server. See below. And also don't print [NONE] when the interface list is empty as it is not clear what 'NONE' referred to. Servers: 1) ConnectionId: 0x1 Hostname: localhost Number of credits: 389 Dialect 0x311 TCP status: 1 Instance: 1 Local Users To Server: 1 SecMode: 0x1 Req On Wire: 0 In Send: 0 In MaxReq Wait: 0 Sessions: 1) Address: 127.0.0.1 ... Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/cifs_debug.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c index 1048180f4722..02e007d0939d 100644 --- a/fs/cifs/cifs_debug.c +++ b/fs/cifs/cifs_debug.c @@ -293,6 +293,8 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) seq_printf(m, "\n%d) ConnectionId: 0x%llx ", c, server->conn_id); + if (server->hostname) + seq_printf(m, "Hostname: %s ", server->hostname); #ifdef CONFIG_CIFS_SMB_DIRECT if (!server->rdma) goto skip_rdma; @@ -394,7 +396,7 @@ skip_rdma: if ((ses->serverDomain == NULL) || (ses->serverOS == NULL) || (ses->serverNOS == NULL)) { - seq_printf(m, "\n\t%d) Name: %s Uses: %d Capability: 0x%x\tSession Status: %d ", + seq_printf(m, "\n\t%d) Address: %s Uses: %d Capability: 0x%x\tSession Status: %d ", i, ses->ip_addr, ses->ses_count, ses->capabilities, ses->status); if (ses->session_flags & SMB2_SESSION_FLAG_IS_GUEST) @@ -462,8 +464,6 @@ skip_rdma: if (is_ses_using_iface(ses, iface)) seq_puts(m, "\t\t[CONNECTED]\n"); } - if (j == 0) - seq_printf(m, "\n\t[NONE]"); spin_unlock(&ses->iface_lock); } if (i == 0) -- cgit v1.2.3 From c12ead71e86f47f4715f61e6dee7b7120532bedb Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Sun, 21 Feb 2021 08:21:25 +0000 Subject: cifs: Fix cifsacl ACE mask for group and others. A two line fix which I made while testing my prev fix with cifsacl mode conversions seem to have gone missing in the final fix that was submitted. This is that fix. Signed-off-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/cifsacl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsacl.h b/fs/cifs/cifsacl.h index ff7fd0862e28..d9e704979d99 100644 --- a/fs/cifs/cifsacl.h +++ b/fs/cifs/cifsacl.h @@ -31,8 +31,8 @@ #define EXEC_BIT 0x1 #define ACL_OWNER_MASK 0700 -#define ACL_GROUP_MASK 0770 -#define ACL_EVERYONE_MASK 0777 +#define ACL_GROUP_MASK 0070 +#define ACL_EVERYONE_MASK 0007 #define UBITSHIFT 6 #define GBITSHIFT 3 -- cgit v1.2.3 From f5065508897a922327f32223082325d10b069ebc Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Fri, 12 Feb 2021 04:38:43 -0800 Subject: cifs: Retain old ACEs when converting between mode bits and ACL. When cifsacl mount option is used, retain the ACEs which should not be modified during chmod. Following is the approach taken: 1. Retain all explicit (non-inherited) ACEs, unless the SID is one of owner/group/everyone/authenticated-users. We're going to set new ACEs for these SIDs anyways. 2. At the end of the list of explicit ACEs, place the new list of ACEs obtained by necessary conversion/encoding. 3. Once the converted/encoded ACEs are set, copy all the remaining ACEs (inherited) into the new ACL. Signed-off-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/cifsacl.c | 270 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 223 insertions(+), 47 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsacl.c b/fs/cifs/cifsacl.c index 562913e2b3f2..5d71fe1088bc 100644 --- a/fs/cifs/cifsacl.c +++ b/fs/cifs/cifsacl.c @@ -267,10 +267,11 @@ is_well_known_sid(const struct cifs_sid *psid, uint32_t *puid, bool is_group) return true; /* well known sid found, uid returned */ } -static void +static __u16 cifs_copy_sid(struct cifs_sid *dst, const struct cifs_sid *src) { int i; + __u16 size = 1 + 1 + 6; dst->revision = src->revision; dst->num_subauth = min_t(u8, src->num_subauth, SID_MAX_SUB_AUTHORITIES); @@ -278,6 +279,9 @@ cifs_copy_sid(struct cifs_sid *dst, const struct cifs_sid *src) dst->authority[i] = src->authority[i]; for (i = 0; i < dst->num_subauth; ++i) dst->sub_auth[i] = src->sub_auth[i]; + size += (dst->num_subauth * 4); + + return size; } static int @@ -663,6 +667,19 @@ static void mode_to_access_flags(umode_t mode, umode_t bits_to_use, return; } +static __u16 cifs_copy_ace(struct cifs_ace *dst, struct cifs_ace *src) +{ + __u16 size = 1 + 1 + 2 + 4; + + dst->type = src->type; + dst->flags = src->flags; + dst->size = src->size; + dst->access_req = src->access_req; + size += cifs_copy_sid(&dst->sid, &src->sid); + + return size; +} + static __u16 fill_ace_for_sid(struct cifs_ace *pntace, const struct cifs_sid *psid, __u64 nmode, umode_t bits, __u8 access_type, @@ -907,29 +924,30 @@ unsigned int setup_special_user_owner_ACE(struct cifs_ace *pntace) return ace_size; } -static int set_chmod_dacl(struct cifs_acl *pndacl, struct cifs_sid *pownersid, - struct cifs_sid *pgrpsid, __u64 *pnmode, bool modefromsid) +static void populate_new_aces(char *nacl_base, + struct cifs_sid *pownersid, + struct cifs_sid *pgrpsid, + __u64 *pnmode, u32 *pnum_aces, u16 *pnsize, + bool modefromsid) { - u16 size = 0; - u32 num_aces = 0; - struct cifs_acl *pnndacl; __u64 nmode; + u32 num_aces = 0; + u16 nsize = 0; __u64 user_mode; __u64 group_mode; __u64 other_mode; __u64 deny_user_mode = 0; __u64 deny_group_mode = 0; bool sticky_set = false; - - pnndacl = (struct cifs_acl *)((char *)pndacl + sizeof(struct cifs_acl)); + struct cifs_ace *pnntace = NULL; nmode = *pnmode; + num_aces = *pnum_aces; + nsize = *pnsize; if (modefromsid) { - struct cifs_ace *pntace = - (struct cifs_ace *)((char *)pnndacl + size); - - size += setup_special_mode_ACE(pntace, nmode); + pnntace = (struct cifs_ace *) (nacl_base + nsize); + nsize += setup_special_mode_ACE(pnntace, nmode); num_aces++; goto set_size; } @@ -966,40 +984,170 @@ static int set_chmod_dacl(struct cifs_acl *pndacl, struct cifs_sid *pownersid, sticky_set = true; if (deny_user_mode) { - size += fill_ace_for_sid((struct cifs_ace *)((char *)pnndacl + size), - pownersid, deny_user_mode, 0700, ACCESS_DENIED, false); + pnntace = (struct cifs_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, pownersid, deny_user_mode, + 0700, ACCESS_DENIED, false); num_aces++; } + /* Group DENY ACE does not conflict with owner ALLOW ACE. Keep in preferred order*/ if (deny_group_mode && !(deny_group_mode & (user_mode >> 3))) { - size += fill_ace_for_sid((struct cifs_ace *)((char *)pnndacl + size), - pgrpsid, deny_group_mode, 0070, ACCESS_DENIED, false); + pnntace = (struct cifs_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, pgrpsid, deny_group_mode, + 0070, ACCESS_DENIED, false); num_aces++; } - size += fill_ace_for_sid((struct cifs_ace *) ((char *)pnndacl + size), - pownersid, user_mode, 0700, ACCESS_ALLOWED, true); + + pnntace = (struct cifs_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, pownersid, user_mode, + 0700, ACCESS_ALLOWED, true); num_aces++; + /* Group DENY ACE conflicts with owner ALLOW ACE. So keep it after. */ if (deny_group_mode && (deny_group_mode & (user_mode >> 3))) { - size += fill_ace_for_sid((struct cifs_ace *)((char *)pnndacl + size), - pgrpsid, deny_group_mode, 0070, ACCESS_DENIED, false); + pnntace = (struct cifs_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, pgrpsid, deny_group_mode, + 0070, ACCESS_DENIED, false); num_aces++; } - size += fill_ace_for_sid((struct cifs_ace *)((char *)pnndacl + size), - pgrpsid, group_mode, 0070, ACCESS_ALLOWED, !sticky_set); + + pnntace = (struct cifs_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, pgrpsid, group_mode, + 0070, ACCESS_ALLOWED, !sticky_set); num_aces++; - size += fill_ace_for_sid((struct cifs_ace *)((char *)pnndacl + size), - &sid_everyone, other_mode, 0007, ACCESS_ALLOWED, !sticky_set); + + pnntace = (struct cifs_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, &sid_everyone, other_mode, + 0007, ACCESS_ALLOWED, !sticky_set); num_aces++; set_size: + *pnum_aces = num_aces; + *pnsize = nsize; +} + +static __u16 replace_sids_and_copy_aces(struct cifs_acl *pdacl, struct cifs_acl *pndacl, + struct cifs_sid *pownersid, struct cifs_sid *pgrpsid, + struct cifs_sid *pnownersid, struct cifs_sid *pngrpsid) +{ + int i; + u16 size = 0; + struct cifs_ace *pntace = NULL; + char *acl_base = NULL; + u32 src_num_aces = 0; + u16 nsize = 0; + struct cifs_ace *pnntace = NULL; + char *nacl_base = NULL; + u16 ace_size = 0; + + acl_base = (char *)pdacl; + size = sizeof(struct cifs_acl); + src_num_aces = le32_to_cpu(pdacl->num_aces); + + nacl_base = (char *)pndacl; + nsize = sizeof(struct cifs_acl); + + /* Go through all the ACEs */ + for (i = 0; i < src_num_aces; ++i) { + pntace = (struct cifs_ace *) (acl_base + size); + pnntace = (struct cifs_ace *) (nacl_base + nsize); + + if (pnownersid && compare_sids(&pntace->sid, pownersid) == 0) + ace_size = cifs_copy_ace(pnntace, pntace); + else if (pngrpsid && compare_sids(&pntace->sid, pgrpsid) == 0) + ace_size = cifs_copy_ace(pnntace, pntace); + else + ace_size = cifs_copy_ace(pnntace, pntace); + + size += le16_to_cpu(pntace->size); + nsize += ace_size; + } + + return nsize; +} + +static int set_chmod_dacl(struct cifs_acl *pdacl, struct cifs_acl *pndacl, + struct cifs_sid *pownersid, struct cifs_sid *pgrpsid, + __u64 *pnmode, bool mode_from_sid) +{ + int i; + u16 size = 0; + struct cifs_ace *pntace = NULL; + char *acl_base = NULL; + u32 src_num_aces = 0; + u16 nsize = 0; + struct cifs_ace *pnntace = NULL; + char *nacl_base = NULL; + u32 num_aces = 0; + __u64 nmode; + bool new_aces_set = false; + + /* Assuming that pndacl and pnmode are never NULL */ + nmode = *pnmode; + nacl_base = (char *)pndacl; + nsize = sizeof(struct cifs_acl); + + /* If pdacl is NULL, we don't have a src. Simply populate new ACL. */ + if (!pdacl) { + populate_new_aces(nacl_base, + pownersid, pgrpsid, + pnmode, &num_aces, &nsize, + mode_from_sid); + goto finalize_dacl; + } + + acl_base = (char *)pdacl; + size = sizeof(struct cifs_acl); + src_num_aces = le32_to_cpu(pdacl->num_aces); + + /* Retain old ACEs which we can retain */ + for (i = 0; i < src_num_aces; ++i) { + pntace = (struct cifs_ace *) (acl_base + size); + pnntace = (struct cifs_ace *) (nacl_base + nsize); + + if (!new_aces_set && (pntace->flags & INHERITED_ACE)) { + /* Place the new ACEs in between existing explicit and inherited */ + populate_new_aces(nacl_base, + pownersid, pgrpsid, + pnmode, &num_aces, &nsize, + mode_from_sid); + + new_aces_set = true; + } + + /* If it's any one of the ACE we're replacing, skip! */ + if ((compare_sids(&pntace->sid, &sid_unix_NFS_mode) == 0) || + (compare_sids(&pntace->sid, pownersid) == 0) || + (compare_sids(&pntace->sid, pgrpsid) == 0) || + (compare_sids(&pntace->sid, &sid_everyone) == 0) || + (compare_sids(&pntace->sid, &sid_authusers) == 0)) { + goto next_ace; + } + + nsize += cifs_copy_ace(pnntace, pntace); + num_aces++; + +next_ace: + size += le32_to_cpu(pntace->size); + } + + /* If inherited ACEs are not present, place the new ones at the tail */ + if (!new_aces_set) { + populate_new_aces(nacl_base, + pownersid, pgrpsid, + pnmode, &num_aces, &nsize, + mode_from_sid); + + new_aces_set = true; + } + +finalize_dacl: pndacl->num_aces = cpu_to_le32(num_aces); - pndacl->size = cpu_to_le16(size + sizeof(struct cifs_acl)); + pndacl->size = cpu_to_le16(nsize); return 0; } - static int parse_sid(struct cifs_sid *psid, char *end_of_acl) { /* BB need to add parm so we can store the SID BB */ @@ -1094,7 +1242,7 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, /* Convert permission bits from mode to equivalent CIFS ACL */ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd, - __u32 secdesclen, __u64 *pnmode, kuid_t uid, kgid_t gid, + __u32 secdesclen, __u32 *pnsecdesclen, __u64 *pnmode, kuid_t uid, kgid_t gid, bool mode_from_sid, bool id_from_sid, int *aclflag) { int rc = 0; @@ -1105,6 +1253,7 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd, struct cifs_sid *nowner_sid_ptr, *ngroup_sid_ptr; struct cifs_acl *dacl_ptr = NULL; /* no need for SACL ptr */ struct cifs_acl *ndacl_ptr = NULL; /* no need for SACL ptr */ + char *end_of_acl = ((char *)pntsd) + secdesclen; if (pnmode && *pnmode != NO_CHANGE_64) { /* chmod */ owner_sid_ptr = (struct cifs_sid *)((char *)pntsd + @@ -1112,21 +1261,36 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd, group_sid_ptr = (struct cifs_sid *)((char *)pntsd + le32_to_cpu(pntsd->gsidoffset)); dacloffset = le32_to_cpu(pntsd->dacloffset); - dacl_ptr = (struct cifs_acl *)((char *)pntsd + dacloffset); + if (dacloffset) { + dacl_ptr = (struct cifs_acl *)((char *)pntsd + dacloffset); + if (end_of_acl < (char *)dacl_ptr + le16_to_cpu(dacl_ptr->size)) { + cifs_dbg(VFS, "Existing ACL size is wrong. Discarding old ACL\n"); + dacl_ptr = NULL; + } + } + ndacloffset = sizeof(struct cifs_ntsd); ndacl_ptr = (struct cifs_acl *)((char *)pnntsd + ndacloffset); - ndacl_ptr->revision = dacl_ptr->revision; - ndacl_ptr->size = 0; - ndacl_ptr->num_aces = 0; + ndacl_ptr->revision = + dacloffset ? dacl_ptr->revision : cpu_to_le16(ACL_REVISION); - rc = set_chmod_dacl(ndacl_ptr, owner_sid_ptr, group_sid_ptr, + ndacl_ptr->size = cpu_to_le32(0); + ndacl_ptr->num_aces = cpu_to_le32(0); + + rc = set_chmod_dacl(dacl_ptr, ndacl_ptr, owner_sid_ptr, group_sid_ptr, pnmode, mode_from_sid); sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size); /* copy sec desc control portion & owner and group sids */ copy_sec_desc(pntsd, pnntsd, sidsoffset); + *pnsecdesclen = sidsoffset + (2 * sizeof(struct cifs_sid)); *aclflag = CIFS_ACL_DACL; } else { - memcpy(pnntsd, pntsd, secdesclen); + ndacloffset = sizeof(struct cifs_ntsd); + ndacl_ptr = (struct cifs_acl *)((char *)pnntsd + ndacloffset); + ndacl_ptr->revision = + dacloffset ? dacl_ptr->revision : cpu_to_le16(ACL_REVISION); + ndacl_ptr->num_aces = dacl_ptr->num_aces; + if (uid_valid(uid)) { /* chown */ uid_t id; owner_sid_ptr = (struct cifs_sid *)((char *)pnntsd + @@ -1384,6 +1548,9 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, int rc = 0; int aclflag = CIFS_ACL_DACL; /* default flag to set */ __u32 secdesclen = 0; + __u32 nsecdesclen = 0; + __u32 dacloffset = 0; + struct cifs_acl *dacl_ptr = NULL; struct cifs_ntsd *pntsd = NULL; /* acl obtained from server */ struct cifs_ntsd *pnntsd = NULL; /* modified acl to be sent to server */ struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); @@ -1414,31 +1581,40 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, return rc; } + if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MODE_FROM_SID) + mode_from_sid = true; + else + mode_from_sid = false; + + if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UID_FROM_ACL) + id_from_sid = true; + else + id_from_sid = false; + + /* Potentially, five new ACEs can be added to the ACL for U,G,O mapping */ + nsecdesclen = secdesclen; + if (pnmode && *pnmode != NO_CHANGE_64) { /* chmod */ + if (mode_from_sid) + nsecdesclen += sizeof(struct cifs_ace); + else /* cifsacl */ + nsecdesclen += 5 * sizeof(struct cifs_ace); + } + /* * Add three ACEs for owner, group, everyone getting rid of other ACEs * as chmod disables ACEs and set the security descriptor. Allocate * memory for the smb header, set security descriptor request security * descriptor parameters, and secuirty descriptor itself */ - secdesclen = max_t(u32, secdesclen, DEFAULT_SEC_DESC_LEN); - pnntsd = kmalloc(secdesclen, GFP_KERNEL); + nsecdesclen = max_t(u32, nsecdesclen, DEFAULT_SEC_DESC_LEN); + pnntsd = kmalloc(nsecdesclen, GFP_KERNEL); if (!pnntsd) { kfree(pntsd); cifs_put_tlink(tlink); return -ENOMEM; } - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MODE_FROM_SID) - mode_from_sid = true; - else - mode_from_sid = false; - - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UID_FROM_ACL) - id_from_sid = true; - else - id_from_sid = false; - - rc = build_sec_desc(pntsd, pnntsd, secdesclen, pnmode, uid, gid, + rc = build_sec_desc(pntsd, pnntsd, secdesclen, &nsecdesclen, pnmode, uid, gid, mode_from_sid, id_from_sid, &aclflag); cifs_dbg(NOISY, "build_sec_desc rc: %d\n", rc); @@ -1448,7 +1624,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, if (!rc) { /* Set the security descriptor */ - rc = ops->set_acl(pnntsd, secdesclen, inode, path, aclflag); + rc = ops->set_acl(pnntsd, nsecdesclen, inode, path, aclflag); cifs_dbg(NOISY, "set_cifs_acl rc: %d\n", rc); } cifs_put_tlink(tlink); -- cgit v1.2.3 From bc3e9dd9d104ca1b75644eab87b38ce8a924aef4 Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Thu, 18 Feb 2021 13:03:23 +0000 Subject: cifs: Change SIDs in ACEs while transferring file ownership. With cifsacl, when a file/dir ownership is transferred (chown/chgrp), the ACEs in the DACL for that file will need to replace the old owner SIDs with the new owner SID. Signed-off-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/cifsacl.c | 139 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 92 insertions(+), 47 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsacl.c b/fs/cifs/cifsacl.c index 5d71fe1088bc..d44bfa62c1cd 100644 --- a/fs/cifs/cifsacl.c +++ b/fs/cifs/cifsacl.c @@ -525,8 +525,11 @@ exit_cifs_idmap(void) } /* copy ntsd, owner sid, and group sid from a security descriptor to another */ -static void copy_sec_desc(const struct cifs_ntsd *pntsd, - struct cifs_ntsd *pnntsd, __u32 sidsoffset) +static __u32 copy_sec_desc(const struct cifs_ntsd *pntsd, + struct cifs_ntsd *pnntsd, + __u32 sidsoffset, + struct cifs_sid *pownersid, + struct cifs_sid *pgrpsid) { struct cifs_sid *owner_sid_ptr, *group_sid_ptr; struct cifs_sid *nowner_sid_ptr, *ngroup_sid_ptr; @@ -540,19 +543,25 @@ static void copy_sec_desc(const struct cifs_ntsd *pntsd, pnntsd->gsidoffset = cpu_to_le32(sidsoffset + sizeof(struct cifs_sid)); /* copy owner sid */ - owner_sid_ptr = (struct cifs_sid *)((char *)pntsd + + if (pownersid) + owner_sid_ptr = pownersid; + else + owner_sid_ptr = (struct cifs_sid *)((char *)pntsd + le32_to_cpu(pntsd->osidoffset)); nowner_sid_ptr = (struct cifs_sid *)((char *)pnntsd + sidsoffset); cifs_copy_sid(nowner_sid_ptr, owner_sid_ptr); /* copy group sid */ - group_sid_ptr = (struct cifs_sid *)((char *)pntsd + + if (pgrpsid) + group_sid_ptr = pgrpsid; + else + group_sid_ptr = (struct cifs_sid *)((char *)pntsd + le32_to_cpu(pntsd->gsidoffset)); ngroup_sid_ptr = (struct cifs_sid *)((char *)pnntsd + sidsoffset + sizeof(struct cifs_sid)); cifs_copy_sid(ngroup_sid_ptr, group_sid_ptr); - return; + return sidsoffset + (2 * sizeof(struct cifs_sid)); } @@ -667,15 +676,21 @@ static void mode_to_access_flags(umode_t mode, umode_t bits_to_use, return; } -static __u16 cifs_copy_ace(struct cifs_ace *dst, struct cifs_ace *src) +static __u16 cifs_copy_ace(struct cifs_ace *dst, struct cifs_ace *src, struct cifs_sid *psid) { __u16 size = 1 + 1 + 2 + 4; dst->type = src->type; dst->flags = src->flags; - dst->size = src->size; dst->access_req = src->access_req; - size += cifs_copy_sid(&dst->sid, &src->sid); + + /* Check if there's a replacement sid specified */ + if (psid) + size += cifs_copy_sid(&dst->sid, psid); + else + size += cifs_copy_sid(&dst->sid, &src->sid); + + dst->size = cpu_to_le16(size); return size; } @@ -1053,11 +1068,11 @@ static __u16 replace_sids_and_copy_aces(struct cifs_acl *pdacl, struct cifs_acl pnntace = (struct cifs_ace *) (nacl_base + nsize); if (pnownersid && compare_sids(&pntace->sid, pownersid) == 0) - ace_size = cifs_copy_ace(pnntace, pntace); + ace_size = cifs_copy_ace(pnntace, pntace, pnownersid); else if (pngrpsid && compare_sids(&pntace->sid, pgrpsid) == 0) - ace_size = cifs_copy_ace(pnntace, pntace); + ace_size = cifs_copy_ace(pnntace, pntace, pngrpsid); else - ace_size = cifs_copy_ace(pnntace, pntace); + ace_size = cifs_copy_ace(pnntace, pntace, NULL); size += le16_to_cpu(pntace->size); nsize += ace_size; @@ -1124,7 +1139,7 @@ static int set_chmod_dacl(struct cifs_acl *pdacl, struct cifs_acl *pndacl, goto next_ace; } - nsize += cifs_copy_ace(pnntace, pntace); + nsize += cifs_copy_ace(pnntace, pntace, NULL); num_aces++; next_ace: @@ -1250,25 +1265,27 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd, __u32 ndacloffset; __u32 sidsoffset; struct cifs_sid *owner_sid_ptr, *group_sid_ptr; - struct cifs_sid *nowner_sid_ptr, *ngroup_sid_ptr; + struct cifs_sid *nowner_sid_ptr = NULL, *ngroup_sid_ptr = NULL; struct cifs_acl *dacl_ptr = NULL; /* no need for SACL ptr */ struct cifs_acl *ndacl_ptr = NULL; /* no need for SACL ptr */ char *end_of_acl = ((char *)pntsd) + secdesclen; + u16 size = 0; - if (pnmode && *pnmode != NO_CHANGE_64) { /* chmod */ - owner_sid_ptr = (struct cifs_sid *)((char *)pntsd + - le32_to_cpu(pntsd->osidoffset)); - group_sid_ptr = (struct cifs_sid *)((char *)pntsd + - le32_to_cpu(pntsd->gsidoffset)); - dacloffset = le32_to_cpu(pntsd->dacloffset); - if (dacloffset) { - dacl_ptr = (struct cifs_acl *)((char *)pntsd + dacloffset); - if (end_of_acl < (char *)dacl_ptr + le16_to_cpu(dacl_ptr->size)) { - cifs_dbg(VFS, "Existing ACL size is wrong. Discarding old ACL\n"); - dacl_ptr = NULL; - } + dacloffset = le32_to_cpu(pntsd->dacloffset); + if (dacloffset) { + dacl_ptr = (struct cifs_acl *)((char *)pntsd + dacloffset); + if (end_of_acl < (char *)dacl_ptr + le16_to_cpu(dacl_ptr->size)) { + cifs_dbg(VFS, "Existing ACL size is wrong. Discarding old ACL\n"); + dacl_ptr = NULL; } + } + + owner_sid_ptr = (struct cifs_sid *)((char *)pntsd + + le32_to_cpu(pntsd->osidoffset)); + group_sid_ptr = (struct cifs_sid *)((char *)pntsd + + le32_to_cpu(pntsd->gsidoffset)); + if (pnmode && *pnmode != NO_CHANGE_64) { /* chmod */ ndacloffset = sizeof(struct cifs_ntsd); ndacl_ptr = (struct cifs_acl *)((char *)pnntsd + ndacloffset); ndacl_ptr->revision = @@ -1279,11 +1296,13 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd, rc = set_chmod_dacl(dacl_ptr, ndacl_ptr, owner_sid_ptr, group_sid_ptr, pnmode, mode_from_sid); + sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size); - /* copy sec desc control portion & owner and group sids */ - copy_sec_desc(pntsd, pnntsd, sidsoffset); - *pnsecdesclen = sidsoffset + (2 * sizeof(struct cifs_sid)); - *aclflag = CIFS_ACL_DACL; + /* copy the non-dacl portion of secdesc */ + *pnsecdesclen = copy_sec_desc(pntsd, pnntsd, sidsoffset, + NULL, NULL); + + *aclflag |= CIFS_ACL_DACL; } else { ndacloffset = sizeof(struct cifs_ntsd); ndacl_ptr = (struct cifs_acl *)((char *)pnntsd + ndacloffset); @@ -1293,12 +1312,12 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd, if (uid_valid(uid)) { /* chown */ uid_t id; - owner_sid_ptr = (struct cifs_sid *)((char *)pnntsd + - le32_to_cpu(pnntsd->osidoffset)); nowner_sid_ptr = kmalloc(sizeof(struct cifs_sid), GFP_KERNEL); - if (!nowner_sid_ptr) - return -ENOMEM; + if (!nowner_sid_ptr) { + rc = -ENOMEM; + goto chown_chgrp_exit; + } id = from_kuid(&init_user_ns, uid); if (id_from_sid) { struct owner_sid *osid = (struct owner_sid *)nowner_sid_ptr; @@ -1309,27 +1328,25 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd, osid->SubAuthorities[0] = cpu_to_le32(88); osid->SubAuthorities[1] = cpu_to_le32(1); osid->SubAuthorities[2] = cpu_to_le32(id); + } else { /* lookup sid with upcall */ rc = id_to_sid(id, SIDOWNER, nowner_sid_ptr); if (rc) { cifs_dbg(FYI, "%s: Mapping error %d for owner id %d\n", __func__, rc, id); - kfree(nowner_sid_ptr); - return rc; + goto chown_chgrp_exit; } } - cifs_copy_sid(owner_sid_ptr, nowner_sid_ptr); - kfree(nowner_sid_ptr); - *aclflag = CIFS_ACL_OWNER; + *aclflag |= CIFS_ACL_OWNER; } if (gid_valid(gid)) { /* chgrp */ gid_t id; - group_sid_ptr = (struct cifs_sid *)((char *)pnntsd + - le32_to_cpu(pnntsd->gsidoffset)); ngroup_sid_ptr = kmalloc(sizeof(struct cifs_sid), GFP_KERNEL); - if (!ngroup_sid_ptr) - return -ENOMEM; + if (!ngroup_sid_ptr) { + rc = -ENOMEM; + goto chown_chgrp_exit; + } id = from_kgid(&init_user_ns, gid); if (id_from_sid) { struct owner_sid *gsid = (struct owner_sid *)ngroup_sid_ptr; @@ -1340,19 +1357,35 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd, gsid->SubAuthorities[0] = cpu_to_le32(88); gsid->SubAuthorities[1] = cpu_to_le32(2); gsid->SubAuthorities[2] = cpu_to_le32(id); + } else { /* lookup sid with upcall */ rc = id_to_sid(id, SIDGROUP, ngroup_sid_ptr); if (rc) { cifs_dbg(FYI, "%s: Mapping error %d for group id %d\n", __func__, rc, id); - kfree(ngroup_sid_ptr); - return rc; + goto chown_chgrp_exit; } } - cifs_copy_sid(group_sid_ptr, ngroup_sid_ptr); - kfree(ngroup_sid_ptr); - *aclflag = CIFS_ACL_GROUP; + *aclflag |= CIFS_ACL_GROUP; } + + if (dacloffset) { + /* Replace ACEs for old owner with new one */ + size = replace_sids_and_copy_aces(dacl_ptr, ndacl_ptr, + owner_sid_ptr, group_sid_ptr, + nowner_sid_ptr, ngroup_sid_ptr); + ndacl_ptr->size = cpu_to_le16(size); + } + + sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size); + /* copy the non-dacl portion of secdesc */ + *pnsecdesclen = copy_sec_desc(pntsd, pnntsd, sidsoffset, + nowner_sid_ptr, ngroup_sid_ptr); + +chown_chgrp_exit: + /* errors could jump here. So make sure we return soon after this */ + kfree(nowner_sid_ptr); + kfree(ngroup_sid_ptr); } return rc; @@ -1598,6 +1631,18 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, nsecdesclen += sizeof(struct cifs_ace); else /* cifsacl */ nsecdesclen += 5 * sizeof(struct cifs_ace); + } else { /* chown */ + /* When ownership changes, changes new owner sid length could be different */ + nsecdesclen = sizeof(struct cifs_ntsd) + (sizeof(struct cifs_sid) * 2); + dacloffset = le32_to_cpu(pntsd->dacloffset); + if (dacloffset) { + dacl_ptr = (struct cifs_acl *)((char *)pntsd + dacloffset); + if (mode_from_sid) + nsecdesclen += + le16_to_cpu(dacl_ptr->num_aces) * sizeof(struct cifs_ace); + else /* cifsacl */ + nsecdesclen += le16_to_cpu(dacl_ptr->size); + } } /* -- cgit v1.2.3 From 23bda5e6511083ec1d76377e8075388e02639147 Mon Sep 17 00:00:00 2001 From: Steve French Date: Mon, 22 Feb 2021 14:40:43 -0600 Subject: cifs: cleanup a few le16 vs. le32 uses in cifsacl.c Cleanup some minor sparse warnings in cifsacl.c Signed-off-by: Steve French --- fs/cifs/cifsacl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsacl.c b/fs/cifs/cifsacl.c index d44bfa62c1cd..0806ae784061 100644 --- a/fs/cifs/cifsacl.c +++ b/fs/cifs/cifsacl.c @@ -1143,7 +1143,7 @@ static int set_chmod_dacl(struct cifs_acl *pdacl, struct cifs_acl *pndacl, num_aces++; next_ace: - size += le32_to_cpu(pntace->size); + size += le16_to_cpu(pntace->size); } /* If inherited ACEs are not present, place the new ones at the tail */ @@ -1291,7 +1291,7 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd, ndacl_ptr->revision = dacloffset ? dacl_ptr->revision : cpu_to_le16(ACL_REVISION); - ndacl_ptr->size = cpu_to_le32(0); + ndacl_ptr->size = cpu_to_le16(0); ndacl_ptr->num_aces = cpu_to_le32(0); rc = set_chmod_dacl(dacl_ptr, ndacl_ptr, owner_sid_ptr, group_sid_ptr, @@ -1639,7 +1639,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, dacl_ptr = (struct cifs_acl *)((char *)pntsd + dacloffset); if (mode_from_sid) nsecdesclen += - le16_to_cpu(dacl_ptr->num_aces) * sizeof(struct cifs_ace); + le32_to_cpu(dacl_ptr->num_aces) * sizeof(struct cifs_ace); else /* cifsacl */ nsecdesclen += le16_to_cpu(dacl_ptr->size); } -- cgit v1.2.3 From 9e550b085206544bd03a8b1dd58a5414e9508351 Mon Sep 17 00:00:00 2001 From: Rohith Surabattula Date: Tue, 16 Feb 2021 10:40:45 +0000 Subject: TCON Reconnect during STATUS_NETWORK_NAME_DELETED When server returns error STATUS_NETWORK_NAME_DELETED, TCON must be marked for reconnect. So, subsequent IO does the tree connect again. Signed-off-by: Rohith Surabattula Reviewed-by: Pavel Shilovsky Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/cifsglob.h | 2 ++ fs/cifs/connect.c | 4 ++++ fs/cifs/smb2ops.c | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) (limited to 'fs') diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index fb904236f07f..3de3c5908a72 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -505,6 +505,8 @@ struct smb_version_operations { loff_t (*llseek)(struct file *, struct cifs_tcon *, loff_t, int); /* Check for STATUS_IO_TIMEOUT */ bool (*is_status_io_timeout)(char *buf); + /* Check for STATUS_NETWORK_NAME_DELETED */ + void (*is_network_name_deleted)(char *buf, struct TCP_Server_Info *srv); }; struct smb_version_values { diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 139e306305df..cd6dbeaf2166 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -995,6 +995,10 @@ next_pdu: if (mids[i] != NULL) { mids[i]->resp_buf_size = server->pdu_size; + if (bufs[i] && server->ops->is_network_name_deleted) + server->ops->is_network_name_deleted(bufs[i], + server); + if (!mids[i]->multiRsp || mids[i]->multiEnd) mids[i]->callback(mids[i]); diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index fe171ccbe8e3..807ecd444c16 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -2451,6 +2451,33 @@ smb2_is_status_io_timeout(char *buf) return false; } +static void +smb2_is_network_name_deleted(char *buf, struct TCP_Server_Info *server) +{ + struct smb2_sync_hdr *shdr = (struct smb2_sync_hdr *)buf; + struct list_head *tmp, *tmp1; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + + if (shdr->Status == STATUS_NETWORK_NAME_DELETED) { + spin_lock(&cifs_tcp_ses_lock); + list_for_each(tmp, &server->smb_ses_list) { + ses = list_entry(tmp, struct cifs_ses, smb_ses_list); + list_for_each(tmp1, &ses->tcon_list) { + tcon = list_entry(tmp1, struct cifs_tcon, tcon_list); + if (tcon->tid == shdr->TreeId) { + tcon->need_reconnect = true; + spin_unlock(&cifs_tcp_ses_lock); + pr_warn_once("Server share %s deleted.\n", + tcon->treeName); + return; + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + } +} + static int smb2_oplock_response(struct cifs_tcon *tcon, struct cifs_fid *fid, struct cifsInodeInfo *cinode) @@ -4638,6 +4665,10 @@ static void smb2_decrypt_offload(struct work_struct *work) #ifdef CONFIG_CIFS_STATS2 mid->when_received = jiffies; #endif + if (dw->server->ops->is_network_name_deleted) + dw->server->ops->is_network_name_deleted(dw->buf, + dw->server); + mid->callback(mid); } else { spin_lock(&GlobalMid_Lock); @@ -4756,6 +4787,12 @@ non_offloaded_decrypt: rc = handle_read_data(server, *mid, buf, server->vals->read_rsp_size, pages, npages, len, false); + if (rc >= 0) { + if (server->ops->is_network_name_deleted) { + server->ops->is_network_name_deleted(buf, + server); + } + } } free_pages: @@ -5105,6 +5142,7 @@ struct smb_version_operations smb20_operations = { .fiemap = smb3_fiemap, .llseek = smb3_llseek, .is_status_io_timeout = smb2_is_status_io_timeout, + .is_network_name_deleted = smb2_is_network_name_deleted, }; struct smb_version_operations smb21_operations = { @@ -5206,6 +5244,7 @@ struct smb_version_operations smb21_operations = { .fiemap = smb3_fiemap, .llseek = smb3_llseek, .is_status_io_timeout = smb2_is_status_io_timeout, + .is_network_name_deleted = smb2_is_network_name_deleted, }; struct smb_version_operations smb30_operations = { @@ -5319,6 +5358,7 @@ struct smb_version_operations smb30_operations = { .fiemap = smb3_fiemap, .llseek = smb3_llseek, .is_status_io_timeout = smb2_is_status_io_timeout, + .is_network_name_deleted = smb2_is_network_name_deleted, }; struct smb_version_operations smb311_operations = { @@ -5432,6 +5472,7 @@ struct smb_version_operations smb311_operations = { .fiemap = smb3_fiemap, .llseek = smb3_llseek, .is_status_io_timeout = smb2_is_status_io_timeout, + .is_network_name_deleted = smb2_is_network_name_deleted, }; struct smb_version_values smb20_values = { -- cgit v1.2.3 From f1a08655cc4f6f2233448e11e2499321728f0849 Mon Sep 17 00:00:00 2001 From: Steve French Date: Sat, 20 Feb 2021 18:52:15 -0600 Subject: cifs: minor simplification to smb2_is_network_name_deleted Trivial change to clarify code in smb2_is_network_name_deleted Suggested-by: Pavel Shilovsky Reviewed-by: Pavel Shilovsky Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/smb2ops.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index 807ecd444c16..f5087295424c 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -2459,23 +2459,24 @@ smb2_is_network_name_deleted(char *buf, struct TCP_Server_Info *server) struct cifs_ses *ses; struct cifs_tcon *tcon; - if (shdr->Status == STATUS_NETWORK_NAME_DELETED) { - spin_lock(&cifs_tcp_ses_lock); - list_for_each(tmp, &server->smb_ses_list) { - ses = list_entry(tmp, struct cifs_ses, smb_ses_list); - list_for_each(tmp1, &ses->tcon_list) { - tcon = list_entry(tmp1, struct cifs_tcon, tcon_list); - if (tcon->tid == shdr->TreeId) { - tcon->need_reconnect = true; - spin_unlock(&cifs_tcp_ses_lock); - pr_warn_once("Server share %s deleted.\n", - tcon->treeName); - return; - } + if (shdr->Status != STATUS_NETWORK_NAME_DELETED) + return; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each(tmp, &server->smb_ses_list) { + ses = list_entry(tmp, struct cifs_ses, smb_ses_list); + list_for_each(tmp1, &ses->tcon_list) { + tcon = list_entry(tmp1, struct cifs_tcon, tcon_list); + if (tcon->tid == shdr->TreeId) { + tcon->need_reconnect = true; + spin_unlock(&cifs_tcp_ses_lock); + pr_warn_once("Server share %s deleted.\n", + tcon->treeName); + return; } } - spin_unlock(&cifs_tcp_ses_lock); } + spin_unlock(&cifs_tcp_ses_lock); } static int -- cgit v1.2.3 From 9652c73246b980b9f2387916c35e02638d163472 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 31 Jan 2021 14:40:40 -0500 Subject: 9p: fix misuse of sscanf() in v9fs_stat2inode() 1) sscanf() return value needs to be checked, damnit 2) sscanf() is perfectly capable of checking for fixed prefix, no need for that %13s + strncmp with constant string. 3) st->extension is a valid string; no need for voodoo with str*cpy() there. Signed-off-by: Al Viro --- fs/9p/vfs_inode.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index 4a937fac1acb..58f6b56ef145 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -1137,9 +1137,6 @@ v9fs_stat2inode(struct p9_wstat *stat, struct inode *inode, struct super_block *sb, unsigned int flags) { umode_t mode; - char ext[32]; - char tag_name[14]; - unsigned int i_nlink; struct v9fs_session_info *v9ses = sb->s_fs_info; struct v9fs_inode *v9inode = V9FS_I(inode); @@ -1157,18 +1154,18 @@ v9fs_stat2inode(struct p9_wstat *stat, struct inode *inode, inode->i_gid = stat->n_gid; } if ((S_ISREG(inode->i_mode)) || (S_ISDIR(inode->i_mode))) { - if (v9fs_proto_dotu(v9ses) && (stat->extension[0] != '\0')) { + if (v9fs_proto_dotu(v9ses)) { + unsigned int i_nlink; /* - * Hadlink support got added later to - * to the .u extension. So there can be - * server out there that doesn't support - * this even with .u extension. So check - * for non NULL stat->extension + * Hadlink support got added later to the .u extension. + * So there can be a server out there that doesn't + * support this even with .u extension. That would + * just leave us with stat->extension being an empty + * string, though. */ - strlcpy(ext, stat->extension, sizeof(ext)); /* HARDLINKCOUNT %u */ - sscanf(ext, "%13s %u", tag_name, &i_nlink); - if (!strncmp(tag_name, "HARDLINKCOUNT", 13)) + if (sscanf(stat->extension, + " HARDLINKCOUNT %u", &i_nlink) == 1) set_nlink(inode, i_nlink); } } -- cgit v1.2.3 From 6f24784f00f2b5862b367caeecc5cca22a77faa3 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 31 Jan 2021 19:23:55 -0500 Subject: whack-a-mole: don't open-code iminor/imajor several instances creeped back into the tree... Signed-off-by: Al Viro --- arch/sh/boards/mach-landisk/gio.c | 6 ++---- drivers/block/loop.c | 2 +- drivers/dax/super.c | 2 +- drivers/rtc/rtc-m41t80.c | 4 ++-- drivers/s390/char/vmur.c | 2 +- drivers/staging/vme/devices/vme_user.c | 12 ++++++------ fs/gfs2/inode.c | 4 ++-- 7 files changed, 15 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/arch/sh/boards/mach-landisk/gio.c b/arch/sh/boards/mach-landisk/gio.c index 1c0da99dfc60..ff2200fec29a 100644 --- a/arch/sh/boards/mach-landisk/gio.c +++ b/arch/sh/boards/mach-landisk/gio.c @@ -27,11 +27,10 @@ static int openCnt; static int gio_open(struct inode *inode, struct file *filp) { - int minor; + int minor = iminor(inode); int ret = -ENOENT; preempt_disable(); - minor = MINOR(inode->i_rdev); if (minor < DEVCOUNT) { if (openCnt > 0) { ret = -EALREADY; @@ -46,9 +45,8 @@ static int gio_open(struct inode *inode, struct file *filp) static int gio_close(struct inode *inode, struct file *filp) { - int minor; + int minor = iminor(inode); - minor = MINOR(inode->i_rdev); if (minor < DEVCOUNT) { openCnt--; } diff --git a/drivers/block/loop.c b/drivers/block/loop.c index e5ff328f0917..b51330017ce1 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -663,7 +663,7 @@ static inline int is_loop_device(struct file *file) { struct inode *i = file->f_mapping->host; - return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR; + return i && S_ISBLK(i->i_mode) && imajor(i) == LOOP_MAJOR; } static int loop_validate_file(struct file *file, struct block_device *bdev) diff --git a/drivers/dax/super.c b/drivers/dax/super.c index cadbd0a1a1ef..5fa6ae9dbc8b 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -480,7 +480,7 @@ static void dax_free_inode(struct inode *inode) kfree(dax_dev->host); dax_dev->host = NULL; if (inode->i_rdev) - ida_simple_remove(&dax_minor_ida, MINOR(inode->i_rdev)); + ida_simple_remove(&dax_minor_ida, iminor(inode)); kmem_cache_free(dax_cache, dax_dev); } diff --git a/drivers/rtc/rtc-m41t80.c b/drivers/rtc/rtc-m41t80.c index 160dcf68e64e..1e5873261e7e 100644 --- a/drivers/rtc/rtc-m41t80.c +++ b/drivers/rtc/rtc-m41t80.c @@ -783,7 +783,7 @@ static long wdt_unlocked_ioctl(struct file *file, unsigned int cmd, */ static int wdt_open(struct inode *inode, struct file *file) { - if (MINOR(inode->i_rdev) == WATCHDOG_MINOR) { + if (iminor(inode) == WATCHDOG_MINOR) { mutex_lock(&m41t80_rtc_mutex); if (test_and_set_bit(0, &wdt_is_open)) { mutex_unlock(&m41t80_rtc_mutex); @@ -807,7 +807,7 @@ static int wdt_open(struct inode *inode, struct file *file) */ static int wdt_release(struct inode *inode, struct file *file) { - if (MINOR(inode->i_rdev) == WATCHDOG_MINOR) + if (iminor(inode) == WATCHDOG_MINOR) clear_bit(0, &wdt_is_open); return 0; } diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c index 1bbf27b98cf6..68f49e2e964c 100644 --- a/drivers/s390/char/vmur.c +++ b/drivers/s390/char/vmur.c @@ -681,7 +681,7 @@ static int ur_open(struct inode *inode, struct file *file) * We treat the minor number as the devno of the ur device * to find in the driver tree. */ - devno = MINOR(file_inode(file)->i_rdev); + devno = iminor(file_inode(file)); urd = urdev_get_from_devno(devno); if (!urd) { diff --git a/drivers/staging/vme/devices/vme_user.c b/drivers/staging/vme/devices/vme_user.c index fd0ea4dbcb91..568698fc3d3f 100644 --- a/drivers/staging/vme/devices/vme_user.c +++ b/drivers/staging/vme/devices/vme_user.c @@ -175,7 +175,7 @@ static ssize_t buffer_from_user(unsigned int minor, const char __user *buf, static ssize_t vme_user_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { - unsigned int minor = MINOR(file_inode(file)->i_rdev); + unsigned int minor = iminor(file_inode(file)); ssize_t retval; size_t image_size; @@ -218,7 +218,7 @@ static ssize_t vme_user_read(struct file *file, char __user *buf, size_t count, static ssize_t vme_user_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - unsigned int minor = MINOR(file_inode(file)->i_rdev); + unsigned int minor = iminor(file_inode(file)); ssize_t retval; size_t image_size; @@ -260,7 +260,7 @@ static ssize_t vme_user_write(struct file *file, const char __user *buf, static loff_t vme_user_llseek(struct file *file, loff_t off, int whence) { - unsigned int minor = MINOR(file_inode(file)->i_rdev); + unsigned int minor = iminor(file_inode(file)); size_t image_size; loff_t res; @@ -294,7 +294,7 @@ static int vme_user_ioctl(struct inode *inode, struct file *file, struct vme_slave slave; struct vme_irq_id irq_req; unsigned long copied; - unsigned int minor = MINOR(inode->i_rdev); + unsigned int minor = iminor(inode); int retval; dma_addr_t pci_addr; void __user *argp = (void __user *)arg; @@ -412,7 +412,7 @@ vme_user_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { int ret; struct inode *inode = file_inode(file); - unsigned int minor = MINOR(inode->i_rdev); + unsigned int minor = iminor(inode); mutex_lock(&image[minor].mutex); ret = vme_user_ioctl(inode, file, cmd, arg); @@ -481,7 +481,7 @@ static int vme_user_master_mmap(unsigned int minor, struct vm_area_struct *vma) static int vme_user_mmap(struct file *file, struct vm_area_struct *vma) { - unsigned int minor = MINOR(file_inode(file)->i_rdev); + unsigned int minor = iminor(file_inode(file)); if (type[minor] == MASTER_MINOR) return vme_user_master_mmap(minor, vma); diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index c1b77e8d6b1c..6cabe5bba1c8 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -490,8 +490,8 @@ static void init_dinode(struct gfs2_inode *dip, struct gfs2_inode *ip, di = (struct gfs2_dinode *)dibh->b_data; gfs2_dinode_out(ip, di); - di->di_major = cpu_to_be32(MAJOR(ip->i_inode.i_rdev)); - di->di_minor = cpu_to_be32(MINOR(ip->i_inode.i_rdev)); + di->di_major = cpu_to_be32(imajor(&ip->i_inode)); + di->di_minor = cpu_to_be32(iminor(&ip->i_inode)); di->__pad1 = 0; di->__pad2 = 0; di->__pad3 = 0; -- cgit v1.2.3 From 9c7d83ae6ba67d6c6199cce24573983db3b56332 Mon Sep 17 00:00:00 2001 From: Tetsuo Handa Date: Sun, 14 Feb 2021 12:13:07 +0900 Subject: pstore: Fix warning in pstore_kill_sb() syzbot is hitting WARN_ON(pstore_sb != sb) at pstore_kill_sb() [1], for the assumption that pstore_sb != NULL is wrong because pstore_fill_super() will not assign pstore_sb = sb when new_inode() for d_make_root() returned NULL (due to memory allocation fault injection). Since mount_single() calls pstore_kill_sb() when pstore_fill_super() failed, pstore_kill_sb() needs to be aware of such failure path. [1] https://syzkaller.appspot.com/bug?id=6abacb8da5137cb47a416f2bef95719ed60508a0 Reported-by: syzbot Signed-off-by: Tetsuo Handa Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20210214031307.57903-1-penguin-kernel@I-love.SAKURA.ne.jp --- fs/pstore/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/pstore/inode.c b/fs/pstore/inode.c index 93a217e4f563..14658b009f1b 100644 --- a/fs/pstore/inode.c +++ b/fs/pstore/inode.c @@ -467,7 +467,7 @@ static struct dentry *pstore_mount(struct file_system_type *fs_type, static void pstore_kill_sb(struct super_block *sb) { mutex_lock(&pstore_sb_lock); - WARN_ON(pstore_sb != sb); + WARN_ON(pstore_sb && pstore_sb != sb); kill_litter_super(sb); pstore_sb = NULL; -- cgit v1.2.3 From 17d77684088510df84ff8285982d0eed52cd5890 Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Thu, 18 Feb 2021 09:07:21 -0500 Subject: gfs2: Don't get stuck with I/O plugged in gfs2_ail1_flush In gfs2_ail1_flush, we're using I/O plugging to give the block layer a better chance of merging I/O requests. If we're too aggressive here, we can end up waiting on I/O to complete while still plugged. Fix that in a way similar to writeback_sb_inodes, except that we can't use blk_flush_plug because blk_flush_plug_list is not exported. Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index a6fbde9f609f..16937ebb2a3e 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -91,7 +91,7 @@ void gfs2_remove_from_ail(struct gfs2_bufdata *bd) static int gfs2_ail1_start_one(struct gfs2_sbd *sdp, struct writeback_control *wbc, - struct gfs2_trans *tr) + struct gfs2_trans *tr, struct blk_plug *plug) __releases(&sdp->sd_ail_lock) __acquires(&sdp->sd_ail_lock) { @@ -133,6 +133,11 @@ __acquires(&sdp->sd_ail_lock) continue; spin_unlock(&sdp->sd_ail_lock); ret = generic_writepages(mapping, wbc); + if (need_resched()) { + blk_finish_plug(plug); + cond_resched(); + blk_start_plug(plug); + } spin_lock(&sdp->sd_ail_lock); if (ret == -ENODATA) /* if a jdata write into a new hole */ ret = 0; /* ignore it */ @@ -207,7 +212,7 @@ restart: list_for_each_entry_reverse(tr, head, tr_list) { if (wbc->nr_to_write <= 0) break; - ret = gfs2_ail1_start_one(sdp, wbc, tr); + ret = gfs2_ail1_start_one(sdp, wbc, tr, &plug); if (ret) { if (ret == -EBUSY) goto restart; -- cgit v1.2.3 From 37d1e2e3642e2380750d7f35279180826f29660e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 17 Feb 2021 21:03:43 -0700 Subject: io_uring: move SQPOLL thread io-wq forked worker Don't use a kthread for SQPOLL, use a forked worker just like the io-wq workers. With that done, we can drop the various context grabbing we do for SQPOLL, it already has everything it needs. Signed-off-by: Jens Axboe --- fs/io_uring.c | 473 ++++++++++++++++++++++------------------------------------ 1 file changed, 181 insertions(+), 292 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6d851033e48d..378cf79e66c9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -57,7 +57,6 @@ #include #include #include -#include #include #include #include @@ -254,6 +253,11 @@ struct io_restriction { bool registered; }; +enum { + IO_SQ_THREAD_SHOULD_STOP = 0, + IO_SQ_THREAD_SHOULD_PARK, +}; + struct io_sq_data { refcount_t refs; struct mutex lock; @@ -267,6 +271,13 @@ struct io_sq_data { struct wait_queue_head wait; unsigned sq_thread_idle; + int sq_cpu; + pid_t task_pid; + + unsigned long state; + struct completion startup; + struct completion completion; + struct completion exited; }; #define IO_IOPOLL_BATCH 8 @@ -367,18 +378,13 @@ struct io_ring_ctx { struct io_rings *rings; /* - * For SQPOLL usage - we hold a reference to the parent task, so we - * have access to the ->files + * For SQPOLL usage */ struct task_struct *sqo_task; /* Only used for accounting purposes */ struct mm_struct *mm_account; -#ifdef CONFIG_BLK_CGROUP - struct cgroup_subsys_state *sqo_blkcg_css; -#endif - struct io_sq_data *sq_data; /* if using sq thread polling */ struct wait_queue_head sqo_sq_wait; @@ -398,13 +404,6 @@ struct io_ring_ctx { struct user_struct *user; - const struct cred *creds; - -#ifdef CONFIG_AUDIT - kuid_t loginuid; - unsigned int sessionid; -#endif - struct completion ref_comp; struct completion sq_thread_comp; @@ -989,6 +988,7 @@ static const struct io_op_def io_op_defs[] = { static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, struct files_struct *files); +static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx); static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node); static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node( struct io_ring_ctx *ctx); @@ -1100,118 +1100,6 @@ static bool io_match_task(struct io_kiocb *head, return false; } -static void io_sq_thread_drop_mm_files(void) -{ - struct files_struct *files = current->files; - struct mm_struct *mm = current->mm; - - if (mm) { - kthread_unuse_mm(mm); - mmput(mm); - current->mm = NULL; - } - if (files) { - struct nsproxy *nsproxy = current->nsproxy; - - task_lock(current); - current->files = NULL; - current->nsproxy = NULL; - task_unlock(current); - put_files_struct(files); - put_nsproxy(nsproxy); - } -} - -static int __io_sq_thread_acquire_files(struct io_ring_ctx *ctx) -{ - if (!current->files) { - struct files_struct *files; - struct nsproxy *nsproxy; - - task_lock(ctx->sqo_task); - files = ctx->sqo_task->files; - if (!files) { - task_unlock(ctx->sqo_task); - return -EOWNERDEAD; - } - atomic_inc(&files->count); - get_nsproxy(ctx->sqo_task->nsproxy); - nsproxy = ctx->sqo_task->nsproxy; - task_unlock(ctx->sqo_task); - - task_lock(current); - current->files = files; - current->nsproxy = nsproxy; - task_unlock(current); - } - return 0; -} - -static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx) -{ - struct mm_struct *mm; - - if (current->mm) - return 0; - - task_lock(ctx->sqo_task); - mm = ctx->sqo_task->mm; - if (unlikely(!mm || !mmget_not_zero(mm))) - mm = NULL; - task_unlock(ctx->sqo_task); - - if (mm) { - kthread_use_mm(mm); - return 0; - } - - return -EFAULT; -} - -static int __io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, - struct io_kiocb *req) -{ - int ret; - - ret = __io_sq_thread_acquire_mm(ctx); - if (unlikely(ret)) - return ret; - - ret = __io_sq_thread_acquire_files(ctx); - if (unlikely(ret)) - return ret; - - return 0; -} - -static inline int io_sq_thread_acquire_mm_files(struct io_ring_ctx *ctx, - struct io_kiocb *req) -{ - if (!(ctx->flags & IORING_SETUP_SQPOLL)) - return 0; - return __io_sq_thread_acquire_mm_files(ctx, req); -} - -static void io_sq_thread_associate_blkcg(struct io_ring_ctx *ctx, - struct cgroup_subsys_state **cur_css) - -{ -#ifdef CONFIG_BLK_CGROUP - /* puts the old one when swapping */ - if (*cur_css != ctx->sqo_blkcg_css) { - kthread_associate_blkcg(ctx->sqo_blkcg_css); - *cur_css = ctx->sqo_blkcg_css; - } -#endif -} - -static void io_sq_thread_unassociate_blkcg(void) -{ -#ifdef CONFIG_BLK_CGROUP - kthread_associate_blkcg(NULL); -#endif -} - static inline void req_set_fail_links(struct io_kiocb *req) { if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK) @@ -2132,15 +2020,11 @@ static void __io_req_task_submit(struct io_kiocb *req) /* ctx stays valid until unlock, even if we drop all ours ctx->refs */ mutex_lock(&ctx->uring_lock); - if (!ctx->sqo_dead && !(current->flags & PF_EXITING) && - !io_sq_thread_acquire_mm_files(ctx, req)) + if (!ctx->sqo_dead && !(current->flags & PF_EXITING)) __io_queue_sqe(req); else __io_req_task_cancel(req, -EFAULT); mutex_unlock(&ctx->uring_lock); - - if (ctx->flags & IORING_SETUP_SQPOLL) - io_sq_thread_drop_mm_files(); } static void io_req_task_submit(struct callback_head *cb) @@ -2604,7 +2488,6 @@ static bool io_rw_reissue(struct io_kiocb *req) { #ifdef CONFIG_BLOCK umode_t mode = file_inode(req->file)->i_mode; - int ret; if (!S_ISBLK(mode) && !S_ISREG(mode)) return false; @@ -2613,9 +2496,7 @@ static bool io_rw_reissue(struct io_kiocb *req) lockdep_assert_held(&req->ctx->uring_lock); - ret = io_sq_thread_acquire_mm_files(req->ctx, req); - - if (!ret && io_resubmit_prep(req)) { + if (io_resubmit_prep(req)) { refcount_inc(&req->refs); io_queue_async_work(req); return true; @@ -6461,9 +6342,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if (unlikely(req->opcode >= IORING_OP_LAST)) return -EINVAL; - if (unlikely(io_sq_thread_acquire_mm_files(ctx, req))) - return -EFAULT; - if (unlikely(!io_check_restriction(ctx, req, sqe_flags))) return -EACCES; @@ -6779,71 +6657,97 @@ static void io_sqd_init_new(struct io_sq_data *sqd) io_sqd_update_thread_idle(sqd); } +static bool io_sq_thread_should_stop(struct io_sq_data *sqd) +{ + return test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); +} + +static bool io_sq_thread_should_park(struct io_sq_data *sqd) +{ + return test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); +} + +static void io_sq_thread_parkme(struct io_sq_data *sqd) +{ + for (;;) { + /* + * TASK_PARKED is a special state; we must serialize against + * possible pending wakeups to avoid store-store collisions on + * task->state. + * + * Such a collision might possibly result in the task state + * changin from TASK_PARKED and us failing the + * wait_task_inactive() in kthread_park(). + */ + set_special_state(TASK_PARKED); + if (!test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) + break; + + /* + * Thread is going to call schedule(), do not preempt it, + * or the caller of kthread_park() may spend more time in + * wait_task_inactive(). + */ + preempt_disable(); + complete(&sqd->completion); + schedule_preempt_disabled(); + preempt_enable(); + } + __set_current_state(TASK_RUNNING); +} + static int io_sq_thread(void *data) { - struct cgroup_subsys_state *cur_css = NULL; - struct files_struct *old_files = current->files; - struct nsproxy *old_nsproxy = current->nsproxy; - const struct cred *old_cred = NULL; struct io_sq_data *sqd = data; struct io_ring_ctx *ctx; unsigned long timeout = 0; + char buf[TASK_COMM_LEN]; DEFINE_WAIT(wait); - task_lock(current); - current->files = NULL; - current->nsproxy = NULL; - task_unlock(current); + sprintf(buf, "iou-sqp-%d", sqd->task_pid); + set_task_comm(current, buf); + sqd->thread = current; + current->pf_io_worker = NULL; + + if (sqd->sq_cpu != -1) + set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu)); + else + set_cpus_allowed_ptr(current, cpu_online_mask); + current->flags |= PF_NO_SETAFFINITY; + + complete(&sqd->completion); - while (!kthread_should_stop()) { + wait_for_completion(&sqd->startup); + + while (!io_sq_thread_should_stop(sqd)) { int ret; bool cap_entries, sqt_spin, needs_sched; /* * Any changes to the sqd lists are synchronized through the - * kthread parking. This synchronizes the thread vs users, + * thread parking. This synchronizes the thread vs users, * the users are synchronized on the sqd->ctx_lock. */ - if (kthread_should_park()) { - kthread_parkme(); - /* - * When sq thread is unparked, in case the previous park operation - * comes from io_put_sq_data(), which means that sq thread is going - * to be stopped, so here needs to have a check. - */ - if (kthread_should_stop()) - break; + if (io_sq_thread_should_park(sqd)) { + io_sq_thread_parkme(sqd); + continue; } - if (unlikely(!list_empty(&sqd->ctx_new_list))) { io_sqd_init_new(sqd); timeout = jiffies + sqd->sq_thread_idle; } - + if (fatal_signal_pending(current)) + break; sqt_spin = false; cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { - if (current->cred != ctx->creds) { - if (old_cred) - revert_creds(old_cred); - old_cred = override_creds(ctx->creds); - } - io_sq_thread_associate_blkcg(ctx, &cur_css); -#ifdef CONFIG_AUDIT - current->loginuid = ctx->loginuid; - current->sessionid = ctx->sessionid; -#endif - ret = __io_sq_thread(ctx, cap_entries); if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) sqt_spin = true; - - io_sq_thread_drop_mm_files(); } if (sqt_spin || !time_after(jiffies, timeout)) { io_run_task_work(); - io_sq_thread_drop_mm_files(); cond_resched(); if (sqt_spin) timeout = jiffies + sqd->sq_thread_idle; @@ -6864,7 +6768,7 @@ static int io_sq_thread(void *data) } } - if (needs_sched && !kthread_should_park()) { + if (needs_sched && !io_sq_thread_should_park(sqd)) { list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); @@ -6877,22 +6781,21 @@ static int io_sq_thread(void *data) timeout = jiffies + sqd->sq_thread_idle; } - io_run_task_work(); - io_sq_thread_drop_mm_files(); + list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) + io_uring_cancel_sqpoll(ctx); - if (cur_css) - io_sq_thread_unassociate_blkcg(); - if (old_cred) - revert_creds(old_cred); - - task_lock(current); - current->files = old_files; - current->nsproxy = old_nsproxy; - task_unlock(current); + io_run_task_work(); - kthread_parkme(); + /* + * Clear thread under lock so that concurrent parks work correctly + */ + complete_all(&sqd->completion); + mutex_lock(&sqd->lock); + sqd->thread = NULL; + mutex_unlock(&sqd->lock); - return 0; + complete(&sqd->exited); + do_exit(0); } struct io_wait_queue { @@ -7182,20 +7085,73 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) return 0; } +static void io_sq_thread_unpark(struct io_sq_data *sqd) + __releases(&sqd->lock) +{ + if (!sqd->thread) + return; + if (sqd->thread == current) + return; + clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); + wake_up_state(sqd->thread, TASK_PARKED); + mutex_unlock(&sqd->lock); +} + +static bool io_sq_thread_park(struct io_sq_data *sqd) + __acquires(&sqd->lock) +{ + if (sqd->thread == current) + return true; + mutex_lock(&sqd->lock); + if (!sqd->thread) { + mutex_unlock(&sqd->lock); + return false; + } + set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); + wake_up_process(sqd->thread); + wait_for_completion(&sqd->completion); + return true; +} + +static void io_sq_thread_stop(struct io_sq_data *sqd) +{ + if (!sqd->thread) + return; + + set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); + WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)); + wake_up_process(sqd->thread); + wait_for_completion(&sqd->exited); +} + static void io_put_sq_data(struct io_sq_data *sqd) { if (refcount_dec_and_test(&sqd->refs)) { - /* - * The park is a bit of a work-around, without it we get - * warning spews on shutdown with SQPOLL set and affinity - * set to a single CPU. - */ + io_sq_thread_stop(sqd); + kfree(sqd); + } +} + +static void io_sq_thread_finish(struct io_ring_ctx *ctx) +{ + struct io_sq_data *sqd = ctx->sq_data; + + if (sqd) { if (sqd->thread) { - kthread_park(sqd->thread); - kthread_stop(sqd->thread); + wait_for_completion(&ctx->sq_thread_comp); + io_sq_thread_park(sqd); } - kfree(sqd); + mutex_lock(&sqd->ctx_lock); + list_del(&ctx->sqd_list); + io_sqd_update_thread_idle(sqd); + mutex_unlock(&sqd->ctx_lock); + + if (sqd->thread) + io_sq_thread_unpark(sqd); + + io_put_sq_data(sqd); + ctx->sq_data = NULL; } } @@ -7242,58 +7198,12 @@ static struct io_sq_data *io_get_sq_data(struct io_uring_params *p) mutex_init(&sqd->ctx_lock); mutex_init(&sqd->lock); init_waitqueue_head(&sqd->wait); + init_completion(&sqd->startup); + init_completion(&sqd->completion); + init_completion(&sqd->exited); return sqd; } -static void io_sq_thread_unpark(struct io_sq_data *sqd) - __releases(&sqd->lock) -{ - if (!sqd->thread) - return; - kthread_unpark(sqd->thread); - mutex_unlock(&sqd->lock); -} - -static void io_sq_thread_park(struct io_sq_data *sqd) - __acquires(&sqd->lock) -{ - if (!sqd->thread) - return; - mutex_lock(&sqd->lock); - kthread_park(sqd->thread); -} - -static void io_sq_thread_stop(struct io_ring_ctx *ctx) -{ - struct io_sq_data *sqd = ctx->sq_data; - - if (sqd) { - if (sqd->thread) { - /* - * We may arrive here from the error branch in - * io_sq_offload_create() where the kthread is created - * without being waked up, thus wake it up now to make - * sure the wait will complete. - */ - wake_up_process(sqd->thread); - wait_for_completion(&ctx->sq_thread_comp); - - io_sq_thread_park(sqd); - } - - mutex_lock(&sqd->ctx_lock); - list_del(&ctx->sqd_list); - io_sqd_update_thread_idle(sqd); - mutex_unlock(&sqd->ctx_lock); - - if (sqd->thread) - io_sq_thread_unpark(sqd); - - io_put_sq_data(sqd); - ctx->sq_data = NULL; - } -} - #if defined(CONFIG_UNIX) /* * Ensure the UNIX gc is aware of our file set, so we are certain that @@ -7969,17 +7879,20 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, if (!cpu_online(cpu)) goto err; - sqd->thread = kthread_create_on_cpu(io_sq_thread, sqd, - cpu, "io_uring-sq"); + sqd->sq_cpu = cpu; } else { - sqd->thread = kthread_create(io_sq_thread, sqd, - "io_uring-sq"); + sqd->sq_cpu = -1; } - if (IS_ERR(sqd->thread)) { - ret = PTR_ERR(sqd->thread); + + sqd->task_pid = current->pid; + current->flags |= PF_IO_WORKER; + ret = io_wq_fork_thread(io_sq_thread, sqd); + current->flags &= ~PF_IO_WORKER; + if (ret < 0) { sqd->thread = NULL; goto err; } + wait_for_completion(&sqd->completion); ret = io_uring_alloc_task_context(sqd->thread, ctx); if (ret) goto err; @@ -7991,7 +7904,7 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, return 0; err: - io_sq_thread_stop(ctx); + io_sq_thread_finish(ctx); return ret; } @@ -8000,7 +7913,7 @@ static void io_sq_offload_start(struct io_ring_ctx *ctx) struct io_sq_data *sqd = ctx->sq_data; if ((ctx->flags & IORING_SETUP_SQPOLL) && sqd->thread) - wake_up_process(sqd->thread); + complete(&sqd->startup); } static inline void __io_unaccount_mem(struct user_struct *user, @@ -8466,21 +8379,14 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) mutex_lock(&ctx->uring_lock); mutex_unlock(&ctx->uring_lock); - io_sq_thread_stop(ctx); + io_sq_thread_finish(ctx); io_sqe_buffers_unregister(ctx); - if (ctx->sqo_task) { - put_task_struct(ctx->sqo_task); - ctx->sqo_task = NULL; + if (ctx->mm_account) { mmdrop(ctx->mm_account); ctx->mm_account = NULL; } -#ifdef CONFIG_BLK_CGROUP - if (ctx->sqo_blkcg_css) - css_put(ctx->sqo_blkcg_css); -#endif - mutex_lock(&ctx->uring_lock); io_sqe_files_unregister(ctx); mutex_unlock(&ctx->uring_lock); @@ -8500,7 +8406,6 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) percpu_ref_exit(&ctx->refs); free_uid(ctx->user); - put_cred(ctx->creds); io_req_caches_free(ctx, NULL); kfree(ctx->cancel_hash); kfree(ctx); @@ -8793,12 +8698,15 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, struct files_struct *files) { struct task_struct *task = current; + bool did_park = false; if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { io_disable_sqo_submit(ctx); - task = ctx->sq_data->thread; - atomic_inc(&task->io_uring->in_idle); - io_sq_thread_park(ctx->sq_data); + did_park = io_sq_thread_park(ctx->sq_data); + if (did_park) { + task = ctx->sq_data->thread; + atomic_inc(&task->io_uring->in_idle); + } } io_cancel_defer_files(ctx, task, files); @@ -8807,7 +8715,7 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, if (!files) io_uring_try_cancel_requests(ctx, task, NULL); - if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { + if (did_park) { atomic_dec(&task->io_uring->in_idle); io_sq_thread_unpark(ctx->sq_data); } @@ -8907,14 +8815,17 @@ static s64 tctx_inflight(struct io_uring_task *tctx) static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) { + struct io_sq_data *sqd = ctx->sq_data; struct io_uring_task *tctx; s64 inflight; DEFINE_WAIT(wait); - if (!ctx->sq_data) + if (!sqd) return; - tctx = ctx->sq_data->thread->io_uring; io_disable_sqo_submit(ctx); + if (!io_sq_thread_park(sqd)) + return; + tctx = ctx->sq_data->thread->io_uring; atomic_inc(&tctx->in_idle); do { @@ -8935,6 +8846,7 @@ static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) finish_wait(&tctx->wait, &wait); } while (1); atomic_dec(&tctx->in_idle); + io_sq_thread_unpark(sqd); } /* @@ -9511,12 +9423,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, ctx->compat = in_compat_syscall(); ctx->limit_mem = !capable(CAP_IPC_LOCK); ctx->user = user; - ctx->creds = get_current_cred(); -#ifdef CONFIG_AUDIT - ctx->loginuid = current->loginuid; - ctx->sessionid = current->sessionid; -#endif - ctx->sqo_task = get_task_struct(current); + ctx->sqo_task = current; /* * This is just grabbed for accounting purposes. When a process exits, @@ -9527,24 +9434,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, mmgrab(current->mm); ctx->mm_account = current->mm; -#ifdef CONFIG_BLK_CGROUP - /* - * The sq thread will belong to the original cgroup it was inited in. - * If the cgroup goes offline (e.g. disabling the io controller), then - * issued bios will be associated with the closest cgroup later in the - * block layer. - */ - rcu_read_lock(); - ctx->sqo_blkcg_css = blkcg_css(); - ret = css_tryget_online(ctx->sqo_blkcg_css); - rcu_read_unlock(); - if (!ret) { - /* don't init against a dying cgroup, have the user try again */ - ctx->sqo_blkcg_css = NULL; - ret = -ENODEV; - goto err; - } -#endif ret = io_allocate_scq_urings(ctx, p); if (ret) goto err; -- cgit v1.2.3 From 7c977a58dc83366e488c217fd88b1469d242bee5 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 23 Feb 2021 19:17:35 -0700 Subject: io_uring: don't attempt IO reissue from the ring exit path If we're exiting the ring, just let the IO fail with -EAGAIN as nobody will care anyway. It's not the right context to reissue from. Cc: stable@vger.kernel.org Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index bf9ad810c621..275ad84e8227 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2839,6 +2839,13 @@ static bool io_rw_reissue(struct io_kiocb *req) return false; if ((req->flags & REQ_F_NOWAIT) || io_wq_current_is_worker()) return false; + /* + * If ref is dying, we might be running poll reap from the exit work. + * Don't attempt to reissue from that path, just let it fail with + * -EAGAIN. + */ + if (percpu_ref_is_dying(&req->ctx->refs)) + return false; lockdep_assert_held(&req->ctx->uring_lock); -- cgit v1.2.3 From e5547d2c5eb363bfac7632ba789ca834fa829650 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 23 Feb 2021 22:17:20 +0000 Subject: io_uring: fix locked_free_list caches_free() Don't forget to zero locked_free_nr, it's not a disaster but makes it attempting to flush it with extra locking when there is nothing in the list. Also, don't traverse a potentially long list freeing requests under spinlock, splice the list and do it afterwards. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 275ad84e8227..5c8e24274acf 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8708,6 +8708,7 @@ static void io_req_cache_free(struct list_head *list, struct task_struct *tsk) static void io_req_caches_free(struct io_ring_ctx *ctx, struct task_struct *tsk) { struct io_submit_state *submit_state = &ctx->submit_state; + struct io_comp_state *cs = &ctx->submit_state.comp; mutex_lock(&ctx->uring_lock); @@ -8717,12 +8718,13 @@ static void io_req_caches_free(struct io_ring_ctx *ctx, struct task_struct *tsk) submit_state->free_reqs = 0; } - io_req_cache_free(&submit_state->comp.free_list, NULL); - spin_lock_irq(&ctx->completion_lock); - io_req_cache_free(&submit_state->comp.locked_free_list, NULL); + list_splice_init(&cs->locked_free_list, &cs->free_list); + cs->locked_free_nr = 0; spin_unlock_irq(&ctx->completion_lock); + io_req_cache_free(&cs->free_list, NULL); + mutex_unlock(&ctx->uring_lock); } -- cgit v1.2.3 From 9e8d9e829c2142cf1d7756e9ed2e0b4c7569d84c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 15 Feb 2021 13:42:18 -0700 Subject: Revert "proc: don't allow async path resolution of /proc/thread-self components" This reverts commit 0d4370cfe36b7f1719123b621a4ec4d9c7a25f89. No longer needed, as the io-wq worker threads have the right identity. Signed-off-by: Jens Axboe --- fs/proc/self.c | 2 +- fs/proc/thread_self.c | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/proc/self.c b/fs/proc/self.c index a4012154e109..cc71ce3466dc 100644 --- a/fs/proc/self.c +++ b/fs/proc/self.c @@ -20,7 +20,7 @@ static const char *proc_self_get_link(struct dentry *dentry, * Not currently supported. Once we can inherit all of struct pid, * we can allow this. */ - if (current->flags & PF_IO_WORKER) + if (current->flags & PF_KTHREAD) return ERR_PTR(-EOPNOTSUPP); if (!tgid) diff --git a/fs/proc/thread_self.c b/fs/proc/thread_self.c index d56681d86d28..a553273fbd41 100644 --- a/fs/proc/thread_self.c +++ b/fs/proc/thread_self.c @@ -17,13 +17,6 @@ static const char *proc_thread_self_get_link(struct dentry *dentry, pid_t pid = task_pid_nr_ns(current, ns); char *name; - /* - * Not currently supported. Once we can inherit all of struct pid, - * we can allow this. - */ - if (current->flags & PF_IO_WORKER) - return ERR_PTR(-EOPNOTSUPP); - if (!pid) return ERR_PTR(-ENOENT); name = kmalloc(10 + 6 + 10 + 1, dentry ? GFP_KERNEL : GFP_ATOMIC); -- cgit v1.2.3 From 2587890b5e2892dfecaa5e5126bdac8076a4e6f7 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 15 Feb 2021 13:42:42 -0700 Subject: Revert "proc: don't allow async path resolution of /proc/self components" This reverts commit 8d4c3e76e3be11a64df95ddee52e99092d42fc19. No longer needed, as the io-wq worker threads have the right identity. Signed-off-by: Jens Axboe --- fs/proc/self.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'fs') diff --git a/fs/proc/self.c b/fs/proc/self.c index cc71ce3466dc..72cd69bcaf4a 100644 --- a/fs/proc/self.c +++ b/fs/proc/self.c @@ -16,13 +16,6 @@ static const char *proc_self_get_link(struct dentry *dentry, pid_t tgid = task_tgid_nr_ns(current, ns); char *name; - /* - * Not currently supported. Once we can inherit all of struct pid, - * we can allow this. - */ - if (current->flags & PF_KTHREAD) - return ERR_PTR(-EOPNOTSUPP); - if (!tgid) return ERR_PTR(-ENOENT); /* max length of unsigned int in decimal + NULL term */ -- cgit v1.2.3 From 1c0aa1fae1acb77c5f9917adb0e4cb4500b9f3a6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 20 Feb 2021 11:55:28 -0700 Subject: io_uring: flag new native workers with IORING_FEAT_NATIVE_WORKERS A few reasons to do this: - The naming of the manager and worker have changed. That's a user visible change, so makes sense to flag it. - Opening certain files that use ->signal (like /proc/self or /dev/tty) now works, and the flag tells the application upfront that this is the case. - Related to the above, using signalfd will now work as well. Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- include/uapi/linux/io_uring.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 378cf79e66c9..cf9a5fa1ad03 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9467,7 +9467,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL | IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED | - IORING_FEAT_EXT_ARG; + IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS; if (copy_to_user(params, p, sizeof(*p))) { ret = -EFAULT; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index ac4e1738a9af..2514eb6b1cf2 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -262,6 +262,7 @@ struct io_uring_params { #define IORING_FEAT_POLL_32BITS (1U << 6) #define IORING_FEAT_SQPOLL_NONFIXED (1U << 7) #define IORING_FEAT_EXT_ARG (1U << 8) +#define IORING_FEAT_NATIVE_WORKERS (1U << 9) /* * io_uring_register(2) opcodes and arguments -- cgit v1.2.3 From 728f13e730093d0b3a1317d2ada83c2538941f34 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 21 Feb 2021 16:02:53 -0700 Subject: io-wq: remove nr_process accounting We're now just using fork like we would from userspace, so there's no need to try and impose extra restrictions or accounting on the user side of things. That's already being done for us. That also means we don't have to pass in the user_struct anymore, that's correctly inherited through ->creds on fork. Signed-off-by: Jens Axboe --- fs/io-wq.c | 48 +----------------------------------------------- fs/io-wq.h | 2 -- fs/io_uring.c | 1 - 3 files changed, 1 insertion(+), 50 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index b0d09f60200b..b5ae8080a41e 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -172,8 +172,6 @@ static void io_worker_exit(struct io_worker *worker) worker->flags = 0; if (flags & IO_WORKER_F_RUNNING) atomic_dec(&acct->nr_running); - if (!(flags & IO_WORKER_F_BOUND)) - atomic_dec(&wqe->wq->user->processes); worker->flags = 0; preempt_enable(); @@ -299,12 +297,10 @@ static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker, worker->flags |= IO_WORKER_F_BOUND; wqe->acct[IO_WQ_ACCT_UNBOUND].nr_workers--; wqe->acct[IO_WQ_ACCT_BOUND].nr_workers++; - atomic_dec(&wqe->wq->user->processes); } else { worker->flags &= ~IO_WORKER_F_BOUND; wqe->acct[IO_WQ_ACCT_UNBOUND].nr_workers++; wqe->acct[IO_WQ_ACCT_BOUND].nr_workers--; - atomic_inc(&wqe->wq->user->processes); } io_wqe_inc_running(worker); } @@ -575,9 +571,6 @@ static int task_thread(void *data, int index) acct->nr_workers++; raw_spin_unlock_irq(&wqe->lock); - if (index == IO_WQ_ACCT_UNBOUND) - atomic_inc(&wq->user->processes); - io_wqe_worker(data); do_exit(0); } @@ -730,29 +723,6 @@ static int io_wq_manager(void *data) do_exit(0); } -static bool io_wq_can_queue(struct io_wqe *wqe, struct io_wqe_acct *acct, - struct io_wq_work *work) -{ - bool free_worker; - - if (!(work->flags & IO_WQ_WORK_UNBOUND)) - return true; - if (atomic_read(&acct->nr_running)) - return true; - - rcu_read_lock(); - free_worker = !hlist_nulls_empty(&wqe->free_list); - rcu_read_unlock(); - if (free_worker) - return true; - - if (atomic_read(&wqe->wq->user->processes) >= acct->max_workers && - !(capable(CAP_SYS_RESOURCE) || capable(CAP_SYS_ADMIN))) - return false; - - return true; -} - static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe) { struct io_wq *wq = wqe->wq; @@ -790,17 +760,6 @@ static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) int work_flags; unsigned long flags; - /* - * Do early check to see if we need a new unbound worker, and if we do, - * if we're allowed to do so. This isn't 100% accurate as there's a - * gap between this check and incrementing the value, but that's OK. - * It's close enough to not be an issue, fork() has the same delay. - */ - if (unlikely(!io_wq_can_queue(wqe, acct, work))) { - io_run_cancel(work, wqe); - return; - } - work_flags = work->flags; raw_spin_lock_irqsave(&wqe->lock, flags); io_wqe_insert_work(wqe, work); @@ -978,9 +937,6 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) wq->free_work = data->free_work; wq->do_work = data->do_work; - /* caller must already hold a reference to this */ - wq->user = data->user; - ret = -ENOMEM; for_each_node(node) { struct io_wqe *wqe; @@ -995,10 +951,8 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) wqe->node = alloc_node; wqe->acct[IO_WQ_ACCT_BOUND].max_workers = bounded; atomic_set(&wqe->acct[IO_WQ_ACCT_BOUND].nr_running, 0); - if (wq->user) { - wqe->acct[IO_WQ_ACCT_UNBOUND].max_workers = + wqe->acct[IO_WQ_ACCT_UNBOUND].max_workers = task_rlimit(current, RLIMIT_NPROC); - } atomic_set(&wqe->acct[IO_WQ_ACCT_UNBOUND].nr_running, 0); wqe->wq = wq; raw_spin_lock_init(&wqe->lock); diff --git a/fs/io-wq.h b/fs/io-wq.h index 23f6cbd620f8..86825673be08 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -94,8 +94,6 @@ typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *); typedef void (io_wq_work_fn)(struct io_wq_work *); struct io_wq_data { - struct user_struct *user; - io_wq_work_fn *do_work; free_work_fn *free_work; }; diff --git a/fs/io_uring.c b/fs/io_uring.c index cf9a5fa1ad03..49a636d291cf 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7767,7 +7767,6 @@ static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx) struct io_wq_data data; unsigned int concurrency; - data.user = ctx->user; data.free_work = io_free_work; data.do_work = io_wq_submit_work; -- cgit v1.2.3 From 62e398be275a6c6efefe117b8960ae4e40e047cd Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 21 Feb 2021 16:19:37 -0700 Subject: io_uring: cleanup ->user usage At this point we're only using it for memory accounting, so there's no need to have an extra ->limit_mem - we can just set ->user if we do the accounting, or leave it at NULL if we don't. Signed-off-by: Jens Axboe --- fs/io_uring.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 49a636d291cf..e62ad6bde569 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -334,7 +334,6 @@ struct io_ring_ctx { struct { unsigned int flags; unsigned int compat: 1; - unsigned int limit_mem: 1; unsigned int cq_overflow_flushed: 1; unsigned int drain_next: 1; unsigned int eventfd_async: 1; @@ -7230,7 +7229,7 @@ static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset) skb->sk = sk; nr_files = 0; - fpl->user = get_uid(ctx->user); + fpl->user = get_uid(current_user()); for (i = 0; i < nr; i++) { struct file *file = io_file_from_index(ctx, i + offset); @@ -7942,7 +7941,7 @@ static inline int __io_account_mem(struct user_struct *user, static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) { - if (ctx->limit_mem) + if (ctx->user) __io_unaccount_mem(ctx->user, nr_pages); if (ctx->mm_account) @@ -7953,7 +7952,7 @@ static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages) { int ret; - if (ctx->limit_mem) { + if (ctx->user) { ret = __io_account_mem(ctx->user, nr_pages); if (ret) return ret; @@ -9370,7 +9369,6 @@ static struct file *io_uring_get_file(struct io_ring_ctx *ctx) static int io_uring_create(unsigned entries, struct io_uring_params *p, struct io_uring_params __user *params) { - struct user_struct *user = NULL; struct io_ring_ctx *ctx; struct file *file; int ret; @@ -9412,16 +9410,12 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, p->cq_entries = 2 * p->sq_entries; } - user = get_uid(current_user()); - ctx = io_ring_ctx_alloc(p); - if (!ctx) { - free_uid(user); + if (!ctx) return -ENOMEM; - } ctx->compat = in_compat_syscall(); - ctx->limit_mem = !capable(CAP_IPC_LOCK); - ctx->user = user; + if (!capable(CAP_IPC_LOCK)) + ctx->user = get_uid(current_user()); ctx->sqo_task = current; /* -- cgit v1.2.3 From 8a378fb096a7f02943c72a428bbfd0029260efb6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 23 Feb 2021 12:27:49 -0700 Subject: io_uring: ensure io-wq context is always destroyed for tasks If the task ends up doing no IO, the context list is empty and we don't call into __io_uring_files_cancel() when the task exits. This can cause a leak of the io-wq structures. Ensure we always call __io_uring_files_cancel(), even if the task context list is empty. Fixes: 5aa75ed5b93f ("io_uring: tie async worker side to the task context") Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 ++++--- include/linux/io_uring.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index e62ad6bde569..0a435a6f265a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8800,9 +8800,10 @@ void __io_uring_files_cancel(struct files_struct *files) if (files) { io_uring_remove_task_files(tctx); - } else if (tctx->io_wq && current->flags & PF_EXITING) { - io_wq_destroy(tctx->io_wq); - tctx->io_wq = NULL; + if (tctx->io_wq) { + io_wq_destroy(tctx->io_wq); + tctx->io_wq = NULL; + } } } diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index c48fcbdc2ea8..51ede771cd99 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -43,7 +43,7 @@ static inline void io_uring_task_cancel(void) } static inline void io_uring_files_cancel(struct files_struct *files) { - if (current->io_uring && !xa_empty(¤t->io_uring->xa)) + if (current->io_uring) __io_uring_files_cancel(files); } static inline void io_uring_free(struct task_struct *tsk) -- cgit v1.2.3 From 8b3e78b5955abb98863832453f5c74eca8f53c3a Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 23 Feb 2021 15:34:06 -0700 Subject: io-wq: fix races around manager/worker creation and task exit These races have always been there, they are just more apparent now that we do early cancel of io-wq when the task exits. Ensure that the io-wq manager sets task state correctly to not miss wakeups for task creation. This is important if we get a wakeup after having marked ourselves as TASK_INTERRUPTIBLE. If we do end up creating workers, then we flip the state back to running, making the subsequent schedule() a no-op. Also increment the wq ref count before forking the thread, to avoid a use-after-free. Signed-off-by: Jens Axboe --- fs/io-wq.c | 57 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 22 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index b5ae8080a41e..0ce5057c3bf7 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -605,6 +605,8 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) struct io_worker *worker; pid_t pid; + __set_current_state(TASK_RUNNING); + worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, wqe->node); if (!worker) return false; @@ -614,15 +616,18 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) worker->wqe = wqe; spin_lock_init(&worker->lock); + refcount_inc(&wq->refs); + if (index == IO_WQ_ACCT_BOUND) pid = io_wq_fork_thread(task_thread_bound, worker); else pid = io_wq_fork_thread(task_thread_unbound, worker); if (pid < 0) { + if (refcount_dec_and_test(&wq->refs)) + complete(&wq->done); kfree(worker); return false; } - refcount_inc(&wq->refs); return true; } @@ -668,6 +673,30 @@ static bool io_wq_worker_wake(struct io_worker *worker, void *data) return false; } +static void io_wq_check_workers(struct io_wq *wq) +{ + int node; + + for_each_node(node) { + struct io_wqe *wqe = wq->wqes[node]; + bool fork_worker[2] = { false, false }; + + if (!node_online(node)) + continue; + + raw_spin_lock_irq(&wqe->lock); + if (io_wqe_need_worker(wqe, IO_WQ_ACCT_BOUND)) + fork_worker[IO_WQ_ACCT_BOUND] = true; + if (io_wqe_need_worker(wqe, IO_WQ_ACCT_UNBOUND)) + fork_worker[IO_WQ_ACCT_UNBOUND] = true; + raw_spin_unlock_irq(&wqe->lock); + if (fork_worker[IO_WQ_ACCT_BOUND]) + create_io_worker(wq, wqe, IO_WQ_ACCT_BOUND); + if (fork_worker[IO_WQ_ACCT_UNBOUND]) + create_io_worker(wq, wqe, IO_WQ_ACCT_UNBOUND); + } +} + /* * Manager thread. Tasked with creating new workers, if we need them. */ @@ -684,30 +713,15 @@ static int io_wq_manager(void *data) complete(&wq->done); - while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { - for_each_node(node) { - struct io_wqe *wqe = wq->wqes[node]; - bool fork_worker[2] = { false, false }; - - if (!node_online(node)) - continue; - - raw_spin_lock_irq(&wqe->lock); - if (io_wqe_need_worker(wqe, IO_WQ_ACCT_BOUND)) - fork_worker[IO_WQ_ACCT_BOUND] = true; - if (io_wqe_need_worker(wqe, IO_WQ_ACCT_UNBOUND)) - fork_worker[IO_WQ_ACCT_UNBOUND] = true; - raw_spin_unlock_irq(&wqe->lock); - if (fork_worker[IO_WQ_ACCT_BOUND]) - create_io_worker(wq, wqe, IO_WQ_ACCT_BOUND); - if (fork_worker[IO_WQ_ACCT_UNBOUND]) - create_io_worker(wq, wqe, IO_WQ_ACCT_UNBOUND); - } + do { set_current_state(TASK_INTERRUPTIBLE); + io_wq_check_workers(wq); schedule_timeout(HZ); if (fatal_signal_pending(current)) set_bit(IO_WQ_BIT_EXIT, &wq->state); - } + } while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)); + + io_wq_check_workers(wq); if (refcount_dec_and_test(&wq->refs)) { complete(&wq->done); @@ -970,7 +984,6 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) current->flags &= ~PF_IO_WORKER; if (ret >= 0) { wait_for_completion(&wq->done); - reinit_completion(&wq->done); return wq; } -- cgit v1.2.3 From eb2de9418d56b5e6ebf27bad51dbce3e22ee109b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 23 Feb 2021 19:59:06 -0700 Subject: io-wq: fix race around io_worker grabbing There's a small window between lookup dropping the reference to the worker and calling wake_up_process() on the worker task, where the worker itself could have exited. We ensure that the worker struct itself is valid, but worker->task may very well be gone by the time we issue the wakeup. Fix the race by using a completion triggered by the reference going to zero, and having exit wait for that completion before proceeding. Signed-off-by: Jens Axboe --- fs/io-wq.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 0ce5057c3bf7..a53df2b3762a 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -56,6 +56,8 @@ struct io_worker { const struct cred *cur_creds; const struct cred *saved_creds; + struct completion ref_done; + struct rcu_head rcu; }; @@ -129,7 +131,7 @@ static bool io_worker_get(struct io_worker *worker) static void io_worker_release(struct io_worker *worker) { if (refcount_dec_and_test(&worker->ref)) - wake_up_process(worker->task); + complete(&worker->ref_done); } static inline struct io_wqe_acct *io_work_get_acct(struct io_wqe *wqe, @@ -157,14 +159,9 @@ static void io_worker_exit(struct io_worker *worker) struct io_wqe_acct *acct = io_wqe_get_acct(worker); unsigned flags; - /* - * If we're not at zero, someone else is holding a brief reference - * to the worker. Wait for that to go away. - */ - set_current_state(TASK_INTERRUPTIBLE); - if (!refcount_dec_and_test(&worker->ref)) - schedule(); - __set_current_state(TASK_RUNNING); + if (refcount_dec_and_test(&worker->ref)) + complete(&worker->ref_done); + wait_for_completion(&worker->ref_done); preempt_disable(); current->flags &= ~PF_IO_WORKER; @@ -615,6 +612,7 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) worker->nulls_node.pprev = NULL; worker->wqe = wqe; spin_lock_init(&worker->lock); + init_completion(&worker->ref_done); refcount_inc(&wq->refs); @@ -724,6 +722,7 @@ static int io_wq_manager(void *data) io_wq_check_workers(wq); if (refcount_dec_and_test(&wq->refs)) { + wq->manager = NULL; complete(&wq->done); do_exit(0); } @@ -734,6 +733,7 @@ static int io_wq_manager(void *data) io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); rcu_read_unlock(); } + wq->manager = NULL; do_exit(0); } -- cgit v1.2.3 From f1ebe48dd3490adeff63b2d86ac7567aa018bd5d Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Wed, 24 Feb 2021 15:04:02 +0000 Subject: cifs: If a corrupted DACL is returned by the server, bail out. Static code analysis reported a possible null pointer dereference in my last commit: cifs: Retain old ACEs when converting between mode bits and ACL. This could happen if the DACL returned by the server is corrupted. We were trying to continue by assuming that the file has empty DACL. We should bail out with an error instead. Signed-off-by: Shyam Prasad N Reported-by: Colin Ian King Reviewed-by: Rohith Surabattula Signed-off-by: Steve French --- fs/cifs/cifsacl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsacl.c b/fs/cifs/cifsacl.c index 0806ae784061..9d29eb9660c2 100644 --- a/fs/cifs/cifsacl.c +++ b/fs/cifs/cifsacl.c @@ -1275,8 +1275,8 @@ static int build_sec_desc(struct cifs_ntsd *pntsd, struct cifs_ntsd *pnntsd, if (dacloffset) { dacl_ptr = (struct cifs_acl *)((char *)pntsd + dacloffset); if (end_of_acl < (char *)dacl_ptr + le16_to_cpu(dacl_ptr->size)) { - cifs_dbg(VFS, "Existing ACL size is wrong. Discarding old ACL\n"); - dacl_ptr = NULL; + cifs_dbg(VFS, "Server returned illegal ACL size\n"); + return -EINVAL; } } -- cgit v1.2.3 From 89e0eb8c13bb842e224b27d7e071262cd84717cb Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 12 Feb 2021 09:14:47 -0800 Subject: xfs: restore speculative_cow_prealloc_lifetime sysctl In commit 9669f51de5c0 I tried to get rid of the undocumented cow gc lifetime knob. The knob's function was never documented and it now doesn't really have a function since eof and cow gc have been consolidated. Regrettably, xfs/231 relies on it and regresses on for-next. I did not succeed at getting far enough through fstests patch review for the fixup to land in time. Restore the sysctl knob, document what it did (does?), put it on the deprecation schedule, and rip out a redundant function. Fixes: 9669f51de5c0 ("xfs: consolidate the eofblocks and cowblocks workers") Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner Reviewed-by: Christoph Hellwig --- Documentation/admin-guide/xfs.rst | 16 ++++++++++------ fs/xfs/xfs_sysctl.c | 35 ++++++++++++++--------------------- 2 files changed, 24 insertions(+), 27 deletions(-) (limited to 'fs') diff --git a/Documentation/admin-guide/xfs.rst b/Documentation/admin-guide/xfs.rst index 6178153d3320..5422407a96d7 100644 --- a/Documentation/admin-guide/xfs.rst +++ b/Documentation/admin-guide/xfs.rst @@ -284,6 +284,9 @@ The following sysctls are available for the XFS filesystem: removes unused preallocation from clean inodes and releases the unused space back to the free pool. + fs.xfs.speculative_cow_prealloc_lifetime + This is an alias for speculative_prealloc_lifetime. + fs.xfs.error_level (Min: 0 Default: 3 Max: 11) A volume knob for error reporting when internal errors occur. This will generate detailed messages & backtraces for filesystem @@ -356,12 +359,13 @@ The following sysctls are available for the XFS filesystem: Deprecated Sysctls ================== -=========================== ================ - Name Removal Schedule -=========================== ================ -fs.xfs.irix_sgid_inherit September 2025 -fs.xfs.irix_symlink_mode September 2025 -=========================== ================ +=========================================== ================ + Name Removal Schedule +=========================================== ================ +fs.xfs.irix_sgid_inherit September 2025 +fs.xfs.irix_symlink_mode September 2025 +fs.xfs.speculative_cow_prealloc_lifetime September 2025 +=========================================== ================ Removed Sysctls diff --git a/fs/xfs/xfs_sysctl.c b/fs/xfs/xfs_sysctl.c index 145e06c47744..546a6cd96729 100644 --- a/fs/xfs/xfs_sysctl.c +++ b/fs/xfs/xfs_sysctl.c @@ -51,7 +51,7 @@ xfs_panic_mask_proc_handler( #endif /* CONFIG_PROC_FS */ STATIC int -xfs_deprecate_irix_sgid_inherit_proc_handler( +xfs_deprecated_dointvec_minmax( struct ctl_table *ctl, int write, void *buffer, @@ -59,24 +59,8 @@ xfs_deprecate_irix_sgid_inherit_proc_handler( loff_t *ppos) { if (write) { - printk_once(KERN_WARNING - "XFS: " "%s sysctl option is deprecated.\n", - ctl->procname); - } - return proc_dointvec_minmax(ctl, write, buffer, lenp, ppos); -} - -STATIC int -xfs_deprecate_irix_symlink_mode_proc_handler( - struct ctl_table *ctl, - int write, - void *buffer, - size_t *lenp, - loff_t *ppos) -{ - if (write) { - printk_once(KERN_WARNING - "XFS: " "%s sysctl option is deprecated.\n", + printk_ratelimited(KERN_WARNING + "XFS: %s sysctl option is deprecated.\n", ctl->procname); } return proc_dointvec_minmax(ctl, write, buffer, lenp, ppos); @@ -88,7 +72,7 @@ static struct ctl_table xfs_table[] = { .data = &xfs_params.sgid_inherit.val, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = xfs_deprecate_irix_sgid_inherit_proc_handler, + .proc_handler = xfs_deprecated_dointvec_minmax, .extra1 = &xfs_params.sgid_inherit.min, .extra2 = &xfs_params.sgid_inherit.max }, @@ -97,7 +81,7 @@ static struct ctl_table xfs_table[] = { .data = &xfs_params.symlink_mode.val, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = xfs_deprecate_irix_symlink_mode_proc_handler, + .proc_handler = xfs_deprecated_dointvec_minmax, .extra1 = &xfs_params.symlink_mode.min, .extra2 = &xfs_params.symlink_mode.max }, @@ -201,6 +185,15 @@ static struct ctl_table xfs_table[] = { .extra1 = &xfs_params.blockgc_timer.min, .extra2 = &xfs_params.blockgc_timer.max, }, + { + .procname = "speculative_cow_prealloc_lifetime", + .data = &xfs_params.blockgc_timer.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = xfs_deprecated_dointvec_minmax, + .extra1 = &xfs_params.blockgc_timer.min, + .extra2 = &xfs_params.blockgc_timer.max, + }, /* please keep this the last entry */ #ifdef CONFIG_PROC_FS { -- cgit v1.2.3 From 6bbf29010fa90a7ff22ff14e2875b4e6dea8d576 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Wed, 24 Feb 2021 12:00:27 -0800 Subject: ntfs: layout.h: delete duplicated words Drop the repeated words "the" and "in" in comments. Link: https://lkml.kernel.org/r/20210125194937.24627-1-rdunlap@infradead.org Signed-off-by: Randy Dunlap Acked-by: Anton Altaparmakov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ntfs/layout.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/ntfs/layout.h b/fs/ntfs/layout.h index 85422761ff43..5d4bf7a3259f 100644 --- a/fs/ntfs/layout.h +++ b/fs/ntfs/layout.h @@ -703,7 +703,7 @@ typedef struct { /* 14*/ le16 instance; /* The instance of this attribute record. This number is unique within this mft record (see MFT_RECORD/next_attribute_instance notes in - in mft.h for more details). */ + mft.h for more details). */ /* 16*/ union { /* Resident attributes. */ struct { @@ -1838,7 +1838,7 @@ typedef struct { * Also, each security descriptor is stored twice in the $SDS stream with a * fixed offset of 0x40000 bytes (256kib, the Windows cache manager's max size) * between them; i.e. if a SDS_ENTRY specifies an offset of 0x51d0, then the - * the first copy of the security descriptor will be at offset 0x51d0 in the + * first copy of the security descriptor will be at offset 0x51d0 in the * $SDS data stream and the second copy will be at offset 0x451d0. */ typedef struct { -- cgit v1.2.3 From 4dfe6bd94959222e18d512bdf15f6bf9edb9c27c Mon Sep 17 00:00:00 2001 From: Rustam Kovhaev Date: Wed, 24 Feb 2021 12:00:30 -0800 Subject: ntfs: check for valid standard information attribute Mounting a corrupted filesystem with NTFS resulted in a kernel crash. We should check for valid STANDARD_INFORMATION attribute offset and length before trying to access it Link: https://lkml.kernel.org/r/20210217155930.1506815-1-rkovhaev@gmail.com Link: https://syzkaller.appspot.com/bug?extid=c584225dabdea2f71969 Signed-off-by: Rustam Kovhaev Reported-by: syzbot+c584225dabdea2f71969@syzkaller.appspotmail.com Tested-by: syzbot+c584225dabdea2f71969@syzkaller.appspotmail.com Acked-by: Anton Altaparmakov Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ntfs/inode.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'fs') diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index 4435dbbc0b63..f5c058b3192c 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -629,6 +629,12 @@ static int ntfs_read_locked_inode(struct inode *vi) } a = ctx->attr; /* Get the standard information attribute value. */ + if ((u8 *)a + le16_to_cpu(a->data.resident.value_offset) + + le32_to_cpu(a->data.resident.value_length) > + (u8 *)ctx->mrec + vol->mft_record_size) { + ntfs_error(vi->i_sb, "Corrupt standard information attribute in inode."); + goto unm_err_out; + } si = (STANDARD_INFORMATION*)((u8*)a + le16_to_cpu(a->data.resident.value_offset)); -- cgit v1.2.3 From 6efb59499aff080e6a9f1485ff968918c30c5b0c Mon Sep 17 00:00:00 2001 From: Yi Li Date: Wed, 24 Feb 2021 12:00:34 -0800 Subject: ocfs2: remove redundant conditional before iput iput handles NULL pointers gracefully, so there's no need to check the pointer before the call. Link: https://lkml.kernel.org/r/20201231040535.4091761-1-yili@winhong.com Signed-off-by: Yi Li Acked-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ocfs2/super.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index 2febc76e9de7..079f8826993e 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -973,8 +973,6 @@ static void ocfs2_disable_quotas(struct ocfs2_super *osb) * quota files */ dquot_disable(sb, type, DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED); - if (!inode) - continue; iput(inode); } } -- cgit v1.2.3 From 95e126d650391696f7ba8d318634cc018df10ef9 Mon Sep 17 00:00:00 2001 From: guozh Date: Wed, 24 Feb 2021 12:00:38 -0800 Subject: ocfs2: clean up some definitions which are not used any more There are some definitions which is not used anymore in OCFS2 module, so as to be removed. Link: https://lkml.kernel.org/r/2021011916182284700534@chinatelecom.cn Signed-off-by: Guozhonghua Acked-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ocfs2/dlm/dlmast.c | 10 ---------- fs/ocfs2/dlm/dlmcommon.h | 4 ---- 2 files changed, 14 deletions(-) (limited to 'fs') diff --git a/fs/ocfs2/dlm/dlmast.c b/fs/ocfs2/dlm/dlmast.c index 6abaded3ff6b..70a10764f249 100644 --- a/fs/ocfs2/dlm/dlmast.c +++ b/fs/ocfs2/dlm/dlmast.c @@ -165,16 +165,6 @@ void __dlm_queue_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock) spin_unlock(&lock->spinlock); } -void dlm_queue_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock) -{ - BUG_ON(!dlm); - BUG_ON(!lock); - - spin_lock(&dlm->ast_lock); - __dlm_queue_bast(dlm, lock); - spin_unlock(&dlm->ast_lock); -} - static void dlm_update_lvb(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, struct dlm_lock *lock) { diff --git a/fs/ocfs2/dlm/dlmcommon.h b/fs/ocfs2/dlm/dlmcommon.h index c8a444622faa..58d57e25d384 100644 --- a/fs/ocfs2/dlm/dlmcommon.h +++ b/fs/ocfs2/dlm/dlmcommon.h @@ -17,10 +17,7 @@ #define DLM_LOCKID_NAME_MAX 32 -#define DLM_DOMAIN_NAME_MAX_LEN 255 #define DLM_LOCK_RES_OWNER_UNKNOWN O2NM_MAX_NODES -#define DLM_THREAD_SHUFFLE_INTERVAL 5 // flush everything every 5 passes -#define DLM_THREAD_MS 200 // flush at least every 200 ms #define DLM_HASH_SIZE_DEFAULT (1 << 17) #if DLM_HASH_SIZE_DEFAULT < PAGE_SIZE @@ -902,7 +899,6 @@ void __dlm_lockres_grab_inflight_worker(struct dlm_ctxt *dlm, struct dlm_lock_resource *res); void dlm_queue_ast(struct dlm_ctxt *dlm, struct dlm_lock *lock); -void dlm_queue_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock); void __dlm_queue_ast(struct dlm_ctxt *dlm, struct dlm_lock *lock); void __dlm_queue_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock); void dlm_do_local_ast(struct dlm_ctxt *dlm, -- cgit v1.2.3 From c57d117f2b2f2a19b570c36f2819ef8d8210af20 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 24 Feb 2021 12:00:41 -0800 Subject: ocfs2: fix a use after free on error The error handling in this function frees "reg" but it is still on the "o2hb_all_regions" list so it will lead to a use after freew. Joseph Qi points out that we need to clear the bit in the "o2hb_region_bitmap" as well Link: https://lkml.kernel.org/r/YBk4M6HUG8jB/jc7@mwanda Fixes: 1cf257f51191 ("ocfs2: fix memory leak") Signed-off-by: Dan Carpenter Reviewed-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ocfs2/cluster/heartbeat.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c index 0179a73a3fa2..12a7590601dd 100644 --- a/fs/ocfs2/cluster/heartbeat.c +++ b/fs/ocfs2/cluster/heartbeat.c @@ -2042,7 +2042,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g o2hb_nego_timeout_handler, reg, NULL, ®->hr_handler_list); if (ret) - goto free; + goto remove_item; ret = o2net_register_handler(O2HB_NEGO_APPROVE_MSG, reg->hr_key, sizeof(struct o2hb_nego_msg), @@ -2057,6 +2057,12 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g unregister_handler: o2net_unregister_handler_list(®->hr_handler_list); +remove_item: + spin_lock(&o2hb_live_lock); + list_del(®->hr_all_item); + if (o2hb_global_heartbeat_active()) + clear_bit(reg->hr_region_num, o2hb_region_bitmap); + spin_unlock(&o2hb_live_lock); free: kfree(reg); return ERR_PTR(ret); -- cgit v1.2.3 From 7c908aec34733408baa755613141a08b960d8eec Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Wed, 24 Feb 2021 12:00:45 -0800 Subject: ocfs2: simplify the calculation of variables Fix the following coccicheck warnings: fs/ocfs2/refcounttree.c:981:16-18: WARNING !A || A && B is equivalent to !A || B. Link: https://lkml.kernel.org/r/1612235424-80367-1-git-send-email-jiapeng.chong@linux.alibaba.com Signed-off-by: Jiapeng Chong Reported-by: Abaci Robot Reviewed-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ocfs2/refcounttree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index c26937824be1..c19a463fac55 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -978,7 +978,7 @@ static int ocfs2_get_refcount_cpos_end(struct ocfs2_caching_info *ci, return 0; } - if (!eb || (eb && !eb->h_next_leaf_blk)) { + if (!eb || !eb->h_next_leaf_blk) { /* * We are the last extent rec, so any high cpos should * be stored in this leaf refcount block. -- cgit v1.2.3 From 3d742d4b6ebb3348e1d478047cfb18b9b337b8df Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Wed, 24 Feb 2021 12:00:48 -0800 Subject: fs: delete repeated words in comments Delete duplicate words in fs/*.c. The doubled words that are being dropped are: that, be, the, in, and, for Link: https://lkml.kernel.org/r/20201224052810.25315-1-rdunlap@infradead.org Signed-off-by: Randy Dunlap Reviewed-by: Matthew Wilcox (Oracle) Cc: Alexander Viro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/block_dev.c | 2 +- fs/dcache.c | 4 ++-- fs/direct-io.c | 4 ++-- fs/exec.c | 4 ++-- fs/fhandle.c | 2 +- fs/pipe.c | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/block_dev.c b/fs/block_dev.c index ec26179c8062..91ff9bfd7c1a 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -1270,7 +1270,7 @@ rescan: return ret; } /* - * Only exported for for loop and dasd for historic reasons. Don't use in new + * Only exported for loop and dasd for historic reasons. Don't use in new * code! */ EXPORT_SYMBOL_GPL(bdev_disk_changed); diff --git a/fs/dcache.c b/fs/dcache.c index 799d9e4f0bcd..7d24ff7eb206 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -2176,8 +2176,8 @@ EXPORT_SYMBOL(d_obtain_root); * same inode, only the actual correct case is stored in the dcache for * case-insensitive filesystems. * - * For a case-insensitive lookup match and if the the case-exact dentry - * already exists in in the dcache, use it and return it. + * For a case-insensitive lookup match and if the case-exact dentry + * already exists in the dcache, use it and return it. * * If no entry exists with the exact case name, allocate new dentry with * the exact case, and return the spliced entry. diff --git a/fs/direct-io.c b/fs/direct-io.c index aa1083ecd623..0957e1bb8eb2 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -462,7 +462,7 @@ static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) * Wait for the next BIO to complete. Remove it and return it. NULL is * returned once all BIOs have been completed. This must only be called once * all bios have been issued so that dio->refcount can only decrease. This - * requires that that the caller hold a reference on the dio. + * requires that the caller hold a reference on the dio. */ static struct bio *dio_await_one(struct dio *dio) { @@ -1279,7 +1279,7 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode, if (retval == -ENOTBLK) { /* * The remaining part of the request will be - * be handled by buffered I/O when we return + * handled by buffered I/O when we return */ retval = 0; } diff --git a/fs/exec.c b/fs/exec.c index 6f3c02066ce3..18594f11c31f 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1455,7 +1455,7 @@ EXPORT_SYMBOL(finalize_exec); /* * Prepare credentials and lock ->cred_guard_mutex. * setup_new_exec() commits the new creds and drops the lock. - * Or, if exec fails before, free_bprm() should release ->cred and + * Or, if exec fails before, free_bprm() should release ->cred * and unlock. */ static int prepare_bprm_creds(struct linux_binprm *bprm) @@ -1841,7 +1841,7 @@ static int bprm_execve(struct linux_binprm *bprm, out: /* - * If past the point of no return ensure the the code never + * If past the point of no return ensure the code never * returns to the userspace process. Use an existing fatal * signal if present otherwise terminate the process with * SIGSEGV. diff --git a/fs/fhandle.c b/fs/fhandle.c index 01263ffbc4c0..ec6feeccc276 100644 --- a/fs/fhandle.c +++ b/fs/fhandle.c @@ -173,7 +173,7 @@ static int handle_to_path(int mountdirfd, struct file_handle __user *ufh, /* * With handle we don't look at the execute bit on the - * the directory. Ideally we would like CAP_DAC_SEARCH. + * directory. Ideally we would like CAP_DAC_SEARCH. * But we don't have that */ if (!capable(CAP_DAC_READ_SEARCH)) { diff --git a/fs/pipe.c b/fs/pipe.c index 39c96845a72f..bfd946a9ad01 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -171,7 +171,7 @@ EXPORT_SYMBOL(generic_pipe_buf_try_steal); * * Description: * This function grabs an extra reference to @buf. It's used in - * in the tee() system call, when we duplicate the buffers in one + * the tee() system call, when we duplicate the buffers in one * pipe into another. */ bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) -- cgit v1.2.3 From 93da400397445f1110b394caab5558d13971378e Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Wed, 24 Feb 2021 12:00:51 -0800 Subject: ramfs: support O_TMPFILE [akpm@linux-foundation.org: update inode_operations.tmpfile] Link: http://lkml.kernel.org/r/20190206073349.GA15311@avx2 Signed-off-by: Alexey Dobriyan Acked-by: Christian Brauner Cc: Al Viro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ramfs/inode.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'fs') diff --git a/fs/ramfs/inode.c b/fs/ramfs/inode.c index 3c2658c8fde0..9ebd17d7befb 100644 --- a/fs/ramfs/inode.c +++ b/fs/ramfs/inode.c @@ -151,6 +151,18 @@ static int ramfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, return error; } +static int ramfs_tmpfile(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + + inode = ramfs_get_inode(dir->i_sb, dir, mode, 0); + if (!inode) + return -ENOSPC; + d_tmpfile(dentry, inode); + return 0; +} + static const struct inode_operations ramfs_dir_inode_operations = { .create = ramfs_create, .lookup = simple_lookup, @@ -161,6 +173,7 @@ static const struct inode_operations ramfs_dir_inode_operations = { .rmdir = simple_rmdir, .mknod = ramfs_mknod, .rename = simple_rename, + .tmpfile = ramfs_tmpfile, }; /* -- cgit v1.2.3 From 1f7ef657740344541645349a8bece90cbff898f5 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Wed, 24 Feb 2021 12:01:42 -0800 Subject: mm/filemap: remove unused parameter and change to void type for replace_page_cache_page() Since commit 74d609585d8b ("page cache: Add and replace pages using the XArray") was merged, the replace_page_cache_page() can not fail and always return 0, we can remove the redundant return value and void it. Moreover remove the unused gfp_mask. Link: https://lkml.kernel.org/r/609c30e5274ba15d8b90c872fd0d8ac437a9b2bb.1610071401.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Cc: Matthew Wilcox Cc: Miklos Szeredi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/fuse/dev.c | 6 +----- include/linux/pagemap.h | 2 +- mm/filemap.c | 7 +------ 3 files changed, 3 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 588f8d1240aa..c6636b4c4ccf 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -844,11 +844,7 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep) if (WARN_ON(PageMlocked(oldpage))) goto out_fallback_unlock; - err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL); - if (err) { - unlock_page(newpage); - goto out_put_old; - } + replace_page_cache_page(oldpage, newpage); get_page(newpage); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index d5570deff400..74e466e5a2ba 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -757,7 +757,7 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); extern void delete_from_page_cache(struct page *page); extern void __delete_from_page_cache(struct page *page, void *shadow); -int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask); +void replace_page_cache_page(struct page *old, struct page *new); void delete_from_page_cache_batch(struct address_space *mapping, struct pagevec *pvec); diff --git a/mm/filemap.c b/mm/filemap.c index 6ff2a3fb0dc7..7dfed3454a2e 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -777,7 +777,6 @@ EXPORT_SYMBOL(file_write_and_wait_range); * replace_page_cache_page - replace a pagecache page with a new one * @old: page to be replaced * @new: page to replace with - * @gfp_mask: allocation mode * * This function replaces a page in the pagecache with a new one. On * success it acquires the pagecache reference for the new page and @@ -786,10 +785,8 @@ EXPORT_SYMBOL(file_write_and_wait_range); * caller must do that. * * The remove + add is atomic. This function cannot fail. - * - * Return: %0 */ -int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) +void replace_page_cache_page(struct page *old, struct page *new) { struct address_space *mapping = old->mapping; void (*freepage)(struct page *) = mapping->a_ops->freepage; @@ -824,8 +821,6 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) if (freepage) freepage(old); put_page(old); - - return 0; } EXPORT_SYMBOL_GPL(replace_page_cache_page); -- cgit v1.2.3 From 87fa0f3eb267eed966ee194907bc15376c1b758f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 24 Feb 2021 12:02:42 -0800 Subject: mm/filemap: rename generic_file_buffered_read to filemap_read Rename generic_file_buffered_read to match the naming of filemap_fault, also update the written parameter to a more descriptive name and improve the kerneldoc comment. Link: https://lkml.kernel.org/r/20210122160140.223228-18-willy@infradead.org Signed-off-by: Christoph Hellwig Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/btrfs/file.c | 2 +- include/linux/fs.h | 4 ++-- mm/filemap.c | 35 ++++++++++++++++------------------- 3 files changed, 19 insertions(+), 22 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index be9e3900cce8..bf2c51a9607a 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -3634,7 +3634,7 @@ static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) return ret; } - return generic_file_buffered_read(iocb, to, ret); + return filemap_read(iocb, to, ret); } const struct file_operations btrfs_file_operations = { diff --git a/include/linux/fs.h b/include/linux/fs.h index 418b772d6eca..ec8f3ddf4a6a 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3080,8 +3080,8 @@ extern ssize_t generic_write_checks(struct kiocb *, struct iov_iter *); extern int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count); extern int generic_file_rw_checks(struct file *file_in, struct file *file_out); -extern ssize_t generic_file_buffered_read(struct kiocb *iocb, - struct iov_iter *to, ssize_t already_read); +ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *to, + ssize_t already_read); extern ssize_t generic_file_read_iter(struct kiocb *, struct iov_iter *); extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *); extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *); diff --git a/mm/filemap.c b/mm/filemap.c index 1358cb061fd6..28c290da22c1 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2394,23 +2394,20 @@ err: } /** - * generic_file_buffered_read - generic file read routine - * @iocb: the iocb to read - * @iter: data destination - * @written: already copied + * filemap_read - Read data from the page cache. + * @iocb: The iocb to read. + * @iter: Destination for the data. + * @already_read: Number of bytes already read by the caller. * - * This is a generic file read routine, and uses the - * mapping->a_ops->readpage() function for the actual low-level stuff. + * Copies data from the page cache. If the data is not currently present, + * uses the readahead and readpage address_space operations to fetch it. * - * This is really ugly. But the goto's actually try to clarify some - * of the logic when it comes to error handling etc. - * - * Return: - * * total number of bytes copied, including those the were already @written - * * negative error code if nothing was copied + * Return: Total number of bytes copied, including those already read by + * the caller. If an error happens before any bytes are copied, returns + * a negative error number. */ -ssize_t generic_file_buffered_read(struct kiocb *iocb, - struct iov_iter *iter, ssize_t written) +ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter, + ssize_t already_read) { struct file *filp = iocb->ki_filp; struct file_ra_state *ra = &filp->f_ra; @@ -2437,7 +2434,7 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, * can no longer safely return -EIOCBQUEUED. Hence mark * an async read NOWAIT at that point. */ - if ((iocb->ki_flags & IOCB_WAITQ) && written) + if ((iocb->ki_flags & IOCB_WAITQ) && already_read) iocb->ki_flags |= IOCB_NOWAIT; error = filemap_get_pages(iocb, iter, &pvec); @@ -2497,7 +2494,7 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, copied = copy_page_to_iter(page, offset, bytes, iter); - written += copied; + already_read += copied; iocb->ki_pos += copied; ra->prev_pos = iocb->ki_pos; @@ -2514,9 +2511,9 @@ put_pages: file_accessed(filp); - return written ? written : error; + return already_read ? already_read : error; } -EXPORT_SYMBOL_GPL(generic_file_buffered_read); +EXPORT_SYMBOL_GPL(filemap_read); /** * generic_file_read_iter - generic filesystem read routine @@ -2591,7 +2588,7 @@ generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) goto out; } - retval = generic_file_buffered_read(iocb, iter, retval); + retval = filemap_read(iocb, iter, retval); out: return retval; } -- cgit v1.2.3 From 4ebd3aec3842662300979dacd6fb38e3e8edf7f4 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 24 Feb 2021 12:02:48 -0800 Subject: fs/buffer.c: add checking buffer head stat before clear clear_buffer_new() is used to clear buffer new stat. When PAGE_SIZE is 64K, most buffer heads in the list are not needed to clear. clear_buffer_new() has an enpensive atomic modification operation, Let's add checking buffer head before clear it as __block_write_begin_int does which is good for performance. Link: https://lkml.kernel.org/r/1612332890-57918-1-git-send-email-zhangshaokun@hisilicon.com Signed-off-by: Yang Guo Signed-off-by: Shaokun Zhang Cc: Alexander Viro Cc: Nick Piggin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/buffer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/buffer.c b/fs/buffer.c index 32647d2011df..f1c3a5b27a90 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2083,7 +2083,8 @@ static int __block_commit_write(struct inode *inode, struct page *page, set_buffer_uptodate(bh); mark_buffer_dirty(bh); } - clear_buffer_new(bh); + if (buffer_new(bh)) + clear_buffer_new(bh); block_start = block_end; bh = bh->b_this_page; -- cgit v1.2.3 From 69473e5de87389be6c0fa4a5d574a50c8f904fb3 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:23 -0800 Subject: mm: memcontrol: convert NR_ANON_THPS account to pages Currently we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. So we convert the NR_ANON_THPS account to pages. This patch is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Doing this also can make the unit of vmstat counters more unified. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. Link: https://lkml.kernel.org/r/20201228164110.2838-3-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Greg Kroah-Hartman Cc: Rafael. J. Wysocki Cc: Alexey Dobriyan Cc: Johannes Weiner Cc: Vladimir Davydov Cc: Hugh Dickins Cc: Shakeel Butt Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Feng Tang Cc: NeilBrown Cc: Joonsoo Kim Cc: Randy Dunlap Cc: Michal Hocko Cc: Pankaj Gupta Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/node.c | 15 +++++++++------ fs/proc/meminfo.c | 2 +- include/linux/mmzone.h | 13 +++++++++++++ mm/huge_memory.c | 3 ++- mm/memcontrol.c | 20 ++++++-------------- mm/page_alloc.c | 2 +- mm/rmap.c | 6 +++--- mm/vmstat.c | 11 +++++++++-- 8 files changed, 44 insertions(+), 28 deletions(-) (limited to 'fs') diff --git a/drivers/base/node.c b/drivers/base/node.c index 04f71c7bc3f8..6da0c3508bc9 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -461,8 +461,7 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(sunreclaimable) #ifdef CONFIG_TRANSPARENT_HUGEPAGE , - nid, K(node_page_state(pgdat, NR_ANON_THPS) * - HPAGE_PMD_NR), + nid, K(node_page_state(pgdat, NR_ANON_THPS)), nid, K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * @@ -519,10 +518,14 @@ static ssize_t node_read_vmstat(struct device *dev, sum_zone_numa_state(nid, i)); #endif - for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) - len += sysfs_emit_at(buf, len, "%s %lu\n", - node_stat_name(i), - node_page_state_pages(pgdat, i)); + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { + unsigned long pages = node_page_state_pages(pgdat, i); + + if (vmstat_item_print_in_thp(i)) + pages /= HPAGE_PMD_NR; + len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i), + pages); + } return len; } diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index d6fc74619625..a635c8a84ddf 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -129,7 +129,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) #ifdef CONFIG_TRANSPARENT_HUGEPAGE show_val_kb(m, "AnonHugePages: ", - global_node_page_state(NR_ANON_THPS) * HPAGE_PMD_NR); + global_node_page_state(NR_ANON_THPS)); show_val_kb(m, "ShmemHugePages: ", global_node_page_state(NR_SHMEM_THPS) * HPAGE_PMD_NR); show_val_kb(m, "ShmemPmdMapped: ", diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index b593316bff3d..67d50ef5dd20 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -209,6 +209,19 @@ enum node_stat_item { NR_VM_NODE_STAT_ITEMS }; +/* + * Returns true if the item should be printed in THPs (/proc/vmstat + * currently prints number of anon, file and shmem THPs. But the item + * is charged in pages). + */ +static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item) +{ + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) + return false; + + return item == NR_ANON_THPS; +} + /* * Returns true if the value is measured in bytes (most vmstat values are * measured in pages). This defines the API part, the internal representation diff --git a/mm/huge_memory.c b/mm/huge_memory.c index f655137536eb..3691e863070a 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2176,7 +2176,8 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, lock_page_memcg(page); if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { /* Last compound_mapcount is gone. */ - __dec_lruvec_page_state(page, NR_ANON_THPS); + __mod_lruvec_page_state(page, NR_ANON_THPS, + -HPAGE_PMD_NR); if (TestClearPageDoubleMap(page)) { /* No need in mapcount reference anymore */ for (i = 0; i < HPAGE_PMD_NR; i++) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e31e47e7bab2..b2405f049006 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1533,7 +1533,7 @@ static struct memory_stat memory_stats[] = { * on some architectures, the macro of HPAGE_PMD_SIZE is not * constant(e.g. powerpc). */ - { "anon_thp", 0, NR_ANON_THPS }, + { "anon_thp", PAGE_SIZE, NR_ANON_THPS }, { "file_thp", 0, NR_FILE_THPS }, { "shmem_thp", 0, NR_SHMEM_THPS }, #endif @@ -1566,8 +1566,7 @@ static int __init memory_stats_init(void) for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (memory_stats[i].idx == NR_ANON_THPS || - memory_stats[i].idx == NR_FILE_THPS || + if (memory_stats[i].idx == NR_FILE_THPS || memory_stats[i].idx == NR_SHMEM_THPS) memory_stats[i].ratio = HPAGE_PMD_SIZE; #endif @@ -4087,10 +4086,6 @@ static int memcg_stat_show(struct seq_file *m, void *v) if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) continue; nr = memcg_page_state_local(memcg, memcg1_stats[i]); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (memcg1_stats[i] == NR_ANON_THPS) - nr *= HPAGE_PMD_NR; -#endif seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE); } @@ -4121,10 +4116,6 @@ static int memcg_stat_show(struct seq_file *m, void *v) if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) continue; nr = memcg_page_state(memcg, memcg1_stats[i]); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (memcg1_stats[i] == NR_ANON_THPS) - nr *= HPAGE_PMD_NR; -#endif seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i], (u64)nr * PAGE_SIZE); } @@ -5652,10 +5643,11 @@ static int mem_cgroup_move_account(struct page *page, __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages); __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages); if (PageTransHuge(page)) { - __dec_lruvec_state(from_vec, NR_ANON_THPS); - __inc_lruvec_state(to_vec, NR_ANON_THPS); + __mod_lruvec_state(from_vec, NR_ANON_THPS, + -nr_pages); + __mod_lruvec_state(to_vec, NR_ANON_THPS, + nr_pages); } - } } else { __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ef5070fed76b..2e2e47f8714b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5587,7 +5587,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR), - K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), + K(node_page_state(pgdat, NR_ANON_THPS)), #endif K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), node_page_state(pgdat, NR_KERNEL_STACK_KB), diff --git a/mm/rmap.c b/mm/rmap.c index 08c56aaf72eb..c4d5c63cfd29 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1144,7 +1144,7 @@ void do_page_add_anon_rmap(struct page *page, * disabled. */ if (compound) - __inc_lruvec_page_state(page, NR_ANON_THPS); + __mod_lruvec_page_state(page, NR_ANON_THPS, nr); __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr); } @@ -1186,7 +1186,7 @@ void page_add_new_anon_rmap(struct page *page, if (hpage_pincount_available(page)) atomic_set(compound_pincount_ptr(page), 0); - __inc_lruvec_page_state(page, NR_ANON_THPS); + __mod_lruvec_page_state(page, NR_ANON_THPS, nr); } else { /* Anon THP always mapped first with PMD */ VM_BUG_ON_PAGE(PageTransCompound(page), page); @@ -1292,7 +1292,7 @@ static void page_remove_anon_compound_rmap(struct page *page) if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) return; - __dec_lruvec_page_state(page, NR_ANON_THPS); + __mod_lruvec_page_state(page, NR_ANON_THPS, -thp_nr_pages(page)); if (TestClearPageDoubleMap(page)) { /* diff --git a/mm/vmstat.c b/mm/vmstat.c index f8942160fc95..07dc0af50cf0 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1619,8 +1619,12 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, if (is_zone_first_populated(pgdat, zone)) { seq_printf(m, "\n per-node stats"); for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { + unsigned long pages = node_page_state_pages(pgdat, i); + + if (vmstat_item_print_in_thp(i)) + pages /= HPAGE_PMD_NR; seq_printf(m, "\n %-12s %lu", node_stat_name(i), - node_page_state_pages(pgdat, i)); + pages); } } seq_printf(m, @@ -1740,8 +1744,11 @@ static void *vmstat_start(struct seq_file *m, loff_t *pos) v += NR_VM_NUMA_STAT_ITEMS; #endif - for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { v[i] = global_node_page_state_pages(i); + if (vmstat_item_print_in_thp(i)) + v[i] /= HPAGE_PMD_NR; + } v += NR_VM_NODE_STAT_ITEMS; global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, -- cgit v1.2.3 From bf9ecead53c89d3d2cf60acbc460174ebbcf0027 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:27 -0800 Subject: mm: memcontrol: convert NR_FILE_THPS account to pages Currently we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with if hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. So we convert the NR_FILE_THPS account to pages. This patch is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Doing this also can make the unit of vmstat counters more unified. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. Link: https://lkml.kernel.org/r/20201228164110.2838-4-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Alexey Dobriyan Cc: Feng Tang Cc: Greg Kroah-Hartman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Michal Hocko Cc: NeilBrown Cc: Pankaj Gupta Cc: Rafael. J. Wysocki Cc: Randy Dunlap Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Shakeel Butt Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/node.c | 3 +-- fs/proc/meminfo.c | 2 +- include/linux/mmzone.h | 3 ++- mm/filemap.c | 2 +- mm/huge_memory.c | 5 ++++- mm/khugepaged.c | 4 +++- mm/memcontrol.c | 5 ++--- 7 files changed, 14 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/drivers/base/node.c b/drivers/base/node.c index 6da0c3508bc9..d5952f754911 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -466,8 +466,7 @@ static ssize_t node_read_meminfo(struct device *dev, HPAGE_PMD_NR), nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_FILE_THPS) * - HPAGE_PMD_NR), + nid, K(node_page_state(pgdat, NR_FILE_THPS)), nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) * HPAGE_PMD_NR) #endif diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index a635c8a84ddf..7ea4679880c8 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -135,7 +135,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "ShmemPmdMapped: ", global_node_page_state(NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR); show_val_kb(m, "FileHugePages: ", - global_node_page_state(NR_FILE_THPS) * HPAGE_PMD_NR); + global_node_page_state(NR_FILE_THPS)); show_val_kb(m, "FilePmdMapped: ", global_node_page_state(NR_FILE_PMDMAPPED) * HPAGE_PMD_NR); #endif diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 67d50ef5dd20..b751a9898bb6 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -219,7 +219,8 @@ static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item) if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) return false; - return item == NR_ANON_THPS; + return item == NR_ANON_THPS || + item == NR_FILE_THPS; } /* diff --git a/mm/filemap.c b/mm/filemap.c index 820d81901eae..9efe059e2b58 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -208,7 +208,7 @@ static void unaccount_page_cache_page(struct address_space *mapping, if (PageTransHuge(page)) __dec_lruvec_page_state(page, NR_SHMEM_THPS); } else if (PageTransHuge(page)) { - __dec_lruvec_page_state(page, NR_FILE_THPS); + __mod_lruvec_page_state(page, NR_FILE_THPS, -nr); filemap_nr_thps_dec(mapping); } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 3691e863070a..77181faa2340 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2752,10 +2752,13 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) } spin_unlock(&ds_queue->split_queue_lock); if (mapping) { + int nr = thp_nr_pages(head); + if (PageSwapBacked(head)) __dec_lruvec_page_state(head, NR_SHMEM_THPS); else - __dec_lruvec_page_state(head, NR_FILE_THPS); + __mod_lruvec_page_state(head, NR_FILE_THPS, + -nr); } __split_huge_page(page, list, end); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index fb0fdaec34d5..c427fe2ca7ff 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1643,6 +1643,7 @@ static void collapse_file(struct mm_struct *mm, XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); int nr_none = 0, result = SCAN_SUCCEED; bool is_shmem = shmem_file(file); + int nr; VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); @@ -1854,11 +1855,12 @@ out_unlock: put_page(page); goto xa_unlocked; } + nr = thp_nr_pages(new_page); if (is_shmem) __inc_lruvec_page_state(new_page, NR_SHMEM_THPS); else { - __inc_lruvec_page_state(new_page, NR_FILE_THPS); + __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); filemap_nr_thps_inc(mapping); } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b2405f049006..fc552f57d3fb 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1534,7 +1534,7 @@ static struct memory_stat memory_stats[] = { * constant(e.g. powerpc). */ { "anon_thp", PAGE_SIZE, NR_ANON_THPS }, - { "file_thp", 0, NR_FILE_THPS }, + { "file_thp", PAGE_SIZE, NR_FILE_THPS }, { "shmem_thp", 0, NR_SHMEM_THPS }, #endif { "inactive_anon", PAGE_SIZE, NR_INACTIVE_ANON }, @@ -1566,8 +1566,7 @@ static int __init memory_stats_init(void) for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (memory_stats[i].idx == NR_FILE_THPS || - memory_stats[i].idx == NR_SHMEM_THPS) + if (memory_stats[i].idx == NR_SHMEM_THPS) memory_stats[i].ratio = HPAGE_PMD_SIZE; #endif VM_BUG_ON(!memory_stats[i].ratio); -- cgit v1.2.3 From 57b2847d3c1dc154923578efb47a12302a57d700 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:31 -0800 Subject: mm: memcontrol: convert NR_SHMEM_THPS account to pages Currently we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. So we convert the NR_SHMEM_THPS account to pages. This patch is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Doing this also can make the unit of vmstat counters more unified. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. Link: https://lkml.kernel.org/r/20201228164110.2838-5-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Alexey Dobriyan Cc: Feng Tang Cc: Greg Kroah-Hartman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Michal Hocko Cc: NeilBrown Cc: Pankaj Gupta Cc: Rafael. J. Wysocki Cc: Randy Dunlap Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Shakeel Butt Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/node.c | 3 +-- fs/proc/meminfo.c | 2 +- include/linux/mmzone.h | 3 ++- mm/filemap.c | 2 +- mm/huge_memory.c | 3 ++- mm/khugepaged.c | 2 +- mm/memcontrol.c | 26 ++------------------------ mm/page_alloc.c | 2 +- mm/shmem.c | 2 +- 9 files changed, 12 insertions(+), 33 deletions(-) (limited to 'fs') diff --git a/drivers/base/node.c b/drivers/base/node.c index d5952f754911..6d5ac6ffb6e1 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -462,8 +462,7 @@ static ssize_t node_read_meminfo(struct device *dev, #ifdef CONFIG_TRANSPARENT_HUGEPAGE , nid, K(node_page_state(pgdat, NR_ANON_THPS)), - nid, K(node_page_state(pgdat, NR_SHMEM_THPS) * - HPAGE_PMD_NR), + nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR), nid, K(node_page_state(pgdat, NR_FILE_THPS)), diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index 7ea4679880c8..cfb107eaa3e6 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -131,7 +131,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "AnonHugePages: ", global_node_page_state(NR_ANON_THPS)); show_val_kb(m, "ShmemHugePages: ", - global_node_page_state(NR_SHMEM_THPS) * HPAGE_PMD_NR); + global_node_page_state(NR_SHMEM_THPS)); show_val_kb(m, "ShmemPmdMapped: ", global_node_page_state(NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR); show_val_kb(m, "FileHugePages: ", diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index b751a9898bb6..788837f40b38 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -220,7 +220,8 @@ static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item) return false; return item == NR_ANON_THPS || - item == NR_FILE_THPS; + item == NR_FILE_THPS || + item == NR_SHMEM_THPS; } /* diff --git a/mm/filemap.c b/mm/filemap.c index 9efe059e2b58..46a8b9e82434 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -206,7 +206,7 @@ static void unaccount_page_cache_page(struct address_space *mapping, if (PageSwapBacked(page)) { __mod_lruvec_page_state(page, NR_SHMEM, -nr); if (PageTransHuge(page)) - __dec_lruvec_page_state(page, NR_SHMEM_THPS); + __mod_lruvec_page_state(page, NR_SHMEM_THPS, -nr); } else if (PageTransHuge(page)) { __mod_lruvec_page_state(page, NR_FILE_THPS, -nr); filemap_nr_thps_dec(mapping); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 77181faa2340..86a3015ba54f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2755,7 +2755,8 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) int nr = thp_nr_pages(head); if (PageSwapBacked(head)) - __dec_lruvec_page_state(head, NR_SHMEM_THPS); + __mod_lruvec_page_state(head, NR_SHMEM_THPS, + -nr); else __mod_lruvec_page_state(head, NR_FILE_THPS, -nr); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index c427fe2ca7ff..75e246f680f4 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1858,7 +1858,7 @@ out_unlock: nr = thp_nr_pages(new_page); if (is_shmem) - __inc_lruvec_page_state(new_page, NR_SHMEM_THPS); + __mod_lruvec_page_state(new_page, NR_SHMEM_THPS, nr); else { __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); filemap_nr_thps_inc(mapping); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index fc552f57d3fb..d3a0c59210e7 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1516,7 +1516,7 @@ struct memory_stat { unsigned int idx; }; -static struct memory_stat memory_stats[] = { +static const struct memory_stat memory_stats[] = { { "anon", PAGE_SIZE, NR_ANON_MAPPED }, { "file", PAGE_SIZE, NR_FILE_PAGES }, { "kernel_stack", 1024, NR_KERNEL_STACK_KB }, @@ -1528,14 +1528,9 @@ static struct memory_stat memory_stats[] = { { "file_dirty", PAGE_SIZE, NR_FILE_DIRTY }, { "file_writeback", PAGE_SIZE, NR_WRITEBACK }, #ifdef CONFIG_TRANSPARENT_HUGEPAGE - /* - * The ratio will be initialized in memory_stats_init(). Because - * on some architectures, the macro of HPAGE_PMD_SIZE is not - * constant(e.g. powerpc). - */ { "anon_thp", PAGE_SIZE, NR_ANON_THPS }, { "file_thp", PAGE_SIZE, NR_FILE_THPS }, - { "shmem_thp", 0, NR_SHMEM_THPS }, + { "shmem_thp", PAGE_SIZE, NR_SHMEM_THPS }, #endif { "inactive_anon", PAGE_SIZE, NR_INACTIVE_ANON }, { "active_anon", PAGE_SIZE, NR_ACTIVE_ANON }, @@ -1560,23 +1555,6 @@ static struct memory_stat memory_stats[] = { { "workingset_nodereclaim", 1, WORKINGSET_NODERECLAIM }, }; -static int __init memory_stats_init(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (memory_stats[i].idx == NR_SHMEM_THPS) - memory_stats[i].ratio = HPAGE_PMD_SIZE; -#endif - VM_BUG_ON(!memory_stats[i].ratio); - VM_BUG_ON(memory_stats[i].idx >= MEMCG_NR_STAT); - } - - return 0; -} -pure_initcall(memory_stats_init); - static char *memory_stat_format(struct mem_cgroup *memcg) { struct seq_buf s; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 2e2e47f8714b..df292d8e659b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5584,7 +5584,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) K(node_page_state(pgdat, NR_WRITEBACK)), K(node_page_state(pgdat, NR_SHMEM)), #ifdef CONFIG_TRANSPARENT_HUGEPAGE - K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), + K(node_page_state(pgdat, NR_SHMEM_THPS)), K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR), K(node_page_state(pgdat, NR_ANON_THPS)), diff --git a/mm/shmem.c b/mm/shmem.c index 7924b3bf46fb..ff741d229701 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -713,7 +713,7 @@ next: } if (PageTransHuge(page)) { count_vm_event(THP_FILE_ALLOC); - __inc_lruvec_page_state(page, NR_SHMEM_THPS); + __mod_lruvec_page_state(page, NR_SHMEM_THPS, nr); } mapping->nrpages += nr; __mod_lruvec_page_state(page, NR_FILE_PAGES, nr); -- cgit v1.2.3 From a1528e21f8915e16252cda1137fe29672c918361 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:35 -0800 Subject: mm: memcontrol: convert NR_SHMEM_PMDMAPPED account to pages Currently we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. So we convert the NR_SHMEM_PMDMAPPED account to pages. This patch is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Doing this also can make the unit of vmstat counters more unified. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. Link: https://lkml.kernel.org/r/20201228164110.2838-6-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Alexey Dobriyan Cc: Feng Tang Cc: Greg Kroah-Hartman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Michal Hocko Cc: NeilBrown Cc: Pankaj Gupta Cc: Rafael. J. Wysocki Cc: Randy Dunlap Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Shakeel Butt Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/node.c | 3 +-- fs/proc/meminfo.c | 2 +- include/linux/mmzone.h | 3 ++- mm/page_alloc.c | 3 +-- mm/rmap.c | 14 ++++++++++---- 5 files changed, 15 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/drivers/base/node.c b/drivers/base/node.c index 6d5ac6ffb6e1..7a66aefe4e46 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -463,8 +463,7 @@ static ssize_t node_read_meminfo(struct device *dev, , nid, K(node_page_state(pgdat, NR_ANON_THPS)), nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), - nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * - HPAGE_PMD_NR), + nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), nid, K(node_page_state(pgdat, NR_FILE_THPS)), nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) * HPAGE_PMD_NR) diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index cfb107eaa3e6..c61f440570f9 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -133,7 +133,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "ShmemHugePages: ", global_node_page_state(NR_SHMEM_THPS)); show_val_kb(m, "ShmemPmdMapped: ", - global_node_page_state(NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR); + global_node_page_state(NR_SHMEM_PMDMAPPED)); show_val_kb(m, "FileHugePages: ", global_node_page_state(NR_FILE_THPS)); show_val_kb(m, "FilePmdMapped: ", diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 788837f40b38..7bdbfeeb5c8c 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -221,7 +221,8 @@ static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item) return item == NR_ANON_THPS || item == NR_FILE_THPS || - item == NR_SHMEM_THPS; + item == NR_SHMEM_THPS || + item == NR_SHMEM_PMDMAPPED; } /* diff --git a/mm/page_alloc.c b/mm/page_alloc.c index df292d8e659b..069561aadc7b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5585,8 +5585,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) K(node_page_state(pgdat, NR_SHMEM)), #ifdef CONFIG_TRANSPARENT_HUGEPAGE K(node_page_state(pgdat, NR_SHMEM_THPS)), - K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) - * HPAGE_PMD_NR), + K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), K(node_page_state(pgdat, NR_ANON_THPS)), #endif K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), diff --git a/mm/rmap.c b/mm/rmap.c index c4d5c63cfd29..1c1b576c0627 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1211,14 +1211,17 @@ void page_add_file_rmap(struct page *page, bool compound) VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page); lock_page_memcg(page); if (compound && PageTransHuge(page)) { - for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { + int nr_pages = thp_nr_pages(page); + + for (i = 0, nr = 0; i < nr_pages; i++) { if (atomic_inc_and_test(&page[i]._mapcount)) nr++; } if (!atomic_inc_and_test(compound_mapcount_ptr(page))) goto out; if (PageSwapBacked(page)) - __inc_node_page_state(page, NR_SHMEM_PMDMAPPED); + __mod_lruvec_page_state(page, NR_SHMEM_PMDMAPPED, + nr_pages); else __inc_node_page_state(page, NR_FILE_PMDMAPPED); } else { @@ -1252,14 +1255,17 @@ static void page_remove_file_rmap(struct page *page, bool compound) /* page still mapped by someone else? */ if (compound && PageTransHuge(page)) { - for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { + int nr_pages = thp_nr_pages(page); + + for (i = 0, nr = 0; i < nr_pages; i++) { if (atomic_add_negative(-1, &page[i]._mapcount)) nr++; } if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) return; if (PageSwapBacked(page)) - __dec_node_page_state(page, NR_SHMEM_PMDMAPPED); + __mod_lruvec_page_state(page, NR_SHMEM_PMDMAPPED, + -nr_pages); else __dec_node_page_state(page, NR_FILE_PMDMAPPED); } else { -- cgit v1.2.3 From 380780e71895ae301505ffcec8f954ab3666a4c7 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:39 -0800 Subject: mm: memcontrol: convert NR_FILE_PMDMAPPED account to pages Currently we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. So we convert the NR_FILE_PMDMAPPED account to pages. This patch is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Doing this also can make the unit of vmstat counters more unified. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. Link: https://lkml.kernel.org/r/20201228164110.2838-7-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Alexey Dobriyan Cc: Feng Tang Cc: Greg Kroah-Hartman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Michal Hocko Cc: NeilBrown Cc: Pankaj Gupta Cc: Rafael. J. Wysocki Cc: Randy Dunlap Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Shakeel Butt Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/node.c | 3 +-- fs/proc/meminfo.c | 2 +- include/linux/mmzone.h | 3 ++- mm/rmap.c | 6 ++++-- 4 files changed, 8 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/drivers/base/node.c b/drivers/base/node.c index 7a66aefe4e46..d02d86aec19f 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -465,8 +465,7 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), nid, K(node_page_state(pgdat, NR_FILE_THPS)), - nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) * - HPAGE_PMD_NR) + nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED)) #endif ); len += hugetlb_report_node_meminfo(buf, len, nid); diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index c61f440570f9..6fa761c9cc78 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -137,7 +137,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "FileHugePages: ", global_node_page_state(NR_FILE_THPS)); show_val_kb(m, "FilePmdMapped: ", - global_node_page_state(NR_FILE_PMDMAPPED) * HPAGE_PMD_NR); + global_node_page_state(NR_FILE_PMDMAPPED)); #endif #ifdef CONFIG_CMA diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 7bdbfeeb5c8c..66d68e5d5a0f 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -222,7 +222,8 @@ static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item) return item == NR_ANON_THPS || item == NR_FILE_THPS || item == NR_SHMEM_THPS || - item == NR_SHMEM_PMDMAPPED; + item == NR_SHMEM_PMDMAPPED || + item == NR_FILE_PMDMAPPED; } /* diff --git a/mm/rmap.c b/mm/rmap.c index 1c1b576c0627..5ebf16fae4b9 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1223,7 +1223,8 @@ void page_add_file_rmap(struct page *page, bool compound) __mod_lruvec_page_state(page, NR_SHMEM_PMDMAPPED, nr_pages); else - __inc_node_page_state(page, NR_FILE_PMDMAPPED); + __mod_lruvec_page_state(page, NR_FILE_PMDMAPPED, + nr_pages); } else { if (PageTransCompound(page) && page_mapping(page)) { VM_WARN_ON_ONCE(!PageLocked(page)); @@ -1267,7 +1268,8 @@ static void page_remove_file_rmap(struct page *page, bool compound) __mod_lruvec_page_state(page, NR_SHMEM_PMDMAPPED, -nr_pages); else - __dec_node_page_state(page, NR_FILE_PMDMAPPED); + __mod_lruvec_page_state(page, NR_FILE_PMDMAPPED, + -nr_pages); } else { if (!atomic_add_negative(-1, &page->_mapcount)) return; -- cgit v1.2.3 From 6eeb104e114cb6b7391c2d69ff873403858c1f35 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 24 Feb 2021 12:04:15 -0800 Subject: fs: buffer: use raw page_memcg() on locked page alloc_page_buffers() currently uses get_mem_cgroup_from_page() for charging the buffers to the page owner, which does an rcu-protected page->memcg lookup and acquires a reference. But buffer allocation has the page lock held throughout, which pins the page to the memcg and thereby the memcg - neither rcu nor holding an extra reference during the allocation are necessary. Use a raw page_memcg() instead. This was the last user of get_mem_cgroup_from_page(), delete it. Link: https://lkml.kernel.org/r/20210209190126.97842-1-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Reported-by: Muchun Song Reviewed-by: Shakeel Butt Acked-by: Roman Gushchin Acked-by: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/buffer.c | 4 ++-- include/linux/memcontrol.h | 7 ------- mm/memcontrol.c | 23 ----------------------- 3 files changed, 2 insertions(+), 32 deletions(-) (limited to 'fs') diff --git a/fs/buffer.c b/fs/buffer.c index f1c3a5b27a90..0cb7ffd4977c 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -847,7 +847,8 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, if (retry) gfp |= __GFP_NOFAIL; - memcg = get_mem_cgroup_from_page(page); + /* The page lock pins the memcg */ + memcg = page_memcg(page); old_memcg = set_active_memcg(memcg); head = NULL; @@ -868,7 +869,6 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, } out: set_active_memcg(old_memcg); - mem_cgroup_put(memcg); return head; /* * In case anything failed, we just free everything we got. diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 7a38a1517a05..e6dc793d587d 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -680,8 +680,6 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p); struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm); -struct mem_cgroup *get_mem_cgroup_from_page(struct page *page); - struct lruvec *lock_page_lruvec(struct page *page); struct lruvec *lock_page_lruvec_irq(struct page *page); struct lruvec *lock_page_lruvec_irqsave(struct page *page, @@ -1191,11 +1189,6 @@ static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) return NULL; } -static inline struct mem_cgroup *get_mem_cgroup_from_page(struct page *page) -{ - return NULL; -} - static inline void mem_cgroup_put(struct mem_cgroup *memcg) { } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 8c035846c7a4..7fdc001ce15f 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1047,29 +1047,6 @@ struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) } EXPORT_SYMBOL(get_mem_cgroup_from_mm); -/** - * get_mem_cgroup_from_page: Obtain a reference on given page's memcg. - * @page: page from which memcg should be extracted. - * - * Obtain a reference on page->memcg and returns it if successful. Otherwise - * root_mem_cgroup is returned. - */ -struct mem_cgroup *get_mem_cgroup_from_page(struct page *page) -{ - struct mem_cgroup *memcg = page_memcg(page); - - if (mem_cgroup_disabled()) - return NULL; - - rcu_read_lock(); - /* Page should not get uncharged and freed memcg under us. */ - if (!memcg || WARN_ON_ONCE(!css_tryget(&memcg->css))) - memcg = root_mem_cgroup; - rcu_read_unlock(); - return memcg; -} -EXPORT_SYMBOL(get_mem_cgroup_from_page); - static __always_inline struct mem_cgroup *active_memcg(void) { if (in_interrupt()) -- cgit v1.2.3 From fb9bf0484af4770240342f4d1b3dd054889cc31e Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 24 Feb 2021 12:05:00 -0800 Subject: vmalloc: remove redundant NULL check Fix below warnings reported by coccicheck: fs/proc/vmcore.c:1503:2-7: WARNING: NULL check before some freeing functions is not needed. Link: https://lkml.kernel.org/r/1611216753-44598-1-git-send-email-abaci-bugfix@linux.alibaba.com Signed-off-by: Yang Li Reported-by: Abaci Robot Acked-by: Baoquan He Cc: Dave Young Cc: Vivek Goyal Cc: Alexey Dobriyan Cc: "Uladzislau Rezki (Sony)" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/vmcore.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c index c3a345c28a93..9a15334da208 100644 --- a/fs/proc/vmcore.c +++ b/fs/proc/vmcore.c @@ -1503,11 +1503,8 @@ int vmcore_add_device_dump(struct vmcoredd_data *data) return 0; out_err: - if (buf) - vfree(buf); - - if (dump) - vfree(dump); + vfree(buf); + vfree(dump); return ret; } -- cgit v1.2.3 From d6995da311221a05c8aef3bda2629e5cb14c7302 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:08:51 -0800 Subject: hugetlb: use page.private for hugetlb specific page flags Patch series "create hugetlb flags to consolidate state", v3. While discussing a series of hugetlb fixes in [1], it became evident that the hugetlb specific page state information is stored in a somewhat haphazard manner. Code dealing with state information would be easier to read, understand and maintain if this information was stored in a consistent manner. This series uses page.private of the hugetlb head page for storing a set of hugetlb specific page flags. Routines are priovided for test, set and clear of the flags. [1] https://lore.kernel.org/r/20210106084739.63318-1-songmuchun@bytedance.com This patch (of 4): As hugetlbfs evolved, state information about hugetlb pages was added. One 'convenient' way of doing this was to use available fields in tail pages. Over time, it has become difficult to know the meaning or contents of fields simply by looking at a small bit of code. Sometimes, the naming is just confusing. For example: The PagePrivate flag indicates a huge page reservation was consumed and needs to be restored if an error is encountered and the page is freed before it is instantiated. The page.private field contains the pointer to a subpool if the page is associated with one. In an effort to make the code more readable, use page.private to contain hugetlb specific page flags. These flags will have test, set and clear functions similar to those used for 'normal' page flags. More importantly, an enum of flag values will be created with names that actually reflect their purpose. In this patch, - Create infrastructure for hugetlb specific page flag functions - Move subpool pointer to page[1].private to make way for flags Create routines with meaningful names to modify subpool field - Use new HPageRestoreReserve flag instead of PagePrivate Conversion of other state information will happen in subsequent patches. Link: https://lkml.kernel.org/r/20210122195231.324857-1-mike.kravetz@oracle.com Link: https://lkml.kernel.org/r/20210122195231.324857-2-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Reviewed-by: Oscar Salvador Acked-by: Michal Hocko Cc: Naoya Horiguchi Cc: Muchun Song Cc: David Hildenbrand Cc: Matthew Wilcox Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 12 +++------ include/linux/hugetlb.h | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ mm/hugetlb.c | 48 +++++++++++++++++----------------- 3 files changed, 96 insertions(+), 32 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index b7a72f577aab..907f6405e805 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -973,15 +973,9 @@ static int hugetlbfs_migrate_page(struct address_space *mapping, if (rc != MIGRATEPAGE_SUCCESS) return rc; - /* - * page_private is subpool pointer in hugetlb pages. Transfer to - * new page. PagePrivate is not associated with page_private for - * hugetlb pages and can not be set here as only page_huge_active - * pages can be migrated. - */ - if (page_private(page)) { - set_page_private(newpage, page_private(page)); - set_page_private(page, 0); + if (hugetlb_page_subpool(page)) { + hugetlb_set_page_subpool(newpage, hugetlb_page_subpool(page)); + hugetlb_set_page_subpool(page, NULL); } if (mode != MIGRATE_SYNC_NO_COPY) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index ac3cb239a7ec..249c65e1d8ca 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -472,6 +472,60 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long flags); #endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */ +/* + * huegtlb page specific state flags. These flags are located in page.private + * of the hugetlb head page. Functions created via the below macros should be + * used to manipulate these flags. + * + * HPG_restore_reserve - Set when a hugetlb page consumes a reservation at + * allocation time. Cleared when page is fully instantiated. Free + * routine checks flag to restore a reservation on error paths. + */ +enum hugetlb_page_flags { + HPG_restore_reserve = 0, + __NR_HPAGEFLAGS, +}; + +/* + * Macros to create test, set and clear function definitions for + * hugetlb specific page flags. + */ +#ifdef CONFIG_HUGETLB_PAGE +#define TESTHPAGEFLAG(uname, flname) \ +static inline int HPage##uname(struct page *page) \ + { return test_bit(HPG_##flname, &(page->private)); } + +#define SETHPAGEFLAG(uname, flname) \ +static inline void SetHPage##uname(struct page *page) \ + { set_bit(HPG_##flname, &(page->private)); } + +#define CLEARHPAGEFLAG(uname, flname) \ +static inline void ClearHPage##uname(struct page *page) \ + { clear_bit(HPG_##flname, &(page->private)); } +#else +#define TESTHPAGEFLAG(uname, flname) \ +static inline int HPage##uname(struct page *page) \ + { return 0; } + +#define SETHPAGEFLAG(uname, flname) \ +static inline void SetHPage##uname(struct page *page) \ + { } + +#define CLEARHPAGEFLAG(uname, flname) \ +static inline void ClearHPage##uname(struct page *page) \ + { } +#endif + +#define HPAGEFLAG(uname, flname) \ + TESTHPAGEFLAG(uname, flname) \ + SETHPAGEFLAG(uname, flname) \ + CLEARHPAGEFLAG(uname, flname) \ + +/* + * Create functions associated with hugetlb page flags + */ +HPAGEFLAG(RestoreReserve, restore_reserve) + #ifdef CONFIG_HUGETLB_PAGE #define HSTATE_NAME_LEN 32 @@ -531,6 +585,20 @@ extern unsigned int default_hstate_idx; #define default_hstate (hstates[default_hstate_idx]) +/* + * hugetlb page subpool pointer located in hpage[1].private + */ +static inline struct hugepage_subpool *hugetlb_page_subpool(struct page *hpage) +{ + return (struct hugepage_subpool *)(hpage+1)->private; +} + +static inline void hugetlb_set_page_subpool(struct page *hpage, + struct hugepage_subpool *subpool) +{ + set_page_private(hpage+1, (unsigned long)subpool); +} + static inline struct hstate *hstate_file(struct file *f) { return hstate_inode(file_inode(f)); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 8a1d1f85e21e..f5e85dabb7a3 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1143,7 +1143,7 @@ static struct page *dequeue_huge_page_vma(struct hstate *h, nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask); if (page && !avoid_reserve && vma_has_reserves(vma, chg)) { - SetPagePrivate(page); + SetHPageRestoreReserve(page); h->resv_huge_pages--; } @@ -1418,20 +1418,19 @@ static void __free_huge_page(struct page *page) */ struct hstate *h = page_hstate(page); int nid = page_to_nid(page); - struct hugepage_subpool *spool = - (struct hugepage_subpool *)page_private(page); + struct hugepage_subpool *spool = hugetlb_page_subpool(page); bool restore_reserve; VM_BUG_ON_PAGE(page_count(page), page); VM_BUG_ON_PAGE(page_mapcount(page), page); - set_page_private(page, 0); + hugetlb_set_page_subpool(page, NULL); page->mapping = NULL; - restore_reserve = PagePrivate(page); - ClearPagePrivate(page); + restore_reserve = HPageRestoreReserve(page); + ClearHPageRestoreReserve(page); /* - * If PagePrivate() was set on page, page allocation consumed a + * If HPageRestoreReserve was set on page, page allocation consumed a * reservation. If the page was associated with a subpool, there * would have been a page reserved in the subpool before allocation * via hugepage_subpool_get_pages(). Since we are 'restoring' the @@ -2263,24 +2262,24 @@ static long vma_add_reservation(struct hstate *h, * This routine is called to restore a reservation on error paths. In the * specific error paths, a huge page was allocated (via alloc_huge_page) * and is about to be freed. If a reservation for the page existed, - * alloc_huge_page would have consumed the reservation and set PagePrivate - * in the newly allocated page. When the page is freed via free_huge_page, - * the global reservation count will be incremented if PagePrivate is set. - * However, free_huge_page can not adjust the reserve map. Adjust the - * reserve map here to be consistent with global reserve count adjustments - * to be made by free_huge_page. + * alloc_huge_page would have consumed the reservation and set + * HPageRestoreReserve in the newly allocated page. When the page is freed + * via free_huge_page, the global reservation count will be incremented if + * HPageRestoreReserve is set. However, free_huge_page can not adjust the + * reserve map. Adjust the reserve map here to be consistent with global + * reserve count adjustments to be made by free_huge_page. */ static void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, unsigned long address, struct page *page) { - if (unlikely(PagePrivate(page))) { + if (unlikely(HPageRestoreReserve(page))) { long rc = vma_needs_reservation(h, vma, address); if (unlikely(rc < 0)) { /* * Rare out of memory condition in reserve map - * manipulation. Clear PagePrivate so that + * manipulation. Clear HPageRestoreReserve so that * global reserve count will not be incremented * by free_huge_page. This will make it appear * as though the reservation for this page was @@ -2289,7 +2288,7 @@ static void restore_reserve_on_error(struct hstate *h, * is better than inconsistent global huge page * accounting of reserve counts. */ - ClearPagePrivate(page); + ClearHPageRestoreReserve(page); } else if (rc) { rc = vma_add_reservation(h, vma, address); if (unlikely(rc < 0)) @@ -2297,7 +2296,7 @@ static void restore_reserve_on_error(struct hstate *h, * See above comment about rare out of * memory condition. */ - ClearPagePrivate(page); + ClearHPageRestoreReserve(page); } else vma_end_reservation(h, vma, address); } @@ -2378,7 +2377,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, if (!page) goto out_uncharge_cgroup; if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { - SetPagePrivate(page); + SetHPageRestoreReserve(page); h->resv_huge_pages--; } spin_lock(&hugetlb_lock); @@ -2396,7 +2395,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, spin_unlock(&hugetlb_lock); - set_page_private(page, (unsigned long)spool); + hugetlb_set_page_subpool(page, spool); map_commit = vma_commit_reservation(h, vma, addr); if (unlikely(map_chg > map_commit)) { @@ -3170,6 +3169,9 @@ static int __init hugetlb_init(void) { int i; + BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE < + __NR_HPAGEFLAGS); + if (!hugepages_supported()) { if (hugetlb_max_hstate || default_hstate_max_huge_pages) pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n"); @@ -4207,7 +4209,7 @@ retry_avoidcopy: spin_lock(ptl); ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { - ClearPagePrivate(new_page); + ClearHPageRestoreReserve(new_page); /* Break COW */ huge_ptep_clear_flush(vma, haddr, ptep); @@ -4274,7 +4276,7 @@ int huge_add_to_page_cache(struct page *page, struct address_space *mapping, if (err) return err; - ClearPagePrivate(page); + ClearHPageRestoreReserve(page); /* * set page dirty so that it will not be removed from cache/file @@ -4436,7 +4438,7 @@ retry: goto backout; if (anon_rmap) { - ClearPagePrivate(page); + ClearHPageRestoreReserve(page); hugepage_add_new_anon_rmap(page, vma, haddr); } else page_dup_rmap(page, true); @@ -4750,7 +4752,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, if (vm_shared) { page_dup_rmap(page, true); } else { - ClearPagePrivate(page); + ClearHPageRestoreReserve(page); hugepage_add_new_anon_rmap(page, dst_vma, dst_addr); } -- cgit v1.2.3 From 8f251a3d5ce3bdea73bd045ed35db64f32e0d0d9 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:08:56 -0800 Subject: hugetlb: convert page_huge_active() HPageMigratable flag Use the new hugetlb page specific flag HPageMigratable to replace the page_huge_active interfaces. By it's name, page_huge_active implied that a huge page was on the active list. However, that is not really what code checking the flag wanted to know. It really wanted to determine if the huge page could be migrated. This happens when the page is actually added to the page cache and/or task page table. This is the reasoning behind the name change. The VM_BUG_ON_PAGE() calls in the *_huge_active() interfaces are not really necessary as we KNOW the page is a hugetlb page. Therefore, they are removed. The routine page_huge_active checked for PageHeadHuge before testing the active bit. This is unnecessary in the case where we hold a reference or lock and know it is a hugetlb head page. page_huge_active is also called without holding a reference or lock (scan_movable_pages), and can race with code freeing the page. The extra check in page_huge_active shortened the race window, but did not prevent the race. Offline code calling scan_movable_pages already deals with these races, so removing the check is acceptable. Add comment to racy code. [songmuchun@bytedance.com: remove set_page_huge_active() declaration from include/linux/hugetlb.h] Link: https://lkml.kernel.org/r/CAMZfGtUda+KoAZscU0718TN61cSFwp4zy=y2oZ=+6Z2TAZZwng@mail.gmail.com Link: https://lkml.kernel.org/r/20210122195231.324857-3-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Reviewed-by: Oscar Salvador Reviewed-by: Muchun Song Reviewed-by: Miaohe Lin Acked-by: Michal Hocko Cc: David Hildenbrand Cc: Matthew Wilcox Cc: Naoya Horiguchi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 2 +- include/linux/hugetlb.h | 7 +++++-- include/linux/page-flags.h | 6 ------ mm/hugetlb.c | 45 +++++++++++---------------------------------- mm/memory_hotplug.c | 9 ++++++++- 5 files changed, 25 insertions(+), 44 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 907f6405e805..5a8ed6bd4f87 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -735,7 +735,7 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, mutex_unlock(&hugetlb_fault_mutex_table[hash]); - set_page_huge_active(page); + SetHPageMigratable(page); /* * unlock_page because locked by add_to_page_cache() * put_page() due to reference from alloc_huge_page() diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 249c65e1d8ca..77f2a032fe32 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -480,9 +480,13 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, * HPG_restore_reserve - Set when a hugetlb page consumes a reservation at * allocation time. Cleared when page is fully instantiated. Free * routine checks flag to restore a reservation on error paths. + * HPG_migratable - Set after a newly allocated page is added to the page + * cache and/or page tables. Indicates the page is a candidate for + * migration. */ enum hugetlb_page_flags { HPG_restore_reserve = 0, + HPG_migratable, __NR_HPAGEFLAGS, }; @@ -525,6 +529,7 @@ static inline void ClearHPage##uname(struct page *page) \ * Create functions associated with hugetlb page flags */ HPAGEFLAG(RestoreReserve, restore_reserve) +HPAGEFLAG(Migratable, migratable) #ifdef CONFIG_HUGETLB_PAGE @@ -838,8 +843,6 @@ static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, } #endif -void set_page_huge_active(struct page *page); - #else /* CONFIG_HUGETLB_PAGE */ struct hstate {}; diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index ec5d0290e0ee..db914477057b 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -592,15 +592,9 @@ static inline void ClearPageCompound(struct page *page) #ifdef CONFIG_HUGETLB_PAGE int PageHuge(struct page *page); int PageHeadHuge(struct page *page); -bool page_huge_active(struct page *page); #else TESTPAGEFLAG_FALSE(Huge) TESTPAGEFLAG_FALSE(HeadHuge) - -static inline bool page_huge_active(struct page *page) -{ - return 0; -} #endif diff --git a/mm/hugetlb.c b/mm/hugetlb.c index f5e85dabb7a3..727c09713627 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1364,30 +1364,6 @@ struct hstate *size_to_hstate(unsigned long size) return NULL; } -/* - * Test to determine whether the hugepage is "active/in-use" (i.e. being linked - * to hstate->hugepage_activelist.) - * - * This function can be called for tail pages, but never returns true for them. - */ -bool page_huge_active(struct page *page) -{ - return PageHeadHuge(page) && PagePrivate(&page[1]); -} - -/* never called for tail page */ -void set_page_huge_active(struct page *page) -{ - VM_BUG_ON_PAGE(!PageHeadHuge(page), page); - SetPagePrivate(&page[1]); -} - -static void clear_page_huge_active(struct page *page) -{ - VM_BUG_ON_PAGE(!PageHeadHuge(page), page); - ClearPagePrivate(&page[1]); -} - /* * Internal hugetlb specific page flag. Do not use outside of the hugetlb * code @@ -1449,7 +1425,7 @@ static void __free_huge_page(struct page *page) } spin_lock(&hugetlb_lock); - clear_page_huge_active(page); + ClearHPageMigratable(page); hugetlb_cgroup_uncharge_page(hstate_index(h), pages_per_huge_page(h), page); hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h), @@ -4218,7 +4194,7 @@ retry_avoidcopy: make_huge_pte(vma, new_page, 1)); page_remove_rmap(old_page, true); hugepage_add_new_anon_rmap(new_page, vma, haddr); - set_page_huge_active(new_page); + SetHPageMigratable(new_page); /* Make the old page be freed below */ new_page = old_page; } @@ -4455,12 +4431,12 @@ retry: spin_unlock(ptl); /* - * Only make newly allocated pages active. Existing pages found - * in the pagecache could be !page_huge_active() if they have been - * isolated for migration. + * Only set HPageMigratable in newly allocated pages. Existing pages + * found in the pagecache may not have HPageMigratableset if they have + * been isolated for migration. */ if (new_page) - set_page_huge_active(page); + SetHPageMigratable(page); unlock_page(page); out: @@ -4771,7 +4747,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, update_mmu_cache(dst_vma, dst_addr, dst_pte); spin_unlock(ptl); - set_page_huge_active(page); + SetHPageMigratable(page); if (vm_shared) unlock_page(page); ret = 0; @@ -5610,12 +5586,13 @@ bool isolate_huge_page(struct page *page, struct list_head *list) bool ret = true; spin_lock(&hugetlb_lock); - if (!PageHeadHuge(page) || !page_huge_active(page) || + if (!PageHeadHuge(page) || + !HPageMigratable(page) || !get_page_unless_zero(page)) { ret = false; goto unlock; } - clear_page_huge_active(page); + ClearHPageMigratable(page); list_move_tail(&page->lru, list); unlock: spin_unlock(&hugetlb_lock); @@ -5625,7 +5602,7 @@ unlock: void putback_active_hugepage(struct page *page) { spin_lock(&hugetlb_lock); - set_page_huge_active(page); + SetHPageMigratable(page); list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); spin_unlock(&hugetlb_lock); put_page(page); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index ddcb1cd24c60..abe43c1ae920 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1260,7 +1260,14 @@ static int scan_movable_pages(unsigned long start, unsigned long end, if (!PageHuge(page)) continue; head = compound_head(page); - if (page_huge_active(head)) + /* + * This test is racy as we hold no reference or lock. The + * hugetlb page could have been free'ed and head is no longer + * a hugetlb page before the following check. In such unlikely + * cases false positives and negatives are possible. Calling + * code must deal with these scenarios. + */ + if (HPageMigratable(head)) goto found; skip = compound_nr(head) - (page - head); pfn += skip - 1; -- cgit v1.2.3 From 33b8f84a4ee78491a8f4f9e4c5520c9da4a10983 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:09:54 -0800 Subject: mm/hugetlb: change hugetlb_reserve_pages() to type bool While reviewing a bug in hugetlb_reserve_pages, it was noticed that all callers ignore the return value. Any failure is considered an ENOMEM error by the callers. Change the function to be of type bool. The function will return true if the reservation was successful, false otherwise. Callers currently assume a zero return code indicates success. Change the callers to look for true to indicate success. No functional change, only code cleanup. Link: https://lkml.kernel.org/r/20201221192542.15732-1-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Reviewed-by: Matthew Wilcox (Oracle) Cc: David Hildenbrand Cc: Dan Carpenter Cc: Michal Hocko Cc: Davidlohr Bueso Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 4 ++-- include/linux/hugetlb.h | 2 +- mm/hugetlb.c | 37 ++++++++++++++----------------------- 3 files changed, 17 insertions(+), 26 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 5a8ed6bd4f87..3eca85a4d940 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -171,7 +171,7 @@ static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma) file_accessed(file); ret = -ENOMEM; - if (hugetlb_reserve_pages(inode, + if (!hugetlb_reserve_pages(inode, vma->vm_pgoff >> huge_page_order(h), len >> huge_page_shift(h), vma, vma->vm_flags)) @@ -1493,7 +1493,7 @@ struct file *hugetlb_file_setup(const char *name, size_t size, inode->i_size = size; clear_nlink(inode); - if (hugetlb_reserve_pages(inode, 0, + if (!hugetlb_reserve_pages(inode, 0, size >> huge_page_shift(hstate_inode(inode)), NULL, acctflag)) file = ERR_PTR(-ENOMEM); diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 3a541c6cf5c3..cccd1aab69dd 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -139,7 +139,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte, unsigned long dst_addr, unsigned long src_addr, struct page **pagep); -int hugetlb_reserve_pages(struct inode *inode, long from, long to, +bool hugetlb_reserve_pages(struct inode *inode, long from, long to, struct vm_area_struct *vma, vm_flags_t vm_flags); long hugetlb_unreserve_pages(struct inode *inode, long start, long end, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 8e5f494291c6..8fb42c6dd74b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5016,12 +5016,13 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, return pages << h->order; } -int hugetlb_reserve_pages(struct inode *inode, +/* Return true if reservation was successful, false otherwise. */ +bool hugetlb_reserve_pages(struct inode *inode, long from, long to, struct vm_area_struct *vma, vm_flags_t vm_flags) { - long ret, chg, add = -1; + long chg, add = -1; struct hstate *h = hstate_inode(inode); struct hugepage_subpool *spool = subpool_inode(inode); struct resv_map *resv_map; @@ -5031,7 +5032,7 @@ int hugetlb_reserve_pages(struct inode *inode, /* This should never happen */ if (from > to) { VM_WARN(1, "%s called with a negative range\n", __func__); - return -EINVAL; + return false; } /* @@ -5040,7 +5041,7 @@ int hugetlb_reserve_pages(struct inode *inode, * without using reserves */ if (vm_flags & VM_NORESERVE) - return 0; + return true; /* * Shared mappings base their reservation on the number of pages that @@ -5062,7 +5063,7 @@ int hugetlb_reserve_pages(struct inode *inode, /* Private mapping. */ resv_map = resv_map_alloc(); if (!resv_map) - return -ENOMEM; + return false; chg = to - from; @@ -5070,18 +5071,12 @@ int hugetlb_reserve_pages(struct inode *inode, set_vma_resv_flags(vma, HPAGE_RESV_OWNER); } - if (chg < 0) { - ret = chg; + if (chg < 0) goto out_err; - } - ret = hugetlb_cgroup_charge_cgroup_rsvd( - hstate_index(h), chg * pages_per_huge_page(h), &h_cg); - - if (ret < 0) { - ret = -ENOMEM; + if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h), + chg * pages_per_huge_page(h), &h_cg) < 0) goto out_err; - } if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) { /* For private mappings, the hugetlb_cgroup uncharge info hangs @@ -5096,19 +5091,15 @@ int hugetlb_reserve_pages(struct inode *inode, * reservations already in place (gbl_reserve). */ gbl_reserve = hugepage_subpool_get_pages(spool, chg); - if (gbl_reserve < 0) { - ret = -ENOSPC; + if (gbl_reserve < 0) goto out_uncharge_cgroup; - } /* * Check enough hugepages are available for the reservation. * Hand the pages back to the subpool if there are not */ - ret = hugetlb_acct_memory(h, gbl_reserve); - if (ret < 0) { + if (hugetlb_acct_memory(h, gbl_reserve) < 0) goto out_put_pages; - } /* * Account for the reservations made. Shared mappings record regions @@ -5126,7 +5117,6 @@ int hugetlb_reserve_pages(struct inode *inode, if (unlikely(add < 0)) { hugetlb_acct_memory(h, -gbl_reserve); - ret = add; goto out_put_pages; } else if (unlikely(chg > add)) { /* @@ -5147,7 +5137,8 @@ int hugetlb_reserve_pages(struct inode *inode, hugetlb_acct_memory(h, -rsv_adjust); } } - return 0; + return true; + out_put_pages: /* put back original number of pages, chg */ (void)hugepage_subpool_put_pages(spool, chg); @@ -5163,7 +5154,7 @@ out_err: region_abort(resv_map, from, to, regions_needed); if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) kref_put(&resv_map->refs, resv_map_release); - return ret; + return false; } long hugetlb_unreserve_pages(struct inode *inode, long start, long end, -- cgit v1.2.3 From a4fa34cdcd18296c097e2648fe894d28c5cf9709 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:09:58 -0800 Subject: hugetlbfs: remove special hugetlbfs_set_page_dirty() Matthew Wilcox noticed that hugetlbfs_set_page_dirty always returns 0. Instead, it should return 1 or 0 depending on the previous state of the dirty bit. In addition, the call to compound_head is redundant as it is also performed in calling routine set_page_dirty. Replace the hugetlbfs specific routine hugetlbfs_set_page_dirty with __set_page_dirty_no_writeback as it addresses both of these issues. Link: https://lkml.kernel.org/r/20201221192542.15732-2-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Suggested-by: Matthew Wilcox Cc: Dan Carpenter Cc: David Hildenbrand Cc: Davidlohr Bueso Cc: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 3eca85a4d940..dc4aceffa2c7 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -952,17 +952,6 @@ static int hugetlbfs_symlink(struct user_namespace *mnt_userns, return error; } -/* - * mark the head page dirty - */ -static int hugetlbfs_set_page_dirty(struct page *page) -{ - struct page *head = compound_head(page); - - SetPageDirty(head); - return 0; -} - static int hugetlbfs_migrate_page(struct address_space *mapping, struct page *newpage, struct page *page, enum migrate_mode mode) @@ -1150,7 +1139,7 @@ static void hugetlbfs_destroy_inode(struct inode *inode) static const struct address_space_operations hugetlbfs_aops = { .write_begin = hugetlbfs_write_begin, .write_end = hugetlbfs_write_end, - .set_page_dirty = hugetlbfs_set_page_dirty, + .set_page_dirty = __set_page_dirty_no_writeback, .migratepage = hugetlbfs_migrate_page, .error_remove_page = hugetlbfs_error_remove_page, }; -- cgit v1.2.3 From d0146756a0993d3a01407b38cd87d965ccda72c6 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:01 -0800 Subject: hugetlbfs: remove useless BUG_ON(!inode) in hugetlbfs_setattr() When we reach here with inode = NULL, we should have crashed as inode has already been dereferenced via hstate_inode. So this BUG_ON(!inode) does not take effect and should be removed. Link: https://lkml.kernel.org/r/20210118110700.52506-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index dc4aceffa2c7..71a00dbfa612 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -761,8 +761,6 @@ static int hugetlbfs_setattr(struct user_namespace *mnt_userns, unsigned int ia_valid = attr->ia_valid; struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode); - BUG_ON(!inode); - error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; -- cgit v1.2.3 From 3b2275a8d83a29e579b4f96f4c431d824e5f4a16 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:04 -0800 Subject: hugetlbfs: use helper macro default_hstate in init_hugetlbfs_fs Since commit e5ff215941d5 ("hugetlb: multiple hstates for multiple page sizes"), we can use macro default_hstate to get the struct hstate which we use by default. But init_hugetlbfs_fs() forgot to use it. Link: https://lkml.kernel.org/r/20210116091827.20982-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: David Hildenbrand Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 71a00dbfa612..32c8ae4e1e48 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1543,7 +1543,7 @@ static int __init init_hugetlbfs_fs(void) goto out_free; /* default hstate mount is required */ - mnt = mount_one_hugetlbfs(&hstates[default_hstate_idx]); + mnt = mount_one_hugetlbfs(&default_hstate); if (IS_ERR(mnt)) { error = PTR_ERR(mnt); goto out_unreg; -- cgit v1.2.3 From c7e285e31f76453bc958006ebe5311a6cca909e3 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:08 -0800 Subject: hugetlbfs: correct obsolete function name in hugetlbfs_read_iter() Since commit 36e789144267 ("kill do_generic_mapping_read"), the function do_generic_mapping_read() is renamed to do_generic_file_read(). And then commit 47c27bc46946 ("fs: pass iocb to do_generic_file_read") renamed it to generic_file_buffered_read(). So replace do_generic_mapping_read() with generic_file_buffered_read() to keep comment uptodate. Link: https://lkml.kernel.org/r/20210118063210.47118-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 32c8ae4e1e48..974eee6736d2 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -310,7 +310,7 @@ hugetlbfs_read_actor(struct page *page, unsigned long offset, /* * Support for read() - Find the page attached to f_mapping and copy out the - * data. Its *very* similar to do_generic_mapping_read(), we can't use that + * data. Its *very* similar to generic_file_buffered_read(), we can't use that * since it has PAGE_SIZE assumptions. */ static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to) -- cgit v1.2.3 From 88ce3fef47f3f382985ecefe8f290b6ff05b4335 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:11 -0800 Subject: hugetlbfs: remove meaningless variable avoid_reserve The variable avoid_reserve is meaningless because we never changed its value and just passed it to alloc_huge_page(). So remove it to make code more clear that in hugetlbfs_fallocate, we never avoid reserve when alloc hugepage yet. Also add a comment offered by Mike Kravetz to explain this. Link: https://lkml.kernel.org/r/20210120071508.9078-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: David Hildenbrand Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 974eee6736d2..7982adc3d98d 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -680,7 +680,6 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, */ struct page *page; unsigned long addr; - int avoid_reserve = 0; cond_resched(); @@ -716,8 +715,15 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, continue; } - /* Allocate page and add to page cache */ - page = alloc_huge_page(&pseudo_vma, addr, avoid_reserve); + /* + * Allocate page without setting the avoid_reserve argument. + * There certainly are no reserves associated with the + * pseudo_vma. However, there could be shared mappings with + * reserves for the file at the inode level. If we fallocate + * pages in these areas, we need to consume the reserves + * to keep reservation accounting consistent. + */ + page = alloc_huge_page(&pseudo_vma, addr, 0); hugetlb_drop_vma_policy(&pseudo_vma); if (IS_ERR(page)) { mutex_unlock(&hugetlb_fault_mutex_table[hash]); -- cgit v1.2.3 From a25fddced835ae53d18eb4bddabd719b4cebf624 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:14 -0800 Subject: hugetlbfs: make hugepage size conversion more readable The calculation 1U << (h->order + PAGE_SHIFT - 10) is actually equal to (PAGE_SHIFT << (h->order)) >> 10. So we can make it more readable by replace it with huge_page_size(h) >> 10. Link: https://lkml.kernel.org/r/20210122083141.24548-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 7982adc3d98d..46fad3d9265b 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1520,8 +1520,8 @@ static struct vfsmount *__init mount_one_hugetlbfs(struct hstate *h) put_fs_context(fc); } if (IS_ERR(mnt)) - pr_err("Cannot mount internal hugetlbfs for page size %uK", - 1U << (h->order + PAGE_SHIFT - 10)); + pr_err("Cannot mount internal hugetlbfs for page size %luK", + huge_page_size(h) >> 10); return mnt; } -- cgit v1.2.3 From 398c0da7364c907ccc662416585c19c5523cf678 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:18 -0800 Subject: hugetlbfs: correct some obsolete comments about inode i_mutex Since commit 9902af79c01a ("parallel lookups: actual switch to rwsem"), i_mutex of inode is converted to i_rwsem. So replace i_mutex with i_rwsem to make comments up to date. Link: https://lkml.kernel.org/r/20210127093111.36672-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Cc: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 46fad3d9265b..f5758ba372ae 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -604,7 +604,7 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) inode_lock(inode); - /* protected by i_mutex */ + /* protected by i_rwsem */ if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) { inode_unlock(inode); return -EPERM; @@ -777,7 +777,7 @@ static int hugetlbfs_setattr(struct user_namespace *mnt_userns, if (newsize & ~huge_page_mask(h)) return -EINVAL; - /* protected by i_mutex */ + /* protected by i_rwsem */ if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || (newsize > oldsize && (info->seals & F_SEAL_GROW))) return -EPERM; -- cgit v1.2.3 From 1935ebd3cf6c44038479bb2e7b4dd99bd492b3f2 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:21 -0800 Subject: hugetlbfs: fix some comment typos Fix typos reserv to reserve, minimim to minimum. No functional change intended. Link: https://lkml.kernel.org/r/20210130092351.28072-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Cc: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index f5758ba372ae..394da2ab08ad 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -442,15 +442,15 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end) * * truncation is indicated by end of range being LLONG_MAX * In this case, we first scan the range and release found pages. - * After releasing pages, hugetlb_unreserve_pages cleans up region/reserv + * After releasing pages, hugetlb_unreserve_pages cleans up region/reserve * maps and global counts. Page faults can not race with truncation * in this routine. hugetlb_no_page() holds i_mmap_rwsem and prevents * page faults in the truncated range by checking i_size. i_size is * modified while holding i_mmap_rwsem. * hole punch is indicated if end is not LLONG_MAX * In the hole punch case we scan the range and release found pages. - * Only when releasing a page is the associated region/reserv map - * deleted. The region/reserv map for ranges without associated + * Only when releasing a page is the associated region/reserve map + * deleted. The region/reserve map for ranges without associated * pages are not modified. Page faults can race with hole punch. * This is indicated if we find a mapped page. * Note: If the passed end of range value is beyond the end of file, but @@ -1343,7 +1343,7 @@ hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc) /* * Allocate and initialize subpool if maximum or minimum size is - * specified. Any needed reservations (for minimim size) are taken + * specified. Any needed reservations (for minimum size) are taken * taken when the subpool is created. */ if (ctx->max_hpages != -1 || ctx->min_hpages != -1) { -- cgit v1.2.3 From e5d319dedafd21211fd19ea28a3f50da7368d6ff Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:25 -0800 Subject: hugetlbfs: remove unneeded return value of hugetlb_vmtruncate() The function hugetlb_vmtruncate() is guaranteed to always success since commit 7aa91e104028 ("hugetlb: allow extending ftruncate on hugetlbfs"). So we should remove the unneeded return value which is always 0. Link: https://lkml.kernel.org/r/20210208084637.47789-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 394da2ab08ad..701c82c36138 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -567,7 +567,7 @@ static void hugetlbfs_evict_inode(struct inode *inode) clear_inode(inode); } -static int hugetlb_vmtruncate(struct inode *inode, loff_t offset) +static void hugetlb_vmtruncate(struct inode *inode, loff_t offset) { pgoff_t pgoff; struct address_space *mapping = inode->i_mapping; @@ -582,7 +582,6 @@ static int hugetlb_vmtruncate(struct inode *inode, loff_t offset) hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0); i_mmap_unlock_write(mapping); remove_inode_hugepages(inode, offset, LLONG_MAX); - return 0; } static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) @@ -781,9 +780,7 @@ static int hugetlbfs_setattr(struct user_namespace *mnt_userns, if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || (newsize > oldsize && (info->seals & F_SEAL_GROW))) return -EPERM; - error = hugetlb_vmtruncate(inode, newsize); - if (error) - return error; + hugetlb_vmtruncate(inode, newsize); } setattr_copy(&init_user_ns, inode, attr); -- cgit v1.2.3 From cb5e1b81304e089ee3ca948db4d29f71902eb575 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 25 Feb 2021 07:37:35 -0700 Subject: Revert "io_uring: wait potential ->release() on resurrect" This reverts commit 88f171ab7798a1ed0b9e39867ee16f307466e870. I ran into a case where the ref resurrect now spins, so revert this change for now until we can further investigate why it's broken. The bug seems to indicate spinning on the lock itself, likely there's some ABBA deadlock involved: [<0>] __percpu_ref_switch_mode+0x45/0x180 [<0>] percpu_ref_resurrect+0x46/0x70 [<0>] io_refs_resurrect+0x25/0xa0 [<0>] __io_uring_register+0x135/0x10c0 [<0>] __x64_sys_io_uring_register+0xc2/0x1a0 [<0>] do_syscall_64+0x42/0x110 [<0>] entry_SYSCALL_64_after_hwframe+0x44/0xa9 Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5c8e24274acf..442337b40c9d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1104,21 +1104,6 @@ static inline void io_set_resource_node(struct io_kiocb *req) } } -static bool io_refs_resurrect(struct percpu_ref *ref, struct completion *compl) -{ - if (!percpu_ref_tryget(ref)) { - /* already at zero, wait for ->release() */ - if (!try_wait_for_completion(compl)) - synchronize_rcu(); - return false; - } - - percpu_ref_resurrect(ref); - reinit_completion(compl); - percpu_ref_put(ref); - return true; -} - static bool io_match_task(struct io_kiocb *head, struct task_struct *task, struct files_struct *files) @@ -7353,11 +7338,13 @@ static int io_rsrc_ref_quiesce(struct fixed_rsrc_data *data, flush_delayed_work(&ctx->rsrc_put_work); ret = wait_for_completion_interruptible(&data->done); - if (!ret || !io_refs_resurrect(&data->refs, &data->done)) + if (!ret) break; + percpu_ref_resurrect(&data->refs); io_sqe_rsrc_set_node(ctx, data, backup_node); backup_node = NULL; + reinit_completion(&data->done); mutex_unlock(&ctx->uring_lock); ret = io_run_task_work_sig(); mutex_lock(&ctx->uring_lock); @@ -10096,8 +10083,10 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, mutex_lock(&ctx->uring_lock); - if (ret && io_refs_resurrect(&ctx->refs, &ctx->ref_comp)) - return ret; + if (ret) { + percpu_ref_resurrect(&ctx->refs); + goto out_quiesce; + } } if (ctx->restricted) { @@ -10189,6 +10178,7 @@ out: if (io_register_op_must_quiesce(opcode)) { /* bring the ctx back to life */ percpu_ref_reinit(&ctx->refs); +out_quiesce: reinit_completion(&ctx->ref_comp); } return ret; -- cgit v1.2.3 From 06058bc40534530e617e5623775c53bb24f032cb Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Tue, 23 Feb 2021 10:22:39 -0800 Subject: xfs: don't reuse busy extents on extent trim Freed extents are marked busy from the point the freeing transaction commits until the associated CIL context is checkpointed to the log. This prevents reuse and overwrite of recently freed blocks before the changes are committed to disk, which can lead to corruption after a crash. The exception to this rule is that metadata allocation is allowed to reuse busy extents because metadata changes are also logged. As of commit 97d3ac75e5e0 ("xfs: exact busy extent tracking"), XFS has allowed modification or complete invalidation of outstanding busy extents for metadata allocations. This implementation assumes that use of the associated extent is imminent, which is not always the case. For example, the trimmed extent might not satisfy the minimum length of the allocation request, or the allocation algorithm might be involved in a search for the optimal result based on locality. generic/019 reproduces a corruption caused by this scenario. First, a metadata block (usually a bmbt or symlink block) is freed from an inode. A subsequent bmbt split on an unrelated inode attempts a near mode allocation request that invalidates the busy block during the search, but does not ultimately allocate it. Due to the busy state invalidation, the block is no longer considered busy to subsequent allocation. A direct I/O write request immediately allocates the block and writes to it. Finally, the filesystem crashes while in a state where the initial metadata block free had not committed to the on-disk log. After recovery, the original metadata block is in its original location as expected, but has been corrupted by the aforementioned dio. This demonstrates that it is fundamentally unsafe to modify busy extent state for extents that are not guaranteed to be allocated. This applies to pretty much all of the code paths that currently trim busy extents for one reason or another. Therefore to address this problem, drop the reuse mechanism from the busy extent trim path. This code already knows how to return partial non-busy ranges of the targeted free extent and higher level code tracks the busy state of the allocation attempt. If a block allocation fails where one or more candidate extents is busy, we force the log and retry the allocation. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_extent_busy.c | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_extent_busy.c b/fs/xfs/xfs_extent_busy.c index 3991e59cfd18..ef17c1f6db32 100644 --- a/fs/xfs/xfs_extent_busy.c +++ b/fs/xfs/xfs_extent_busy.c @@ -344,7 +344,6 @@ xfs_extent_busy_trim( ASSERT(*len > 0); spin_lock(&args->pag->pagb_lock); -restart: fbno = *bno; flen = *len; rbp = args->pag->pagb_tree.rb_node; @@ -363,19 +362,6 @@ restart: continue; } - /* - * If this is a metadata allocation, try to reuse the busy - * extent instead of trimming the allocation. - */ - if (!(args->datatype & XFS_ALLOC_USERDATA) && - !(busyp->flags & XFS_EXTENT_BUSY_DISCARDED)) { - if (!xfs_extent_busy_update_extent(args->mp, args->pag, - busyp, fbno, flen, - false)) - goto restart; - continue; - } - if (bbno <= fbno) { /* start overlap */ -- cgit v1.2.3 From 9febcda6f8d1db9f922945d026bb838864b1b6d5 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 19 Feb 2021 09:18:06 -0800 Subject: xfs: don't nest transactions when scanning for eofblocks Brian Foster reported a lockdep warning on xfs/167: ============================================ WARNING: possible recursive locking detected 5.11.0-rc4 #35 Tainted: G W I -------------------------------------------- fsstress/17733 is trying to acquire lock: ffff8e0fd1d90650 (sb_internal){++++}-{0:0}, at: xfs_free_eofblocks+0x104/0x1d0 [xfs] but task is already holding lock: ffff8e0fd1d90650 (sb_internal){++++}-{0:0}, at: xfs_trans_alloc_inode+0x5f/0x160 [xfs] stack backtrace: CPU: 38 PID: 17733 Comm: fsstress Tainted: G W I 5.11.0-rc4 #35 Hardware name: Dell Inc. PowerEdge R740/01KPX8, BIOS 1.6.11 11/20/2018 Call Trace: dump_stack+0x8b/0xb0 __lock_acquire.cold+0x159/0x2ab lock_acquire+0x116/0x370 xfs_trans_alloc+0x1ad/0x310 [xfs] xfs_free_eofblocks+0x104/0x1d0 [xfs] xfs_blockgc_scan_inode+0x24/0x60 [xfs] xfs_inode_walk_ag+0x202/0x4b0 [xfs] xfs_inode_walk+0x66/0xc0 [xfs] xfs_trans_alloc+0x160/0x310 [xfs] xfs_trans_alloc_inode+0x5f/0x160 [xfs] xfs_alloc_file_space+0x105/0x300 [xfs] xfs_file_fallocate+0x270/0x460 [xfs] vfs_fallocate+0x14d/0x3d0 __x64_sys_fallocate+0x3e/0x70 do_syscall_64+0x33/0x40 entry_SYSCALL_64_after_hwframe+0x44/0xa9 The cause of this is the new code that spurs a scan to garbage collect speculative preallocations if we fail to reserve enough blocks while allocating a transaction. While the warning itself is a fairly benign lockdep complaint, it does expose a potential livelock if the rwsem behavior ever changes with regards to nesting read locks when someone's waiting for a write lock. Fix this by freeing the transaction and jumping back to xfs_trans_alloc like this patch in the V4 submission[1]. [1] https://lore.kernel.org/linux-xfs/161142798066.2171939.9311024588681972086.stgit@magnolia/ Fixes: a1a7d05a0576 ("xfs: flush speculative space allocations when we run out of space") Reported-by: Brian Foster Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Allison Henderson Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_trans.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 44f72c09c203..377f3961d7ed 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -260,6 +260,7 @@ xfs_trans_alloc( struct xfs_trans **tpp) { struct xfs_trans *tp; + bool want_retry = true; int error; /* @@ -267,6 +268,7 @@ xfs_trans_alloc( * GFP_NOFS allocation context so that we avoid lockdep false positives * by doing GFP_KERNEL allocations inside sb_start_intwrite(). */ +retry: tp = kmem_cache_zalloc(xfs_trans_zone, GFP_KERNEL | __GFP_NOFAIL); if (!(flags & XFS_TRANS_NO_WRITECOUNT)) sb_start_intwrite(mp->m_super); @@ -289,7 +291,9 @@ xfs_trans_alloc( tp->t_firstblock = NULLFSBLOCK; error = xfs_trans_reserve(tp, resp, blocks, rtextents); - if (error == -ENOSPC) { + if (error == -ENOSPC && want_retry) { + xfs_trans_cancel(tp); + /* * We weren't able to reserve enough space for the transaction. * Flush the other speculative space allocations to free space. @@ -297,8 +301,11 @@ xfs_trans_alloc( * other locks. */ error = xfs_blockgc_free_space(mp, NULL); - if (!error) - error = xfs_trans_reserve(tp, resp, blocks, rtextents); + if (error) + return error; + + want_retry = false; + goto retry; } if (error) { xfs_trans_cancel(tp); -- cgit v1.2.3 From 756b1c343333a5aefcc26b0409f3fd16f72281bf Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Tue, 23 Feb 2021 10:26:06 -0800 Subject: xfs: use current->journal_info for detecting transaction recursion Because the iomap code using PF_MEMALLOC_NOFS to detect transaction recursion in XFS is just wrong. Remove it from the iomap code and replace it with XFS specific internal checks using current->journal_info instead. [djwong: This change also realigns the lifetime of NOFS flag changes to match the incore transaction, instead of the inconsistent scheme we have now.] Fixes: 9070733b4efa ("xfs: abstract PF_FSTRANS to PF_MEMALLOC_NOFS") Signed-off-by: Dave Chinner Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/iomap/buffered-io.c | 7 ------- fs/xfs/libxfs/xfs_btree.c | 12 ++++++++++-- fs/xfs/xfs_aops.c | 17 +++++++++++++++-- fs/xfs/xfs_trans.c | 20 +++++--------------- fs/xfs/xfs_trans.h | 30 ++++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 26 deletions(-) (limited to 'fs') diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 16a1e82e3aeb..fcd4a0d71fc1 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -1458,13 +1458,6 @@ iomap_do_writepage(struct page *page, struct writeback_control *wbc, void *data) PF_MEMALLOC)) goto redirty; - /* - * Given that we do not allow direct reclaim to call us, we should - * never be called in a recursive filesystem reclaim context. - */ - if (WARN_ON_ONCE(current->flags & PF_MEMALLOC_NOFS)) - goto redirty; - /* * Is this page beyond the end of the file? * diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c index b56ff451adce..5b6fcb9b44e2 100644 --- a/fs/xfs/libxfs/xfs_btree.c +++ b/fs/xfs/libxfs/xfs_btree.c @@ -2805,7 +2805,7 @@ xfs_btree_split_worker( struct xfs_btree_split_args *args = container_of(work, struct xfs_btree_split_args, work); unsigned long pflags; - unsigned long new_pflags = PF_MEMALLOC_NOFS; + unsigned long new_pflags = 0; /* * we are in a transaction context here, but may also be doing work @@ -2817,12 +2817,20 @@ xfs_btree_split_worker( new_pflags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD; current_set_flags_nested(&pflags, new_pflags); + xfs_trans_set_context(args->cur->bc_tp); args->result = __xfs_btree_split(args->cur, args->level, args->ptrp, args->key, args->curp, args->stat); - complete(args->done); + xfs_trans_clear_context(args->cur->bc_tp); current_restore_flags_nested(&pflags, new_pflags); + + /* + * Do not access args after complete() has run here. We don't own args + * and the owner may run and free args before we return here. + */ + complete(args->done); + } /* diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index 4304c6416fbb..b4186d666157 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -62,7 +62,7 @@ xfs_setfilesize_trans_alloc( * We hand off the transaction to the completion thread now, so * clear the flag here. */ - current_restore_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS); + xfs_trans_clear_context(tp); return 0; } @@ -125,7 +125,7 @@ xfs_setfilesize_ioend( * thus we need to mark ourselves as being in a transaction manually. * Similarly for freeze protection. */ - current_set_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS); + xfs_trans_set_context(tp); __sb_writers_acquired(VFS_I(ip)->i_sb, SB_FREEZE_FS); /* we abort the update if there was an IO error */ @@ -568,6 +568,12 @@ xfs_vm_writepage( { struct xfs_writepage_ctx wpc = { }; + if (WARN_ON_ONCE(current->journal_info)) { + redirty_page_for_writepage(wbc, page); + unlock_page(page); + return 0; + } + return iomap_writepage(page, wbc, &wpc.ctx, &xfs_writeback_ops); } @@ -578,6 +584,13 @@ xfs_vm_writepages( { struct xfs_writepage_ctx wpc = { }; + /* + * Writing back data in a transaction context can result in recursive + * transactions. This is bad, so issue a warning and get out of here. + */ + if (WARN_ON_ONCE(current->journal_info)) + return 0; + xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED); return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops); } diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 377f3961d7ed..b22a09e9daee 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -72,6 +72,7 @@ xfs_trans_free( xfs_extent_busy_clear(tp->t_mountp, &tp->t_busy, false); trace_xfs_trans_free(tp, _RET_IP_); + xfs_trans_clear_context(tp); if (!(tp->t_flags & XFS_TRANS_NO_WRITECOUNT)) sb_end_intwrite(tp->t_mountp->m_super); xfs_trans_free_dqinfo(tp); @@ -123,7 +124,8 @@ xfs_trans_dup( ntp->t_rtx_res = tp->t_rtx_res - tp->t_rtx_res_used; tp->t_rtx_res = tp->t_rtx_res_used; - ntp->t_pflags = tp->t_pflags; + + xfs_trans_switch_context(tp, ntp); /* move deferred ops over to the new tp */ xfs_defer_move(ntp, tp); @@ -157,9 +159,6 @@ xfs_trans_reserve( int error = 0; bool rsvd = (tp->t_flags & XFS_TRANS_RESERVE) != 0; - /* Mark this thread as being in a transaction */ - current_set_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS); - /* * Attempt to reserve the needed disk blocks by decrementing * the number needed from the number available. This will @@ -167,10 +166,8 @@ xfs_trans_reserve( */ if (blocks > 0) { error = xfs_mod_fdblocks(mp, -((int64_t)blocks), rsvd); - if (error != 0) { - current_restore_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS); + if (error != 0) return -ENOSPC; - } tp->t_blk_res += blocks; } @@ -244,9 +241,6 @@ undo_blocks: xfs_mod_fdblocks(mp, (int64_t)blocks, rsvd); tp->t_blk_res = 0; } - - current_restore_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS); - return error; } @@ -272,6 +266,7 @@ retry: tp = kmem_cache_zalloc(xfs_trans_zone, GFP_KERNEL | __GFP_NOFAIL); if (!(flags & XFS_TRANS_NO_WRITECOUNT)) sb_start_intwrite(mp->m_super); + xfs_trans_set_context(tp); /* * Zero-reservation ("empty") transactions can't modify anything, so @@ -900,7 +895,6 @@ __xfs_trans_commit( xfs_log_commit_cil(mp, tp, &commit_lsn, regrant); - current_restore_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS); xfs_trans_free(tp); /* @@ -932,7 +926,6 @@ out_unreserve: xfs_log_ticket_ungrant(mp->m_log, tp->t_ticket); tp->t_ticket = NULL; } - current_restore_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS); xfs_trans_free_items(tp, !!error); xfs_trans_free(tp); @@ -992,9 +985,6 @@ xfs_trans_cancel( tp->t_ticket = NULL; } - /* mark this thread as no longer being in a transaction */ - current_restore_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS); - xfs_trans_free_items(tp, dirty); xfs_trans_free(tp); } diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h index 8b03fbfe9a1b..9dd745cf77c9 100644 --- a/fs/xfs/xfs_trans.h +++ b/fs/xfs/xfs_trans.h @@ -281,4 +281,34 @@ int xfs_trans_alloc_ichange(struct xfs_inode *ip, struct xfs_dquot *udqp, struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, bool force, struct xfs_trans **tpp); +static inline void +xfs_trans_set_context( + struct xfs_trans *tp) +{ + ASSERT(current->journal_info == NULL); + tp->t_pflags = memalloc_nofs_save(); + current->journal_info = tp; +} + +static inline void +xfs_trans_clear_context( + struct xfs_trans *tp) +{ + if (current->journal_info == tp) { + memalloc_nofs_restore(tp->t_pflags); + current->journal_info = NULL; + } +} + +static inline void +xfs_trans_switch_context( + struct xfs_trans *old_tp, + struct xfs_trans *new_tp) +{ + ASSERT(current->journal_info == old_tp); + new_tp->t_pflags = old_tp->t_pflags; + old_tp->t_pflags = 0; + current->journal_info = new_tp; +} + #endif /* __XFS_TRANS_H__ */ -- cgit v1.2.3 From e941894eae31b52f0fd9bdb3ce20620afa152f45 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 19 Feb 2021 12:33:30 -0700 Subject: io-wq: make buffered file write hashed work map per-ctx Before the io-wq thread change, we maintained a hash work map and lock per-node per-ring. That wasn't ideal, as we really wanted it to be per ring. But now that we have per-task workers, the hash map ends up being just per-task. That'll work just fine for the normal case of having one task use a ring, but if you share the ring between tasks, then it's considerably worse than it was before. Make the hash map per ctx instead, which provides full per-ctx buffered write serialization on hashed writes. Signed-off-by: Jens Axboe --- fs/io-wq.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++-------- fs/io-wq.h | 14 ++++++++++ fs/io_uring.c | 19 +++++++++++++ 3 files changed, 107 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index a53df2b3762a..d28ad66b7f16 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -87,7 +87,6 @@ struct io_wqe { struct { raw_spinlock_t lock; struct io_wq_work_list work_list; - unsigned long hash_map; unsigned flags; } ____cacheline_aligned_in_smp; @@ -97,6 +96,8 @@ struct io_wqe { struct hlist_nulls_head free_list; struct list_head all_list; + struct wait_queue_entry wait; + struct io_wq *wq; struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS]; }; @@ -113,6 +114,9 @@ struct io_wq { struct task_struct *manager; struct user_struct *user; + + struct io_wq_hash *hash; + refcount_t refs; struct completion done; @@ -328,14 +332,31 @@ static inline unsigned int io_get_work_hash(struct io_wq_work *work) return work->flags >> IO_WQ_HASH_SHIFT; } +static void io_wait_on_hash(struct io_wqe *wqe, unsigned int hash) +{ + struct io_wq *wq = wqe->wq; + + spin_lock(&wq->hash->wait.lock); + if (list_empty(&wqe->wait.entry)) { + __add_wait_queue(&wq->hash->wait, &wqe->wait); + if (!test_bit(hash, &wq->hash->map)) { + __set_current_state(TASK_RUNNING); + list_del_init(&wqe->wait.entry); + } + } + spin_unlock(&wq->hash->wait.lock); +} + static struct io_wq_work *io_get_next_work(struct io_wqe *wqe) __must_hold(wqe->lock) { struct io_wq_work_node *node, *prev; struct io_wq_work *work, *tail; - unsigned int hash; + unsigned int stall_hash = -1U; wq_list_for_each(node, prev, &wqe->work_list) { + unsigned int hash; + work = container_of(node, struct io_wq_work, list); /* not hashed, can run anytime */ @@ -344,16 +365,26 @@ static struct io_wq_work *io_get_next_work(struct io_wqe *wqe) return work; } - /* hashed, can run if not already running */ hash = io_get_work_hash(work); - if (!(wqe->hash_map & BIT(hash))) { - wqe->hash_map |= BIT(hash); - /* all items with this hash lie in [work, tail] */ - tail = wqe->hash_tail[hash]; + /* all items with this hash lie in [work, tail] */ + tail = wqe->hash_tail[hash]; + + /* hashed, can run if not already running */ + if (!test_and_set_bit(hash, &wqe->wq->hash->map)) { wqe->hash_tail[hash] = NULL; wq_list_cut(&wqe->work_list, &tail->list, prev); return work; } + if (stall_hash == -1U) + stall_hash = hash; + /* fast forward to a next hash, for-each will fix up @prev */ + node = &tail->list; + } + + if (stall_hash != -1U) { + raw_spin_unlock(&wqe->lock); + io_wait_on_hash(wqe, stall_hash); + raw_spin_lock(&wqe->lock); } return NULL; @@ -421,6 +452,7 @@ get_next: if (!work) break; io_assign_current_work(worker, work); + __set_current_state(TASK_RUNNING); /* handle a whole dependent link */ do { @@ -444,8 +476,10 @@ get_next: io_wqe_enqueue(wqe, linked); if (hash != -1U && !next_hashed) { + clear_bit(hash, &wq->hash->map); + if (wq_has_sleeper(&wq->hash->wait)) + wake_up(&wq->hash->wait); raw_spin_lock_irq(&wqe->lock); - wqe->hash_map &= ~BIT_ULL(hash); wqe->flags &= ~IO_WQE_FLAG_STALLED; /* skip unnecessary unlock-lock wqe->lock */ if (!work) @@ -471,7 +505,6 @@ static int io_wqe_worker(void *data) loop: raw_spin_lock_irq(&wqe->lock); if (io_wqe_run_queue(wqe)) { - __set_current_state(TASK_RUNNING); io_worker_handle_work(worker); goto loop; } @@ -928,6 +961,24 @@ enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, return IO_WQ_CANCEL_NOTFOUND; } +static int io_wqe_hash_wake(struct wait_queue_entry *wait, unsigned mode, + int sync, void *key) +{ + struct io_wqe *wqe = container_of(wait, struct io_wqe, wait); + int ret; + + list_del_init(&wait->entry); + + rcu_read_lock(); + ret = io_wqe_activate_free_worker(wqe); + rcu_read_unlock(); + + if (!ret) + wake_up_process(wqe->wq->manager); + + return 1; +} + struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) { int ret = -ENOMEM, node; @@ -948,6 +999,8 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) if (ret) goto err_wqes; + refcount_inc(&data->hash->refs); + wq->hash = data->hash; wq->free_work = data->free_work; wq->do_work = data->do_work; @@ -968,6 +1021,8 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) wqe->acct[IO_WQ_ACCT_UNBOUND].max_workers = task_rlimit(current, RLIMIT_NPROC); atomic_set(&wqe->acct[IO_WQ_ACCT_UNBOUND].nr_running, 0); + wqe->wait.func = io_wqe_hash_wake; + INIT_LIST_HEAD(&wqe->wait.entry); wqe->wq = wq; raw_spin_lock_init(&wqe->lock); INIT_WQ_LIST(&wqe->work_list); @@ -989,6 +1044,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) if (refcount_dec_and_test(&wq->refs)) complete(&wq->done); + io_wq_put_hash(data->hash); err: cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); for_each_node(node) @@ -1017,8 +1073,15 @@ void io_wq_destroy(struct io_wq *wq) wait_for_completion(&wq->done); - for_each_node(node) - kfree(wq->wqes[node]); + spin_lock_irq(&wq->hash->wait.lock); + for_each_node(node) { + struct io_wqe *wqe = wq->wqes[node]; + + list_del_init(&wqe->wait.entry); + kfree(wqe); + } + spin_unlock_irq(&wq->hash->wait.lock); + io_wq_put_hash(wq->hash); kfree(wq->wqes); kfree(wq); } diff --git a/fs/io-wq.h b/fs/io-wq.h index 86825673be08..3677b39db015 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -1,6 +1,7 @@ #ifndef INTERNAL_IO_WQ_H #define INTERNAL_IO_WQ_H +#include #include struct io_wq; @@ -93,7 +94,20 @@ static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *); typedef void (io_wq_work_fn)(struct io_wq_work *); +struct io_wq_hash { + refcount_t refs; + unsigned long map; + struct wait_queue_head wait; +}; + +static inline void io_wq_put_hash(struct io_wq_hash *hash) +{ + if (refcount_dec_and_test(&hash->refs)) + kfree(hash); +} + struct io_wq_data { + struct io_wq_hash *hash; io_wq_work_fn *do_work; free_work_fn *free_work; }; diff --git a/fs/io_uring.c b/fs/io_uring.c index 0a435a6f265a..fbc85afa9a87 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -360,6 +360,9 @@ struct io_ring_ctx { unsigned cached_cq_overflow; unsigned long sq_check_overflow; + /* hashed buffered write serialization */ + struct io_wq_hash *hash_map; + struct list_head defer_list; struct list_head timeout_list; struct list_head cq_overflow_list; @@ -454,6 +457,8 @@ struct io_ring_ctx { /* exit task_work */ struct callback_head *exit_task_work; + struct wait_queue_head hash_wait; + /* Keep this last, we don't need it for the fast path */ struct work_struct exit_work; }; @@ -7763,9 +7768,21 @@ static struct io_wq_work *io_free_work(struct io_wq_work *work) static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx) { + struct io_wq_hash *hash; struct io_wq_data data; unsigned int concurrency; + hash = ctx->hash_map; + if (!hash) { + hash = kzalloc(sizeof(*hash), GFP_KERNEL); + if (!hash) + return ERR_PTR(-ENOMEM); + refcount_set(&hash->refs, 1); + init_waitqueue_head(&hash->wait); + ctx->hash_map = hash; + } + + data.hash = hash; data.free_work = io_free_work; data.do_work = io_wq_submit_work; @@ -8405,6 +8422,8 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) percpu_ref_exit(&ctx->refs); free_uid(ctx->user); io_req_caches_free(ctx, NULL); + if (ctx->hash_map) + io_wq_put_hash(ctx->hash_map); kfree(ctx->cancel_hash); kfree(ctx); } -- cgit v1.2.3 From eb85890b29e4d7ae1accdcfba35ed8b16ba9fb97 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 25 Feb 2021 10:13:29 -0700 Subject: io_uring: ensure SQPOLL startup is triggered before error shutdown syzbot reports the following hang: INFO: task syz-executor.0:12538 can't die for more than 143 seconds. task:syz-executor.0 state:D stack:28352 pid:12538 ppid: 8423 flags:0x00004004 Call Trace: context_switch kernel/sched/core.c:4324 [inline] __schedule+0x90c/0x21a0 kernel/sched/core.c:5075 schedule+0xcf/0x270 kernel/sched/core.c:5154 schedule_timeout+0x1db/0x250 kernel/time/timer.c:1868 do_wait_for_common kernel/sched/completion.c:85 [inline] __wait_for_common kernel/sched/completion.c:106 [inline] wait_for_common kernel/sched/completion.c:117 [inline] wait_for_completion+0x168/0x270 kernel/sched/completion.c:138 io_sq_thread_finish+0x96/0x580 fs/io_uring.c:7152 io_sq_offload_create fs/io_uring.c:7929 [inline] io_uring_create fs/io_uring.c:9465 [inline] io_uring_setup+0x1fb2/0x2c20 fs/io_uring.c:9550 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xae which is due to exiting after the SQPOLL thread has been created, but hasn't been started yet. Ensure that we always complete the startup side when waiting for it to exit. Reported-by: syzbot+c927c937cba8ef66dd4a@syzkaller.appspotmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index fbc85afa9a87..ef743594d34a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7141,6 +7141,7 @@ static void io_sq_thread_finish(struct io_ring_ctx *ctx) struct io_sq_data *sqd = ctx->sq_data; if (sqd) { + complete(&sqd->startup); if (sqd->thread) { wait_for_completion(&ctx->sq_thread_comp); io_sq_thread_park(sqd); @@ -7927,7 +7928,7 @@ static void io_sq_offload_start(struct io_ring_ctx *ctx) { struct io_sq_data *sqd = ctx->sq_data; - if ((ctx->flags & IORING_SETUP_SQPOLL) && sqd->thread) + if (ctx->flags & IORING_SETUP_SQPOLL) complete(&sqd->startup); } -- cgit v1.2.3 From 4fb6ac326204b3ab81e6e7a914ccd44d957c1d2c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 25 Feb 2021 10:17:09 -0700 Subject: io-wq: improve manager/worker handling over exec exec will cancel any threads, including the ones that io-wq is using. This isn't a problem, in fact we'd prefer it to be that way since it means we know that any async work cancels naturally without having to handle it proactively. But it does mean that we need to setup a new manager, as the manager and workers are gone. Handle this at queue time, and cancel work if we fail. Since the manager can go away without us noticing, ensure that the manager itself holds a reference to the 'wq' as well. Rename io_wq_destroy() to io_wq_put() to reflect that. In the future we can now simplify exec cancelation handling, for now just leave it the same. Signed-off-by: Jens Axboe --- fs/io-wq.c | 62 ++++++++++++++++++++++++++++++++++++++++------------------- fs/io-wq.h | 2 +- fs/io_uring.c | 4 ++-- 3 files changed, 45 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index d28ad66b7f16..a32b81bac8a2 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -189,8 +189,7 @@ static void io_worker_exit(struct io_worker *worker) raw_spin_unlock_irq(&wqe->lock); kfree_rcu(worker, rcu); - if (refcount_dec_and_test(&wqe->wq->refs)) - complete(&wqe->wq->done); + io_wq_put(wqe->wq); } static inline bool io_wqe_run_queue(struct io_wqe *wqe) @@ -654,8 +653,7 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) else pid = io_wq_fork_thread(task_thread_unbound, worker); if (pid < 0) { - if (refcount_dec_and_test(&wq->refs)) - complete(&wq->done); + io_wq_put(wq); kfree(worker); return false; } @@ -754,11 +752,6 @@ static int io_wq_manager(void *data) io_wq_check_workers(wq); - if (refcount_dec_and_test(&wq->refs)) { - wq->manager = NULL; - complete(&wq->done); - do_exit(0); - } /* if ERROR is set and we get here, we have workers to wake */ if (test_bit(IO_WQ_BIT_ERROR, &wq->state)) { rcu_read_lock(); @@ -767,6 +760,7 @@ static int io_wq_manager(void *data) rcu_read_unlock(); } wq->manager = NULL; + io_wq_put(wq); do_exit(0); } @@ -801,12 +795,40 @@ append: wq_list_add_after(&work->list, &tail->list, &wqe->work_list); } +static int io_wq_fork_manager(struct io_wq *wq) +{ + int ret; + + if (wq->manager) + return 0; + + clear_bit(IO_WQ_BIT_EXIT, &wq->state); + refcount_inc(&wq->refs); + current->flags |= PF_IO_WORKER; + ret = io_wq_fork_thread(io_wq_manager, wq); + current->flags &= ~PF_IO_WORKER; + if (ret >= 0) { + wait_for_completion(&wq->done); + return 0; + } + + io_wq_put(wq); + return ret; +} + static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) { struct io_wqe_acct *acct = io_work_get_acct(wqe, work); int work_flags; unsigned long flags; + /* Can only happen if manager creation fails after exec */ + if (unlikely(io_wq_fork_manager(wqe->wq))) { + work->flags |= IO_WQ_WORK_CANCEL; + wqe->wq->do_work(work); + return; + } + work_flags = work->flags; raw_spin_lock_irqsave(&wqe->lock, flags); io_wqe_insert_work(wqe, work); @@ -1034,16 +1056,11 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) init_completion(&wq->done); refcount_set(&wq->refs, 1); - current->flags |= PF_IO_WORKER; - ret = io_wq_fork_thread(io_wq_manager, wq); - current->flags &= ~PF_IO_WORKER; - if (ret >= 0) { - wait_for_completion(&wq->done); + ret = io_wq_fork_manager(wq); + if (!ret) return wq; - } - if (refcount_dec_and_test(&wq->refs)) - complete(&wq->done); + io_wq_put(wq); io_wq_put_hash(data->hash); err: cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); @@ -1056,7 +1073,7 @@ err_wq: return ERR_PTR(ret); } -void io_wq_destroy(struct io_wq *wq) +static void io_wq_destroy(struct io_wq *wq) { int node; @@ -1071,8 +1088,6 @@ void io_wq_destroy(struct io_wq *wq) io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); rcu_read_unlock(); - wait_for_completion(&wq->done); - spin_lock_irq(&wq->hash->wait.lock); for_each_node(node) { struct io_wqe *wqe = wq->wqes[node]; @@ -1084,6 +1099,13 @@ void io_wq_destroy(struct io_wq *wq) io_wq_put_hash(wq->hash); kfree(wq->wqes); kfree(wq); + +} + +void io_wq_put(struct io_wq *wq) +{ + if (refcount_dec_and_test(&wq->refs)) + io_wq_destroy(wq); } static bool io_wq_worker_affinity(struct io_worker *worker, void *data) diff --git a/fs/io-wq.h b/fs/io-wq.h index 3677b39db015..b6ca12b60c35 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -113,7 +113,7 @@ struct io_wq_data { }; struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); -void io_wq_destroy(struct io_wq *wq); +void io_wq_put(struct io_wq *wq); void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); void io_wq_hash_work(struct io_wq_work *work, void *val); diff --git a/fs/io_uring.c b/fs/io_uring.c index ef743594d34a..f66a8137e125 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2024,7 +2024,7 @@ static void __io_req_task_submit(struct io_kiocb *req) /* ctx stays valid until unlock, even if we drop all ours ctx->refs */ mutex_lock(&ctx->uring_lock); - if (!ctx->sqo_dead && !(current->flags & PF_EXITING)) + if (!ctx->sqo_dead && !(current->flags & PF_EXITING) && !current->in_execve) __io_queue_sqe(req); else __io_req_task_cancel(req, -EFAULT); @@ -8821,7 +8821,7 @@ void __io_uring_files_cancel(struct files_struct *files) if (files) { io_uring_remove_task_files(tctx); if (tctx->io_wq) { - io_wq_destroy(tctx->io_wq); + io_wq_put(tctx->io_wq); tctx->io_wq = NULL; } } -- cgit v1.2.3 From 5f3f26f98ae484a3e187411f9ea8c88c00a65ffc Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 25 Feb 2021 10:17:46 -0700 Subject: io_uring: fix SQPOLL thread handling over exec Just like the changes for io-wq, ensure that we re-fork the SQPOLL thread if the owner execs. Mark the ctx sq thread as sqo_exec if it dies, and the ring as needing a wakeup which will force the task to enter the kernel. When it does, setup the new thread and proceed as usual. Signed-off-by: Jens Axboe --- fs/io_uring.c | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index f66a8137e125..4d79732d7d6b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -339,6 +339,7 @@ struct io_ring_ctx { unsigned int eventfd_async: 1; unsigned int restricted: 1; unsigned int sqo_dead: 1; + unsigned int sqo_exec: 1; /* * Ring buffer of indices into array of io_uring_sqe, which is @@ -6796,6 +6797,10 @@ static int io_sq_thread(void *data) complete_all(&sqd->completion); mutex_lock(&sqd->lock); sqd->thread = NULL; + list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { + ctx->sqo_exec = 1; + io_ring_set_wakeup_flag(ctx); + } mutex_unlock(&sqd->lock); complete(&sqd->exited); @@ -7840,6 +7845,25 @@ void __io_uring_free(struct task_struct *tsk) tsk->io_uring = NULL; } +static int io_sq_thread_fork(struct io_sq_data *sqd, struct io_ring_ctx *ctx) +{ + int ret; + + clear_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); + reinit_completion(&sqd->completion); + ctx->sqo_dead = ctx->sqo_exec = 0; + sqd->task_pid = current->pid; + current->flags |= PF_IO_WORKER; + ret = io_wq_fork_thread(io_sq_thread, sqd); + current->flags &= ~PF_IO_WORKER; + if (ret < 0) { + sqd->thread = NULL; + return ret; + } + wait_for_completion(&sqd->completion); + return io_uring_alloc_task_context(sqd->thread, ctx); +} + static int io_sq_offload_create(struct io_ring_ctx *ctx, struct io_uring_params *p) { @@ -9128,6 +9152,12 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, if (ctx->flags & IORING_SETUP_SQPOLL) { io_cqring_overflow_flush(ctx, false, NULL, NULL); + if (unlikely(ctx->sqo_exec)) { + ret = io_sq_thread_fork(ctx->sq_data, ctx); + if (ret) + goto out; + ctx->sqo_exec = 0; + } ret = -EOWNERDEAD; if (unlikely(ctx->sqo_dead)) goto out; @@ -9229,8 +9259,11 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) */ has_lock = mutex_trylock(&ctx->uring_lock); - if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) + if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) { sq = ctx->sq_data; + if (!sq->thread) + sq = NULL; + } seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1); seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1); -- cgit v1.2.3 From d6ce7f6761bf6d669d9c74ec5d3bd1bfe92380c5 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 25 Feb 2021 10:19:43 -0700 Subject: io-wq: remove now unused IO_WQ_BIT_ERROR This flag is now dead, remove it. Fixes: 1cbd9c2bcf02 ("io-wq: don't create any IO workers upfront") Signed-off-by: Jens Axboe --- fs/io-wq.c | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index a32b81bac8a2..44e20248805a 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -32,7 +32,6 @@ enum { enum { IO_WQ_BIT_EXIT = 0, /* wq exiting */ - IO_WQ_BIT_ERROR = 1, /* error on setup */ }; enum { @@ -733,7 +732,6 @@ static int io_wq_manager(void *data) { struct io_wq *wq = data; char buf[TASK_COMM_LEN]; - int node; sprintf(buf, "iou-mgr-%d", wq->task_pid); set_task_comm(current, buf); @@ -751,14 +749,6 @@ static int io_wq_manager(void *data) } while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)); io_wq_check_workers(wq); - - /* if ERROR is set and we get here, we have workers to wake */ - if (test_bit(IO_WQ_BIT_ERROR, &wq->state)) { - rcu_read_lock(); - for_each_node(node) - io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); - rcu_read_unlock(); - } wq->manager = NULL; io_wq_put(wq); do_exit(0); -- cgit v1.2.3 From 4c9f948142a550af416a2bfb5e56d29ce29e92cf Mon Sep 17 00:00:00 2001 From: Steve French Date: Tue, 23 Feb 2021 15:50:57 -0600 Subject: cifs: Add new mount parameter "acdirmax" to allow caching directory metadata nfs and cifs on Linux currently have a mount parameter "actimeo" to control metadata (attribute) caching but cifs does not have additional mount parameters to allow distinguishing between caching directory metadata (e.g. needed to revalidate paths) and that for files. Add new mount parameter "acdirmax" to allow caching metadata for directories more loosely than file data. NFS adjusts metadata caching from acdirmin to acdirmax (and another two mount parms for files) but to reduce complexity, it is safer to just introduce the one mount parm to allow caching directories longer. The defaults for acdirmax and actimeo (for cifs.ko) are conservative, 1 second (NFS defaults acdirmax to 60 seconds). For many workloads, setting acdirmax to a higher value is safe and will improve performance. This patch leaves unchanged the default values for caching metadata for files and directories but gives the user more flexibility in adjusting them safely for their workload via the new mount parm. Signed-off-by: Steve French Reviewed-by: Ronnie Sahlberg Reviewed-By: Tom Talpey --- fs/cifs/cifsfs.c | 3 ++- fs/cifs/connect.c | 2 ++ fs/cifs/fs_context.c | 9 +++++++++ fs/cifs/fs_context.h | 4 +++- 4 files changed, 16 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index 6f33ff3f625f..4e0b0b26e844 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -637,8 +637,9 @@ cifs_show_options(struct seq_file *s, struct dentry *root) seq_printf(s, ",snapshot=%llu", tcon->snapshot_time); if (tcon->handle_timeout) seq_printf(s, ",handletimeout=%u", tcon->handle_timeout); - /* convert actimeo and display it in seconds */ + /* convert actimeo and directory attribute timeout and display in seconds */ seq_printf(s, ",actimeo=%lu", cifs_sb->ctx->actimeo / HZ); + seq_printf(s, ",acdirmax=%lu", cifs_sb->ctx->acdirmax / HZ); if (tcon->ses->chan_max > 1) seq_printf(s, ",multichannel,max_channels=%zu", diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index cd6dbeaf2166..a9dc39aee9f4 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -2278,6 +2278,8 @@ compare_mount_options(struct super_block *sb, struct cifs_mnt_data *mnt_data) if (old->ctx->actimeo != new->ctx->actimeo) return 0; + if (old->ctx->acdirmax != new->ctx->acdirmax) + return 0; return 1; } diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 7d04f2255624..f3be07f4671d 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -140,6 +140,7 @@ const struct fs_parameter_spec smb3_fs_parameters[] = { fsparam_u32("rsize", Opt_rsize), fsparam_u32("wsize", Opt_wsize), fsparam_u32("actimeo", Opt_actimeo), + fsparam_u32("acdirmax", Opt_acdirmax), fsparam_u32("echo_interval", Opt_echo_interval), fsparam_u32("max_credits", Opt_max_credits), fsparam_u32("handletimeout", Opt_handletimeout), @@ -936,6 +937,13 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, goto cifs_parse_mount_err; } break; + case Opt_acdirmax: + ctx->acdirmax = HZ * result.uint_32; + if (ctx->acdirmax > CIFS_MAX_ACTIMEO) { + cifs_dbg(VFS, "acdirmax too large\n"); + goto cifs_parse_mount_err; + } + break; case Opt_echo_interval: ctx->echo_interval = result.uint_32; break; @@ -1362,6 +1370,7 @@ int smb3_init_fs_context(struct fs_context *fc) ctx->strict_io = true; ctx->actimeo = CIFS_DEF_ACTIMEO; + ctx->acdirmax = CIFS_DEF_ACTIMEO; /* Most clients set timeout to 0, allows server to use its default */ ctx->handle_timeout = 0; /* See MS-SMB2 spec section 2.2.14.2.12 */ diff --git a/fs/cifs/fs_context.h b/fs/cifs/fs_context.h index 1c44a460e2c0..472372fec4e9 100644 --- a/fs/cifs/fs_context.h +++ b/fs/cifs/fs_context.h @@ -118,6 +118,7 @@ enum cifs_param { Opt_rsize, Opt_wsize, Opt_actimeo, + Opt_acdirmax, Opt_echo_interval, Opt_max_credits, Opt_snapshot, @@ -232,7 +233,8 @@ struct smb3_fs_context { unsigned int wsize; unsigned int min_offload; bool sockopt_tcp_nodelay:1; - unsigned long actimeo; /* attribute cache timeout (jiffies) */ + unsigned long actimeo; /* attribute cache timeout for files (jiffies) */ + unsigned long acdirmax; /* attribute cache timeout for directories (jiffies) */ struct smb_version_operations *ops; struct smb_version_values *vals; char *prepath; -- cgit v1.2.3 From ddaf6d4a9253939036fa70d71534e482ee7413f6 Mon Sep 17 00:00:00 2001 From: Steve French Date: Tue, 23 Feb 2021 16:16:09 -0600 Subject: cifs: convert revalidate of directories to using directory metadata cache timeout The new optional mount parm, "acdirmax" allows caching the metadata for a directory longer than file metadata, which can be very helpful for performance. Convert cifs_inode_needs_reval to check acdirmax for revalidating directory metadata. Signed-off-by: Steve French Reviewed-by: Ronnie Sahlberg Reviewed-By: Tom Talpey Signed-off-by: Steve French --- fs/cifs/inode.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index a83b3a8ffaac..cfd31cc4520f 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -2198,12 +2198,23 @@ cifs_inode_needs_reval(struct inode *inode) if (!lookupCacheEnabled) return true; - if (!cifs_sb->ctx->actimeo) - return true; - - if (!time_in_range(jiffies, cifs_i->time, - cifs_i->time + cifs_sb->ctx->actimeo)) - return true; + /* + * depending on inode type, check if attribute caching disabled for + * files or directories + */ + if (S_ISDIR(inode->i_mode)) { + if (!cifs_sb->ctx->acdirmax) + return true; + if (!time_in_range(jiffies, cifs_i->time, + cifs_i->time + cifs_sb->ctx->acdirmax)) + return true; + } else { /* file */ + if (!cifs_sb->ctx->actimeo) + return true; + if (!time_in_range(jiffies, cifs_i->time, + cifs_i->time + cifs_sb->ctx->actimeo)) + return true; + } /* hardlinked files w/ noserverino get "special" treatment */ if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM) && -- cgit v1.2.3 From 5780464614f6abe6026f00cf5a0777aa453ba450 Mon Sep 17 00:00:00 2001 From: Steve French Date: Wed, 24 Feb 2021 12:12:53 -0600 Subject: cifs: Add new parameter "acregmax" for distinct file and directory metadata timeout The new optional mount parameter "acregmax" allows a different timeout for file metadata ("acdirmax" now allows controlling timeout for directory metadata). Setting "actimeo" still works as before, and changes timeout for both files and directories, but specifying "acregmax" or "acdirmax" allows overriding the default more granularly which can be a big performance benefit on some workloads. "acregmax" is already used by NFS as a mount parameter (albeit with a larger default and thus looser caching). Suggested-by: Tom Talpey Reviewed-By: Tom Talpey Reviewed-by: Ronnie Sahlberg Signed-off-by: Steve French --- fs/cifs/cifsfs.c | 15 ++++++++++++--- fs/cifs/connect.c | 2 +- fs/cifs/fs_context.c | 23 ++++++++++++++++++----- fs/cifs/fs_context.h | 6 ++++-- fs/cifs/inode.c | 4 ++-- 5 files changed, 37 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index 4e0b0b26e844..3b61f09f3e1b 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -637,9 +637,18 @@ cifs_show_options(struct seq_file *s, struct dentry *root) seq_printf(s, ",snapshot=%llu", tcon->snapshot_time); if (tcon->handle_timeout) seq_printf(s, ",handletimeout=%u", tcon->handle_timeout); - /* convert actimeo and directory attribute timeout and display in seconds */ - seq_printf(s, ",actimeo=%lu", cifs_sb->ctx->actimeo / HZ); - seq_printf(s, ",acdirmax=%lu", cifs_sb->ctx->acdirmax / HZ); + + /* + * Display file and directory attribute timeout in seconds. + * If file and directory attribute timeout the same then actimeo + * was likely specified on mount + */ + if (cifs_sb->ctx->acdirmax == cifs_sb->ctx->acregmax) + seq_printf(s, ",actimeo=%lu", cifs_sb->ctx->acregmax / HZ); + else { + seq_printf(s, ",acdirmax=%lu", cifs_sb->ctx->acdirmax / HZ); + seq_printf(s, ",acregmax=%lu", cifs_sb->ctx->acregmax / HZ); + } if (tcon->ses->chan_max > 1) seq_printf(s, ",multichannel,max_channels=%zu", diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index a9dc39aee9f4..9ecd8098c2b6 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -2276,7 +2276,7 @@ compare_mount_options(struct super_block *sb, struct cifs_mnt_data *mnt_data) if (strcmp(old->local_nls->charset, new->local_nls->charset)) return 0; - if (old->ctx->actimeo != new->ctx->actimeo) + if (old->ctx->acregmax != new->ctx->acregmax) return 0; if (old->ctx->acdirmax != new->ctx->acdirmax) return 0; diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index f3be07f4671d..14c955a30006 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -141,6 +141,7 @@ const struct fs_parameter_spec smb3_fs_parameters[] = { fsparam_u32("wsize", Opt_wsize), fsparam_u32("actimeo", Opt_actimeo), fsparam_u32("acdirmax", Opt_acdirmax), + fsparam_u32("acregmax", Opt_acregmax), fsparam_u32("echo_interval", Opt_echo_interval), fsparam_u32("max_credits", Opt_max_credits), fsparam_u32("handletimeout", Opt_handletimeout), @@ -930,10 +931,10 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, ctx->wsize = result.uint_32; ctx->got_wsize = true; break; - case Opt_actimeo: - ctx->actimeo = HZ * result.uint_32; - if (ctx->actimeo > CIFS_MAX_ACTIMEO) { - cifs_dbg(VFS, "attribute cache timeout too large\n"); + case Opt_acregmax: + ctx->acregmax = HZ * result.uint_32; + if (ctx->acregmax > CIFS_MAX_ACTIMEO) { + cifs_dbg(VFS, "acregmax too large\n"); goto cifs_parse_mount_err; } break; @@ -944,6 +945,18 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, goto cifs_parse_mount_err; } break; + case Opt_actimeo: + if (HZ * result.uint_32 > CIFS_MAX_ACTIMEO) { + cifs_dbg(VFS, "timeout too large\n"); + goto cifs_parse_mount_err; + } + if ((ctx->acdirmax != CIFS_DEF_ACTIMEO) || + (ctx->acregmax != CIFS_DEF_ACTIMEO)) { + cifs_dbg(VFS, "actimeo ignored since acregmax or acdirmax specified\n"); + break; + } + ctx->acdirmax = ctx->acregmax = HZ * result.uint_32; + break; case Opt_echo_interval: ctx->echo_interval = result.uint_32; break; @@ -1369,7 +1382,7 @@ int smb3_init_fs_context(struct fs_context *fc) /* default is to use strict cifs caching semantics */ ctx->strict_io = true; - ctx->actimeo = CIFS_DEF_ACTIMEO; + ctx->acregmax = CIFS_DEF_ACTIMEO; ctx->acdirmax = CIFS_DEF_ACTIMEO; /* Most clients set timeout to 0, allows server to use its default */ diff --git a/fs/cifs/fs_context.h b/fs/cifs/fs_context.h index 472372fec4e9..87dd1f7168f2 100644 --- a/fs/cifs/fs_context.h +++ b/fs/cifs/fs_context.h @@ -119,6 +119,7 @@ enum cifs_param { Opt_wsize, Opt_actimeo, Opt_acdirmax, + Opt_acregmax, Opt_echo_interval, Opt_max_credits, Opt_snapshot, @@ -233,8 +234,9 @@ struct smb3_fs_context { unsigned int wsize; unsigned int min_offload; bool sockopt_tcp_nodelay:1; - unsigned long actimeo; /* attribute cache timeout for files (jiffies) */ - unsigned long acdirmax; /* attribute cache timeout for directories (jiffies) */ + /* attribute cache timemout for files and directories in jiffies */ + unsigned long acregmax; + unsigned long acdirmax; struct smb_version_operations *ops; struct smb_version_values *vals; char *prepath; diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index cfd31cc4520f..0b0b01ef3ecb 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -2209,10 +2209,10 @@ cifs_inode_needs_reval(struct inode *inode) cifs_i->time + cifs_sb->ctx->acdirmax)) return true; } else { /* file */ - if (!cifs_sb->ctx->actimeo) + if (!cifs_sb->ctx->acregmax) return true; if (!time_in_range(jiffies, cifs_i->time, - cifs_i->time + cifs_sb->ctx->actimeo)) + cifs_i->time + cifs_sb->ctx->acregmax)) return true; } -- cgit v1.2.3 From d08395a3f2f473c6ceeb316a1aeb7fad5b43014f Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Thu, 25 Feb 2021 17:36:27 +1000 Subject: cifs: fix handling of escaped ',' in the password mount argument Passwords can contain ',' which are also used as the separator between mount options. Mount.cifs will escape all ',' characters as the string ",,". Update parsing of the mount options to detect ",," and treat it as a single 'c' character. Fixes: 24e0a1eff9e2 ("cifs: switch to new mount api") Cc: stable@vger.kernel.org # 5.11 Reported-by: Simon Taylor Tested-by: Simon Taylor Signed-off-by: Ronnie Sahlberg Signed-off-by: Steve French --- fs/cifs/fs_context.c | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 14c955a30006..892f51a21278 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -544,20 +544,37 @@ static int smb3_fs_context_parse_monolithic(struct fs_context *fc, /* BB Need to add support for sep= here TBD */ while ((key = strsep(&options, ",")) != NULL) { - if (*key) { - size_t v_len = 0; - char *value = strchr(key, '='); - - if (value) { - if (value == key) - continue; - *value++ = 0; - v_len = strlen(value); - } - ret = vfs_parse_fs_string(fc, key, value, v_len); - if (ret < 0) - break; + size_t len; + char *value; + + if (*key == 0) + break; + + /* Check if following character is the deliminator If yes, + * we have encountered a double deliminator reset the NULL + * character to the deliminator + */ + while (options && options[0] == ',') { + len = strlen(key); + strcpy(key + len, options); + options = strchr(options, ','); + if (options) + *options++ = 0; } + + + len = 0; + value = strchr(key, '='); + if (value) { + if (value == key) + continue; + *value++ = 0; + len = strlen(value); + } + + ret = vfs_parse_fs_string(fc, key, value, len); + if (ret < 0) + break; } return ret; -- cgit v1.2.3 From d01132ae50207bb6fd94e08e80c2d7b839408086 Mon Sep 17 00:00:00 2001 From: Paulo Alcantara Date: Wed, 24 Feb 2021 20:59:24 -0300 Subject: cifs: fix nodfs mount option Skip DFS resolving when mounting with 'nodfs' even if CONFIG_CIFS_DFS_UPCALL is enabled. Signed-off-by: Paulo Alcantara (SUSE) Cc: stable@vger.kernel.org # 5.11 Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/connect.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 9ecd8098c2b6..afb0154e8cb0 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -3379,15 +3379,15 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) rc = mount_get_conns(ctx, cifs_sb, &xid, &server, &ses, &tcon); /* - * Unconditionally try to get an DFS referral (even cached) to determine whether it is an - * DFS mount. + * If called with 'nodfs' mount option, then skip DFS resolving. Otherwise unconditionally + * try to get an DFS referral (even cached) to determine whether it is an DFS mount. * * Skip prefix path to provide support for DFS referrals from w2k8 servers which don't seem * to respond with PATH_NOT_COVERED to requests that include the prefix. */ - if (dfs_cache_find(xid, ses, cifs_sb->local_nls, cifs_remap(cifs_sb), ctx->UNC + 1, NULL, + if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_DFS) || + dfs_cache_find(xid, ses, cifs_sb->local_nls, cifs_remap(cifs_sb), ctx->UNC + 1, NULL, NULL)) { - /* No DFS referral was returned. Looks like a regular share. */ if (rc) goto error; /* Check if it is fully accessible and then mount it */ -- cgit v1.2.3 From 8513222b9ef2709ba40cbda07b55d5fbcfdd4bc7 Mon Sep 17 00:00:00 2001 From: Paulo Alcantara Date: Wed, 24 Feb 2021 20:59:21 -0300 Subject: cifs: fix DFS failover In do_dfs_failover(), the mount_get_conns() function requires the full fs context in order to get new connection to server, so clone the original context and change it accordingly when retrying the DFS targets in the referral. If failover was successful, then update original context with the new UNC, prefix path and ip address. Signed-off-by: Paulo Alcantara (SUSE) Cc: stable@vger.kernel.org # 5.11 Signed-off-by: Steve French --- fs/cifs/connect.c | 123 ++++++++++++++++++++++++++---------------------------- 1 file changed, 59 insertions(+), 64 deletions(-) (limited to 'fs') diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index afb0154e8cb0..820aaaa48c57 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -3047,96 +3047,91 @@ static int update_vol_info(const struct dfs_cache_tgt_iterator *tgt_it, return 0; } -static int setup_dfs_tgt_conn(const char *path, const char *full_path, - const struct dfs_cache_tgt_iterator *tgt_it, - struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx, - unsigned int *xid, struct TCP_Server_Info **server, - struct cifs_ses **ses, struct cifs_tcon **tcon) -{ - int rc; - struct dfs_info3_param ref = {0}; - char *mdata = NULL; - struct smb3_fs_context fake_ctx = {NULL}; - char *fake_devname = NULL; - - cifs_dbg(FYI, "%s: dfs path: %s\n", __func__, path); - - rc = dfs_cache_get_tgt_referral(path, tgt_it, &ref); - if (rc) - return rc; - - mdata = cifs_compose_mount_options(cifs_sb->ctx->mount_options, - full_path + 1, &ref, - &fake_devname); - free_dfs_info_param(&ref); - - if (IS_ERR(mdata)) { - rc = PTR_ERR(mdata); - mdata = NULL; - } else - rc = cifs_setup_volume_info(&fake_ctx, mdata, fake_devname); - - kfree(mdata); - kfree(fake_devname); - - if (!rc) { - /* - * We use a 'fake_ctx' here because we need pass it down to the - * mount_{get,put} functions to test connection against new DFS - * targets. - */ - mount_put_conns(cifs_sb, *xid, *server, *ses, *tcon); - rc = mount_get_conns(&fake_ctx, cifs_sb, xid, server, ses, - tcon); - if (!rc || (*server && *ses)) { - /* - * We were able to connect to new target server. - * Update current context with new target server. - */ - rc = update_vol_info(tgt_it, &fake_ctx, ctx); - } - } - smb3_cleanup_fs_context_contents(&fake_ctx); - return rc; -} - static int do_dfs_failover(const char *path, const char *full_path, struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx, struct cifs_ses *root_ses, unsigned int *xid, struct TCP_Server_Info **server, struct cifs_ses **ses, struct cifs_tcon **tcon) { int rc; - struct dfs_cache_tgt_list tgt_list; + struct dfs_cache_tgt_list tgt_list = {0}; struct dfs_cache_tgt_iterator *tgt_it = NULL; + struct smb3_fs_context tmp_ctx = {NULL}; if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_DFS) return -EOPNOTSUPP; + cifs_dbg(FYI, "%s: path=%s full_path=%s\n", __func__, path, full_path); + rc = dfs_cache_noreq_find(path, NULL, &tgt_list); if (rc) return rc; + /* + * We use a 'tmp_ctx' here because we need pass it down to the mount_{get,put} functions to + * test connection against new DFS targets. + */ + rc = smb3_fs_context_dup(&tmp_ctx, ctx); + if (rc) + goto out; for (;;) { + struct dfs_info3_param ref = {0}; + char *fake_devname = NULL, *mdata = NULL; + /* Get next DFS target server - if any */ rc = get_next_dfs_tgt(path, &tgt_list, &tgt_it); if (rc) break; - /* Connect to next DFS target */ - rc = setup_dfs_tgt_conn(path, full_path, tgt_it, cifs_sb, ctx, xid, server, ses, - tcon); - if (!rc || (*server && *ses)) + + rc = dfs_cache_get_tgt_referral(path, tgt_it, &ref); + if (rc) + break; + + cifs_dbg(FYI, "%s: old ctx: UNC=%s prepath=%s\n", __func__, tmp_ctx.UNC, + tmp_ctx.prepath); + + mdata = cifs_compose_mount_options(cifs_sb->ctx->mount_options, full_path + 1, &ref, + &fake_devname); + free_dfs_info_param(&ref); + + if (IS_ERR(mdata)) { + rc = PTR_ERR(mdata); + mdata = NULL; + } else + rc = cifs_setup_volume_info(&tmp_ctx, mdata, fake_devname); + + kfree(mdata); + kfree(fake_devname); + + if (rc) + break; + + cifs_dbg(FYI, "%s: new ctx: UNC=%s prepath=%s\n", __func__, tmp_ctx.UNC, + tmp_ctx.prepath); + + mount_put_conns(cifs_sb, *xid, *server, *ses, *tcon); + rc = mount_get_conns(&tmp_ctx, cifs_sb, xid, server, ses, tcon); + if (!rc || (*server && *ses)) { + /* + * We were able to connect to new target server. Update current context with + * new target server. + */ + rc = update_vol_info(tgt_it, &tmp_ctx, ctx); break; + } } if (!rc) { + cifs_dbg(FYI, "%s: final ctx: UNC=%s prepath=%s\n", __func__, tmp_ctx.UNC, + tmp_ctx.prepath); /* - * Update DFS target hint in DFS referral cache with the target - * server we successfully reconnected to. + * Update DFS target hint in DFS referral cache with the target server we + * successfully reconnected to. */ - rc = dfs_cache_update_tgthint(*xid, root_ses ? root_ses : *ses, - cifs_sb->local_nls, - cifs_remap(cifs_sb), path, - tgt_it); + rc = dfs_cache_update_tgthint(*xid, root_ses ? root_ses : *ses, cifs_sb->local_nls, + cifs_remap(cifs_sb), path, tgt_it); } + +out: + smb3_cleanup_fs_context_contents(&tmp_ctx); dfs_cache_free_tgts(&tgt_list); return rc; } -- cgit v1.2.3 From ff2c54a04097dee0b8899c485360719844d923f8 Mon Sep 17 00:00:00 2001 From: Paulo Alcantara Date: Wed, 24 Feb 2021 20:59:22 -0300 Subject: cifs: check all path components in resolved dfs target Handle the case where a resolved target share is like //server/users/dir, and the user "foo" has no read permission to access the parent folder "users" but has access to the final path component "dir". is_path_remote() already implements that, so call it directly. Signed-off-by: Paulo Alcantara (SUSE) Cc: stable@vger.kernel.org # 5.11 Signed-off-by: Steve French --- fs/cifs/connect.c | 93 ++++++++++++++++++++----------------------------------- 1 file changed, 33 insertions(+), 60 deletions(-) (limited to 'fs') diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 820aaaa48c57..1a6d6e1e2c71 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -3289,73 +3289,46 @@ static void put_root_ses(struct cifs_ses *ses) cifs_put_smb_ses(ses); } -/* Check if a path component is remote and then update @dfs_path accordingly */ -static int check_dfs_prepath(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx, - const unsigned int xid, struct TCP_Server_Info *server, - struct cifs_tcon *tcon, char **dfs_path) +/* Set up next dfs prefix path in @dfs_path */ +static int next_dfs_prepath(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx, + const unsigned int xid, struct TCP_Server_Info *server, + struct cifs_tcon *tcon, char **dfs_path) { - char *path, *s; - char sep = CIFS_DIR_SEP(cifs_sb), tmp; - char *npath; - int rc = 0; - int added_treename = tcon->Flags & SMB_SHARE_IS_IN_DFS; - int skip = added_treename; + char *path, *npath; + int added_treename = is_tcon_dfs(tcon); + int rc; path = cifs_build_path_to_root(ctx, cifs_sb, tcon, added_treename); if (!path) return -ENOMEM; - /* - * Walk through the path components in @path and check if they're accessible. In case any of - * the components is -EREMOTE, then update @dfs_path with the next DFS referral request path - * (NOT including the remaining components). - */ - s = path; - do { - /* skip separators */ - while (*s && *s == sep) - s++; - if (!*s) - break; - /* next separator */ - while (*s && *s != sep) - s++; - /* - * if the treename is added, we then have to skip the first - * part within the separators - */ - if (skip) { - skip = 0; - continue; + rc = is_path_remote(cifs_sb, ctx, xid, server, tcon); + if (rc == -EREMOTE) { + struct smb3_fs_context v = {NULL}; + /* if @path contains a tree name, skip it in the prefix path */ + if (added_treename) { + rc = smb3_parse_devname(path, &v); + if (rc) + goto out; + npath = build_unc_path_to_root(&v, cifs_sb, true); + smb3_cleanup_fs_context_contents(&v); + } else { + v.UNC = ctx->UNC; + v.prepath = path + 1; + npath = build_unc_path_to_root(&v, cifs_sb, true); } - tmp = *s; - *s = 0; - rc = server->ops->is_path_accessible(xid, tcon, cifs_sb, path); - if (rc && rc == -EREMOTE) { - struct smb3_fs_context v = {NULL}; - /* if @path contains a tree name, skip it in the prefix path */ - if (added_treename) { - rc = smb3_parse_devname(path, &v); - if (rc) - break; - rc = -EREMOTE; - npath = build_unc_path_to_root(&v, cifs_sb, true); - smb3_cleanup_fs_context_contents(&v); - } else { - v.UNC = ctx->UNC; - v.prepath = path + 1; - npath = build_unc_path_to_root(&v, cifs_sb, true); - } - if (IS_ERR(npath)) { - rc = PTR_ERR(npath); - break; - } - kfree(*dfs_path); - *dfs_path = npath; + + if (IS_ERR(npath)) { + rc = PTR_ERR(npath); + goto out; } - *s = tmp; - } while (rc == 0); + kfree(*dfs_path); + *dfs_path = npath; + rc = -EREMOTE; + } + +out: kfree(path); return rc; } @@ -3441,8 +3414,8 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) put_root_ses(root_ses); set_root_ses(cifs_sb, ses, &root_ses); } - /* Check for remaining path components and then continue chasing them (-EREMOTE) */ - rc = check_dfs_prepath(cifs_sb, ctx, xid, server, tcon, &ref_path); + /* Get next dfs path and then continue chasing them if -EREMOTE */ + rc = next_dfs_prepath(cifs_sb, ctx, xid, server, tcon, &ref_path); /* Prevent recursion on broken link referrals */ if (rc == -EREMOTE && ++count > MAX_NESTED_LINKS) rc = -ELOOP; -- cgit v1.2.3 From 5ff2836ed3a5c24420a7235be25a462594cdc4ea Mon Sep 17 00:00:00 2001 From: Paulo Alcantara Date: Wed, 24 Feb 2021 20:59:23 -0300 Subject: cifs: introduce helper for finding referral server to improve DFS target resolution Some servers seem to mistakenly report different values for capabilities and share flags, so we can't always rely on those values to decide whether the resolved target can handle any new DFS referrals. Add a new helper is_referral_server() to check if all resolved targets can handle new DFS referrals by directly looking at the GET_DFS_REFERRAL.ReferralHeaderFlags value as specified in MS-DFSC 2.2.4 RESP_GET_DFS_REFERRAL in addition to is_tcon_dfs(). Signed-off-by: Paulo Alcantara (SUSE) Cc: stable@vger.kernel.org # 5.11 Signed-off-by: Steve French --- fs/cifs/connect.c | 35 ++++++++++++++++++++++++++++++++++- fs/cifs/dfs_cache.c | 33 +++++++++++++++++---------------- 2 files changed, 51 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 1a6d6e1e2c71..b2447cea45ae 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -3333,6 +3333,33 @@ out: return rc; } +/* Check if resolved targets can handle any DFS referrals */ +static int is_referral_server(const char *ref_path, struct cifs_tcon *tcon, bool *ref_server) +{ + int rc; + struct dfs_info3_param ref = {0}; + + if (is_tcon_dfs(tcon)) { + *ref_server = true; + } else { + cifs_dbg(FYI, "%s: ref_path=%s\n", __func__, ref_path); + + rc = dfs_cache_noreq_find(ref_path, &ref, NULL); + if (rc) { + cifs_dbg(VFS, "%s: dfs_cache_noreq_find: failed (rc=%d)\n", __func__, rc); + return rc; + } + cifs_dbg(FYI, "%s: ref.flags=0x%x\n", __func__, ref.flags); + /* + * Check if all targets are capable of handling DFS referrals as per + * MS-DFSC 2.2.4 RESP_GET_DFS_REFERRAL. + */ + *ref_server = !!(ref.flags & DFSREF_REFERRAL_SERVER); + free_dfs_info_param(&ref); + } + return 0; +} + int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) { int rc = 0; @@ -3344,6 +3371,7 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) char *ref_path = NULL, *full_path = NULL; char *oldmnt = NULL; char *mntdata = NULL; + bool ref_server = false; rc = mount_get_conns(ctx, cifs_sb, &xid, &server, &ses, &tcon); /* @@ -3409,11 +3437,16 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) break; if (!tcon) continue; + /* Make sure that requests go through new root servers */ - if (is_tcon_dfs(tcon)) { + rc = is_referral_server(ref_path + 1, tcon, &ref_server); + if (rc) + break; + if (ref_server) { put_root_ses(root_ses); set_root_ses(cifs_sb, ses, &root_ses); } + /* Get next dfs path and then continue chasing them if -EREMOTE */ rc = next_dfs_prepath(cifs_sb, ctx, xid, server, tcon, &ref_path); /* Prevent recursion on broken link referrals */ diff --git a/fs/cifs/dfs_cache.c b/fs/cifs/dfs_cache.c index 4950ab0486ae..098b4bc8da59 100644 --- a/fs/cifs/dfs_cache.c +++ b/fs/cifs/dfs_cache.c @@ -37,11 +37,12 @@ struct cache_dfs_tgt { struct cache_entry { struct hlist_node hlist; const char *path; - int ttl; - int srvtype; - int flags; + int hdr_flags; /* RESP_GET_DFS_REFERRAL.ReferralHeaderFlags */ + int ttl; /* DFS_REREFERRAL_V3.TimeToLive */ + int srvtype; /* DFS_REREFERRAL_V3.ServerType */ + int ref_flags; /* DFS_REREFERRAL_V3.ReferralEntryFlags */ struct timespec64 etime; - int path_consumed; + int path_consumed; /* RESP_GET_DFS_REFERRAL.PathConsumed */ int numtgts; struct list_head tlist; struct cache_dfs_tgt *tgthint; @@ -166,14 +167,11 @@ static int dfscache_proc_show(struct seq_file *m, void *v) continue; seq_printf(m, - "cache entry: path=%s,type=%s,ttl=%d,etime=%ld," - "interlink=%s,path_consumed=%d,expired=%s\n", - ce->path, - ce->srvtype == DFS_TYPE_ROOT ? "root" : "link", - ce->ttl, ce->etime.tv_nsec, - IS_INTERLINK_SET(ce->flags) ? "yes" : "no", - ce->path_consumed, - cache_entry_expired(ce) ? "yes" : "no"); + "cache entry: path=%s,type=%s,ttl=%d,etime=%ld,hdr_flags=0x%x,ref_flags=0x%x,interlink=%s,path_consumed=%d,expired=%s\n", + ce->path, ce->srvtype == DFS_TYPE_ROOT ? "root" : "link", + ce->ttl, ce->etime.tv_nsec, ce->ref_flags, ce->hdr_flags, + IS_INTERLINK_SET(ce->hdr_flags) ? "yes" : "no", + ce->path_consumed, cache_entry_expired(ce) ? "yes" : "no"); list_for_each_entry(t, &ce->tlist, list) { seq_printf(m, " %s%s\n", @@ -236,11 +234,12 @@ static inline void dump_tgts(const struct cache_entry *ce) static inline void dump_ce(const struct cache_entry *ce) { - cifs_dbg(FYI, "cache entry: path=%s,type=%s,ttl=%d,etime=%ld,interlink=%s,path_consumed=%d,expired=%s\n", + cifs_dbg(FYI, "cache entry: path=%s,type=%s,ttl=%d,etime=%ld,hdr_flags=0x%x,ref_flags=0x%x,interlink=%s,path_consumed=%d,expired=%s\n", ce->path, ce->srvtype == DFS_TYPE_ROOT ? "root" : "link", ce->ttl, ce->etime.tv_nsec, - IS_INTERLINK_SET(ce->flags) ? "yes" : "no", + ce->hdr_flags, ce->ref_flags, + IS_INTERLINK_SET(ce->hdr_flags) ? "yes" : "no", ce->path_consumed, cache_entry_expired(ce) ? "yes" : "no"); dump_tgts(ce); @@ -381,7 +380,8 @@ static int copy_ref_data(const struct dfs_info3_param *refs, int numrefs, ce->ttl = refs[0].ttl; ce->etime = get_expire_time(ce->ttl); ce->srvtype = refs[0].server_type; - ce->flags = refs[0].ref_flag; + ce->hdr_flags = refs[0].flags; + ce->ref_flags = refs[0].ref_flag; ce->path_consumed = refs[0].path_consumed; for (i = 0; i < numrefs; i++) { @@ -799,7 +799,8 @@ static int setup_referral(const char *path, struct cache_entry *ce, ref->path_consumed = ce->path_consumed; ref->ttl = ce->ttl; ref->server_type = ce->srvtype; - ref->ref_flag = ce->flags; + ref->ref_flag = ce->ref_flags; + ref->flags = ce->hdr_flags; return 0; -- cgit v1.2.3 From cf0604a686b11175d8beae60281c4ccc95aaa5c2 Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 4 Feb 2021 00:15:21 -0600 Subject: cifs: use discard iterator to discard unneeded network data more efficiently The iterator, ITER_DISCARD, that can only be used in READ mode and just discards any data copied to it, was added to allow a network filesystem to discard any unwanted data sent by a server. Convert cifs_discard_from_socket() to use this. Signed-off-by: David Howells Signed-off-by: Steve French --- fs/cifs/cifsproto.h | 2 ++ fs/cifs/cifssmb.c | 6 +++--- fs/cifs/connect.c | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsproto.h b/fs/cifs/cifsproto.h index 32f7a013402e..75ce6f742b8d 100644 --- a/fs/cifs/cifsproto.h +++ b/fs/cifs/cifsproto.h @@ -232,6 +232,8 @@ extern unsigned int setup_special_user_owner_ACE(struct cifs_ace *pace); extern void dequeue_mid(struct mid_q_entry *mid, bool malformed); extern int cifs_read_from_socket(struct TCP_Server_Info *server, char *buf, unsigned int to_read); +extern ssize_t cifs_discard_from_socket(struct TCP_Server_Info *server, + size_t to_read); extern int cifs_read_page_from_socket(struct TCP_Server_Info *server, struct page *page, unsigned int page_offset, diff --git a/fs/cifs/cifssmb.c b/fs/cifs/cifssmb.c index 0496934feecb..c279527aae92 100644 --- a/fs/cifs/cifssmb.c +++ b/fs/cifs/cifssmb.c @@ -1451,9 +1451,9 @@ cifs_discard_remaining_data(struct TCP_Server_Info *server) while (remaining > 0) { int length; - length = cifs_read_from_socket(server, server->bigbuf, - min_t(unsigned int, remaining, - CIFSMaxBufSize + MAX_HEADER_SIZE(server))); + length = cifs_discard_from_socket(server, + min_t(size_t, remaining, + CIFSMaxBufSize + MAX_HEADER_SIZE(server))); if (length < 0) return length; server->total_read += length; diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index b2447cea45ae..112692300fb6 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -564,6 +564,23 @@ cifs_read_from_socket(struct TCP_Server_Info *server, char *buf, return cifs_readv_from_socket(server, &smb_msg); } +ssize_t +cifs_discard_from_socket(struct TCP_Server_Info *server, size_t to_read) +{ + struct msghdr smb_msg; + + /* + * iov_iter_discard already sets smb_msg.type and count and iov_offset + * and cifs_readv_from_socket sets msg_control and msg_controllen + * so little to initialize in struct msghdr + */ + smb_msg.msg_name = NULL; + smb_msg.msg_namelen = 0; + iov_iter_discard(&smb_msg.msg_iter, READ, to_read); + + return cifs_readv_from_socket(server, &smb_msg); +} + int cifs_read_page_from_socket(struct TCP_Server_Info *server, struct page *page, unsigned int page_offset, unsigned int to_read) -- cgit v1.2.3 From 8369dfd7841e70711c53a065ffb8029f24520200 Mon Sep 17 00:00:00 2001 From: Steve French Date: Mon, 15 Feb 2021 23:58:58 -0600 Subject: cifs: update internal version number To 2.31 Signed-off-by: Steve French --- fs/cifs/cifsfs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/cifs/cifsfs.h b/fs/cifs/cifsfs.h index 2307bb0f6147..766e38862870 100644 --- a/fs/cifs/cifsfs.h +++ b/fs/cifs/cifsfs.h @@ -160,5 +160,5 @@ extern struct dentry *cifs_smb3_do_mount(struct file_system_type *fs_type, extern const struct export_operations cifs_export_ops; #endif /* CONFIG_CIFS_NFSD_EXPORT */ -#define CIFS_VERSION "2.30" +#define CIFS_VERSION "2.31" #endif /* _CIFSFS_H */ -- cgit v1.2.3 From 3590ec58991bcf0f3512c4353a786079a6619758 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Tue, 9 Feb 2021 22:22:19 -0800 Subject: btrfs: use memcpy_[to|from]_page() and kmap_local_page() There are many places where the pattern kmap/memcpy/kunmap occurs. This pattern was lifted to the core common functions memcpy_[to|from]_page(). Use these new functions to reduce the code, eliminate direct uses of kmap, and leverage the new core functions use of kmap_local_page(). Also, there is 1 place where a kmap/memcpy is followed by an optional memset. Here we leave the kmap open coded to avoid remapping the page but use kmap_local_page() directly. Development of this patch was aided by the coccinelle script: // // SPDX-License-Identifier: GPL-2.0-only // Find kmap/memcpy/kunmap pattern and replace with memcpy*page calls // // NOTE: Offsets and other expressions may be more complex than what the script // will automatically generate. Therefore a catchall rule is provided to find // the pattern which then must be evaluated by hand. // // Confidence: Low // Copyright: (C) 2021 Intel Corporation // URL: http://coccinelle.lip6.fr/ // Comments: // Options: // // simple memcpy version // @ memcpy_rule1 @ expression page, T, F, B, Off; identifier ptr; type VP; @@ ( -VP ptr = kmap(page); | -ptr = kmap(page); | -VP ptr = kmap_atomic(page); | -ptr = kmap_atomic(page); ) <+... ( -memcpy(ptr + Off, F, B); +memcpy_to_page(page, Off, F, B); | -memcpy(ptr, F, B); +memcpy_to_page(page, 0, F, B); | -memcpy(T, ptr + Off, B); +memcpy_from_page(T, page, Off, B); | -memcpy(T, ptr, B); +memcpy_from_page(T, page, 0, B); ) ...+> ( -kunmap(page); | -kunmap_atomic(ptr); ) // Remove any pointers left unused @ depends on memcpy_rule1 @ identifier memcpy_rule1.ptr; type VP, VP1; @@ -VP ptr; ... when != ptr; ? VP1 ptr; // // Some callers kmap without a temp pointer // @ memcpy_rule2 @ expression page, T, Off, F, B; @@ <+... ( -memcpy(kmap(page) + Off, F, B); +memcpy_to_page(page, Off, F, B); | -memcpy(kmap(page), F, B); +memcpy_to_page(page, 0, F, B); | -memcpy(T, kmap(page) + Off, B); +memcpy_from_page(T, page, Off, B); | -memcpy(T, kmap(page), B); +memcpy_from_page(T, page, 0, B); ) ...+> -kunmap(page); // No need for the ptr variable removal // // Catch all // @ memcpy_rule3 @ expression page; expression GenTo, GenFrom, GenSize; identifier ptr; type VP; @@ ( -VP ptr = kmap(page); | -ptr = kmap(page); | -VP ptr = kmap_atomic(page); | -ptr = kmap_atomic(page); ) <+... ( // // Some call sites have complex expressions within the memcpy // match a catch all to be evaluated by hand. // -memcpy(GenTo, GenFrom, GenSize); +memcpy_to_pageExtra(page, GenTo, GenFrom, GenSize); +memcpy_from_pageExtra(GenTo, page, GenFrom, GenSize); ) ...+> ( -kunmap(page); | -kunmap_atomic(ptr); ) // Remove any pointers left unused @ depends on memcpy_rule3 @ identifier memcpy_rule3.ptr; type VP, VP1; @@ -VP ptr; ... when != ptr; ? VP1 ptr; // Reviewed-by: Christoph Hellwig Signed-off-by: Ira Weiny Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/compression.c | 6 ++---- fs/btrfs/lzo.c | 4 ++-- fs/btrfs/reflink.c | 6 +----- fs/btrfs/send.c | 7 ++----- fs/btrfs/zlib.c | 5 ++--- fs/btrfs/zstd.c | 6 ++---- 6 files changed, 11 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 5ae3fa0386b7..047b632b4139 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -1231,7 +1231,6 @@ int btrfs_decompress_buf2page(const char *buf, unsigned long buf_start, unsigned long prev_start_byte; unsigned long working_bytes = total_out - buf_start; unsigned long bytes; - char *kaddr; struct bio_vec bvec = bio_iter_iovec(bio, bio->bi_iter); /* @@ -1262,9 +1261,8 @@ int btrfs_decompress_buf2page(const char *buf, unsigned long buf_start, PAGE_SIZE - (buf_offset % PAGE_SIZE)); bytes = min(bytes, working_bytes); - kaddr = kmap_atomic(bvec.bv_page); - memcpy(kaddr + bvec.bv_offset, buf + buf_offset, bytes); - kunmap_atomic(kaddr); + memcpy_to_page(bvec.bv_page, bvec.bv_offset, buf + buf_offset, + bytes); flush_dcache_page(bvec.bv_page); buf_offset += bytes; diff --git a/fs/btrfs/lzo.c b/fs/btrfs/lzo.c index aa9cd11f4b78..9084a950dc09 100644 --- a/fs/btrfs/lzo.c +++ b/fs/btrfs/lzo.c @@ -467,7 +467,7 @@ int lzo_decompress(struct list_head *ws, unsigned char *data_in, destlen = min_t(unsigned long, destlen, PAGE_SIZE); bytes = min_t(unsigned long, destlen, out_len - start_byte); - kaddr = kmap_atomic(dest_page); + kaddr = kmap_local_page(dest_page); memcpy(kaddr, workspace->buf + start_byte, bytes); /* @@ -477,7 +477,7 @@ int lzo_decompress(struct list_head *ws, unsigned char *data_in, */ if (bytes < destlen) memset(kaddr+bytes, 0, destlen-bytes); - kunmap_atomic(kaddr); + kunmap_local(kaddr); out: return ret; } diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index b03e7891394e..74c62e49c0c9 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -103,12 +103,8 @@ static int copy_inline_to_page(struct btrfs_inode *inode, set_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &inode->runtime_flags); if (comp_type == BTRFS_COMPRESS_NONE) { - char *map; - - map = kmap(page); - memcpy(map, data_start, datal); + memcpy_to_page(page, 0, data_start, datal); flush_dcache_page(page); - kunmap(page); } else { ret = btrfs_decompress(comp_type, data_start, page, 0, inline_size, datal); diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 78a35374d492..83982b3b7057 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -4948,7 +4948,6 @@ static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len) struct btrfs_fs_info *fs_info = root->fs_info; struct inode *inode; struct page *page; - char *addr; pgoff_t index = offset >> PAGE_SHIFT; pgoff_t last_index; unsigned pg_offset = offset_in_page(offset); @@ -5001,10 +5000,8 @@ static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len) } } - addr = kmap(page); - memcpy(sctx->send_buf + sctx->send_size, addr + pg_offset, - cur_len); - kunmap(page); + memcpy_from_page(sctx->send_buf + sctx->send_size, page, + pg_offset, cur_len); unlock_page(page); put_page(page); index++; diff --git a/fs/btrfs/zlib.c b/fs/btrfs/zlib.c index 05615a1099db..d524acf7b3e5 100644 --- a/fs/btrfs/zlib.c +++ b/fs/btrfs/zlib.c @@ -432,9 +432,8 @@ int zlib_decompress(struct list_head *ws, unsigned char *data_in, PAGE_SIZE - (buf_offset % PAGE_SIZE)); bytes = min(bytes, bytes_left); - kaddr = kmap_atomic(dest_page); - memcpy(kaddr + pg_offset, workspace->buf + buf_offset, bytes); - kunmap_atomic(kaddr); + memcpy_to_page(dest_page, pg_offset, + workspace->buf + buf_offset, bytes); pg_offset += bytes; bytes_left -= bytes; diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c index 9a4871636c6c..8e9626d63976 100644 --- a/fs/btrfs/zstd.c +++ b/fs/btrfs/zstd.c @@ -688,10 +688,8 @@ int zstd_decompress(struct list_head *ws, unsigned char *data_in, bytes = min_t(unsigned long, destlen - pg_offset, workspace->out_buf.size - buf_offset); - kaddr = kmap_atomic(dest_page); - memcpy(kaddr + pg_offset, workspace->out_buf.dst + buf_offset, - bytes); - kunmap_atomic(kaddr); + memcpy_to_page(dest_page, pg_offset, + workspace->out_buf.dst + buf_offset, bytes); pg_offset += bytes; } -- cgit v1.2.3 From 80cc83842394e5ad3e93487359106aab3420bcb7 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Tue, 9 Feb 2021 22:22:20 -0800 Subject: btrfs: use copy_highpage() instead of 2 kmaps() There are many places where kmap/memove/kunmap patterns occur. This pattern exists in the core common function copy_highpage(). Use copy_highpage to avoid open coding the use of kmap and leverages the core functions use of kmap_local_page(). Development of this patch was aided by the following coccinelle script: // // SPDX-License-Identifier: GPL-2.0-only // Find kmap/copypage/kunmap pattern and replace with copy_highpage calls // // NOTE: The expressions in the copy page version of this kmap pattern are // overly complex and so these all need individual attention. // // Confidence: Low // Copyright: (C) 2021 Intel Corporation // URL: http://coccinelle.lip6.fr/ // Comments: // Options: // // Then a copy_page where we have 2 pages involved. // @ copy_page_rule @ expression page, page2, To, From, Size; identifier ptr, ptr2; type VP, VP2; @@ /* kmap */ ( -VP ptr = kmap(page); ... -VP2 ptr2 = kmap(page2); | -VP ptr = kmap_atomic(page); ... -VP2 ptr2 = kmap_atomic(page2); | -ptr = kmap(page); ... -ptr2 = kmap(page2); | -ptr = kmap_atomic(page); ... -ptr2 = kmap_atomic(page2); ) // 1 or more copy versions of the entire page <+... ( -copy_page(To, From); +copy_highpage(To, From); | -memmove(To, From, Size); +memmoveExtra(To, From, Size); ) ...+> /* kunmap */ ( -kunmap(page2); ... -kunmap(page); | -kunmap(page); ... -kunmap(page2); | -kmap_atomic(ptr2); ... -kmap_atomic(ptr); ) // Remove any pointers left unused @ depends on copy_page_rule @ identifier copy_page_rule.ptr; identifier copy_page_rule.ptr2; type VP, VP1; type VP2, VP21; @@ -VP ptr; ... when != ptr; ? VP1 ptr; -VP2 ptr2; ... when != ptr2; ? VP21 ptr2; // Reviewed-by: Christoph Hellwig Signed-off-by: Ira Weiny Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 93fbf87bdc8d..c86aff9c7daa 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -250,8 +250,6 @@ int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info) static void cache_rbio_pages(struct btrfs_raid_bio *rbio) { int i; - char *s; - char *d; int ret; ret = alloc_rbio_pages(rbio); @@ -262,13 +260,7 @@ static void cache_rbio_pages(struct btrfs_raid_bio *rbio) if (!rbio->bio_pages[i]) continue; - s = kmap(rbio->bio_pages[i]); - d = kmap(rbio->stripe_pages[i]); - - copy_page(d, s); - - kunmap(rbio->bio_pages[i]); - kunmap(rbio->stripe_pages[i]); + copy_highpage(rbio->stripe_pages[i], rbio->bio_pages[i]); SetPageUptodate(rbio->stripe_pages[i]); } set_bit(RBIO_CACHE_READY_BIT, &rbio->flags); -- cgit v1.2.3 From 54fa39ac2e00b1b8c2a7fe72e648773ffa48f76d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 25 Feb 2021 17:15:52 -0800 Subject: iomap: use mapping_seek_hole_data Enhance mapping_seek_hole_data() to handle partially uptodate pages and convert the iomap seek code to call it. Link: https://lkml.kernel.org/r/20201112212641.27837-9-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: Christoph Hellwig Cc: Dave Chinner Cc: Hugh Dickins Cc: Jan Kara Cc: Johannes Weiner Cc: Kirill A. Shutemov Cc: William Kucharski Cc: Yang Shi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/iomap/seek.c | 125 +++++--------------------------------------------------- mm/filemap.c | 37 ++++++++++++++--- 2 files changed, 43 insertions(+), 119 deletions(-) (limited to 'fs') diff --git a/fs/iomap/seek.c b/fs/iomap/seek.c index 107ee80c3568..dab1b02eba5b 100644 --- a/fs/iomap/seek.c +++ b/fs/iomap/seek.c @@ -10,122 +10,17 @@ #include #include -/* - * Seek for SEEK_DATA / SEEK_HOLE within @page, starting at @lastoff. - * Returns true if found and updates @lastoff to the offset in file. - */ -static bool -page_seek_hole_data(struct inode *inode, struct page *page, loff_t *lastoff, - int whence) -{ - const struct address_space_operations *ops = inode->i_mapping->a_ops; - unsigned int bsize = i_blocksize(inode), off; - bool seek_data = whence == SEEK_DATA; - loff_t poff = page_offset(page); - - if (WARN_ON_ONCE(*lastoff >= poff + PAGE_SIZE)) - return false; - - if (*lastoff < poff) { - /* - * Last offset smaller than the start of the page means we found - * a hole: - */ - if (whence == SEEK_HOLE) - return true; - *lastoff = poff; - } - - /* - * Just check the page unless we can and should check block ranges: - */ - if (bsize == PAGE_SIZE || !ops->is_partially_uptodate) - return PageUptodate(page) == seek_data; - - lock_page(page); - if (unlikely(page->mapping != inode->i_mapping)) - goto out_unlock_not_found; - - for (off = 0; off < PAGE_SIZE; off += bsize) { - if (offset_in_page(*lastoff) >= off + bsize) - continue; - if (ops->is_partially_uptodate(page, off, bsize) == seek_data) { - unlock_page(page); - return true; - } - *lastoff = poff + off + bsize; - } - -out_unlock_not_found: - unlock_page(page); - return false; -} - -/* - * Seek for SEEK_DATA / SEEK_HOLE in the page cache. - * - * Within unwritten extents, the page cache determines which parts are holes - * and which are data: uptodate buffer heads count as data; everything else - * counts as a hole. - * - * Returns the resulting offset on successs, and -ENOENT otherwise. - */ static loff_t -page_cache_seek_hole_data(struct inode *inode, loff_t offset, loff_t length, - int whence) -{ - pgoff_t index = offset >> PAGE_SHIFT; - pgoff_t end = DIV_ROUND_UP(offset + length, PAGE_SIZE); - loff_t lastoff = offset; - struct pagevec pvec; - - if (length <= 0) - return -ENOENT; - - pagevec_init(&pvec); - - do { - unsigned nr_pages, i; - - nr_pages = pagevec_lookup_range(&pvec, inode->i_mapping, &index, - end - 1); - if (nr_pages == 0) - break; - - for (i = 0; i < nr_pages; i++) { - struct page *page = pvec.pages[i]; - - if (page_seek_hole_data(inode, page, &lastoff, whence)) - goto check_range; - lastoff = page_offset(page) + PAGE_SIZE; - } - pagevec_release(&pvec); - } while (index < end); - - /* When no page at lastoff and we are not done, we found a hole. */ - if (whence != SEEK_HOLE) - goto not_found; - -check_range: - if (lastoff < offset + length) - goto out; -not_found: - lastoff = -ENOENT; -out: - pagevec_release(&pvec); - return lastoff; -} - - -static loff_t -iomap_seek_hole_actor(struct inode *inode, loff_t offset, loff_t length, +iomap_seek_hole_actor(struct inode *inode, loff_t start, loff_t length, void *data, struct iomap *iomap, struct iomap *srcmap) { + loff_t offset = start; + switch (iomap->type) { case IOMAP_UNWRITTEN: - offset = page_cache_seek_hole_data(inode, offset, length, - SEEK_HOLE); - if (offset < 0) + offset = mapping_seek_hole_data(inode->i_mapping, start, + start + length, SEEK_HOLE); + if (offset == start + length) return length; fallthrough; case IOMAP_HOLE: @@ -164,15 +59,17 @@ iomap_seek_hole(struct inode *inode, loff_t offset, const struct iomap_ops *ops) EXPORT_SYMBOL_GPL(iomap_seek_hole); static loff_t -iomap_seek_data_actor(struct inode *inode, loff_t offset, loff_t length, +iomap_seek_data_actor(struct inode *inode, loff_t start, loff_t length, void *data, struct iomap *iomap, struct iomap *srcmap) { + loff_t offset = start; + switch (iomap->type) { case IOMAP_HOLE: return length; case IOMAP_UNWRITTEN: - offset = page_cache_seek_hole_data(inode, offset, length, - SEEK_DATA); + offset = mapping_seek_hole_data(inode->i_mapping, start, + start + length, SEEK_DATA); if (offset < 0) return length; fallthrough; diff --git a/mm/filemap.c b/mm/filemap.c index eff3006be12a..6a34f94adf3b 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2553,11 +2553,36 @@ generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) } EXPORT_SYMBOL(generic_file_read_iter); -static inline bool page_seek_match(struct page *page, bool seek_data) +static inline loff_t page_seek_hole_data(struct xa_state *xas, + struct address_space *mapping, struct page *page, + loff_t start, loff_t end, bool seek_data) { + const struct address_space_operations *ops = mapping->a_ops; + size_t offset, bsz = i_blocksize(mapping->host); + if (xa_is_value(page) || PageUptodate(page)) - return seek_data; - return !seek_data; + return seek_data ? start : end; + if (!ops->is_partially_uptodate) + return seek_data ? end : start; + + xas_pause(xas); + rcu_read_unlock(); + lock_page(page); + if (unlikely(page->mapping != mapping)) + goto unlock; + + offset = offset_in_thp(page, start) & ~(bsz - 1); + + do { + if (ops->is_partially_uptodate(page, offset, bsz) == seek_data) + break; + start = (start + bsz) & ~(bsz - 1); + offset += bsz; + } while (offset < thp_size(page)); +unlock: + unlock_page(page); + rcu_read_lock(); + return start; } static inline @@ -2607,9 +2632,11 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, start = pos; } - if (page_seek_match(page, seek_data)) + pos += seek_page_size(&xas, page); + start = page_seek_hole_data(&xas, mapping, page, start, pos, + seek_data); + if (start < pos) goto unlock; - start = pos + seek_page_size(&xas, page); if (!xa_is_value(page)) put_page(page); } -- cgit v1.2.3 From 152c432b128cb043fc107e8f211195fe94b2159c Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Thu, 25 Feb 2021 17:20:45 -0800 Subject: proc/wchan: use printk format instead of lookup_symbol_name() To resolve the symbol fuction name for wchan, use the printk format specifier %ps instead of manually looking up the symbol function name via lookup_symbol_name(). Link: https://lkml.kernel.org/r/20201217165413.GA1959@ls3530.fritz.box Signed-off-by: Helge Deller Cc: Alexey Dobriyan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/base.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/proc/base.c b/fs/proc/base.c index 56bf14316122..3851bfcdba56 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -67,7 +67,6 @@ #include #include #include -#include #include #include #include @@ -386,19 +385,17 @@ static int proc_pid_wchan(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *task) { unsigned long wchan; - char symname[KSYM_NAME_LEN]; - if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) - goto print0; + if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) + wchan = get_wchan(task); + else + wchan = 0; - wchan = get_wchan(task); - if (wchan && !lookup_symbol_name(wchan, symname)) { - seq_puts(m, symname); - return 0; - } + if (wchan) + seq_printf(m, "%ps", (void *) wchan); + else + seq_putc(m, '0'); -print0: - seq_putc(m, '0'); return 0; } #endif /* CONFIG_KALLSYMS */ -- cgit v1.2.3 From 4508943794efdd94171549c0bd52810e2f4ad9fe Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 25 Feb 2021 17:20:49 -0800 Subject: proc: use kvzalloc for our kernel buffer Since sysctl: pass kernel pointers to ->proc_handler we have been pre-allocating a buffer to copy the data from the proc handlers into, and then copying that to userspace. The problem is this just blindly kzalloc()'s the buffer size passed in from the read, which in the case of our 'cat' binary was 64kib. Order-4 allocations are not awesome, and since we can potentially allocate up to our maximum order, so use kvzalloc for these buffers. [willy@infradead.org: changelog tweaks] Link: https://lkml.kernel.org/r/6345270a2c1160b89dd5e6715461f388176899d1.1612972413.git.josef@toxicpanda.com Fixes: 32927393dc1c ("sysctl: pass kernel pointers to ->proc_handler") Signed-off-by: Josef Bacik Reviewed-by: Christoph Hellwig Acked-by: Vlastimil Babka Cc: Al Viro Cc: Alexey Dobriyan CC: Matthew Wilcox Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/proc_sysctl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 656ba24c317d..984e42f8cb11 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -571,7 +571,7 @@ static ssize_t proc_sys_call_handler(struct kiocb *iocb, struct iov_iter *iter, error = -ENOMEM; if (count >= KMALLOC_MAX_SIZE) goto out; - kbuf = kzalloc(count + 1, GFP_KERNEL); + kbuf = kvzalloc(count + 1, GFP_KERNEL); if (!kbuf) goto out; @@ -600,7 +600,7 @@ static ssize_t proc_sys_call_handler(struct kiocb *iocb, struct iov_iter *iter, error = count; out_free_buf: - kfree(kbuf); + kvfree(kbuf); out: sysctl_head_finish(head); -- cgit v1.2.3 From 3159ed57792be7453793bda27297a423e1c63d6c Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 25 Feb 2021 17:22:22 -0800 Subject: fs/coredump: use kmap_local_page() In dump_user_range() there is no reason for the mapping to be global. Use kmap_local_page() rather than kmap. Link: https://lkml.kernel.org/r/20210203223328.558945-1-ira.weiny@intel.com Signed-off-by: Ira Weiny Cc: Alexander Viro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/coredump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/coredump.c b/fs/coredump.c index ae778937a1ff..1c0fdc1aa70b 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -897,10 +897,10 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, */ page = get_dump_page(addr); if (page) { - void *kaddr = kmap(page); + void *kaddr = kmap_local_page(page); stop = !dump_emit(cprm, kaddr, PAGE_SIZE); - kunmap(page); + kunmap_local(kaddr); put_page(page); } else { stop = !dump_skip(cprm, PAGE_SIZE); -- cgit v1.2.3 From 5f7136db82996089cdfb2939c7664b29e9da141d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 29 Jan 2021 04:38:57 +0000 Subject: block: Add bio_max_segs It's often inconvenient to use BIO_MAX_PAGES due to min() requiring the sign to be the same. Introduce bio_max_segs() and change BIO_MAX_PAGES to be unsigned to make it easier for the users. Reviewed-by: Chaitanya Kulkarni Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Jens Axboe --- block/blk-map.c | 4 +--- drivers/block/xen-blkback/blkback.c | 4 +--- drivers/md/dm-io.c | 4 ++-- drivers/md/dm-log-writes.c | 10 +++++----- drivers/nvme/target/io-cmd-bdev.c | 8 ++++---- drivers/nvme/target/passthru.c | 4 ++-- drivers/target/target_core_iblock.c | 9 +++------ drivers/target/target_core_pscsi.c | 2 +- fs/block_dev.c | 10 +++++----- fs/direct-io.c | 2 +- fs/erofs/data.c | 4 +--- fs/ext4/readpage.c | 3 +-- fs/f2fs/data.c | 3 +-- fs/f2fs/node.c | 2 +- fs/iomap/buffered-io.c | 4 ++-- fs/mpage.c | 4 +--- fs/nfs/blocklayout/blocklayout.c | 6 +++--- fs/xfs/xfs_bio_io.c | 2 +- fs/xfs/xfs_buf.c | 4 ++-- include/linux/bio.h | 7 ++++++- 20 files changed, 44 insertions(+), 52 deletions(-) (limited to 'fs') diff --git a/block/blk-map.c b/block/blk-map.c index 21630dccac62..369e204d14d0 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -150,9 +150,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, bmd->is_our_pages = !map_data; bmd->is_null_mapped = (map_data && map_data->null_mapped); - nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); - if (nr_pages > BIO_MAX_PAGES) - nr_pages = BIO_MAX_PAGES; + nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); ret = -ENOMEM; bio = bio_kmalloc(gfp_mask, nr_pages); diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index da16121140ca..1cdf09ff67b6 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c @@ -1326,9 +1326,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, pages[i]->page, seg[i].nsec << 9, seg[i].offset) == 0)) { - - int nr_iovecs = min_t(int, (nseg-i), BIO_MAX_PAGES); - bio = bio_alloc(GFP_KERNEL, nr_iovecs); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(nseg - i)); if (unlikely(bio == NULL)) goto fail_put_bio; diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index 4312007d2d34..2d3cda0acacb 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c @@ -341,8 +341,8 @@ static void do_region(int op, int op_flags, unsigned region, num_bvecs = 1; break; default: - num_bvecs = min_t(int, BIO_MAX_PAGES, - dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT))); + num_bvecs = bio_max_segs(dm_sector_div_up(remaining, + (PAGE_SIZE >> SECTOR_SHIFT))); } bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, &io->client->bios); diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c index e3d35c6c9f71..57882654ffee 100644 --- a/drivers/md/dm-log-writes.c +++ b/drivers/md/dm-log-writes.c @@ -264,15 +264,14 @@ static int write_inline_data(struct log_writes_c *lc, void *entry, size_t entrylen, void *data, size_t datalen, sector_t sector) { - int num_pages, bio_pages, pg_datalen, pg_sectorlen, i; + int bio_pages, pg_datalen, pg_sectorlen, i; struct page *page; struct bio *bio; size_t ret; void *ptr; while (datalen) { - num_pages = ALIGN(datalen, PAGE_SIZE) >> PAGE_SHIFT; - bio_pages = min(num_pages, BIO_MAX_PAGES); + bio_pages = bio_max_segs(DIV_ROUND_UP(datalen, PAGE_SIZE)); atomic_inc(&lc->io_blocks); @@ -364,7 +363,7 @@ static int log_one_block(struct log_writes_c *lc, goto out; atomic_inc(&lc->io_blocks); - bio = bio_alloc(GFP_KERNEL, min(block->vec_cnt, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(block->vec_cnt)); if (!bio) { DMERR("Couldn't alloc log bio"); goto error; @@ -386,7 +385,8 @@ static int log_one_block(struct log_writes_c *lc, if (ret != block->vecs[i].bv_len) { atomic_inc(&lc->io_blocks); submit_bio(bio); - bio = bio_alloc(GFP_KERNEL, min(block->vec_cnt - i, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, + bio_max_segs(block->vec_cnt - i)); if (!bio) { DMERR("Couldn't alloc log bio"); goto error; diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 3d9a5d3ed9cd..9a8b3726a37c 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -185,7 +185,7 @@ static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio, } bip = bio_integrity_alloc(bio, GFP_NOIO, - min_t(unsigned int, req->metadata_sg_cnt, BIO_MAX_PAGES)); + bio_max_segs(req->metadata_sg_cnt)); if (IS_ERR(bip)) { pr_err("Unable to allocate bio_integrity_payload\n"); return PTR_ERR(bip); @@ -225,7 +225,7 @@ static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio, static void nvmet_bdev_execute_rw(struct nvmet_req *req) { - int sg_cnt = req->sg_cnt; + unsigned int sg_cnt = req->sg_cnt; struct bio *bio; struct scatterlist *sg; struct blk_plug plug; @@ -262,7 +262,7 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) bio = &req->b.inline_bio; bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec)); } else { - bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(sg_cnt)); } bio_set_dev(bio, req->ns->bdev); bio->bi_iter.bi_sector = sector; @@ -289,7 +289,7 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) } } - bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(sg_cnt)); bio_set_dev(bio, req->ns->bdev); bio->bi_iter.bi_sector = sector; bio->bi_opf = op; diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index f50c7b2bf21c..26c587ccd152 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -26,7 +26,7 @@ static u16 nvmet_passthru_override_id_ctrl(struct nvmet_req *req) struct nvme_ctrl *pctrl = ctrl->subsys->passthru_ctrl; u16 status = NVME_SC_SUCCESS; struct nvme_id_ctrl *id; - int max_hw_sectors; + unsigned int max_hw_sectors; int page_shift; id = kzalloc(sizeof(*id), GFP_KERNEL); @@ -198,7 +198,7 @@ static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq) bio = &req->p.inline_bio; bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec)); } else { - bio = bio_alloc(GFP_KERNEL, min(req->sg_cnt, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(req->sg_cnt)); bio->bi_end_io = bio_put; } bio->bi_opf = req_op(rq); diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index 8ed93fd205c7..ee3d52061281 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c @@ -315,10 +315,8 @@ iblock_get_bio(struct se_cmd *cmd, sector_t lba, u32 sg_num, int op, * Only allocate as many vector entries as the bio code allows us to, * we'll loop later on until we have handled the whole request. */ - if (sg_num > BIO_MAX_PAGES) - sg_num = BIO_MAX_PAGES; - - bio = bio_alloc_bioset(GFP_NOIO, sg_num, &ib_dev->ibd_bio_set); + bio = bio_alloc_bioset(GFP_NOIO, bio_max_segs(sg_num), + &ib_dev->ibd_bio_set); if (!bio) { pr_err("Unable to allocate memory for bio\n"); return NULL; @@ -638,8 +636,7 @@ iblock_alloc_bip(struct se_cmd *cmd, struct bio *bio, return -ENODEV; } - bip = bio_integrity_alloc(bio, GFP_NOIO, - min_t(unsigned int, cmd->t_prot_nents, BIO_MAX_PAGES)); + bip = bio_integrity_alloc(bio, GFP_NOIO, bio_max_segs(cmd->t_prot_nents)); if (IS_ERR(bip)) { pr_err("Unable to allocate bio_integrity_payload\n"); return PTR_ERR(bip); diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c index 33770e5808ce..3cbc074992bc 100644 --- a/drivers/target/target_core_pscsi.c +++ b/drivers/target/target_core_pscsi.c @@ -881,7 +881,7 @@ pscsi_map_sg(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, if (!bio) { new_bio: - nr_vecs = min_t(int, BIO_MAX_PAGES, nr_pages); + nr_vecs = bio_max_segs(nr_pages); nr_pages -= nr_vecs; /* * Calls bio_kmalloc() and sets bio->bi_end_io() diff --git a/fs/block_dev.c b/fs/block_dev.c index ec26179c8062..0f95ff343d6b 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -221,7 +221,7 @@ static void blkdev_bio_end_io_simple(struct bio *bio) static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter, - int nr_pages) + unsigned int nr_pages) { struct file *file = iocb->ki_filp; struct block_device *bdev = I_BDEV(bdev_file_inode(file)); @@ -355,8 +355,8 @@ static void blkdev_bio_end_io(struct bio *bio) } } -static ssize_t -__blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) +static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, + unsigned int nr_pages) { struct file *file = iocb->ki_filp; struct inode *inode = bdev_file_inode(file); @@ -486,7 +486,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) { - int nr_pages; + unsigned int nr_pages; if (!iov_iter_count(iter)) return 0; @@ -495,7 +495,7 @@ blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES) return __blkdev_direct_IO_simple(iocb, iter, nr_pages); - return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES)); + return __blkdev_direct_IO(iocb, iter, bio_max_segs(nr_pages)); } static __init int blkdev_init(void) diff --git a/fs/direct-io.c b/fs/direct-io.c index aa1083ecd623..c9639b4166c2 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -695,7 +695,7 @@ static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, if (ret) goto out; sector = start_sector << (sdio->blkbits - 9); - nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES); + nr_pages = bio_max_segs(sdio->pages_in_io); BUG_ON(nr_pages <= 0); dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); sdio->boundary = 0; diff --git a/fs/erofs/data.c b/fs/erofs/data.c index ea4f693bee22..f88851c5c250 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -215,10 +215,8 @@ submit_bio_retry: /* max # of continuous pages */ if (nblocks > DIV_ROUND_UP(map.m_plen, PAGE_SIZE)) nblocks = DIV_ROUND_UP(map.m_plen, PAGE_SIZE); - if (nblocks > BIO_MAX_PAGES) - nblocks = BIO_MAX_PAGES; - bio = bio_alloc(GFP_NOIO, nblocks); + bio = bio_alloc(GFP_NOIO, bio_max_segs(nblocks)); bio->bi_end_io = erofs_readendio; bio_set_dev(bio, sb->s_bdev); diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index f014c5e473a9..3db923403505 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -371,8 +371,7 @@ int ext4_mpage_readpages(struct inode *inode, * bio_alloc will _always_ be able to allocate a bio if * __GFP_DIRECT_RECLAIM is set, see bio_alloc_bioset(). */ - bio = bio_alloc(GFP_KERNEL, - min_t(int, nr_pages, BIO_MAX_PAGES)); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(nr_pages)); fscrypt_set_bio_crypt_ctx(bio, inode, next_block, GFP_KERNEL); ext4_set_bio_post_read_ctx(bio, inode, page->index); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index b9721c8f116c..7c95818639a6 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -969,8 +969,7 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, unsigned int post_read_steps = 0; bio = bio_alloc_bioset(for_write ? GFP_NOIO : GFP_KERNEL, - min_t(int, nr_pages, BIO_MAX_PAGES), - &f2fs_bioset); + bio_max_segs(nr_pages), &f2fs_bioset); if (!bio) return ERR_PTR(-ENOMEM); diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index a8a0fb890e8d..4b0e2e3c2c88 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -2747,7 +2747,7 @@ int f2fs_restore_node_summary(struct f2fs_sb_info *sbi, sum_entry = &sum->entries[0]; for (i = 0; i < last_offset; i += nrpages, addr += nrpages) { - nrpages = min(last_offset - i, BIO_MAX_PAGES); + nrpages = bio_max_segs(last_offset - i); /* readahead node pages */ f2fs_ra_meta_pages(sbi, addr, nrpages, META_POR, true); diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 16a1e82e3aeb..0d9d1a6a947e 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -278,14 +278,14 @@ iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data, if (!is_contig || bio_full(ctx->bio, plen)) { gfp_t gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL); gfp_t orig_gfp = gfp; - int nr_vecs = (length + PAGE_SIZE - 1) >> PAGE_SHIFT; + unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE); if (ctx->bio) submit_bio(ctx->bio); if (ctx->rac) /* same as readahead_gfp_mask */ gfp |= __GFP_NORETRY | __GFP_NOWARN; - ctx->bio = bio_alloc(gfp, min(BIO_MAX_PAGES, nr_vecs)); + ctx->bio = bio_alloc(gfp, bio_max_segs(nr_vecs)); /* * If the bio_alloc fails, try it again for a single page to * avoid having to deal with partial page reads. This emulates diff --git a/fs/mpage.c b/fs/mpage.c index 830e6cc2a9e7..961234d68779 100644 --- a/fs/mpage.c +++ b/fs/mpage.c @@ -304,9 +304,7 @@ alloc_new: goto out; } args->bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9), - min_t(int, args->nr_pages, - BIO_MAX_PAGES), - gfp); + bio_max_segs(args->nr_pages), gfp); if (args->bio == NULL) goto confused; } diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c index 1a96ce28efb0..fe860c538747 100644 --- a/fs/nfs/blocklayout/blocklayout.c +++ b/fs/nfs/blocklayout/blocklayout.c @@ -115,13 +115,13 @@ bl_submit_bio(struct bio *bio) return NULL; } -static struct bio * -bl_alloc_init_bio(int npg, struct block_device *bdev, sector_t disk_sector, +static struct bio *bl_alloc_init_bio(unsigned int npg, + struct block_device *bdev, sector_t disk_sector, bio_end_io_t end_io, struct parallel_io *par) { struct bio *bio; - npg = min(npg, BIO_MAX_PAGES); + npg = bio_max_segs(npg); bio = bio_alloc(GFP_NOIO, npg); if (bio) { bio->bi_iter.bi_sector = disk_sector; diff --git a/fs/xfs/xfs_bio_io.c b/fs/xfs/xfs_bio_io.c index e2148f2d5d6b..17f36db2f792 100644 --- a/fs/xfs/xfs_bio_io.c +++ b/fs/xfs/xfs_bio_io.c @@ -6,7 +6,7 @@ static inline unsigned int bio_max_vecs(unsigned int count) { - return min_t(unsigned, howmany(count, PAGE_SIZE), BIO_MAX_PAGES); + return bio_max_segs(howmany(count, PAGE_SIZE)); } int diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index f6e5235df7c9..37a1d12762d8 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1480,7 +1480,7 @@ xfs_buf_ioapply_map( int op) { int page_index; - int total_nr_pages = bp->b_page_count; + unsigned int total_nr_pages = bp->b_page_count; int nr_pages; struct bio *bio; sector_t sector = bp->b_maps[map].bm_bn; @@ -1505,7 +1505,7 @@ xfs_buf_ioapply_map( next_chunk: atomic_inc(&bp->b_io_remaining); - nr_pages = min(total_nr_pages, BIO_MAX_PAGES); + nr_pages = bio_max_segs(total_nr_pages); bio = bio_alloc(GFP_NOIO, nr_pages); bio_set_dev(bio, bp->b_target->bt_bdev); diff --git a/include/linux/bio.h b/include/linux/bio.h index 5b468f2242ff..983ed2fe7c85 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -20,7 +20,12 @@ #define BIO_BUG_ON #endif -#define BIO_MAX_PAGES 256 +#define BIO_MAX_PAGES 256U + +static inline unsigned int bio_max_segs(unsigned int nr_segs) +{ + return min(nr_segs, BIO_MAX_PAGES); +} #define bio_prio(bio) (bio)->bi_ioprio #define bio_set_prio(bio, prio) ((bio)->bi_ioprio = prio) -- cgit v1.2.3 From 65d43023171edc0d27208f6ac7a1a73732950cf7 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 26 Feb 2021 09:50:55 -0700 Subject: io-wq: wait for worker startup when forking a new one We need to have our worker count updated before continuing, to avoid cases where we repeatedly think we need a new worker, but a fork is already in progress. Signed-off-by: Jens Axboe --- fs/io-wq.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 44e20248805a..965022fe9961 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -56,6 +56,7 @@ struct io_worker { const struct cred *saved_creds; struct completion ref_done; + struct completion started; struct rcu_head rcu; }; @@ -267,6 +268,7 @@ static void io_worker_start(struct io_worker *worker) { worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); io_wqe_inc_running(worker); + complete(&worker->started); } /* @@ -644,6 +646,7 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) worker->wqe = wqe; spin_lock_init(&worker->lock); init_completion(&worker->ref_done); + init_completion(&worker->started); refcount_inc(&wq->refs); @@ -656,6 +659,7 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) kfree(worker); return false; } + wait_for_completion(&worker->started); return true; } -- cgit v1.2.3 From aedb9d9089ceb1c86be495bcc70e6021c01f92ff Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 18 Feb 2021 22:54:17 -0800 Subject: btrfs: ref-verify: use 'inline void' keyword ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix build warnings of function signature when CONFIG_STACKTRACE is not enabled by reordering the 'inline' and 'void' keywords. ../fs/btrfs/ref-verify.c:221:1: warning: ‘inline’ is not at beginning of declaration [-Wold-style-declaration] static void inline __save_stack_trace(struct ref_action *ra) ../fs/btrfs/ref-verify.c:225:1: warning: ‘inline’ is not at beginning of declaration [-Wold-style-declaration] static void inline __print_stack_trace(struct btrfs_fs_info *fs_info, Signed-off-by: Randy Dunlap Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ref-verify.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/ref-verify.c b/fs/btrfs/ref-verify.c index 2b490becbe67..8e026de74c44 100644 --- a/fs/btrfs/ref-verify.c +++ b/fs/btrfs/ref-verify.c @@ -218,11 +218,11 @@ static void __print_stack_trace(struct btrfs_fs_info *fs_info, stack_trace_print(ra->trace, ra->trace_len, 2); } #else -static void inline __save_stack_trace(struct ref_action *ra) +static inline void __save_stack_trace(struct ref_action *ra) { } -static void inline __print_stack_trace(struct btrfs_fs_info *fs_info, +static inline void __print_stack_trace(struct btrfs_fs_info *fs_info, struct ref_action *ra) { btrfs_err(fs_info, " ref-verify: no stacktrace support"); -- cgit v1.2.3 From 4f6a49de64fd1b1dba5229c02047376da7cf24fd Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Tue, 23 Feb 2021 15:20:42 +0200 Subject: btrfs: unlock extents in btrfs_zero_range in case of quota reservation errors If btrfs_qgroup_reserve_data returns an error (i.e quota limit reached) the handling logic directly goes to the 'out' label without first unlocking the extent range between lockstart, lockend. This results in deadlocks as other processes try to lock the same extent. Fixes: a7f8b1c2ac21 ("btrfs: file: reserve qgroup space after the hole punch range is locked") CC: stable@vger.kernel.org # 5.10+ Reviewed-by: Qu Wenruo Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/file.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 01a72f53fb5d..2c282664c4b8 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -3260,8 +3260,11 @@ reserve_space: goto out; ret = btrfs_qgroup_reserve_data(BTRFS_I(inode), &data_reserved, alloc_start, bytes_to_reserve); - if (ret) + if (ret) { + unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, + lockend, &cached_state); goto out; + } ret = btrfs_prealloc_file_range(inode, mode, alloc_start, alloc_end - alloc_start, i_blocksize(inode), -- cgit v1.2.3 From 5011c5a663b9c6d6aff3d394f11049b371199627 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 17 Feb 2021 09:04:34 +0300 Subject: btrfs: validate qgroup inherit for SNAP_CREATE_V2 ioctl The problem is we're copying "inherit" from user space but we don't necessarily know that we're copying enough data for a 64 byte struct. Then the next problem is that 'inherit' has a variable size array at the end, and we have to verify that array is the size we expected. Fixes: 6f72c7e20dba ("Btrfs: add qgroup inheritance") CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Dan Carpenter Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ioctl.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index a8c60d46d19c..1b837c08ca90 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -1935,7 +1935,10 @@ static noinline int btrfs_ioctl_snap_create_v2(struct file *file, if (vol_args->flags & BTRFS_SUBVOL_RDONLY) readonly = true; if (vol_args->flags & BTRFS_SUBVOL_QGROUP_INHERIT) { - if (vol_args->size > PAGE_SIZE) { + u64 nums; + + if (vol_args->size < sizeof(*inherit) || + vol_args->size > PAGE_SIZE) { ret = -EINVAL; goto free_args; } @@ -1944,6 +1947,20 @@ static noinline int btrfs_ioctl_snap_create_v2(struct file *file, ret = PTR_ERR(inherit); goto free_args; } + + if (inherit->num_qgroups > PAGE_SIZE || + inherit->num_ref_copies > PAGE_SIZE || + inherit->num_excl_copies > PAGE_SIZE) { + ret = -EINVAL; + goto free_inherit; + } + + nums = inherit->num_qgroups + 2 * inherit->num_ref_copies + + 2 * inherit->num_excl_copies; + if (vol_args->size != struct_size(inherit, qgroups, nums)) { + ret = -EINVAL; + goto free_inherit; + } } ret = __btrfs_ioctl_snap_create(file, vol_args->name, vol_args->fd, -- cgit v1.2.3 From c55a4319c4f2c3ba0a385b1ebc454fa283cfe920 Mon Sep 17 00:00:00 2001 From: Boris Burkov Date: Tue, 23 Feb 2021 10:22:32 -0800 Subject: btrfs: fix spurious free_space_tree remount warning The intended logic of the check is to catch cases where the desired free_space_tree setting doesn't match the mounted setting, and the remount is anything but ro->rw. However, it makes the mistake of checking equality on a masked integer (btrfs_test_opt) against a boolean (btrfs_fs_compat_ro). If you run the reproducer: $ mount -o space_cache=v2 dev mnt $ mount -o remount,ro mnt you would expect no warning, because the remount is not attempting to change the free space tree setting, but we do see the warning. To fix this, add explicit bool type casts to the condition. I tested a variety of transitions: sudo mount -o space_cache=v2 /dev/vg0/lv0 mnt/lol (fst enabled) mount -o remount,ro mnt/lol (no warning, no fst change) sudo mount -o remount,rw,space_cache=v1,clear_cache (no warning, ro->rw) sudo mount -o remount,rw,space_cache=v2 mnt (warning, rw->rw with change) sudo mount -o remount,ro mnt (no warning, no fst change) sudo mount -o remount,rw,space_cache=v2 mnt (no warning, no fst change) Reported-by: Chris Murphy CC: stable@vger.kernel.org # 5.11 Signed-off-by: Boris Burkov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/super.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index f8435641b912..f7a4ad86adee 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -1918,8 +1918,8 @@ static int btrfs_remount(struct super_block *sb, int *flags, char *data) btrfs_resize_thread_pool(fs_info, fs_info->thread_pool_size, old_thread_pool_size); - if (btrfs_test_opt(fs_info, FREE_SPACE_TREE) != - btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) && + if ((bool)btrfs_test_opt(fs_info, FREE_SPACE_TREE) != + (bool)btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) && (!sb_rdonly(sb) || (*flags & SB_RDONLY))) { btrfs_warn(fs_info, "remount supports changing free space tree only from ro to rw"); -- cgit v1.2.3 From 0f9c03d824f6f522d3bc43629635c9765546ebc5 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 22 Feb 2021 18:40:42 +0200 Subject: btrfs: free correct amount of space in btrfs_delayed_inode_reserve_metadata Following commit f218ea6c4792 ("btrfs: delayed-inode: Remove wrong qgroup meta reservation calls") this function now reserves num_bytes, rather than the fixed amount of nodesize. As such this requires the same amount to be freed in case of failure. Fix this by adjusting the amount we are freeing. Fixes: f218ea6c4792 ("btrfs: delayed-inode: Remove wrong qgroup meta reservation calls") CC: stable@vger.kernel.org # 4.19+ Reviewed-by: Qu Wenruo Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/delayed-inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index ec0b50b8c5d6..ac9966e76a2f 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -649,7 +649,7 @@ static int btrfs_delayed_inode_reserve_metadata( btrfs_ino(inode), num_bytes, 1); } else { - btrfs_qgroup_free_meta_prealloc(root, fs_info->nodesize); + btrfs_qgroup_free_meta_prealloc(root, num_bytes); } return ret; } -- cgit v1.2.3 From 80e9baed722c853056e0c5374f51524593cb1031 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 22 Feb 2021 18:40:43 +0200 Subject: btrfs: export and rename qgroup_reserve_meta Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/qgroup.c | 8 ++++---- fs/btrfs/qgroup.h | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 808370ada888..14ff388fd3bd 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -3841,8 +3841,8 @@ static int sub_root_meta_rsv(struct btrfs_root *root, int num_bytes, return num_bytes; } -static int qgroup_reserve_meta(struct btrfs_root *root, int num_bytes, - enum btrfs_qgroup_rsv_type type, bool enforce) +int btrfs_qgroup_reserve_meta(struct btrfs_root *root, int num_bytes, + enum btrfs_qgroup_rsv_type type, bool enforce) { struct btrfs_fs_info *fs_info = root->fs_info; int ret; @@ -3873,14 +3873,14 @@ int __btrfs_qgroup_reserve_meta(struct btrfs_root *root, int num_bytes, { int ret; - ret = qgroup_reserve_meta(root, num_bytes, type, enforce); + ret = btrfs_qgroup_reserve_meta(root, num_bytes, type, enforce); if (ret <= 0 && ret != -EDQUOT) return ret; ret = try_flush_qgroup(root); if (ret < 0) return ret; - return qgroup_reserve_meta(root, num_bytes, type, enforce); + return btrfs_qgroup_reserve_meta(root, num_bytes, type, enforce); } void btrfs_qgroup_free_meta_all_pertrans(struct btrfs_root *root) diff --git a/fs/btrfs/qgroup.h b/fs/btrfs/qgroup.h index 50dea9a2d8fb..7283e4f549af 100644 --- a/fs/btrfs/qgroup.h +++ b/fs/btrfs/qgroup.h @@ -361,6 +361,8 @@ int btrfs_qgroup_release_data(struct btrfs_inode *inode, u64 start, u64 len); int btrfs_qgroup_free_data(struct btrfs_inode *inode, struct extent_changeset *reserved, u64 start, u64 len); +int btrfs_qgroup_reserve_meta(struct btrfs_root *root, int num_bytes, + enum btrfs_qgroup_rsv_type type, bool enforce); int __btrfs_qgroup_reserve_meta(struct btrfs_root *root, int num_bytes, enum btrfs_qgroup_rsv_type type, bool enforce); /* Reserve metadata space for pertrans and prealloc type */ -- cgit v1.2.3 From 4d14c5cde5c268a2bc26addecf09489cb953ef64 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 22 Feb 2021 18:40:44 +0200 Subject: btrfs: don't flush from btrfs_delayed_inode_reserve_metadata Calling btrfs_qgroup_reserve_meta_prealloc from btrfs_delayed_inode_reserve_metadata can result in flushing delalloc while holding a transaction and delayed node locks. This is deadlock prone. In the past multiple commits: * ae5e070eaca9 ("btrfs: qgroup: don't try to wait flushing if we're already holding a transaction") * 6f23277a49e6 ("btrfs: qgroup: don't commit transaction when we already hold the handle") Tried to solve various aspects of this but this was always a whack-a-mole game. Unfortunately those 2 fixes don't solve a deadlock scenario involving btrfs_delayed_node::mutex. Namely, one thread can call btrfs_dirty_inode as a result of reading a file and modifying its atime: PID: 6963 TASK: ffff8c7f3f94c000 CPU: 2 COMMAND: "test" #0 __schedule at ffffffffa529e07d #1 schedule at ffffffffa529e4ff #2 schedule_timeout at ffffffffa52a1bdd #3 wait_for_completion at ffffffffa529eeea <-- sleeps with delayed node mutex held #4 start_delalloc_inodes at ffffffffc0380db5 #5 btrfs_start_delalloc_snapshot at ffffffffc0393836 #6 try_flush_qgroup at ffffffffc03f04b2 #7 __btrfs_qgroup_reserve_meta at ffffffffc03f5bb6 <-- tries to reserve space and starts delalloc inodes. #8 btrfs_delayed_update_inode at ffffffffc03e31aa <-- acquires delayed node mutex #9 btrfs_update_inode at ffffffffc0385ba8 #10 btrfs_dirty_inode at ffffffffc038627b <-- TRANSACTIION OPENED #11 touch_atime at ffffffffa4cf0000 #12 generic_file_read_iter at ffffffffa4c1f123 #13 new_sync_read at ffffffffa4ccdc8a #14 vfs_read at ffffffffa4cd0849 #15 ksys_read at ffffffffa4cd0bd1 #16 do_syscall_64 at ffffffffa4a052eb #17 entry_SYSCALL_64_after_hwframe at ffffffffa540008c This will cause an asynchronous work to flush the delalloc inodes to happen which can try to acquire the same delayed_node mutex: PID: 455 TASK: ffff8c8085fa4000 CPU: 5 COMMAND: "kworker/u16:30" #0 __schedule at ffffffffa529e07d #1 schedule at ffffffffa529e4ff #2 schedule_preempt_disabled at ffffffffa529e80a #3 __mutex_lock at ffffffffa529fdcb <-- goes to sleep, never wakes up. #4 btrfs_delayed_update_inode at ffffffffc03e3143 <-- tries to acquire the mutex #5 btrfs_update_inode at ffffffffc0385ba8 <-- this is the same inode that pid 6963 is holding #6 cow_file_range_inline.constprop.78 at ffffffffc0386be7 #7 cow_file_range at ffffffffc03879c1 #8 btrfs_run_delalloc_range at ffffffffc038894c #9 writepage_delalloc at ffffffffc03a3c8f #10 __extent_writepage at ffffffffc03a4c01 #11 extent_write_cache_pages at ffffffffc03a500b #12 extent_writepages at ffffffffc03a6de2 #13 do_writepages at ffffffffa4c277eb #14 __filemap_fdatawrite_range at ffffffffa4c1e5bb #15 btrfs_run_delalloc_work at ffffffffc0380987 <-- starts running delayed nodes #16 normal_work_helper at ffffffffc03b706c #17 process_one_work at ffffffffa4aba4e4 #18 worker_thread at ffffffffa4aba6fd #19 kthread at ffffffffa4ac0a3d #20 ret_from_fork at ffffffffa54001ff To fully address those cases the complete fix is to never issue any flushing while holding the transaction or the delayed node lock. This patch achieves it by calling qgroup_reserve_meta directly which will either succeed without flushing or will fail and return -EDQUOT. In the latter case that return value is going to be propagated to btrfs_dirty_inode which will fallback to start a new transaction. That's fine as the majority of time we expect the inode will have BTRFS_DELAYED_NODE_INODE_DIRTY flag set which will result in directly copying the in-memory state. Fixes: c53e9653605d ("btrfs: qgroup: try to flush qgroup space when we get -EDQUOT") CC: stable@vger.kernel.org # 5.10+ Reviewed-by: Qu Wenruo Signed-off-by: Nikolay Borisov Signed-off-by: David Sterba --- fs/btrfs/delayed-inode.c | 3 ++- fs/btrfs/inode.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index ac9966e76a2f..bf25401c9768 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -627,7 +627,8 @@ static int btrfs_delayed_inode_reserve_metadata( */ if (!src_rsv || (!trans->bytes_reserved && src_rsv->type != BTRFS_BLOCK_RSV_DELALLOC)) { - ret = btrfs_qgroup_reserve_meta_prealloc(root, num_bytes, true); + ret = btrfs_qgroup_reserve_meta(root, num_bytes, + BTRFS_QGROUP_RSV_META_PREALLOC, true); if (ret < 0) return ret; ret = btrfs_block_rsv_add(root, dst_rsv, num_bytes, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 4f2f1e932751..c35b724a5611 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -6081,7 +6081,7 @@ static int btrfs_dirty_inode(struct inode *inode) return PTR_ERR(trans); ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); - if (ret && ret == -ENOSPC) { + if (ret && (ret == -ENOSPC || ret == -EDQUOT)) { /* whoops, lets try again with the full transaction */ btrfs_end_transaction(trans); trans = btrfs_start_transaction(root, 1); -- cgit v1.2.3 From fd57a98d6f0c98fa295813087f13afb26c224e73 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 26 Feb 2021 17:51:44 +0000 Subject: btrfs: fix warning when creating a directory with smack enabled When we have smack enabled, during the creation of a directory smack may attempt to add a "smack transmute" xattr on the inode, which results in the following warning and trace: WARNING: CPU: 3 PID: 2548 at fs/btrfs/transaction.c:537 start_transaction+0x489/0x4f0 Modules linked in: nft_objref nf_conntrack_netbios_ns (...) CPU: 3 PID: 2548 Comm: mkdir Not tainted 5.9.0-rc2smack+ #81 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.13.0-2.fc32 04/01/2014 RIP: 0010:start_transaction+0x489/0x4f0 Code: e9 be fc ff ff (...) RSP: 0018:ffffc90001887d10 EFLAGS: 00010202 RAX: ffff88816f1e0000 RBX: 0000000000000201 RCX: 0000000000000003 RDX: 0000000000000201 RSI: 0000000000000002 RDI: ffff888177849000 RBP: ffff888177849000 R08: 0000000000000001 R09: 0000000000000004 R10: ffffffff825e8f7a R11: 0000000000000003 R12: ffffffffffffffe2 R13: 0000000000000000 R14: ffff88803d884270 R15: ffff8881680d8000 FS: 00007f67317b8440(0000) GS:ffff88817bcc0000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f67247a22a8 CR3: 000000004bfbc002 CR4: 0000000000370ee0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: ? slab_free_freelist_hook+0xea/0x1b0 ? trace_hardirqs_on+0x1c/0xe0 btrfs_setxattr_trans+0x3c/0xf0 __vfs_setxattr+0x63/0x80 smack_d_instantiate+0x2d3/0x360 security_d_instantiate+0x29/0x40 d_instantiate_new+0x38/0x90 btrfs_mkdir+0x1cf/0x1e0 vfs_mkdir+0x14f/0x200 do_mkdirat+0x6d/0x110 do_syscall_64+0x2d/0x40 entry_SYSCALL_64_after_hwframe+0x44/0xa9 RIP: 0033:0x7f673196ae6b Code: 8b 05 11 (...) RSP: 002b:00007ffc3c679b18 EFLAGS: 00000246 ORIG_RAX: 0000000000000053 RAX: ffffffffffffffda RBX: 00000000000001ff RCX: 00007f673196ae6b RDX: 0000000000000000 RSI: 00000000000001ff RDI: 00007ffc3c67a30d RBP: 00007ffc3c67a30d R08: 00000000000001ff R09: 0000000000000000 R10: 000055d3e39fe930 R11: 0000000000000246 R12: 0000000000000000 R13: 00007ffc3c679cd8 R14: 00007ffc3c67a30d R15: 00007ffc3c679ce0 irq event stamp: 11029 hardirqs last enabled at (11037): [] console_unlock+0x486/0x670 hardirqs last disabled at (11044): [] console_unlock+0xa1/0x670 softirqs last enabled at (8864): [] asm_call_on_stack+0xf/0x20 softirqs last disabled at (8851): [] asm_call_on_stack+0xf/0x20 This happens because at btrfs_mkdir() we call d_instantiate_new() while holding a transaction handle, which results in the following call chain: btrfs_mkdir() trans = btrfs_start_transaction(root, 5); d_instantiate_new() smack_d_instantiate() __vfs_setxattr() btrfs_setxattr_trans() btrfs_start_transaction() start_transaction() WARN_ON() --> a tansaction start has TRANS_EXTWRITERS set in its type h->orig_rsv = h->block_rsv h->block_rsv = NULL btrfs_end_transaction(trans) Besides the warning triggered at start_transaction, we set the handle's block_rsv to NULL which may cause some surprises later on. So fix this by making btrfs_setxattr_trans() not start a transaction when we already have a handle on one, stored in current->journal_info, and use that handle. We are good to use the handle because at btrfs_mkdir() we did reserve space for the xattr and the inode item. Reported-by: Casey Schaufler CC: stable@vger.kernel.org # 5.4+ Acked-by: Casey Schaufler Tested-by: Casey Schaufler Link: https://lore.kernel.org/linux-btrfs/434d856f-bd7b-4889-a6ec-e81aaebfa735@schaufler-ca.com/ Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/xattr.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/xattr.c b/fs/btrfs/xattr.c index af6246f36a9e..03135dbb318a 100644 --- a/fs/btrfs/xattr.c +++ b/fs/btrfs/xattr.c @@ -229,11 +229,33 @@ int btrfs_setxattr_trans(struct inode *inode, const char *name, { struct btrfs_root *root = BTRFS_I(inode)->root; struct btrfs_trans_handle *trans; + const bool start_trans = (current->journal_info == NULL); int ret; - trans = btrfs_start_transaction(root, 2); - if (IS_ERR(trans)) - return PTR_ERR(trans); + if (start_trans) { + /* + * 1 unit for inserting/updating/deleting the xattr + * 1 unit for the inode item update + */ + trans = btrfs_start_transaction(root, 2); + if (IS_ERR(trans)) + return PTR_ERR(trans); + } else { + /* + * This can happen when smack is enabled and a directory is being + * created. It happens through d_instantiate_new(), which calls + * smack_d_instantiate(), which in turn calls __vfs_setxattr() to + * set the transmute xattr (XATTR_NAME_SMACKTRANSMUTE) on the + * inode. We have already reserved space for the xattr and inode + * update at btrfs_mkdir(), so just use the transaction handle. + * We don't join or start a transaction, as that will reset the + * block_rsv of the handle and trigger a warning for the start + * case. + */ + ASSERT(strncmp(name, XATTR_SECURITY_PREFIX, + XATTR_SECURITY_PREFIX_LEN) == 0); + trans = current->journal_info; + } ret = btrfs_setxattr(trans, inode, name, value, size, flags); if (ret) @@ -244,7 +266,8 @@ int btrfs_setxattr_trans(struct inode *inode, const char *name, ret = btrfs_update_inode(trans, root, BTRFS_I(inode)); BUG_ON(ret); out: - btrfs_end_transaction(trans); + if (start_trans) + btrfs_end_transaction(trans); return ret; } -- cgit v1.2.3 From c28ea613fafad910d08f67efe76ae552b1434e44 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 1 Mar 2021 16:44:22 +0800 Subject: btrfs: subpage: fix the false data csum mismatch error [BUG] When running fstresss, we can hit strange data csum mismatch where the on-disk data is in fact correct (passes scrub). With some extra debug info added, we have the following traces: 0482us: btrfs_do_readpage: root=5 ino=284 offset=393216, submit force=0 pgoff=0 iosize=8192 0494us: btrfs_do_readpage: root=5 ino=284 offset=401408, submit force=0 pgoff=8192 iosize=4096 0498us: btrfs_submit_data_bio: root=5 ino=284 bio first bvec=393216 len=8192 0591us: btrfs_do_readpage: root=5 ino=284 offset=405504, submit force=0 pgoff=12288 iosize=36864 0594us: btrfs_submit_data_bio: root=5 ino=284 bio first bvec=401408 len=4096 0863us: btrfs_submit_data_bio: root=5 ino=284 bio first bvec=405504 len=36864 0933us: btrfs_verify_data_csum: root=5 ino=284 offset=393216 len=8192 0967us: btrfs_do_readpage: root=5 ino=284 offset=442368, skip beyond isize pgoff=49152 iosize=16384 1047us: btrfs_verify_data_csum: root=5 ino=284 offset=401408 len=4096 1163us: btrfs_verify_data_csum: root=5 ino=284 offset=405504 len=36864 1290us: check_data_csum: !!! root=5 ino=284 offset=438272 pg_off=45056 !!! 7387us: end_bio_extent_readpage: root=5 ino=284 before pending_read_bios=0 [CAUSE] Normally we expect all submitted bio reads to only touch the range we specified, and under subpage context, it means we should only touch the range specified in each bvec. But in data read path, inside end_bio_extent_readpage(), we have page zeroing which only takes regular page size into consideration. This means for subpage if we have an inode whose content looks like below: 0 16K 32K 48K 64K |///////| |///////| | |//| = data needs to be read from disk | | = hole And i_size is 64K initially. Then the following race can happen: T1 | T2 --------------------------------+-------------------------------- btrfs_do_readpage() | |- isize = 64K; | | At this time, the isize is | | 64K | | | |- submit_extent_page() | | submit previous assembled bio| | assemble bio for [0, 16K) | | | |- submit_extent_page() | submit read bio for [0, 16K) | assemble read bio for | [32K, 48K) | | | btrfs_setsize() | |- i_size_write(, 16K); | Now i_size is only 16K end_io() for [0K, 16K) | |- end_bio_extent_readpage() | |- btrfs_verify_data_csum() | | No csum error | |- i_size = 16K; | |- zero_user_segment(16K, | PAGE_SIZE); | !!! We zeroed range | !!! [32K, 48K) | | end_io for [32K, 48K) | |- end_bio_extent_readpage() | |- btrfs_verify_data_csum() | ! CSUM MISMATCH ! | ! As the range is zeroed now ! [FIX] To fix the problem, make end_bio_extent_readpage() to only zero the range of bvec. The bug only affects subpage read-write support, as for full read-only mount we can't change i_size thus won't hit the race condition. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 4dfb3ead1175..4671c99d468d 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -3008,12 +3008,23 @@ readpage_ok: if (likely(uptodate)) { loff_t i_size = i_size_read(inode); pgoff_t end_index = i_size >> PAGE_SHIFT; - unsigned off; - /* Zero out the end if this page straddles i_size */ - off = offset_in_page(i_size); - if (page->index == end_index && off) - zero_user_segment(page, off, PAGE_SIZE); + /* + * Zero out the remaining part if this range straddles + * i_size. + * + * Here we should only zero the range inside the bvec, + * not touch anything else. + * + * NOTE: i_size is exclusive while end is inclusive. + */ + if (page->index == end_index && i_size <= end) { + u32 zero_start = max(offset_in_page(i_size), + offset_in_page(end)); + + zero_user_segment(page, zero_start, + offset_in_page(end) + 1); + } } ASSERT(bio_offset + len > bio_offset); bio_offset += len; -- cgit v1.2.3 From 7db688e99c0f770ae73e0f1f3fb67f9b64266445 Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Tue, 2 Mar 2021 12:58:50 +0300 Subject: pstore/ram: Rate-limit "uncorrectable error in header" message There is a quite huge "uncorrectable error in header" flood in KMSG on a clean system boot since there is no pstore buffer saved in RAM. Let's silence the redundant noisy messages by rate-limiting the printk message. Now there are maximum 10 messages printed repeatedly instead of 35+. Signed-off-by: Dmitry Osipenko Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20210302095850.30894-1-digetx@gmail.com --- fs/pstore/ram_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c index aa8e0b65ff1a..fff363bfd484 100644 --- a/fs/pstore/ram_core.c +++ b/fs/pstore/ram_core.c @@ -246,7 +246,7 @@ static int persistent_ram_init_ecc(struct persistent_ram_zone *prz, pr_info("error in header, %d\n", numerr); prz->corrected_bytes += numerr; } else if (numerr < 0) { - pr_info("uncorrectable error in header\n"); + pr_info_ratelimited("uncorrectable error in header\n"); prz->bad_blocks++; } -- cgit v1.2.3 From 775c5033a0d164622d9d10dd0f0a5531639ed3ed Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Thu, 4 Mar 2021 11:09:12 +0200 Subject: fuse: fix live lock in fuse_iget() Commit 5d069dbe8aaf ("fuse: fix bad inode") replaced make_bad_inode() in fuse_iget() with a private implementation fuse_make_bad(). The private implementation fails to remove the bad inode from inode cache, so the retry loop with iget5_locked() finds the same bad inode and marks it bad forever. kmsg snip: [ ] rcu: INFO: rcu_sched self-detected stall on CPU ... [ ] ? bit_wait_io+0x50/0x50 [ ] ? fuse_init_file_inode+0x70/0x70 [ ] ? find_inode.isra.32+0x60/0xb0 [ ] ? fuse_init_file_inode+0x70/0x70 [ ] ilookup5_nowait+0x65/0x90 [ ] ? fuse_init_file_inode+0x70/0x70 [ ] ilookup5.part.36+0x2e/0x80 [ ] ? fuse_init_file_inode+0x70/0x70 [ ] ? fuse_inode_eq+0x20/0x20 [ ] iget5_locked+0x21/0x80 [ ] ? fuse_inode_eq+0x20/0x20 [ ] fuse_iget+0x96/0x1b0 Fixes: 5d069dbe8aaf ("fuse: fix bad inode") Cc: stable@vger.kernel.org # 5.10+ Signed-off-by: Amir Goldstein Signed-off-by: Miklos Szeredi --- fs/fuse/fuse_i.h | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 68cca8d4db6e..63d97a15ffde 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -863,6 +863,7 @@ static inline u64 fuse_get_attr_version(struct fuse_conn *fc) static inline void fuse_make_bad(struct inode *inode) { + remove_inode_hash(inode); set_bit(FUSE_I_BAD, &get_fuse_inode(inode)->state); } -- cgit v1.2.3 From fb3a1f6c745ccd896afadf6e2d6f073e871d38ba Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 26 Feb 2021 09:47:20 -0700 Subject: io-wq: have manager wait for all workers to exit Instead of having to wait separately on workers and manager, just have the manager wait on the workers. We use an atomic_t for the reference here, as we need to start at 0 and allow increment from that. Since the number of workers is naturally capped by the allowed nr of processes, and that uses an int, there is no risk of overflow. Signed-off-by: Jens Axboe --- fs/io-wq.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 965022fe9961..1d01edada8aa 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -120,6 +120,9 @@ struct io_wq { refcount_t refs; struct completion done; + atomic_t worker_refs; + struct completion worker_done; + struct hlist_node cpuhp_node; pid_t task_pid; @@ -189,7 +192,8 @@ static void io_worker_exit(struct io_worker *worker) raw_spin_unlock_irq(&wqe->lock); kfree_rcu(worker, rcu); - io_wq_put(wqe->wq); + if (atomic_dec_and_test(&wqe->wq->worker_refs)) + complete(&wqe->wq->worker_done); } static inline bool io_wqe_run_queue(struct io_wqe *wqe) @@ -648,14 +652,15 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) init_completion(&worker->ref_done); init_completion(&worker->started); - refcount_inc(&wq->refs); + atomic_inc(&wq->worker_refs); if (index == IO_WQ_ACCT_BOUND) pid = io_wq_fork_thread(task_thread_bound, worker); else pid = io_wq_fork_thread(task_thread_unbound, worker); if (pid < 0) { - io_wq_put(wq); + if (atomic_dec_and_test(&wq->worker_refs)) + complete(&wq->worker_done); kfree(worker); return false; } @@ -736,6 +741,7 @@ static int io_wq_manager(void *data) { struct io_wq *wq = data; char buf[TASK_COMM_LEN]; + int node; sprintf(buf, "iou-mgr-%d", wq->task_pid); set_task_comm(current, buf); @@ -753,6 +759,15 @@ static int io_wq_manager(void *data) } while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)); io_wq_check_workers(wq); + + rcu_read_lock(); + for_each_node(node) + io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); + rcu_read_unlock(); + + /* we might not ever have created any workers */ + if (atomic_read(&wq->worker_refs)) + wait_for_completion(&wq->worker_done); wq->manager = NULL; io_wq_put(wq); do_exit(0); @@ -796,6 +811,7 @@ static int io_wq_fork_manager(struct io_wq *wq) if (wq->manager) return 0; + reinit_completion(&wq->worker_done); clear_bit(IO_WQ_BIT_EXIT, &wq->state); refcount_inc(&wq->refs); current->flags |= PF_IO_WORKER; @@ -1050,6 +1066,9 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) init_completion(&wq->done); refcount_set(&wq->refs, 1); + init_completion(&wq->worker_done); + atomic_set(&wq->worker_refs, 0); + ret = io_wq_fork_manager(wq); if (!ret) return wq; @@ -1077,11 +1096,6 @@ static void io_wq_destroy(struct io_wq *wq) if (wq->manager) wake_up_process(wq->manager); - rcu_read_lock(); - for_each_node(node) - io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); - rcu_read_unlock(); - spin_lock_irq(&wq->hash->wait.lock); for_each_node(node) { struct io_wqe *wqe = wq->wqes[node]; -- cgit v1.2.3 From 613eeb600e3e636a1d3b3711dddaf2b134d5a32c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 26 Feb 2021 09:52:02 -0700 Subject: io-wq: don't ask for a new worker if we're exiting If we're in the process of shutting down the async context, then don't create new workers if we already have at least the fixed one. Signed-off-by: Jens Axboe --- fs/io-wq.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 1d01edada8aa..2f9d7ee12ee1 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -673,6 +673,8 @@ static inline bool io_wqe_need_worker(struct io_wqe *wqe, int index) { struct io_wqe_acct *acct = &wqe->acct[index]; + if (acct->nr_workers && test_bit(IO_WQ_BIT_EXIT, &wqe->wq->state)) + return false; /* if we have available workers or no work, no need */ if (!hlist_nulls_empty(&wqe->free_list) || !io_wqe_run_queue(wqe)) return false; -- cgit v1.2.3 From dbf996202e28c6b1eb30afad534abe45a691499e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 26 Feb 2021 09:53:17 -0700 Subject: io-wq: rename wq->done completion to wq->started This is a leftover from a different use cases, it's used to wait for the manager to startup. Rename it as such. Signed-off-by: Jens Axboe --- fs/io-wq.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 2f9d7ee12ee1..1e5b41614bd6 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -118,7 +118,7 @@ struct io_wq { struct io_wq_hash *hash; refcount_t refs; - struct completion done; + struct completion started; atomic_t worker_refs; struct completion worker_done; @@ -750,7 +750,7 @@ static int io_wq_manager(void *data) current->flags |= PF_IO_WORKER; wq->manager = current; - complete(&wq->done); + complete(&wq->started); do { set_current_state(TASK_INTERRUPTIBLE); @@ -820,7 +820,7 @@ static int io_wq_fork_manager(struct io_wq *wq) ret = io_wq_fork_thread(io_wq_manager, wq); current->flags &= ~PF_IO_WORKER; if (ret >= 0) { - wait_for_completion(&wq->done); + wait_for_completion(&wq->started); return 0; } @@ -1065,7 +1065,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) } wq->task_pid = current->pid; - init_completion(&wq->done); + init_completion(&wq->started); refcount_set(&wq->refs, 1); init_completion(&wq->worker_done); -- cgit v1.2.3 From d364d9e5db41678b77ed95c41e3ccaad9ab99ba0 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 26 Feb 2021 09:56:32 -0700 Subject: io-wq: wait for manager exit on wq destroy The manager waits for the workers, hence the manager is always valid if workers are running. Now also have wq destroy wait for the manager on exit, so we now everything is gone. Signed-off-by: Jens Axboe --- fs/io-wq.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 1e5b41614bd6..7a1d51c1aca9 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -119,6 +119,7 @@ struct io_wq { refcount_t refs; struct completion started; + struct completion exited; atomic_t worker_refs; struct completion worker_done; @@ -771,6 +772,7 @@ static int io_wq_manager(void *data) if (atomic_read(&wq->worker_refs)) wait_for_completion(&wq->worker_done); wq->manager = NULL; + complete(&wq->exited); io_wq_put(wq); do_exit(0); } @@ -1066,6 +1068,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) wq->task_pid = current->pid; init_completion(&wq->started); + init_completion(&wq->exited); refcount_set(&wq->refs, 1); init_completion(&wq->worker_done); @@ -1095,8 +1098,10 @@ static void io_wq_destroy(struct io_wq *wq) cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); set_bit(IO_WQ_BIT_EXIT, &wq->state); - if (wq->manager) + if (wq->manager) { wake_up_process(wq->manager); + wait_for_completion(&wq->exited); + } spin_lock_irq(&wq->hash->wait.lock); for_each_node(node) { -- cgit v1.2.3 From 470ec4ed8c91b4db398ad607c700e9ce88365202 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 26 Feb 2021 10:20:34 -0700 Subject: io-wq: fix double put of 'wq' in error path We are already freeing the wq struct in both spots, so don't put it and get it freed twice. Reported-by: syzbot+7bf785eedca35ca05501@syzkaller.appspotmail.com Fixes: 4fb6ac326204 ("io-wq: improve manager/worker handling over exec") Signed-off-by: Jens Axboe --- fs/io-wq.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 7a1d51c1aca9..f0b7e9ff63fa 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -826,7 +826,6 @@ static int io_wq_fork_manager(struct io_wq *wq) return 0; } - io_wq_put(wq); return ret; } @@ -1078,7 +1077,6 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) if (!ret) return wq; - io_wq_put(wq); io_wq_put_hash(data->hash); err: cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); -- cgit v1.2.3 From e54945ae947fb881212a4b97d5599a01bba6ad06 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 26 Feb 2021 11:27:15 -0700 Subject: io_uring: SQPOLL stop error handling fixes If we fail to fork an SQPOLL worker, we can hit cancel, and hence attempted thread stop, with the thread already being stopped. Ensure we check for that. Also guard thread stop fully by the sqd mutex, just like we do for park. Signed-off-by: Jens Axboe --- fs/io_uring.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 4a088581b0f2..d55c9ab6314a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6793,9 +6793,9 @@ static int io_sq_thread(void *data) ctx->sqo_exec = 1; io_ring_set_wakeup_flag(ctx); } - mutex_unlock(&sqd->lock); complete(&sqd->exited); + mutex_unlock(&sqd->lock); do_exit(0); } @@ -7118,13 +7118,19 @@ static bool io_sq_thread_park(struct io_sq_data *sqd) static void io_sq_thread_stop(struct io_sq_data *sqd) { - if (!sqd->thread) + if (test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) return; - - set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); - WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)); - wake_up_process(sqd->thread); - wait_for_completion(&sqd->exited); + mutex_lock(&sqd->lock); + if (sqd->thread) { + set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); + WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)); + wake_up_process(sqd->thread); + mutex_unlock(&sqd->lock); + wait_for_completion(&sqd->exited); + WARN_ON_ONCE(sqd->thread); + } else { + mutex_unlock(&sqd->lock); + } } static void io_put_sq_data(struct io_sq_data *sqd) @@ -8867,6 +8873,11 @@ static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) if (!io_sq_thread_park(sqd)) return; tctx = ctx->sq_data->thread->io_uring; + /* can happen on fork/alloc failure, just ignore that state */ + if (!tctx) { + io_sq_thread_unpark(sqd); + return; + } atomic_inc(&tctx->in_idle); do { -- cgit v1.2.3 From ba50a036f23c44608b1d903c34644a1acd5d21fa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 26 Feb 2021 15:47:56 +0000 Subject: io_uring: run fallback on cancellation io_uring_try_cancel_requests() matches not only current's requests, but also of other exiting tasks, so we need to actively cancel them and not just wait, especially since the function can be called on flush during do_exit() -> exit_files(). Even if it's not a problem for now, it's much nicer to know that the function tries to cancel everything it can. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index d55c9ab6314a..9d6696ff5748 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8518,9 +8518,10 @@ static int io_remove_personalities(int id, void *p, void *data) return 0; } -static void io_run_ctx_fallback(struct io_ring_ctx *ctx) +static bool io_run_ctx_fallback(struct io_ring_ctx *ctx) { struct callback_head *work, *head, *next; + bool executed = false; do { do { @@ -8537,7 +8538,10 @@ static void io_run_ctx_fallback(struct io_ring_ctx *ctx) work = next; cond_resched(); } while (work); + executed = true; } while (1); + + return executed; } static void io_ring_exit_work(struct work_struct *work) @@ -8677,6 +8681,7 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, ret |= io_poll_remove_all(ctx, task, files); ret |= io_kill_timeouts(ctx, task, files); ret |= io_run_task_work(); + ret |= io_run_ctx_fallback(ctx); io_cqring_overflow_flush(ctx, true, task, files); if (!ret) break; -- cgit v1.2.3 From 8629397e6e2753bb4cc62ba48a12e1d4d912b6a4 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 26 Feb 2021 13:46:49 -0700 Subject: io_uring: don't use complete_all() on SQPOLL thread exit We want to reuse this completion, and a single complete should do just fine. Ensure that we park ourselves first if requested, as that is what lead to the initial deadlock in this area. If we've got someone attempting to park us, then we can't proceed without having them finish first. Fixes: 37d1e2e3642e ("io_uring: move SQPOLL thread io-wq forked worker") Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 9d6696ff5748..904bf0fecc36 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6783,10 +6783,13 @@ static int io_sq_thread(void *data) io_run_task_work(); + if (io_sq_thread_should_park(sqd)) + io_sq_thread_parkme(sqd); + /* * Clear thread under lock so that concurrent parks work correctly */ - complete_all(&sqd->completion); + complete(&sqd->completion); mutex_lock(&sqd->lock); sqd->thread = NULL; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { -- cgit v1.2.3 From afcc4015d1bf5659b8c722aff679e9b8c41ee156 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 26 Feb 2021 13:48:19 -0700 Subject: io-wq: provide an io_wq_put_and_exit() helper If we put the io-wq from io_uring, we really want it to exit. Provide a helper that does that for us. Couple that with not having the manager hold a reference to the 'wq' and the normal SQPOLL exit will tear down the io-wq context appropriate. On the io-wq side, our wq context is per task, so only the task itself is manipulating ->manager and hence it's safe to check and clear without any extra locking. We just need to ensure that the manager task stays around, in case it exits. Signed-off-by: Jens Axboe --- fs/io-wq.c | 29 +++++++++++++++++++---------- fs/io-wq.h | 1 + fs/io_uring.c | 2 +- 3 files changed, 21 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index f0b7e9ff63fa..1407ba74ffc3 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -749,7 +749,7 @@ static int io_wq_manager(void *data) sprintf(buf, "iou-mgr-%d", wq->task_pid); set_task_comm(current, buf); current->flags |= PF_IO_WORKER; - wq->manager = current; + wq->manager = get_task_struct(current); complete(&wq->started); @@ -771,9 +771,7 @@ static int io_wq_manager(void *data) /* we might not ever have created any workers */ if (atomic_read(&wq->worker_refs)) wait_for_completion(&wq->worker_done); - wq->manager = NULL; complete(&wq->exited); - io_wq_put(wq); do_exit(0); } @@ -816,8 +814,6 @@ static int io_wq_fork_manager(struct io_wq *wq) return 0; reinit_completion(&wq->worker_done); - clear_bit(IO_WQ_BIT_EXIT, &wq->state); - refcount_inc(&wq->refs); current->flags |= PF_IO_WORKER; ret = io_wq_fork_thread(io_wq_manager, wq); current->flags &= ~PF_IO_WORKER; @@ -1089,6 +1085,16 @@ err_wq: return ERR_PTR(ret); } +static void io_wq_destroy_manager(struct io_wq *wq) +{ + if (wq->manager) { + wake_up_process(wq->manager); + wait_for_completion(&wq->exited); + put_task_struct(wq->manager); + wq->manager = NULL; + } +} + static void io_wq_destroy(struct io_wq *wq) { int node; @@ -1096,10 +1102,7 @@ static void io_wq_destroy(struct io_wq *wq) cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); set_bit(IO_WQ_BIT_EXIT, &wq->state); - if (wq->manager) { - wake_up_process(wq->manager); - wait_for_completion(&wq->exited); - } + io_wq_destroy_manager(wq); spin_lock_irq(&wq->hash->wait.lock); for_each_node(node) { @@ -1112,7 +1115,6 @@ static void io_wq_destroy(struct io_wq *wq) io_wq_put_hash(wq->hash); kfree(wq->wqes); kfree(wq); - } void io_wq_put(struct io_wq *wq) @@ -1121,6 +1123,13 @@ void io_wq_put(struct io_wq *wq) io_wq_destroy(wq); } +void io_wq_put_and_exit(struct io_wq *wq) +{ + set_bit(IO_WQ_BIT_EXIT, &wq->state); + io_wq_destroy_manager(wq); + io_wq_put(wq); +} + static bool io_wq_worker_affinity(struct io_worker *worker, void *data) { struct task_struct *task = worker->task; diff --git a/fs/io-wq.h b/fs/io-wq.h index b6ca12b60c35..f6ef433df8a8 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -114,6 +114,7 @@ struct io_wq_data { struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); void io_wq_put(struct io_wq *wq); +void io_wq_put_and_exit(struct io_wq *wq); void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); void io_wq_hash_work(struct io_wq_work *work, void *val); diff --git a/fs/io_uring.c b/fs/io_uring.c index 904bf0fecc36..cb65e54c1b09 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8857,7 +8857,7 @@ void __io_uring_files_cancel(struct files_struct *files) if (files) { io_uring_remove_task_files(tctx); if (tctx->io_wq) { - io_wq_put(tctx->io_wq); + io_wq_put_and_exit(tctx->io_wq); tctx->io_wq = NULL; } } -- cgit v1.2.3 From 1d5f360dd1a3c04e00a52af74dd84fdb0e1d454b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 26 Feb 2021 14:54:16 -0700 Subject: io_uring: fix race condition in task_work add and clear We clear the bit marking the ctx task_work as active after having run the queued work, but we really should be clearing it before. Otherwise we can hit a tiny race ala: CPU0 CPU1 io_task_work_add() tctx_task_work() run_work add_to_list test_and_set_bit clear_bit already set and CPU0 will return thinking the task_work is queued, while in reality it's already being run. If we hit the condition after __tctx_task_work() found no more work, but before we've cleared the bit, then we'll end up thinking it's queued and will be run. In reality it is queued, but we didn't queue the ctx task_work to ensure that it gets run. Fixes: 7cbf1722d5fc ("io_uring: provide FIFO ordering for task_work") Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index cb65e54c1b09..83973f6b3c0a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1893,10 +1893,10 @@ static void tctx_task_work(struct callback_head *cb) { struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work); + clear_bit(0, &tctx->task_state); + while (__tctx_task_work(tctx)) cond_resched(); - - clear_bit(0, &tctx->task_state); } static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req, -- cgit v1.2.3 From ef8eaa4e65facb1f51a64dbb4f5500134622c67c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 27 Feb 2021 11:16:45 +0000 Subject: io_uring: warn on not destroyed io-wq Make sure that we killed an io-wq by the time a task is dead. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 83973f6b3c0a..796b6d1f72f9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7843,6 +7843,8 @@ void __io_uring_free(struct task_struct *tsk) struct io_uring_task *tctx = tsk->io_uring; WARN_ON_ONCE(!xa_empty(&tctx->xa)); + WARN_ON_ONCE(tctx->io_wq); + percpu_counter_destroy(&tctx->inflight); kfree(tctx); tsk->io_uring = NULL; -- cgit v1.2.3 From 8452d4a674b0e59bd53baef0b30b018690dde594 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 27 Feb 2021 11:16:46 +0000 Subject: io_uring: destroy io-wq on exec Destroy current's io-wq backend and tctx on __io_uring_task_cancel(), aka exec(). Looks it's not strictly necessary, because it will be done at some point when the task dies and changes of creds/files/etc. are handled, but better to do that earlier to free io-wq and not potentially lock previous mm and other resources for the time being. It's safe to do because we wait for all requests of the current task to complete, so no request will use tctx afterwards. Note, that io_uring_files_cancel() may leave some requests for later reaping, so it leaves tctx intact, that's ok as the task is dying anyway. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 19 ++++++++++--------- include/linux/io_uring.h | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 796b6d1f72f9..73d1bd8db1bb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8835,13 +8835,17 @@ static void io_uring_del_task_file(struct file *file) fput(file); } -static void io_uring_remove_task_files(struct io_uring_task *tctx) +static void io_uring_clean_tctx(struct io_uring_task *tctx) { struct file *file; unsigned long index; xa_for_each(&tctx->xa, index, file) io_uring_del_task_file(file); + if (tctx->io_wq) { + io_wq_put_and_exit(tctx->io_wq); + tctx->io_wq = NULL; + } } void __io_uring_files_cancel(struct files_struct *files) @@ -8856,13 +8860,8 @@ void __io_uring_files_cancel(struct files_struct *files) io_uring_cancel_task_requests(file->private_data, files); atomic_dec(&tctx->in_idle); - if (files) { - io_uring_remove_task_files(tctx); - if (tctx->io_wq) { - io_wq_put_and_exit(tctx->io_wq); - tctx->io_wq = NULL; - } - } + if (files) + io_uring_clean_tctx(tctx); } static s64 tctx_inflight(struct io_uring_task *tctx) @@ -8954,7 +8953,9 @@ void __io_uring_task_cancel(void) atomic_dec(&tctx->in_idle); - io_uring_remove_task_files(tctx); + io_uring_clean_tctx(tctx); + /* all current's requests should be gone, we can kill tctx */ + __io_uring_free(current); } static int io_uring_flush(struct file *file, void *data) diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 51ede771cd99..7cb7bd0e334c 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -38,7 +38,7 @@ void __io_uring_free(struct task_struct *tsk); static inline void io_uring_task_cancel(void) { - if (current->io_uring && !xa_empty(¤t->io_uring->xa)) + if (current->io_uring) __io_uring_task_cancel(); } static inline void io_uring_files_cancel(struct files_struct *files) -- cgit v1.2.3 From 4010fec41fd9fc5ca6956b958d14b32e41aded48 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 27 Feb 2021 15:04:18 -0700 Subject: io_uring: remove unused argument 'tsk' from io_req_caches_free() We prune the full cache regardless, get rid of the dead argument. Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 73d1bd8db1bb..acbf3c7264b5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8395,7 +8395,7 @@ static void io_req_cache_free(struct list_head *list, struct task_struct *tsk) } } -static void io_req_caches_free(struct io_ring_ctx *ctx, struct task_struct *tsk) +static void io_req_caches_free(struct io_ring_ctx *ctx) { struct io_submit_state *submit_state = &ctx->submit_state; struct io_comp_state *cs = &ctx->submit_state.comp; @@ -8455,7 +8455,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) percpu_ref_exit(&ctx->refs); free_uid(ctx->user); - io_req_caches_free(ctx, NULL); + io_req_caches_free(ctx); if (ctx->hash_map) io_wq_put_hash(ctx->hash_map); kfree(ctx->cancel_hash); @@ -8969,7 +8969,7 @@ static int io_uring_flush(struct file *file, void *data) if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) { io_uring_cancel_task_requests(ctx, NULL); - io_req_caches_free(ctx, current); + io_req_caches_free(ctx); } io_run_ctx_fallback(ctx); -- cgit v1.2.3 From 1575f21a09206e914b81dace0add693346d97594 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 27 Feb 2021 15:20:49 -0700 Subject: io_uring: kill unnecessary REQ_F_WORK_INITIALIZED checks We're no longer checking anything that requires the work item to be initialized, as we're not carrying any file related state there. Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index acbf3c7264b5..1dd30a15ea6a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1080,8 +1080,6 @@ static bool io_match_task(struct io_kiocb *head, return true; io_for_each_link(req, head) { - if (!(req->flags & REQ_F_WORK_INITIALIZED)) - continue; if (req->file && req->file->f_op == &io_uring_fops) return true; if (req->task->files == files) @@ -1800,15 +1798,7 @@ static void io_fail_links(struct io_kiocb *req) trace_io_uring_fail_link(req, link); io_cqring_fill_event(link, -ECANCELED); - /* - * It's ok to free under spinlock as they're not linked anymore, - * but avoid REQ_F_WORK_INITIALIZED because it may deadlock on - * work.fs->lock. - */ - if (link->flags & REQ_F_WORK_INITIALIZED) - io_put_req_deferred(link, 2); - else - io_double_put_req(link); + io_put_req_deferred(link, 2); link = nxt; } io_commit_cqring(ctx); -- cgit v1.2.3 From 5730b27e84fdb37353c7cc2b11c24a4f9d73626e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 27 Feb 2021 15:57:30 -0700 Subject: io_uring: move cred assignment into io_issue_sqe() If we move it in there, then we no longer have to care about it in io-wq. This means we can drop the cred handling in io-wq, and we can drop the REQ_F_WORK_INITIALIZED flag and async init functions as that was the last user of it since we moved to the new workers. Then we can also drop io_wq_work->creds, and just hold the personality u16 in there instead. Suggested-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io-wq.c | 26 --------------------- fs/io-wq.h | 2 +- fs/io_uring.c | 75 +++++++++++++++++------------------------------------------ 3 files changed, 22 insertions(+), 81 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 1407ba74ffc3..946826beefe6 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -52,9 +52,6 @@ struct io_worker { struct io_wq_work *cur_work; spinlock_t lock; - const struct cred *cur_creds; - const struct cred *saved_creds; - struct completion ref_done; struct completion started; @@ -180,11 +177,6 @@ static void io_worker_exit(struct io_worker *worker) worker->flags = 0; preempt_enable(); - if (worker->saved_creds) { - revert_creds(worker->saved_creds); - worker->cur_creds = worker->saved_creds = NULL; - } - raw_spin_lock_irq(&wqe->lock); if (flags & IO_WORKER_F_FREE) hlist_nulls_del_rcu(&worker->nulls_node); @@ -326,10 +318,6 @@ static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) worker->flags |= IO_WORKER_F_FREE; hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); } - if (worker->saved_creds) { - revert_creds(worker->saved_creds); - worker->cur_creds = worker->saved_creds = NULL; - } } static inline unsigned int io_get_work_hash(struct io_wq_work *work) @@ -404,18 +392,6 @@ static void io_flush_signals(void) } } -static void io_wq_switch_creds(struct io_worker *worker, - struct io_wq_work *work) -{ - const struct cred *old_creds = override_creds(work->creds); - - worker->cur_creds = work->creds; - if (worker->saved_creds) - put_cred(old_creds); /* creds set by previous switch */ - else - worker->saved_creds = old_creds; -} - static void io_assign_current_work(struct io_worker *worker, struct io_wq_work *work) { @@ -465,8 +441,6 @@ get_next: unsigned int hash = io_get_work_hash(work); next_hashed = wq_next_work(work); - if (work->creds && worker->cur_creds != work->creds) - io_wq_switch_creds(worker, work); wq->do_work(work); io_assign_current_work(worker, NULL); diff --git a/fs/io-wq.h b/fs/io-wq.h index f6ef433df8a8..42f0be64a84d 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -79,8 +79,8 @@ static inline void wq_list_del(struct io_wq_work_list *list, struct io_wq_work { struct io_wq_work_node list; - const struct cred *creds; unsigned flags; + unsigned short personality; }; static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) diff --git a/fs/io_uring.c b/fs/io_uring.c index 1dd30a15ea6a..d48be0ccc590 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -688,7 +688,6 @@ enum { REQ_F_POLLED_BIT, REQ_F_BUFFER_SELECTED_BIT, REQ_F_NO_FILE_TABLE_BIT, - REQ_F_WORK_INITIALIZED_BIT, REQ_F_LTIMEOUT_ACTIVE_BIT, REQ_F_COMPLETE_INLINE_BIT, @@ -730,8 +729,6 @@ enum { REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT), /* doesn't need file table for this request */ REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT), - /* io_wq_work is initialized */ - REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT), /* linked timeout is active, i.e. prepared by link's head */ REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT), /* completion is deferred through io_comp_state */ @@ -1094,24 +1091,6 @@ static inline void req_set_fail_links(struct io_kiocb *req) req->flags |= REQ_F_FAIL_LINK; } -static inline void __io_req_init_async(struct io_kiocb *req) -{ - memset(&req->work, 0, sizeof(req->work)); - req->flags |= REQ_F_WORK_INITIALIZED; -} - -/* - * Note: must call io_req_init_async() for the first time you - * touch any members of io_wq_work. - */ -static inline void io_req_init_async(struct io_kiocb *req) -{ - if (req->flags & REQ_F_WORK_INITIALIZED) - return; - - __io_req_init_async(req); -} - static void io_ring_ctx_ref_free(struct percpu_ref *ref) { struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); @@ -1196,13 +1175,6 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) static void io_req_clean_work(struct io_kiocb *req) { - if (!(req->flags & REQ_F_WORK_INITIALIZED)) - return; - - if (req->work.creds) { - put_cred(req->work.creds); - req->work.creds = NULL; - } if (req->flags & REQ_F_INFLIGHT) { struct io_ring_ctx *ctx = req->ctx; struct io_uring_task *tctx = req->task->io_uring; @@ -1215,8 +1187,6 @@ static void io_req_clean_work(struct io_kiocb *req) if (atomic_read(&tctx->in_idle)) wake_up(&tctx->wait); } - - req->flags &= ~REQ_F_WORK_INITIALIZED; } static void io_req_track_inflight(struct io_kiocb *req) @@ -1224,7 +1194,6 @@ static void io_req_track_inflight(struct io_kiocb *req) struct io_ring_ctx *ctx = req->ctx; if (!(req->flags & REQ_F_INFLIGHT)) { - io_req_init_async(req); req->flags |= REQ_F_INFLIGHT; spin_lock_irq(&ctx->inflight_lock); @@ -1238,8 +1207,6 @@ static void io_prep_async_work(struct io_kiocb *req) const struct io_op_def *def = &io_op_defs[req->opcode]; struct io_ring_ctx *ctx = req->ctx; - io_req_init_async(req); - if (req->flags & REQ_F_FORCE_ASYNC) req->work.flags |= IO_WQ_WORK_CONCURRENT; @@ -1250,8 +1217,6 @@ static void io_prep_async_work(struct io_kiocb *req) if (def->unbound_nonreg_file) req->work.flags |= IO_WQ_WORK_UNBOUND; } - if (!req->work.creds) - req->work.creds = get_current_cred(); } static void io_prep_async_link(struct io_kiocb *req) @@ -3578,7 +3543,6 @@ static int __io_splice_prep(struct io_kiocb *req, * Splice operation will be punted aync, and here need to * modify io_wq_work.flags, so initialize io_wq_work firstly. */ - io_req_init_async(req); req->work.flags |= IO_WQ_WORK_UNBOUND; } @@ -5935,8 +5899,22 @@ static void __io_clean_op(struct io_kiocb *req) static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; + const struct cred *creds = NULL; int ret; + if (req->work.personality) { + const struct cred *new_creds; + + if (!(issue_flags & IO_URING_F_NONBLOCK)) + mutex_lock(&ctx->uring_lock); + new_creds = idr_find(&ctx->personality_idr, req->work.personality); + if (!(issue_flags & IO_URING_F_NONBLOCK)) + mutex_unlock(&ctx->uring_lock); + if (!new_creds) + return -EINVAL; + creds = override_creds(new_creds); + } + switch (req->opcode) { case IORING_OP_NOP: ret = io_nop(req, issue_flags); @@ -6043,6 +6021,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) break; } + if (creds) + revert_creds(creds); + if (ret) return ret; @@ -6206,18 +6187,10 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) static void __io_queue_sqe(struct io_kiocb *req) { struct io_kiocb *linked_timeout = io_prep_linked_timeout(req); - const struct cred *old_creds = NULL; int ret; - if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds && - req->work.creds != current_cred()) - old_creds = override_creds(req->work.creds); - ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); - if (old_creds) - revert_creds(old_creds); - /* * We async punt it if the file wasn't marked NOWAIT, or if the file * doesn't support non-blocking read/write attempts @@ -6304,7 +6277,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, { struct io_submit_state *state; unsigned int sqe_flags; - int id, ret = 0; + int ret = 0; req->opcode = READ_ONCE(sqe->opcode); /* same numerical values with corresponding REQ_F_*, safe to copy */ @@ -6336,15 +6309,9 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, !io_op_defs[req->opcode].buffer_select) return -EOPNOTSUPP; - id = READ_ONCE(sqe->personality); - if (id) { - __io_req_init_async(req); - req->work.creds = idr_find(&ctx->personality_idr, id); - if (unlikely(!req->work.creds)) - return -EINVAL; - get_cred(req->work.creds); - } - + req->work.list.next = NULL; + req->work.flags = 0; + req->work.personality = READ_ONCE(sqe->personality); state = &ctx->submit_state; /* -- cgit v1.2.3 From 914390bcfdd6351a4d308da7f43294476ea7d3bf Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 28 Feb 2021 12:22:37 -0700 Subject: io_uring: kill unnecessary io_run_ctx_fallback() in io_ring_exit_work() We already run the fallback task_work in io_uring_try_cancel_requests(), no need to duplicate at ring exit explicitly. Signed-off-by: Jens Axboe --- fs/io_uring.c | 1 - 1 file changed, 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index d48be0ccc590..7cad82b51eca 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8519,7 +8519,6 @@ static void io_ring_exit_work(struct work_struct *work) */ do { io_uring_try_cancel_requests(ctx, NULL, NULL); - io_run_ctx_fallback(ctx); } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)); io_ring_ctx_free(ctx); } -- cgit v1.2.3 From 0d30b3e7eea94cc818fadf2ac0dd189c616028f8 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 28 Feb 2021 12:23:27 -0700 Subject: io_uring: kill io_uring_flush() This was always a weird work-around or file referencing, and we don't need it anymore. Get rid of it. Signed-off-by: Jens Axboe --- fs/io_uring.c | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 7cad82b51eca..766df21769a8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8914,52 +8914,6 @@ void __io_uring_task_cancel(void) __io_uring_free(current); } -static int io_uring_flush(struct file *file, void *data) -{ - struct io_uring_task *tctx = current->io_uring; - struct io_ring_ctx *ctx = file->private_data; - - /* Ignore helper thread files exit */ - if (current->flags & PF_IO_WORKER) - return 0; - - if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) { - io_uring_cancel_task_requests(ctx, NULL); - io_req_caches_free(ctx); - } - - io_run_ctx_fallback(ctx); - - if (!tctx) - return 0; - - /* we should have cancelled and erased it before PF_EXITING */ - WARN_ON_ONCE((current->flags & PF_EXITING) && - xa_load(&tctx->xa, (unsigned long)file)); - - /* - * fput() is pending, will be 2 if the only other ref is our potential - * task file note. If the task is exiting, drop regardless of count. - */ - if (atomic_long_read(&file->f_count) != 2) - return 0; - - if (ctx->flags & IORING_SETUP_SQPOLL) { - /* there is only one file note, which is owned by sqo_task */ - WARN_ON_ONCE(ctx->sqo_task != current && - xa_load(&tctx->xa, (unsigned long)file)); - /* sqo_dead check is for when this happens after cancellation */ - WARN_ON_ONCE(ctx->sqo_task == current && !ctx->sqo_dead && - !xa_load(&tctx->xa, (unsigned long)file)); - - io_disable_sqo_submit(ctx); - } - - if (!(ctx->flags & IORING_SETUP_SQPOLL) || ctx->sqo_task == current) - io_uring_del_task_file(file); - return 0; -} - static void *io_uring_validate_mmap_request(struct file *file, loff_t pgoff, size_t sz) { @@ -9291,7 +9245,6 @@ static void io_uring_show_fdinfo(struct seq_file *m, struct file *f) static const struct file_operations io_uring_fops = { .release = io_uring_release, - .flush = io_uring_flush, .mmap = io_uring_mmap, #ifndef CONFIG_MMU .get_unmapped_area = io_uring_nommu_get_unmapped_area, -- cgit v1.2.3 From 2c32395d8111037ae2cb8cab883e80bcdbb70713 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 28 Feb 2021 22:04:53 +0000 Subject: io_uring: fix __tctx_task_work() ctx race There is an unlikely but possible race using a freed context. That's because req->task_work.func() can free a request, but we won't necessarily find a completion in submit_state.comp and so all ctx refs may be put by the time we do mutex_lock(&ctx->uring_ctx); There are several reasons why it can miss going through submit_state.comp: 1) req->task_work.func() didn't complete it itself, but punted to iowq (e.g. reissue) and it got freed later, or a similar situation with it overflowing and getting flushed by someone else, or being submitted to IRQ completion, 2) As we don't hold the uring_lock, someone else can do io_submit_flush_completions() and put our ref. 3) Bugs and code obscurities, e.g. failing to propagate issue_flags properly. One example is as follows CPU1 | CPU2 ======================================================================= @req->task_work.func() | -> @req overflwed, | so submit_state.comp,nr==0 | | flush overflows, and free @req | ctx refs == 0, free it ctx is dead, but we do | lock + flush + unlock | So take a ctx reference for each new ctx we see in __tctx_task_work(), and do release it until we do all our flushing. Fixes: 65453d1efbd2 ("io_uring: enable req cache for task_work items") Reported-by: syzbot+a157ac7c03a56397f553@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov [axboe: fold in my one-liner and fix ref mismatch] Signed-off-by: Jens Axboe --- fs/io_uring.c | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 766df21769a8..62a73543ab86 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1800,6 +1800,18 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) return __io_req_find_next(req); } +static void ctx_flush_and_put(struct io_ring_ctx *ctx) +{ + if (!ctx) + return; + if (ctx->submit_state.comp.nr) { + mutex_lock(&ctx->uring_lock); + io_submit_flush_completions(&ctx->submit_state.comp, ctx); + mutex_unlock(&ctx->uring_lock); + } + percpu_ref_put(&ctx->refs); +} + static bool __tctx_task_work(struct io_uring_task *tctx) { struct io_ring_ctx *ctx = NULL; @@ -1817,30 +1829,20 @@ static bool __tctx_task_work(struct io_uring_task *tctx) node = list.first; while (node) { struct io_wq_work_node *next = node->next; - struct io_ring_ctx *this_ctx; struct io_kiocb *req; req = container_of(node, struct io_kiocb, io_task_work.node); - this_ctx = req->ctx; - req->task_work.func(&req->task_work); - node = next; - - if (!ctx) { - ctx = this_ctx; - } else if (ctx != this_ctx) { - mutex_lock(&ctx->uring_lock); - io_submit_flush_completions(&ctx->submit_state.comp, ctx); - mutex_unlock(&ctx->uring_lock); - ctx = this_ctx; + if (req->ctx != ctx) { + ctx_flush_and_put(ctx); + ctx = req->ctx; + percpu_ref_get(&ctx->refs); } - } - if (ctx && ctx->submit_state.comp.nr) { - mutex_lock(&ctx->uring_lock); - io_submit_flush_completions(&ctx->submit_state.comp, ctx); - mutex_unlock(&ctx->uring_lock); + req->task_work.func(&req->task_work); + node = next; } + ctx_flush_and_put(ctx); return list.first != NULL; } -- cgit v1.2.3 From 28c4721b80a702462fb77373c23428ee698fa5dd Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 28 Feb 2021 22:04:54 +0000 Subject: io_uring: replace cmpxchg in fallback with xchg io_run_ctx_fallback() can use xchg() instead of cmpxchg(). It's simpler and faster. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 62a73543ab86..57c7833ba62b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8484,15 +8484,11 @@ static int io_remove_personalities(int id, void *p, void *data) static bool io_run_ctx_fallback(struct io_ring_ctx *ctx) { - struct callback_head *work, *head, *next; + struct callback_head *work, *next; bool executed = false; do { - do { - head = NULL; - work = READ_ONCE(ctx->exit_task_work); - } while (cmpxchg(&ctx->exit_task_work, work, head) != work); - + work = xchg(&ctx->exit_task_work, NULL); if (!work) break; -- cgit v1.2.3 From 3ebba796fa251d042be42b929a2d916ee5c34a49 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 28 Feb 2021 15:32:18 -0700 Subject: io_uring: ensure that SQPOLL thread is started for exit If we create it in a disabled state because IORING_SETUP_R_DISABLED is set on ring creation, we need to ensure that we've kicked the thread if we're exiting before it's been explicitly disabled. Otherwise we can run into a deadlock where exit is waiting go park the SQPOLL thread, but the SQPOLL thread itself is waiting to get a signal to start. That results in the below trace of both tasks hung, waiting on each other: INFO: task syz-executor458:8401 blocked for more than 143 seconds. Not tainted 5.11.0-next-20210226-syzkaller #0 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. task:syz-executor458 state:D stack:27536 pid: 8401 ppid: 8400 flags:0x00004004 Call Trace: context_switch kernel/sched/core.c:4324 [inline] __schedule+0x90c/0x21a0 kernel/sched/core.c:5075 schedule+0xcf/0x270 kernel/sched/core.c:5154 schedule_timeout+0x1db/0x250 kernel/time/timer.c:1868 do_wait_for_common kernel/sched/completion.c:85 [inline] __wait_for_common kernel/sched/completion.c:106 [inline] wait_for_common kernel/sched/completion.c:117 [inline] wait_for_completion+0x168/0x270 kernel/sched/completion.c:138 io_sq_thread_park fs/io_uring.c:7115 [inline] io_sq_thread_park+0xd5/0x130 fs/io_uring.c:7103 io_uring_cancel_task_requests+0x24c/0xd90 fs/io_uring.c:8745 __io_uring_files_cancel+0x110/0x230 fs/io_uring.c:8840 io_uring_files_cancel include/linux/io_uring.h:47 [inline] do_exit+0x299/0x2a60 kernel/exit.c:780 do_group_exit+0x125/0x310 kernel/exit.c:922 __do_sys_exit_group kernel/exit.c:933 [inline] __se_sys_exit_group kernel/exit.c:931 [inline] __x64_sys_exit_group+0x3a/0x50 kernel/exit.c:931 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x43e899 RSP: 002b:00007ffe89376d48 EFLAGS: 00000246 ORIG_RAX: 00000000000000e7 RAX: ffffffffffffffda RBX: 00000000004af2f0 RCX: 000000000043e899 RDX: 000000000000003c RSI: 00000000000000e7 RDI: 0000000000000000 RBP: 0000000000000000 R08: ffffffffffffffc0 R09: 0000000010000000 R10: 0000000000008011 R11: 0000000000000246 R12: 00000000004af2f0 R13: 0000000000000001 R14: 0000000000000000 R15: 0000000000000001 INFO: task iou-sqp-8401:8402 can't die for more than 143 seconds. task:iou-sqp-8401 state:D stack:30272 pid: 8402 ppid: 8400 flags:0x00004004 Call Trace: context_switch kernel/sched/core.c:4324 [inline] __schedule+0x90c/0x21a0 kernel/sched/core.c:5075 schedule+0xcf/0x270 kernel/sched/core.c:5154 schedule_timeout+0x1db/0x250 kernel/time/timer.c:1868 do_wait_for_common kernel/sched/completion.c:85 [inline] __wait_for_common kernel/sched/completion.c:106 [inline] wait_for_common kernel/sched/completion.c:117 [inline] wait_for_completion+0x168/0x270 kernel/sched/completion.c:138 io_sq_thread+0x27d/0x1ae0 fs/io_uring.c:6717 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:294 INFO: task iou-sqp-8401:8402 blocked for more than 143 seconds. Reported-by: syzbot+fb5458330b4442f2090d@syzkaller.appspotmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 57c7833ba62b..94b1a0f48fed 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7916,6 +7916,7 @@ static void io_sq_offload_start(struct io_ring_ctx *ctx) { struct io_sq_data *sqd = ctx->sq_data; + ctx->flags &= ~IORING_SETUP_R_DISABLED; if (ctx->flags & IORING_SETUP_SQPOLL) complete(&sqd->startup); } @@ -8692,6 +8693,8 @@ static void io_disable_sqo_submit(struct io_ring_ctx *ctx) { mutex_lock(&ctx->uring_lock); ctx->sqo_dead = 1; + if (ctx->flags & IORING_SETUP_R_DISABLED) + io_sq_offload_start(ctx); mutex_unlock(&ctx->uring_lock); /* make sure callers enter the ring to get error */ @@ -9637,10 +9640,7 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx) if (ctx->restrictions.registered) ctx->restricted = 1; - ctx->flags &= ~IORING_SETUP_R_DISABLED; - io_sq_offload_start(ctx); - return 0; } -- cgit v1.2.3 From 1c3b3e6527e57156bf4082f11c2151957560fe6a Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 28 Feb 2021 16:07:30 -0700 Subject: io_uring: ignore double poll add on the same waitqueue head syzbot reports a deadlock, attempting to lock the same spinlock twice: ============================================ WARNING: possible recursive locking detected 5.11.0-syzkaller #0 Not tainted -------------------------------------------- swapper/1/0 is trying to acquire lock: ffff88801b2b1130 (&runtime->sleep){..-.}-{2:2}, at: spin_lock include/linux/spinlock.h:354 [inline] ffff88801b2b1130 (&runtime->sleep){..-.}-{2:2}, at: io_poll_double_wake+0x25f/0x6a0 fs/io_uring.c:4960 but task is already holding lock: ffff88801b2b3130 (&runtime->sleep){..-.}-{2:2}, at: __wake_up_common_lock+0xb4/0x130 kernel/sched/wait.c:137 other info that might help us debug this: Possible unsafe locking scenario: CPU0 ---- lock(&runtime->sleep); lock(&runtime->sleep); *** DEADLOCK *** May be due to missing lock nesting notation 2 locks held by swapper/1/0: #0: ffff888147474908 (&group->lock){..-.}-{2:2}, at: _snd_pcm_stream_lock_irqsave+0x9f/0xd0 sound/core/pcm_native.c:170 #1: ffff88801b2b3130 (&runtime->sleep){..-.}-{2:2}, at: __wake_up_common_lock+0xb4/0x130 kernel/sched/wait.c:137 stack backtrace: CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.11.0-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:79 [inline] dump_stack+0xfa/0x151 lib/dump_stack.c:120 print_deadlock_bug kernel/locking/lockdep.c:2829 [inline] check_deadlock kernel/locking/lockdep.c:2872 [inline] validate_chain kernel/locking/lockdep.c:3661 [inline] __lock_acquire.cold+0x14c/0x3b4 kernel/locking/lockdep.c:4900 lock_acquire kernel/locking/lockdep.c:5510 [inline] lock_acquire+0x1ab/0x730 kernel/locking/lockdep.c:5475 __raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline] _raw_spin_lock+0x2a/0x40 kernel/locking/spinlock.c:151 spin_lock include/linux/spinlock.h:354 [inline] io_poll_double_wake+0x25f/0x6a0 fs/io_uring.c:4960 __wake_up_common+0x147/0x650 kernel/sched/wait.c:108 __wake_up_common_lock+0xd0/0x130 kernel/sched/wait.c:138 snd_pcm_update_state+0x46a/0x540 sound/core/pcm_lib.c:203 snd_pcm_update_hw_ptr0+0xa75/0x1a50 sound/core/pcm_lib.c:464 snd_pcm_period_elapsed+0x160/0x250 sound/core/pcm_lib.c:1805 dummy_hrtimer_callback+0x94/0x1b0 sound/drivers/dummy.c:378 __run_hrtimer kernel/time/hrtimer.c:1519 [inline] __hrtimer_run_queues+0x609/0xe40 kernel/time/hrtimer.c:1583 hrtimer_run_softirq+0x17b/0x360 kernel/time/hrtimer.c:1600 __do_softirq+0x29b/0x9f6 kernel/softirq.c:345 invoke_softirq kernel/softirq.c:221 [inline] __irq_exit_rcu kernel/softirq.c:422 [inline] irq_exit_rcu+0x134/0x200 kernel/softirq.c:434 sysvec_apic_timer_interrupt+0x93/0xc0 arch/x86/kernel/apic/apic.c:1100 asm_sysvec_apic_timer_interrupt+0x12/0x20 arch/x86/include/asm/idtentry.h:632 RIP: 0010:native_save_fl arch/x86/include/asm/irqflags.h:29 [inline] RIP: 0010:arch_local_save_flags arch/x86/include/asm/irqflags.h:70 [inline] RIP: 0010:arch_irqs_disabled arch/x86/include/asm/irqflags.h:137 [inline] RIP: 0010:acpi_safe_halt drivers/acpi/processor_idle.c:111 [inline] RIP: 0010:acpi_idle_do_entry+0x1c9/0x250 drivers/acpi/processor_idle.c:516 Code: dd 38 6e f8 84 db 75 ac e8 54 32 6e f8 e8 0f 1c 74 f8 e9 0c 00 00 00 e8 45 32 6e f8 0f 00 2d 4e 4a c5 00 e8 39 32 6e f8 fb f4 <9c> 5b 81 e3 00 02 00 00 fa 31 ff 48 89 de e8 14 3a 6e f8 48 85 db RSP: 0018:ffffc90000d47d18 EFLAGS: 00000293 RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000 RDX: ffff8880115c3780 RSI: ffffffff89052537 RDI: 0000000000000000 RBP: ffff888141127064 R08: 0000000000000001 R09: 0000000000000001 R10: ffffffff81794168 R11: 0000000000000000 R12: 0000000000000001 R13: ffff888141127000 R14: ffff888141127064 R15: ffff888143331804 acpi_idle_enter+0x361/0x500 drivers/acpi/processor_idle.c:647 cpuidle_enter_state+0x1b1/0xc80 drivers/cpuidle/cpuidle.c:237 cpuidle_enter+0x4a/0xa0 drivers/cpuidle/cpuidle.c:351 call_cpuidle kernel/sched/idle.c:158 [inline] cpuidle_idle_call kernel/sched/idle.c:239 [inline] do_idle+0x3e1/0x590 kernel/sched/idle.c:300 cpu_startup_entry+0x14/0x20 kernel/sched/idle.c:397 start_secondary+0x274/0x350 arch/x86/kernel/smpboot.c:272 secondary_startup_64_no_verify+0xb0/0xbb which is due to the driver doing poll_wait() twice on the same wait_queue_head. That is perfectly valid, but from checking the rest of the kernel tree, it's the only driver that does this. We can handle this just fine, we just need to ignore the second addition as we'll get woken just fine on the first one. Cc: stable@vger.kernel.org # 5.8+ Fixes: 18bceab101ad ("io_uring: allow POLL_ADD with double poll_wait() users") Reported-by: syzbot+28abd693db9e92c160d8@syzkaller.appspotmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 94b1a0f48fed..eb4bc8967178 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4959,6 +4959,9 @@ static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, pt->error = -EINVAL; return; } + /* double add on the same waitqueue head, ignore */ + if (poll->head == head) + return; poll = kmalloc(sizeof(*poll), GFP_ATOMIC); if (!poll) { pt->error = -ENOMEM; -- cgit v1.2.3 From 70aacfe66136809d7f080f89c492c278298719f4 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Mar 2021 13:02:15 +0000 Subject: io_uring: kill sqo_dead and sqo submission halting As SQPOLL task doesn't poke into ->sqo_task anymore, there is no need to kill the sqo when the master task exits. Before it was necessary to avoid races accessing sqo_task->files with removing them. Signed-off-by: Pavel Begunkov [axboe: don't forget to enable SQPOLL before exit, if started disabled] Signed-off-by: Jens Axboe --- fs/io_uring.c | 45 ++++++++------------------------------------- 1 file changed, 8 insertions(+), 37 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index eb4bc8967178..6090a380e903 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -338,7 +338,6 @@ struct io_ring_ctx { unsigned int drain_next: 1; unsigned int eventfd_async: 1; unsigned int restricted: 1; - unsigned int sqo_dead: 1; unsigned int sqo_exec: 1; /* @@ -1967,7 +1966,7 @@ static void __io_req_task_submit(struct io_kiocb *req) /* ctx stays valid until unlock, even if we drop all ours ctx->refs */ mutex_lock(&ctx->uring_lock); - if (!ctx->sqo_dead && !(current->flags & PF_EXITING) && !current->in_execve) + if (!(current->flags & PF_EXITING) && !current->in_execve) __io_queue_sqe(req); else __io_req_task_cancel(req, -EFAULT); @@ -6578,8 +6577,7 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) if (!list_empty(&ctx->iopoll_list)) io_do_iopoll(ctx, &nr_events, 0); - if (to_submit && !ctx->sqo_dead && - likely(!percpu_ref_is_dying(&ctx->refs))) + if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs))) ret = io_submit_sqes(ctx, to_submit); mutex_unlock(&ctx->uring_lock); } @@ -7818,7 +7816,7 @@ static int io_sq_thread_fork(struct io_sq_data *sqd, struct io_ring_ctx *ctx) clear_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); reinit_completion(&sqd->completion); - ctx->sqo_dead = ctx->sqo_exec = 0; + ctx->sqo_exec = 0; sqd->task_pid = current->pid; current->flags |= PF_IO_WORKER; ret = io_wq_fork_thread(io_sq_thread, sqd); @@ -8529,10 +8527,6 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) { mutex_lock(&ctx->uring_lock); percpu_ref_kill(&ctx->refs); - - if (WARN_ON_ONCE((ctx->flags & IORING_SETUP_SQPOLL) && !ctx->sqo_dead)) - ctx->sqo_dead = 1; - /* if force is set, the ring is going away. always drop after that */ ctx->cq_overflow_flushed = 1; if (ctx->rings) @@ -8692,19 +8686,6 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, } } -static void io_disable_sqo_submit(struct io_ring_ctx *ctx) -{ - mutex_lock(&ctx->uring_lock); - ctx->sqo_dead = 1; - if (ctx->flags & IORING_SETUP_R_DISABLED) - io_sq_offload_start(ctx); - mutex_unlock(&ctx->uring_lock); - - /* make sure callers enter the ring to get error */ - if (ctx->rings) - io_ring_set_wakeup_flag(ctx); -} - /* * We need to iteratively cancel requests, in case a request has dependent * hard links. These persist even for failure of cancelations, hence keep @@ -8717,7 +8698,11 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, bool did_park = false; if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { - io_disable_sqo_submit(ctx); + /* never started, nothing to cancel */ + if (ctx->flags & IORING_SETUP_R_DISABLED) { + io_sq_offload_start(ctx); + return; + } did_park = io_sq_thread_park(ctx->sq_data); if (did_park) { task = ctx->sq_data->thread; @@ -8838,7 +8823,6 @@ static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) if (!sqd) return; - io_disable_sqo_submit(ctx); if (!io_sq_thread_park(sqd)) return; tctx = ctx->sq_data->thread->io_uring; @@ -8883,7 +8867,6 @@ void __io_uring_task_cancel(void) /* make sure overflow events are dropped */ atomic_inc(&tctx->in_idle); - /* trigger io_disable_sqo_submit() */ if (tctx->sqpoll) { struct file *file; unsigned long index; @@ -8996,22 +8979,14 @@ static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) do { if (!io_sqring_full(ctx)) break; - prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE); - if (unlikely(ctx->sqo_dead)) { - ret = -EOWNERDEAD; - goto out; - } - if (!io_sqring_full(ctx)) break; - schedule(); } while (!signal_pending(current)); finish_wait(&ctx->sqo_sq_wait, &wait); -out: return ret; } @@ -9093,8 +9068,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, ctx->sqo_exec = 0; } ret = -EOWNERDEAD; - if (unlikely(ctx->sqo_dead)) - goto out; if (flags & IORING_ENTER_SQ_WAKEUP) wake_up(&ctx->sq_data->wait); if (flags & IORING_ENTER_SQ_WAIT) { @@ -9466,7 +9439,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, */ ret = io_uring_install_fd(ctx, file); if (ret < 0) { - io_disable_sqo_submit(ctx); /* fput will clean it up */ fput(file); return ret; @@ -9475,7 +9447,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags); return ret; err: - io_disable_sqo_submit(ctx); io_ring_ctx_wait_and_kill(ctx); return ret; } -- cgit v1.2.3 From 16270893d71219816513a255e6c3163bc7224ce4 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Mar 2021 13:02:16 +0000 Subject: io_uring: remove sqo_task Now, sqo_task is used only for a warning that is not interesting anymore since sqo_dead is gone, remove all of that including ctx->sqo_task. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6090a380e903..f060dcc1cc86 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -379,11 +379,6 @@ struct io_ring_ctx { struct io_rings *rings; - /* - * For SQPOLL usage - */ - struct task_struct *sqo_task; - /* Only used for accounting purposes */ struct mm_struct *mm_account; @@ -8747,10 +8742,6 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) fput(file); return ret; } - - /* one and only SQPOLL file note, held by sqo_task */ - WARN_ON_ONCE((ctx->flags & IORING_SETUP_SQPOLL) && - current != ctx->sqo_task); } tctx->last = file; } @@ -9376,7 +9367,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, ctx->compat = in_compat_syscall(); if (!capable(CAP_IPC_LOCK)) ctx->user = get_uid(current_user()); - ctx->sqo_task = current; /* * This is just grabbed for accounting purposes. When a process exits, -- cgit v1.2.3 From dc7bbc9ef361bea331bf5258a35abcdef619d44d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 1 Mar 2021 09:09:56 -0700 Subject: io-wq: fix error path leak of buffered write hash map The 'err' path should include the hash put, we already grabbed a reference once we get that far. Fixes: e941894eae31 ("io-wq: make buffered file write hashed work map per-ctx") Signed-off-by: Jens Axboe --- fs/io-wq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 946826beefe6..dc430381b694 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -1047,8 +1047,8 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) if (!ret) return wq; - io_wq_put_hash(data->hash); err: + io_wq_put_hash(data->hash); cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); for_each_node(node) kfree(wq->wqes[node]); -- cgit v1.2.3 From 3e6a0d3c7571ce3ed0d25c5c32543a54a7ebcd75 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 1 Mar 2021 13:56:00 -0700 Subject: io_uring: fix -EAGAIN retry with IOPOLL We no longer revert the iovec on -EIOCBQUEUED, see commit ab2125df921d, and this started causing issues for IOPOLL on devies that run out of request slots. Turns out what outside of needing a revert for those, we also had a bug where we didn't properly setup retry inside the submission path. That could cause re-import of the iovec, if any, and that could lead to spurious results if the application had those allocated on the stack. Catch -EAGAIN retry and make the iovec stable for IOPOLL, just like we do for !IOPOLL retries. Cc: # 5.9+ Reported-by: Abaci Robot Reported-by: Xiaoguang Wang Signed-off-by: Jens Axboe --- fs/io_uring.c | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index f060dcc1cc86..361befaae28f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2423,23 +2423,32 @@ static bool io_resubmit_prep(struct io_kiocb *req) return false; return !io_setup_async_rw(req, iovec, inline_vecs, &iter, false); } -#endif -static bool io_rw_reissue(struct io_kiocb *req) +static bool io_rw_should_reissue(struct io_kiocb *req) { -#ifdef CONFIG_BLOCK umode_t mode = file_inode(req->file)->i_mode; + struct io_ring_ctx *ctx = req->ctx; if (!S_ISBLK(mode) && !S_ISREG(mode)) return false; - if ((req->flags & REQ_F_NOWAIT) || io_wq_current_is_worker()) + if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && + !(ctx->flags & IORING_SETUP_IOPOLL))) return false; /* * If ref is dying, we might be running poll reap from the exit work. * Don't attempt to reissue from that path, just let it fail with * -EAGAIN. */ - if (percpu_ref_is_dying(&req->ctx->refs)) + if (percpu_ref_is_dying(&ctx->refs)) + return false; + return true; +} +#endif + +static bool io_rw_reissue(struct io_kiocb *req) +{ +#ifdef CONFIG_BLOCK + if (!io_rw_should_reissue(req)) return false; lockdep_assert_held(&req->ctx->uring_lock); @@ -2482,6 +2491,19 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); +#ifdef CONFIG_BLOCK + /* Rewind iter, if we have one. iopoll path resubmits as usual */ + if (res == -EAGAIN && io_rw_should_reissue(req)) { + struct io_async_rw *rw = req->async_data; + + if (rw) + iov_iter_revert(&rw->iter, + req->result - iov_iter_count(&rw->iter)); + else if (!io_resubmit_prep(req)) + res = -EIO; + } +#endif + if (kiocb->ki_flags & IOCB_WRITE) kiocb_end_write(req); @@ -3230,6 +3252,8 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) ret = io_iter_do_read(req, iter); if (ret == -EIOCBQUEUED) { + if (req->async_data) + iov_iter_revert(iter, io_size - iov_iter_count(iter)); goto out_free; } else if (ret == -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ @@ -3361,6 +3385,8 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) /* no retry on NONBLOCK nor RWF_NOWAIT */ if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) goto done; + if (ret2 == -EIOCBQUEUED && req->async_data) + iov_iter_revert(iter, io_size - iov_iter_count(iter)); if (!force_nonblock || ret2 != -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) -- cgit v1.2.3 From 64c7212391e778949aa3055fb3863439417ddba9 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Mar 2021 18:20:45 +0000 Subject: io_uring: choose right tctx->io_wq for try cancel When we cancel SQPOLL, @task in io_uring_try_cancel_requests() will differ from current. Use the right tctx from passed in @task, and don't forget that it can be NULL when the io_uring ctx exits. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 361befaae28f..d6c5465b372b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8636,7 +8636,8 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct files_struct *files) { struct io_task_cancel cancel = { .task = task, .files = files, }; - struct io_uring_task *tctx = current->io_uring; + struct task_struct *tctx_task = task ?: current; + struct io_uring_task *tctx = tctx_task->io_uring; while (1) { enum io_wq_cancel cret; -- cgit v1.2.3 From f85c310ac376ce81a954507315ff11be4ddbf214 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Mar 2021 18:20:46 +0000 Subject: io_uring: inline io_req_clean_work() Inline io_req_clean_work(), less code and easier to analyse tctx dependencies and refs usage. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index d6c5465b372b..afa0e91488e6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1167,22 +1167,6 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) return false; } -static void io_req_clean_work(struct io_kiocb *req) -{ - if (req->flags & REQ_F_INFLIGHT) { - struct io_ring_ctx *ctx = req->ctx; - struct io_uring_task *tctx = req->task->io_uring; - unsigned long flags; - - spin_lock_irqsave(&ctx->inflight_lock, flags); - list_del(&req->inflight_entry); - spin_unlock_irqrestore(&ctx->inflight_lock, flags); - req->flags &= ~REQ_F_INFLIGHT; - if (atomic_read(&tctx->in_idle)) - wake_up(&tctx->wait); - } -} - static void io_req_track_inflight(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; @@ -1671,7 +1655,19 @@ static void io_dismantle_req(struct io_kiocb *req) io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE)); if (req->fixed_rsrc_refs) percpu_ref_put(req->fixed_rsrc_refs); - io_req_clean_work(req); + + if (req->flags & REQ_F_INFLIGHT) { + struct io_ring_ctx *ctx = req->ctx; + struct io_uring_task *tctx = req->task->io_uring; + unsigned long flags; + + spin_lock_irqsave(&ctx->inflight_lock, flags); + list_del(&req->inflight_entry); + spin_unlock_irqrestore(&ctx->inflight_lock, flags); + req->flags &= ~REQ_F_INFLIGHT; + if (atomic_read(&tctx->in_idle)) + wake_up(&tctx->wait); + } } static inline void io_put_task(struct task_struct *task, int nr) -- cgit v1.2.3 From ebf936670721be805a9cb87781a5ee9271ba4633 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Mar 2021 18:20:47 +0000 Subject: io_uring: inline __io_queue_async_work() __io_queue_async_work() is only called from io_queue_async_work(), inline it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index afa0e91488e6..840b73db9c3d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1205,7 +1205,7 @@ static void io_prep_async_link(struct io_kiocb *req) io_prep_async_work(cur); } -static struct io_kiocb *__io_queue_async_work(struct io_kiocb *req) +static void io_queue_async_work(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; struct io_kiocb *link = io_prep_linked_timeout(req); @@ -1216,18 +1216,9 @@ static struct io_kiocb *__io_queue_async_work(struct io_kiocb *req) trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req, &req->work, req->flags); - io_wq_enqueue(tctx->io_wq, &req->work); - return link; -} - -static void io_queue_async_work(struct io_kiocb *req) -{ - struct io_kiocb *link; - /* init ->work of the whole link before punting */ io_prep_async_link(req); - link = __io_queue_async_work(req); - + io_wq_enqueue(tctx->io_wq, &req->work); if (link) io_queue_linked_timeout(link); } -- cgit v1.2.3 From b23fcf477f85164f3b33b2e8c2c99b2ec61ba902 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 1 Mar 2021 18:20:48 +0000 Subject: io_uring: remove extra in_idle wake up io_dismantle_req() is always followed by io_put_task(), which already do proper in_idle wake ups, so we can skip waking the owner task in io_dismantle_req(). The rules are simpler now, do io_put_task() shortly after ending a request, and it will be fine. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 840b73db9c3d..26e83cabf3bf 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1649,18 +1649,16 @@ static void io_dismantle_req(struct io_kiocb *req) if (req->flags & REQ_F_INFLIGHT) { struct io_ring_ctx *ctx = req->ctx; - struct io_uring_task *tctx = req->task->io_uring; unsigned long flags; spin_lock_irqsave(&ctx->inflight_lock, flags); list_del(&req->inflight_entry); spin_unlock_irqrestore(&ctx->inflight_lock, flags); req->flags &= ~REQ_F_INFLIGHT; - if (atomic_read(&tctx->in_idle)) - wake_up(&tctx->wait); } } +/* must to be called somewhat shortly after putting a request */ static inline void io_put_task(struct task_struct *task, int nr) { struct io_uring_task *tctx = task->io_uring; -- cgit v1.2.3 From e4b4a13f494120c475580927864cc1dd96f595d1 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 1 Mar 2021 18:36:25 -0700 Subject: io_uring: ensure that threads freeze on suspend Alex reports that his system fails to suspend using 5.12-rc1, with the following dump: [ 240.650300] PM: suspend entry (deep) [ 240.650748] Filesystems sync: 0.000 seconds [ 240.725605] Freezing user space processes ... [ 260.739483] Freezing of tasks failed after 20.013 seconds (3 tasks refusing to freeze, wq_busy=0): [ 260.739497] task:iou-mgr-446 state:S stack: 0 pid: 516 ppid: 439 flags:0x00004224 [ 260.739504] Call Trace: [ 260.739507] ? sysvec_apic_timer_interrupt+0xb/0x81 [ 260.739515] ? pick_next_task_fair+0x197/0x1cde [ 260.739519] ? sysvec_reschedule_ipi+0x2f/0x6a [ 260.739522] ? asm_sysvec_reschedule_ipi+0x12/0x20 [ 260.739525] ? __schedule+0x57/0x6d6 [ 260.739529] ? del_timer_sync+0xb9/0x115 [ 260.739533] ? schedule+0x63/0xd5 [ 260.739536] ? schedule_timeout+0x219/0x356 [ 260.739540] ? __next_timer_interrupt+0xf1/0xf1 [ 260.739544] ? io_wq_manager+0x73/0xb1 [ 260.739549] ? io_wq_create+0x262/0x262 [ 260.739553] ? ret_from_fork+0x22/0x30 [ 260.739557] task:iou-mgr-517 state:S stack: 0 pid: 522 ppid: 439 flags:0x00004224 [ 260.739561] Call Trace: [ 260.739563] ? sysvec_apic_timer_interrupt+0xb/0x81 [ 260.739566] ? pick_next_task_fair+0x16f/0x1cde [ 260.739569] ? sysvec_apic_timer_interrupt+0xb/0x81 [ 260.739571] ? asm_sysvec_apic_timer_interrupt+0x12/0x20 [ 260.739574] ? __schedule+0x5b7/0x6d6 [ 260.739578] ? del_timer_sync+0x70/0x115 [ 260.739581] ? schedule_timeout+0x211/0x356 [ 260.739585] ? __next_timer_interrupt+0xf1/0xf1 [ 260.739588] ? io_wq_check_workers+0x15/0x11f [ 260.739592] ? io_wq_manager+0x69/0xb1 [ 260.739596] ? io_wq_create+0x262/0x262 [ 260.739600] ? ret_from_fork+0x22/0x30 [ 260.739603] task:iou-wrk-517 state:S stack: 0 pid: 523 ppid: 439 flags:0x00004224 [ 260.739607] Call Trace: [ 260.739609] ? __schedule+0x5b7/0x6d6 [ 260.739614] ? schedule+0x63/0xd5 [ 260.739617] ? schedule_timeout+0x219/0x356 [ 260.739621] ? __next_timer_interrupt+0xf1/0xf1 [ 260.739624] ? task_thread.isra.0+0x148/0x3af [ 260.739628] ? task_thread_unbound+0xa/0xa [ 260.739632] ? task_thread_bound+0x7/0x7 [ 260.739636] ? ret_from_fork+0x22/0x30 [ 260.739647] OOM killer enabled. [ 260.739648] Restarting tasks ... done. [ 260.740077] PM: suspend exit Play nice and ensure that any thread we create will call try_to_freeze() at an opportune time so that memory suspend can proceed. For the io-wq worker threads, mark them as PF_NOFREEZE. They could potentially be blocked for a long time. Reported-by: Alex Xu (Hello71) Tested-by: Alex Xu (Hello71) Signed-off-by: Jens Axboe --- fs/io-wq.c | 3 +++ fs/io_uring.c | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index dc430381b694..acffc85d1a93 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "../kernel/sched/sched.h" #include "io-wq.h" @@ -263,6 +264,7 @@ static void io_wqe_dec_running(struct io_worker *worker) static void io_worker_start(struct io_worker *worker) { + current->flags |= PF_NOFREEZE; worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); io_wqe_inc_running(worker); complete(&worker->started); @@ -731,6 +733,7 @@ static int io_wq_manager(void *data) set_current_state(TASK_INTERRUPTIBLE); io_wq_check_workers(wq); schedule_timeout(HZ); + try_to_freeze(); if (fatal_signal_pending(current)) set_bit(IO_WQ_BIT_EXIT, &wq->state); } while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)); diff --git a/fs/io_uring.c b/fs/io_uring.c index 26e83cabf3bf..b1734efdc7e8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -74,13 +74,11 @@ #include #include #include -#include #include #include #include #include -#include -#include +#include #define CREATE_TRACE_POINTS #include @@ -6736,6 +6734,7 @@ static int io_sq_thread(void *data) io_ring_set_wakeup_flag(ctx); schedule(); + try_to_freeze(); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_clear_wakeup_flag(ctx); } -- cgit v1.2.3 From f01272541d2cd7b7f24909d63ea2b028a6a66293 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 3 Mar 2021 15:47:04 -0700 Subject: io-wq: ensure all pending work is canceled on exit If we race on shutting down the io-wq, then we should ensure that any work that was queued after workers shutdown is canceled. Harden the add work check a bit too, checking for IO_WQ_BIT_EXIT and cancel if it's set. Add a WARN_ON() for having any work before we kill the io-wq context. Reported-by: syzbot+91b4b56ead187d35c9d3@syzkaller.appspotmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index acffc85d1a93..19f18389ead2 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -129,6 +129,17 @@ struct io_wq { static enum cpuhp_state io_wq_online; +struct io_cb_cancel_data { + work_cancel_fn *fn; + void *data; + int nr_running; + int nr_pending; + bool cancel_all; +}; + +static void io_wqe_cancel_pending_work(struct io_wqe *wqe, + struct io_cb_cancel_data *match); + static bool io_worker_get(struct io_worker *worker) { return refcount_inc_not_zero(&worker->ref); @@ -713,6 +724,23 @@ static void io_wq_check_workers(struct io_wq *wq) } } +static bool io_wq_work_match_all(struct io_wq_work *work, void *data) +{ + return true; +} + +static void io_wq_cancel_pending(struct io_wq *wq) +{ + struct io_cb_cancel_data match = { + .fn = io_wq_work_match_all, + .cancel_all = true, + }; + int node; + + for_each_node(node) + io_wqe_cancel_pending_work(wq->wqes[node], &match); +} + /* * Manager thread. Tasked with creating new workers, if we need them. */ @@ -748,6 +776,8 @@ static int io_wq_manager(void *data) /* we might not ever have created any workers */ if (atomic_read(&wq->worker_refs)) wait_for_completion(&wq->worker_done); + + io_wq_cancel_pending(wq); complete(&wq->exited); do_exit(0); } @@ -809,7 +839,8 @@ static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) unsigned long flags; /* Can only happen if manager creation fails after exec */ - if (unlikely(io_wq_fork_manager(wqe->wq))) { + if (io_wq_fork_manager(wqe->wq) || + test_bit(IO_WQ_BIT_EXIT, &wqe->wq->state)) { work->flags |= IO_WQ_WORK_CANCEL; wqe->wq->do_work(work); return; @@ -845,14 +876,6 @@ void io_wq_hash_work(struct io_wq_work *work, void *val) work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT)); } -struct io_cb_cancel_data { - work_cancel_fn *fn; - void *data; - int nr_running; - int nr_pending; - bool cancel_all; -}; - static bool io_wq_worker_cancel(struct io_worker *worker, void *data) { struct io_cb_cancel_data *match = data; @@ -1086,6 +1109,7 @@ static void io_wq_destroy(struct io_wq *wq) struct io_wqe *wqe = wq->wqes[node]; list_del_init(&wqe->wait.entry); + WARN_ON_ONCE(!wq_list_empty(&wqe->work_list)); kfree(wqe); } spin_unlock_irq(&wq->hash->wait.lock); -- cgit v1.2.3 From d734492a14a2da6e7bcce8cf66436a9cf4e51ddf Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Wed, 3 Mar 2021 17:55:46 +0900 Subject: btrfs: zoned: use sector_t for zone sectors We need to use sector_t for zone_sectors, or it would set the zone size to zero when the size >= 4GB (= 2^24 sectors) by shifting the zone_sectors value by SECTOR_SHIFT. We're assuming zones sizes up to 8GiB. Fixes: 5b316468983d ("btrfs: get zone information of zoned block devices") Reviewed-by: Johannes Thumshirn Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/zoned.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 9a5cf153da89..43948bd40e02 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -269,7 +269,7 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device) sector_t sector = 0; struct blk_zone *zones = NULL; unsigned int i, nreported = 0, nr_zones; - unsigned int zone_sectors; + sector_t zone_sectors; char *model, *emulated; int ret; @@ -658,7 +658,7 @@ int btrfs_sb_log_location_bdev(struct block_device *bdev, int mirror, int rw, u64 *bytenr_ret) { struct blk_zone zones[BTRFS_NR_SB_LOG_ZONES]; - unsigned int zone_sectors; + sector_t zone_sectors; u32 sb_zone; int ret; u8 zone_sectors_shift; -- cgit v1.2.3 From badae9c86979c459bd7d895d6d7ddc7a01131ff7 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Wed, 3 Mar 2021 17:55:48 +0900 Subject: btrfs: zoned: do not account freed region of read-only block group as zone_unusable We migrate zone unusable bytes to read-only bytes when a block group is set to read-only, and account all the free region as bytes_readonly. Thus, we should not increase block_group->zone_unusable when the block group is read-only. Fixes: 169e0da91a21 ("btrfs: zoned: track unusable bytes for zones") Reviewed-by: Johannes Thumshirn Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/free-space-cache.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 711a6a751ae9..9988decd5717 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -2555,7 +2555,12 @@ static int __btrfs_add_free_space_zoned(struct btrfs_block_group *block_group, to_unusable = size - to_free; ctl->free_space += to_free; - block_group->zone_unusable += to_unusable; + /* + * If the block group is read-only, we should account freed space into + * bytes_readonly. + */ + if (!block_group->ro) + block_group->zone_unusable += to_unusable; spin_unlock(&ctl->tree_lock); if (!used) { spin_lock(&block_group->lock); -- cgit v1.2.3 From b05a1bcd40184f12f2cd87db79e871aa8c17563f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Mar 2021 13:59:24 +0000 Subject: io_uring: cancel-match based on flags Instead of going into request internals, like checking req->file->f_op, do match them based on REQ_F_INFLIGHT, it's set only when we want it to be reliably cancelled. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index b1734efdc7e8..fb4abea1e5d6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -703,7 +703,7 @@ enum { /* fail rest of links */ REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT), - /* on inflight list */ + /* on inflight list, should be cancelled and waited on exit reliably */ REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT), /* read/write uses file position */ REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT), @@ -1069,7 +1069,7 @@ static bool io_match_task(struct io_kiocb *head, return true; io_for_each_link(req, head) { - if (req->file && req->file->f_op == &io_uring_fops) + if (req->flags & REQ_F_INFLIGHT) return true; if (req->task->files == files) return true; -- cgit v1.2.3 From dd59a3d595cc10230ded4c8b727b096e16bceeb5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 4 Mar 2021 13:59:25 +0000 Subject: io_uring: reliably cancel linked timeouts Linked timeouts are fired asynchronously (i.e. soft-irq), and use generic cancellation paths to do its stuff, including poking into io-wq. The problem is that it's racy to access tctx->io_wq, as io_uring_task_cancel() and others may be happening at this exact moment. Mark linked timeouts with REQ_F_INLIFGHT for now, making sure there are no timeouts before io-wq destraction. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index fb4abea1e5d6..e55369555e5c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5500,6 +5500,7 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, data->mode = io_translate_timeout_mode(flags); hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode); + io_req_track_inflight(req); return 0; } -- cgit v1.2.3 From 3f9b9efd82a84f27e95d0414f852caf1fa839e83 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Tue, 9 Feb 2021 17:47:54 -0500 Subject: virtiofs: Fail dax mount if device does not support it Right now "mount -t virtiofs -o dax myfs /mnt/virtiofs" succeeds even if filesystem deivce does not have a cache window and hence DAX can't be supported. This gives a false sense to user that they are using DAX with virtiofs but fact of the matter is that they are not. Fix this by returning error if dax can't be supported and user has asked for it. Signed-off-by: Vivek Goyal Reviewed-by: Stefan Hajnoczi Signed-off-by: Miklos Szeredi --- fs/fuse/virtio_fs.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index 8868ac31a3c0..4ee6f734ba83 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -1324,8 +1324,15 @@ static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc) /* virtiofs allocates and installs its own fuse devices */ ctx->fudptr = NULL; - if (ctx->dax) + if (ctx->dax) { + if (!fs->dax_dev) { + err = -EINVAL; + pr_err("virtio-fs: dax can't be enabled as filesystem" + " device does not support it.\n"); + goto err_free_fuse_devs; + } ctx->dax_dev = fs->dax_dev; + } err = fuse_fill_super_common(sb, ctx); if (err < 0) goto err_free_fuse_devs; -- cgit v1.2.3 From 46fe18b16c4656969347fc0a3d83a034e47d9119 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 4 Mar 2021 12:39:36 -0700 Subject: io_uring: move to using create_io_thread() This allows us to do task creation and setup without needing to use completions to try and synchronize with the starting thread. Get rid of the old io_wq_fork_thread() wrapper, and the 'wq' and 'worker' startup completion events - we can now do setup before the task is running. Signed-off-by: Jens Axboe --- fs/io-wq.c | 123 +++++++++++++++++----------------------------------------- fs/io-wq.h | 2 - fs/io_uring.c | 38 +++++++++--------- 3 files changed, 54 insertions(+), 109 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 19f18389ead2..d7cfe8fd282a 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -54,7 +54,6 @@ struct io_worker { spinlock_t lock; struct completion ref_done; - struct completion started; struct rcu_head rcu; }; @@ -116,7 +115,6 @@ struct io_wq { struct io_wq_hash *hash; refcount_t refs; - struct completion started; struct completion exited; atomic_t worker_refs; @@ -199,6 +197,7 @@ static void io_worker_exit(struct io_worker *worker) kfree_rcu(worker, rcu); if (atomic_dec_and_test(&wqe->wq->worker_refs)) complete(&wqe->wq->worker_done); + do_exit(0); } static inline bool io_wqe_run_queue(struct io_wqe *wqe) @@ -273,14 +272,6 @@ static void io_wqe_dec_running(struct io_worker *worker) io_wqe_wake_worker(wqe, acct); } -static void io_worker_start(struct io_worker *worker) -{ - current->flags |= PF_NOFREEZE; - worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); - io_wqe_inc_running(worker); - complete(&worker->started); -} - /* * Worker will start processing some work. Move it to the busy list, if * it's currently on the freelist @@ -489,8 +480,13 @@ static int io_wqe_worker(void *data) struct io_worker *worker = data; struct io_wqe *wqe = worker->wqe; struct io_wq *wq = wqe->wq; + char buf[TASK_COMM_LEN]; - io_worker_start(worker); + worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); + io_wqe_inc_running(worker); + + sprintf(buf, "iou-wrk-%d", wq->task_pid); + set_task_comm(current, buf); while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { set_current_state(TASK_INTERRUPTIBLE); @@ -565,67 +561,11 @@ void io_wq_worker_sleeping(struct task_struct *tsk) raw_spin_unlock_irq(&worker->wqe->lock); } -static int task_thread(void *data, int index) -{ - struct io_worker *worker = data; - struct io_wqe *wqe = worker->wqe; - struct io_wqe_acct *acct = &wqe->acct[index]; - struct io_wq *wq = wqe->wq; - char buf[TASK_COMM_LEN]; - - sprintf(buf, "iou-wrk-%d", wq->task_pid); - set_task_comm(current, buf); - - current->pf_io_worker = worker; - worker->task = current; - - set_cpus_allowed_ptr(current, cpumask_of_node(wqe->node)); - current->flags |= PF_NO_SETAFFINITY; - - raw_spin_lock_irq(&wqe->lock); - hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); - list_add_tail_rcu(&worker->all_list, &wqe->all_list); - worker->flags |= IO_WORKER_F_FREE; - if (index == IO_WQ_ACCT_BOUND) - worker->flags |= IO_WORKER_F_BOUND; - if (!acct->nr_workers && (worker->flags & IO_WORKER_F_BOUND)) - worker->flags |= IO_WORKER_F_FIXED; - acct->nr_workers++; - raw_spin_unlock_irq(&wqe->lock); - - io_wqe_worker(data); - do_exit(0); -} - -static int task_thread_bound(void *data) -{ - return task_thread(data, IO_WQ_ACCT_BOUND); -} - -static int task_thread_unbound(void *data) -{ - return task_thread(data, IO_WQ_ACCT_UNBOUND); -} - -pid_t io_wq_fork_thread(int (*fn)(void *), void *arg) -{ - unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD| - CLONE_IO|SIGCHLD; - struct kernel_clone_args args = { - .flags = ((lower_32_bits(flags) | CLONE_VM | - CLONE_UNTRACED) & ~CSIGNAL), - .exit_signal = (lower_32_bits(flags) & CSIGNAL), - .stack = (unsigned long)fn, - .stack_size = (unsigned long)arg, - }; - - return kernel_clone(&args); -} - static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) { + struct io_wqe_acct *acct = &wqe->acct[index]; struct io_worker *worker; - pid_t pid; + struct task_struct *tsk; __set_current_state(TASK_RUNNING); @@ -638,21 +578,33 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) worker->wqe = wqe; spin_lock_init(&worker->lock); init_completion(&worker->ref_done); - init_completion(&worker->started); atomic_inc(&wq->worker_refs); - if (index == IO_WQ_ACCT_BOUND) - pid = io_wq_fork_thread(task_thread_bound, worker); - else - pid = io_wq_fork_thread(task_thread_unbound, worker); - if (pid < 0) { + tsk = create_io_thread(io_wqe_worker, worker, wqe->node); + if (IS_ERR(tsk)) { if (atomic_dec_and_test(&wq->worker_refs)) complete(&wq->worker_done); kfree(worker); return false; } - wait_for_completion(&worker->started); + + tsk->pf_io_worker = worker; + worker->task = tsk; + set_cpus_allowed_ptr(tsk, cpumask_of_node(wqe->node)); + tsk->flags |= PF_NOFREEZE | PF_NO_SETAFFINITY; + + raw_spin_lock_irq(&wqe->lock); + hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); + list_add_tail_rcu(&worker->all_list, &wqe->all_list); + worker->flags |= IO_WORKER_F_FREE; + if (index == IO_WQ_ACCT_BOUND) + worker->flags |= IO_WORKER_F_BOUND; + if (!acct->nr_workers && (worker->flags & IO_WORKER_F_BOUND)) + worker->flags |= IO_WORKER_F_FIXED; + acct->nr_workers++; + raw_spin_unlock_irq(&wqe->lock); + wake_up_new_task(tsk); return true; } @@ -696,6 +648,7 @@ static bool io_wq_for_each_worker(struct io_wqe *wqe, static bool io_wq_worker_wake(struct io_worker *worker, void *data) { + set_notify_signal(worker->task); wake_up_process(worker->task); return false; } @@ -752,10 +705,6 @@ static int io_wq_manager(void *data) sprintf(buf, "iou-mgr-%d", wq->task_pid); set_task_comm(current, buf); - current->flags |= PF_IO_WORKER; - wq->manager = get_task_struct(current); - - complete(&wq->started); do { set_current_state(TASK_INTERRUPTIBLE); @@ -815,21 +764,20 @@ append: static int io_wq_fork_manager(struct io_wq *wq) { - int ret; + struct task_struct *tsk; if (wq->manager) return 0; reinit_completion(&wq->worker_done); - current->flags |= PF_IO_WORKER; - ret = io_wq_fork_thread(io_wq_manager, wq); - current->flags &= ~PF_IO_WORKER; - if (ret >= 0) { - wait_for_completion(&wq->started); + tsk = create_io_thread(io_wq_manager, wq, NUMA_NO_NODE); + if (!IS_ERR(tsk)) { + wq->manager = get_task_struct(tsk); + wake_up_new_task(tsk); return 0; } - return ret; + return PTR_ERR(tsk); } static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) @@ -1062,7 +1010,6 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) } wq->task_pid = current->pid; - init_completion(&wq->started); init_completion(&wq->exited); refcount_set(&wq->refs, 1); diff --git a/fs/io-wq.h b/fs/io-wq.h index 42f0be64a84d..5fbf7997149e 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -119,8 +119,6 @@ void io_wq_put_and_exit(struct io_wq *wq); void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); void io_wq_hash_work(struct io_wq_work *work, void *val); -pid_t io_wq_fork_thread(int (*fn)(void *), void *arg); - static inline bool io_wq_is_hashed(struct io_wq_work *work) { return work->flags & IO_WQ_WORK_HASHED; diff --git a/fs/io_uring.c b/fs/io_uring.c index e55369555e5c..76e243c056b5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6668,7 +6668,6 @@ static int io_sq_thread(void *data) sprintf(buf, "iou-sqp-%d", sqd->task_pid); set_task_comm(current, buf); - sqd->thread = current; current->pf_io_worker = NULL; if (sqd->sq_cpu != -1) @@ -6677,8 +6676,6 @@ static int io_sq_thread(void *data) set_cpus_allowed_ptr(current, cpu_online_mask); current->flags |= PF_NO_SETAFFINITY; - complete(&sqd->completion); - wait_for_completion(&sqd->startup); while (!io_sq_thread_should_stop(sqd)) { @@ -7818,21 +7815,22 @@ void __io_uring_free(struct task_struct *tsk) static int io_sq_thread_fork(struct io_sq_data *sqd, struct io_ring_ctx *ctx) { + struct task_struct *tsk; int ret; clear_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); reinit_completion(&sqd->completion); ctx->sqo_exec = 0; sqd->task_pid = current->pid; - current->flags |= PF_IO_WORKER; - ret = io_wq_fork_thread(io_sq_thread, sqd); - current->flags &= ~PF_IO_WORKER; - if (ret < 0) { - sqd->thread = NULL; - return ret; - } - wait_for_completion(&sqd->completion); - return io_uring_alloc_task_context(sqd->thread, ctx); + tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE); + if (IS_ERR(tsk)) + return PTR_ERR(tsk); + ret = io_uring_alloc_task_context(tsk, ctx); + if (ret) + set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); + sqd->thread = tsk; + wake_up_new_task(tsk); + return ret; } static int io_sq_offload_create(struct io_ring_ctx *ctx, @@ -7855,6 +7853,7 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, fdput(f); } if (ctx->flags & IORING_SETUP_SQPOLL) { + struct task_struct *tsk; struct io_sq_data *sqd; ret = -EPERM; @@ -7896,15 +7895,16 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, } sqd->task_pid = current->pid; - current->flags |= PF_IO_WORKER; - ret = io_wq_fork_thread(io_sq_thread, sqd); - current->flags &= ~PF_IO_WORKER; - if (ret < 0) { - sqd->thread = NULL; + tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE); + if (IS_ERR(tsk)) { + ret = PTR_ERR(tsk); goto err; } - wait_for_completion(&sqd->completion); - ret = io_uring_alloc_task_context(sqd->thread, ctx); + ret = io_uring_alloc_task_context(tsk, ctx); + if (ret) + set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); + sqd->thread = tsk; + wake_up_new_task(tsk); if (ret) goto err; } else if (p->flags & IORING_SETUP_SQ_AFF) { -- cgit v1.2.3 From ca0a26511c679a797f86589894a4523db36d833e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 4 Mar 2021 17:15:48 -0700 Subject: io_uring: don't keep looping for more events if we can't flush overflow It doesn't make sense to wait for more events to come in, if we can't even flush the overflow we already have to the ring. Return -EBUSY for that condition, just like we do for attempts to submit with overflow pending. Cc: stable@vger.kernel.org # 5.11 Signed-off-by: Jens Axboe --- fs/io_uring.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 76e243c056b5..044170165402 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1451,18 +1451,22 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, return all_flushed; } -static void io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, +static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, struct task_struct *tsk, struct files_struct *files) { + bool ret = true; + if (test_bit(0, &ctx->cq_check_overflow)) { /* iopoll syncs against uring_lock, not completion_lock */ if (ctx->flags & IORING_SETUP_IOPOLL) mutex_lock(&ctx->uring_lock); - __io_cqring_overflow_flush(ctx, force, tsk, files); + ret = __io_cqring_overflow_flush(ctx, force, tsk, files); if (ctx->flags & IORING_SETUP_IOPOLL) mutex_unlock(&ctx->uring_lock); } + + return ret; } static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags) @@ -6883,11 +6887,16 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); trace_io_uring_cqring_wait(ctx, min_events); do { - io_cqring_overflow_flush(ctx, false, NULL, NULL); + /* if we can't even flush overflow, don't wait for more */ + if (!io_cqring_overflow_flush(ctx, false, NULL, NULL)) { + ret = -EBUSY; + break; + } prepare_to_wait_exclusive(&ctx->wait, &iowq.wq, TASK_INTERRUPTIBLE); ret = io_cqring_wait_schedule(ctx, &iowq, &timeout); finish_wait(&ctx->wait, &iowq.wq); + cond_resched(); } while (ret > 0); restore_saved_sigmask_unless(ret == -EINTR); -- cgit v1.2.3 From b5b0ecb736f1ce1e68eb50613c0cfecff10198eb Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 4 Mar 2021 21:02:58 -0700 Subject: io_uring: clear IOCB_WAITQ for non -EIOCBQUEUED return The callback can only be armed, if we get -EIOCBQUEUED returned. It's important that we clear the WAITQ bit for other cases, otherwise we can queue for async retry and filemap will assume that we're armed and return -EAGAIN instead of just blocking for the IO. Cc: stable@vger.kernel.org # 5.9+ Signed-off-by: Jens Axboe --- fs/io_uring.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 044170165402..5762750c666c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3286,6 +3286,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) if (ret == -EIOCBQUEUED) return 0; /* we got some bytes, but not all. retry. */ + kiocb->ki_flags &= ~IOCB_WAITQ; } while (ret > 0 && ret < io_size); done: kiocb_done(kiocb, ret, issue_flags); -- cgit v1.2.3 From 09ca6c40c2024211657fdb2c50522a355610c3b7 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 5 Mar 2021 08:14:08 -0700 Subject: io-wq: kill hashed waitqueue before manager exits If we race with shutting down the io-wq context and someone queueing a hashed entry, then we can exit the manager with it armed. If it then triggers after the manager has exited, we can have a use-after-free where io_wqe_hash_wake() attempts to wake a now gone manager process. Move the killing of the hashed write queue into the manager itself, so that we know we've killed it before the task exits. Fixes: e941894eae31 ("io-wq: make buffered file write hashed work map per-ctx") Signed-off-by: Jens Axboe --- fs/io-wq.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index d7cfe8fd282a..28868eb4cd09 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -726,6 +726,11 @@ static int io_wq_manager(void *data) if (atomic_read(&wq->worker_refs)) wait_for_completion(&wq->worker_done); + spin_lock_irq(&wq->hash->wait.lock); + for_each_node(node) + list_del_init(&wq->wqes[node]->wait.entry); + spin_unlock_irq(&wq->hash->wait.lock); + io_wq_cancel_pending(wq); complete(&wq->exited); do_exit(0); @@ -1051,15 +1056,11 @@ static void io_wq_destroy(struct io_wq *wq) set_bit(IO_WQ_BIT_EXIT, &wq->state); io_wq_destroy_manager(wq); - spin_lock_irq(&wq->hash->wait.lock); for_each_node(node) { struct io_wqe *wqe = wq->wqes[node]; - - list_del_init(&wqe->wait.entry); WARN_ON_ONCE(!wq_list_empty(&wqe->work_list)); kfree(wqe); } - spin_unlock_irq(&wq->hash->wait.lock); io_wq_put_hash(wq->hash); kfree(wq->wqes); kfree(wq); -- cgit v1.2.3 From 86e0d6766cf909813474857bd22fdc04c97c0b36 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 5 Mar 2021 08:44:39 -0700 Subject: io_uring: make SQPOLL thread parking saner We have this weird true/false return from parking, and then some of the callers decide to look at that. It can lead to unbalanced parks and sqd locking. Have the callers check the thread status once it's parked. We know we have the lock at that point, so it's either valid or it's NULL. Fix race with parking on thread exit. We need to be careful here with ordering of the sdq->lock and the IO_SQ_THREAD_SHOULD_PARK bit. Rename sqd->completion to sqd->parked to reflect that this is the only thing this completion event doesn. Signed-off-by: Jens Axboe --- fs/io_uring.c | 65 +++++++++++++++++++++++++++-------------------------------- 1 file changed, 30 insertions(+), 35 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5762750c666c..d30cbf0f7b1c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -274,7 +274,7 @@ struct io_sq_data { unsigned long state; struct completion startup; - struct completion completion; + struct completion parked; struct completion exited; }; @@ -6656,7 +6656,7 @@ static void io_sq_thread_parkme(struct io_sq_data *sqd) * wait_task_inactive(). */ preempt_disable(); - complete(&sqd->completion); + complete(&sqd->parked); schedule_preempt_disabled(); preempt_enable(); } @@ -6751,14 +6751,18 @@ static int io_sq_thread(void *data) io_run_task_work(); - if (io_sq_thread_should_park(sqd)) - io_sq_thread_parkme(sqd); - /* - * Clear thread under lock so that concurrent parks work correctly + * Ensure that we park properly if racing with someone trying to park + * while we're exiting. If we fail to grab the lock, check park and + * park if necessary. The ordering with the park bit and the lock + * ensures that we catch this reliably. */ - complete(&sqd->completion); - mutex_lock(&sqd->lock); + if (!mutex_trylock(&sqd->lock)) { + if (io_sq_thread_should_park(sqd)) + io_sq_thread_parkme(sqd); + mutex_lock(&sqd->lock); + } + sqd->thread = NULL; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { ctx->sqo_exec = 1; @@ -7067,29 +7071,25 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) static void io_sq_thread_unpark(struct io_sq_data *sqd) __releases(&sqd->lock) { - if (!sqd->thread) - return; if (sqd->thread == current) return; clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); - wake_up_state(sqd->thread, TASK_PARKED); + if (sqd->thread) + wake_up_state(sqd->thread, TASK_PARKED); mutex_unlock(&sqd->lock); } -static bool io_sq_thread_park(struct io_sq_data *sqd) +static void io_sq_thread_park(struct io_sq_data *sqd) __acquires(&sqd->lock) { if (sqd->thread == current) - return true; + return; + set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); mutex_lock(&sqd->lock); - if (!sqd->thread) { - mutex_unlock(&sqd->lock); - return false; + if (sqd->thread) { + wake_up_process(sqd->thread); + wait_for_completion(&sqd->parked); } - set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); - wake_up_process(sqd->thread); - wait_for_completion(&sqd->completion); - return true; } static void io_sq_thread_stop(struct io_sq_data *sqd) @@ -7185,7 +7185,7 @@ static struct io_sq_data *io_get_sq_data(struct io_uring_params *p) mutex_init(&sqd->lock); init_waitqueue_head(&sqd->wait); init_completion(&sqd->startup); - init_completion(&sqd->completion); + init_completion(&sqd->parked); init_completion(&sqd->exited); return sqd; } @@ -7829,7 +7829,7 @@ static int io_sq_thread_fork(struct io_sq_data *sqd, struct io_ring_ctx *ctx) int ret; clear_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); - reinit_completion(&sqd->completion); + reinit_completion(&sqd->parked); ctx->sqo_exec = 0; sqd->task_pid = current->pid; tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE); @@ -8712,7 +8712,6 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, struct files_struct *files) { struct task_struct *task = current; - bool did_park = false; if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { /* never started, nothing to cancel */ @@ -8720,11 +8719,10 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, io_sq_offload_start(ctx); return; } - did_park = io_sq_thread_park(ctx->sq_data); - if (did_park) { - task = ctx->sq_data->thread; + io_sq_thread_park(ctx->sq_data); + task = ctx->sq_data->thread; + if (task) atomic_inc(&task->io_uring->in_idle); - } } io_cancel_defer_files(ctx, task, files); @@ -8733,10 +8731,10 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, if (!files) io_uring_try_cancel_requests(ctx, task, NULL); - if (did_park) { + if (task) atomic_dec(&task->io_uring->in_idle); + if (ctx->sq_data) io_sq_thread_unpark(ctx->sq_data); - } } /* @@ -8836,15 +8834,12 @@ static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) if (!sqd) return; - if (!io_sq_thread_park(sqd)) - return; - tctx = ctx->sq_data->thread->io_uring; - /* can happen on fork/alloc failure, just ignore that state */ - if (!tctx) { + io_sq_thread_park(sqd); + if (!sqd->thread || !sqd->thread->io_uring) { io_sq_thread_unpark(sqd); return; } - + tctx = ctx->sq_data->thread->io_uring; atomic_inc(&tctx->in_idle); do { /* read completions before cancelations */ -- cgit v1.2.3 From e45cff58858883290c98f65d409839a7295c95f3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 28 Feb 2021 22:35:14 +0000 Subject: io_uring: don't restrict issue_flags for io_openat 45d189c606292 ("io_uring: replace force_nonblock with flags") did something strange for io_openat() slicing all issue_flags but IO_URING_F_NONBLOCK. Not a bug for now, but better to just forward the flags. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index d30cbf0f7b1c..92c25b5f1349 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3828,7 +3828,7 @@ err: static int io_openat(struct io_kiocb *req, unsigned int issue_flags) { - return io_openat2(req, issue_flags & IO_URING_F_NONBLOCK); + return io_openat2(req, issue_flags); } static int io_remove_buffers_prep(struct io_kiocb *req, -- cgit v1.2.3 From 56887cffe946bb0a90c74429fa94d6110a73119d Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 22 Feb 2021 10:48:09 +0100 Subject: block: Try to handle busy underlying device on discard Commit 384d87ef2c95 ("block: Do not discard buffers under a mounted filesystem") made paths issuing discard or zeroout requests to the underlying device try to grab block device in exclusive mode. If that failed we returned EBUSY to userspace. This however caused unexpected fallout in userspace where e.g. FUSE filesystems issue discard requests from userspace daemons although the device is open exclusively by the kernel. Also shrinking of logical volume by LVM issues discard requests to a device which may be claimed exclusively because there's another LV on the same PV. So to avoid these userspace regressions, fall back to invalidate_inode_pages2_range() instead of returning EBUSY to userspace and return EBUSY only of that call fails as well (meaning that there's indeed someone using the particular device range we are trying to discard). Link: https://bugzilla.kernel.org/show_bug.cgi?id=211167 Fixes: 384d87ef2c95 ("block: Do not discard buffers under a mounted filesystem") CC: stable@vger.kernel.org Signed-off-by: Jan Kara Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- fs/block_dev.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/block_dev.c b/fs/block_dev.c index 4aa1f88d5bf8..03166b3dea4d 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -118,13 +118,22 @@ int truncate_bdev_range(struct block_device *bdev, fmode_t mode, if (!(mode & FMODE_EXCL)) { int err = bd_prepare_to_claim(bdev, truncate_bdev_range); if (err) - return err; + goto invalidate; } truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend); if (!(mode & FMODE_EXCL)) bd_abort_claiming(bdev, truncate_bdev_range); return 0; + +invalidate: + /* + * Someone else has handle exclusively open. Try invalidating instead. + * The 'end' argument is inclusive so the rounding is safe. + */ + return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping, + lstart >> PAGE_SHIFT, + lend >> PAGE_SHIFT); } static void set_init_blocksize(struct block_device *bdev) -- cgit v1.2.3 From efc61345274d6c7a46a0570efbc916fcbe3e927b Mon Sep 17 00:00:00 2001 From: Eric Whitney Date: Thu, 18 Feb 2021 10:11:32 -0500 Subject: ext4: shrink race window in ext4_should_retry_alloc() When generic/371 is run on kvm-xfstests using 5.10 and 5.11 kernels, it fails at significant rates on the two test scenarios that disable delayed allocation (ext3conv and data_journal) and force actual block allocation for the fallocate and pwrite functions in the test. The failure rate on 5.10 for both ext3conv and data_journal on one test system typically runs about 85%. On 5.11, the failure rate on ext3conv sometimes drops to as low as 1% while the rate on data_journal increases to nearly 100%. The observed failures are largely due to ext4_should_retry_alloc() cutting off block allocation retries when s_mb_free_pending (used to indicate that a transaction in progress will free blocks) is 0. However, free space is usually available when this occurs during runs of generic/371. It appears that a thread attempting to allocate blocks is just missing transaction commits in other threads that increase the free cluster count and reset s_mb_free_pending while the allocating thread isn't running. Explicitly testing for free space availability avoids this race. The current code uses a post-increment operator in the conditional expression that determines whether the retry limit has been exceeded. This means that the conditional expression uses the value of the retry counter before it's increased, resulting in an extra retry cycle. The current code actually retries twice before hitting its retry limit rather than once. Increasing the retry limit to 3 from the current actual maximum retry count of 2 in combination with the change described above reduces the observed failure rate to less that 0.1% on both ext3conv and data_journal with what should be limited impact on users sensitive to the overhead caused by retries. A per filesystem percpu counter exported via sysfs is added to allow users or developers to track the number of times the retry limit is exceeded without resorting to debugging methods. This should provide some insight into worst case retry behavior. Signed-off-by: Eric Whitney Link: https://lore.kernel.org/r/20210218151132.19678-1-enwlinux@gmail.com Signed-off-by: Theodore Ts'o --- fs/ext4/balloc.c | 38 ++++++++++++++++++++++++++------------ fs/ext4/ext4.h | 1 + fs/ext4/super.c | 5 +++++ fs/ext4/sysfs.c | 7 +++++++ 4 files changed, 39 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/ext4/balloc.c b/fs/ext4/balloc.c index f45f9feebe59..74a5172c2d83 100644 --- a/fs/ext4/balloc.c +++ b/fs/ext4/balloc.c @@ -626,27 +626,41 @@ int ext4_claim_free_clusters(struct ext4_sb_info *sbi, /** * ext4_should_retry_alloc() - check if a block allocation should be retried - * @sb: super block - * @retries: number of attemps has been made + * @sb: superblock + * @retries: number of retry attempts made so far * - * ext4_should_retry_alloc() is called when ENOSPC is returned, and if - * it is profitable to retry the operation, this function will wait - * for the current or committing transaction to complete, and then - * return TRUE. We will only retry once. + * ext4_should_retry_alloc() is called when ENOSPC is returned while + * attempting to allocate blocks. If there's an indication that a pending + * journal transaction might free some space and allow another attempt to + * succeed, this function will wait for the current or committing transaction + * to complete and then return TRUE. */ int ext4_should_retry_alloc(struct super_block *sb, int *retries) { - if (!ext4_has_free_clusters(EXT4_SB(sb), 1, 0) || - (*retries)++ > 1 || - !EXT4_SB(sb)->s_journal) + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (!sbi->s_journal) return 0; - smp_mb(); - if (EXT4_SB(sb)->s_mb_free_pending == 0) + if (++(*retries) > 3) { + percpu_counter_inc(&sbi->s_sra_exceeded_retry_limit); return 0; + } + /* + * if there's no indication that blocks are about to be freed it's + * possible we just missed a transaction commit that did so + */ + smp_mb(); + if (sbi->s_mb_free_pending == 0) + return ext4_has_free_clusters(sbi, 1, 0); + + /* + * it's possible we've just missed a transaction commit here, + * so ignore the returned status + */ jbd_debug(1, "%s: retrying operation after ENOSPC\n", sb->s_id); - jbd2_journal_force_commit_nested(EXT4_SB(sb)->s_journal); + (void) jbd2_journal_force_commit_nested(sbi->s_journal); return 1; } diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 644fd69185d3..ea3b41579f38 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1484,6 +1484,7 @@ struct ext4_sb_info { struct percpu_counter s_freeinodes_counter; struct percpu_counter s_dirs_counter; struct percpu_counter s_dirtyclusters_counter; + struct percpu_counter s_sra_exceeded_retry_limit; struct blockgroup_lock *s_blockgroup_lock; struct proc_dir_entry *s_proc; struct kobject s_kobj; diff --git a/fs/ext4/super.c b/fs/ext4/super.c index ad34a37278cd..a0a256859662 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1210,6 +1210,7 @@ static void ext4_put_super(struct super_block *sb) percpu_counter_destroy(&sbi->s_freeinodes_counter); percpu_counter_destroy(&sbi->s_dirs_counter); percpu_counter_destroy(&sbi->s_dirtyclusters_counter); + percpu_counter_destroy(&sbi->s_sra_exceeded_retry_limit); percpu_free_rwsem(&sbi->s_writepages_rwsem); #ifdef CONFIG_QUOTA for (i = 0; i < EXT4_MAXQUOTAS; i++) @@ -5011,6 +5012,9 @@ no_journal: if (!err) err = percpu_counter_init(&sbi->s_dirtyclusters_counter, 0, GFP_KERNEL); + if (!err) + err = percpu_counter_init(&sbi->s_sra_exceeded_retry_limit, 0, + GFP_KERNEL); if (!err) err = percpu_init_rwsem(&sbi->s_writepages_rwsem); @@ -5124,6 +5128,7 @@ failed_mount6: percpu_counter_destroy(&sbi->s_freeinodes_counter); percpu_counter_destroy(&sbi->s_dirs_counter); percpu_counter_destroy(&sbi->s_dirtyclusters_counter); + percpu_counter_destroy(&sbi->s_sra_exceeded_retry_limit); percpu_free_rwsem(&sbi->s_writepages_rwsem); failed_mount5: ext4_ext_release(sb); diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index 075aa3a19ff5..a3d08276d441 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -24,6 +24,7 @@ typedef enum { attr_session_write_kbytes, attr_lifetime_write_kbytes, attr_reserved_clusters, + attr_sra_exceeded_retry_limit, attr_inode_readahead, attr_trigger_test_error, attr_first_error_time, @@ -202,6 +203,7 @@ EXT4_ATTR_FUNC(delayed_allocation_blocks, 0444); EXT4_ATTR_FUNC(session_write_kbytes, 0444); EXT4_ATTR_FUNC(lifetime_write_kbytes, 0444); EXT4_ATTR_FUNC(reserved_clusters, 0644); +EXT4_ATTR_FUNC(sra_exceeded_retry_limit, 0444); EXT4_ATTR_OFFSET(inode_readahead_blks, 0644, inode_readahead, ext4_sb_info, s_inode_readahead_blks); @@ -251,6 +253,7 @@ static struct attribute *ext4_attrs[] = { ATTR_LIST(session_write_kbytes), ATTR_LIST(lifetime_write_kbytes), ATTR_LIST(reserved_clusters), + ATTR_LIST(sra_exceeded_retry_limit), ATTR_LIST(inode_readahead_blks), ATTR_LIST(inode_goal), ATTR_LIST(mb_stats), @@ -374,6 +377,10 @@ static ssize_t ext4_attr_show(struct kobject *kobj, return snprintf(buf, PAGE_SIZE, "%llu\n", (unsigned long long) atomic64_read(&sbi->s_resv_clusters)); + case attr_sra_exceeded_retry_limit: + return snprintf(buf, PAGE_SIZE, "%llu\n", + (unsigned long long) + percpu_counter_sum(&sbi->s_sra_exceeded_retry_limit)); case attr_inode_readahead: case attr_pointer_ui: if (!ptr) -- cgit v1.2.3 From 163f0ec1df33cf468509ff38cbcbb5eb0d7fac60 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 22 Feb 2021 18:16:26 +0100 Subject: ext4: add reclaim checks to xattr code Syzbot is reporting that ext4 can enter fs reclaim from kvmalloc() while the transaction is started like: fs_reclaim_acquire+0x117/0x150 mm/page_alloc.c:4340 might_alloc include/linux/sched/mm.h:193 [inline] slab_pre_alloc_hook mm/slab.h:493 [inline] slab_alloc_node mm/slub.c:2817 [inline] __kmalloc_node+0x5f/0x430 mm/slub.c:4015 kmalloc_node include/linux/slab.h:575 [inline] kvmalloc_node+0x61/0xf0 mm/util.c:587 kvmalloc include/linux/mm.h:781 [inline] ext4_xattr_inode_cache_find fs/ext4/xattr.c:1465 [inline] ext4_xattr_inode_lookup_create fs/ext4/xattr.c:1508 [inline] ext4_xattr_set_entry+0x1ce6/0x3780 fs/ext4/xattr.c:1649 ext4_xattr_ibody_set+0x78/0x2b0 fs/ext4/xattr.c:2224 ext4_xattr_set_handle+0x8f4/0x13e0 fs/ext4/xattr.c:2380 ext4_xattr_set+0x13a/0x340 fs/ext4/xattr.c:2493 This should be impossible since transaction start sets PF_MEMALLOC_NOFS. Add some assertions to the code to catch if something isn't working as expected early. Link: https://lore.kernel.org/linux-ext4/000000000000563a0205bafb7970@google.com/ Signed-off-by: Jan Kara Link: https://lore.kernel.org/r/20210222171626.21884-1-jack@suse.cz Signed-off-by: Theodore Ts'o --- fs/ext4/xattr.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'fs') diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 372208500f4e..083c95126781 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -1462,6 +1462,9 @@ ext4_xattr_inode_cache_find(struct inode *inode, const void *value, if (!ce) return NULL; + WARN_ON_ONCE(ext4_handle_valid(journal_current_handle()) && + !(current->flags & PF_MEMALLOC_NOFS)); + ea_data = kvmalloc(value_len, GFP_KERNEL); if (!ea_data) { mb_cache_entry_put(ea_inode_cache, ce); @@ -2327,6 +2330,7 @@ ext4_xattr_set_handle(handle_t *handle, struct inode *inode, int name_index, error = -ENOSPC; goto cleanup; } + WARN_ON_ONCE(!(current->flags & PF_MEMALLOC_NOFS)); } error = ext4_reserve_inode_write(handle, inode, &is.iloc); -- cgit v1.2.3 From f91436d55a279f045987e8b8c1385585dca54be9 Mon Sep 17 00:00:00 2001 From: Sabyrzhan Tasbolatov Date: Wed, 24 Feb 2021 15:58:00 +0600 Subject: fs/ext4: fix integer overflow in s_log_groups_per_flex syzbot found UBSAN: shift-out-of-bounds in ext4_mb_init [1], when 1 << sbi->s_es->s_log_groups_per_flex is bigger than UINT_MAX, where sbi->s_mb_prefetch is unsigned integer type. 32 is the maximum allowed power of s_log_groups_per_flex. Following if check will also trigger UBSAN shift-out-of-bound: if (1 << sbi->s_es->s_log_groups_per_flex >= UINT_MAX) { So I'm checking it against the raw number, perhaps there is another way to calculate UINT_MAX max power. Also use min_t as to make sure it's uint type. [1] UBSAN: shift-out-of-bounds in fs/ext4/mballoc.c:2713:24 shift exponent 60 is too large for 32-bit type 'int' Call Trace: __dump_stack lib/dump_stack.c:79 [inline] dump_stack+0x137/0x1be lib/dump_stack.c:120 ubsan_epilogue lib/ubsan.c:148 [inline] __ubsan_handle_shift_out_of_bounds+0x432/0x4d0 lib/ubsan.c:395 ext4_mb_init_backend fs/ext4/mballoc.c:2713 [inline] ext4_mb_init+0x19bc/0x19f0 fs/ext4/mballoc.c:2898 ext4_fill_super+0xc2ec/0xfbe0 fs/ext4/super.c:4983 Reported-by: syzbot+a8b4b0c60155e87e9484@syzkaller.appspotmail.com Signed-off-by: Sabyrzhan Tasbolatov Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20210224095800.3350002-1-snovitoll@gmail.com Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 99bf091fee10..a02fadf4fc84 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -2709,8 +2709,15 @@ static int ext4_mb_init_backend(struct super_block *sb) } if (ext4_has_feature_flex_bg(sb)) { - /* a single flex group is supposed to be read by a single IO */ - sbi->s_mb_prefetch = min(1 << sbi->s_es->s_log_groups_per_flex, + /* a single flex group is supposed to be read by a single IO. + * 2 ^ s_log_groups_per_flex != UINT_MAX as s_mb_prefetch is + * unsigned integer, so the maximum shift is 32. + */ + if (sbi->s_es->s_log_groups_per_flex >= 32) { + ext4_msg(sb, KERN_ERR, "too many log groups per flexible block group"); + goto err_freesgi; + } + sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex, BLK_MAX_SEGMENT_SIZE >> (sb->s_blocksize_bits - 9)); sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */ } else { -- cgit v1.2.3 From c915fb80eaa6194fa9bd0a4487705cd5b0dda2f1 Mon Sep 17 00:00:00 2001 From: Zhaolong Zhang Date: Tue, 2 Mar 2021 17:42:31 +0800 Subject: ext4: fix bh ref count on error paths __ext4_journalled_writepage should drop bhs' ref count on error paths Signed-off-by: Zhaolong Zhang Link: https://lore.kernel.org/r/1614678151-70481-1-git-send-email-zhangzl2013@126.com Signed-off-by: Theodore Ts'o --- fs/ext4/inode.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 650c5acd2f2d..a79a9ea58c56 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1938,13 +1938,13 @@ static int __ext4_journalled_writepage(struct page *page, if (!ret) ret = err; - if (!ext4_has_inline_data(inode)) - ext4_walk_page_buffers(NULL, page_bufs, 0, len, - NULL, bput_one); ext4_set_inode_state(inode, EXT4_STATE_JDATA); out: unlock_page(page); out_no_pagelock: + if (!inline_data && page_bufs) + ext4_walk_page_buffers(NULL, page_bufs, 0, len, + NULL, bput_one); brelse(inode_bh); return ret; } -- cgit v1.2.3 From a249cc8bc2e2fed680047d326eb9a50756724198 Mon Sep 17 00:00:00 2001 From: Aurelien Aptel Date: Thu, 4 Mar 2021 17:42:21 +0000 Subject: cifs: fix credit accounting for extra channel With multichannel, operations like the queries from "ls -lR" can cause all credits to be used and errors to be returned since max_credits was not being set correctly on the secondary channels and thus the client was requesting 0 credits incorrectly in some cases (which can lead to not having enough credits to perform any operation on that channel). Signed-off-by: Aurelien Aptel CC: # v5.8+ Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/connect.c | 10 +++++----- fs/cifs/sess.c | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 112692300fb6..68642e3d4270 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -1429,6 +1429,11 @@ smbd_connected: tcp_ses->min_offload = ctx->min_offload; tcp_ses->tcpStatus = CifsNeedNegotiate; + if ((ctx->max_credits < 20) || (ctx->max_credits > 60000)) + tcp_ses->max_credits = SMB2_MAX_CREDITS_AVAILABLE; + else + tcp_ses->max_credits = ctx->max_credits; + tcp_ses->nr_targets = 1; tcp_ses->ignore_signature = ctx->ignore_signature; /* thread spawned, put it on the list */ @@ -2832,11 +2837,6 @@ static int mount_get_conns(struct smb3_fs_context *ctx, struct cifs_sb_info *cif *nserver = server; - if ((ctx->max_credits < 20) || (ctx->max_credits > 60000)) - server->max_credits = SMB2_MAX_CREDITS_AVAILABLE; - else - server->max_credits = ctx->max_credits; - /* get a reference to a SMB session */ ses = cifs_get_smb_ses(server, ctx); if (IS_ERR(ses)) { diff --git a/fs/cifs/sess.c b/fs/cifs/sess.c index 183a3a868d7b..63d517b9f2ff 100644 --- a/fs/cifs/sess.c +++ b/fs/cifs/sess.c @@ -230,6 +230,7 @@ cifs_ses_add_channel(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses, ctx.noautotune = ses->server->noautotune; ctx.sockopt_tcp_nodelay = ses->server->tcp_nodelay; ctx.echo_interval = ses->server->echo_interval / HZ; + ctx.max_credits = ses->server->max_credits; /* * This will be used for encoding/decoding user/domain/pw -- cgit v1.2.3 From 88fd98a2306755b965e4f4567f84e73db3b6738c Mon Sep 17 00:00:00 2001 From: Aurelien Aptel Date: Thu, 4 Mar 2021 17:51:48 +0000 Subject: cifs: ask for more credit on async read/write code paths When doing a large read or write workload we only very gradually increase the number of credits which can cause problems with parallelizing large i/o (I/O ramps up more slowly than it should for large read/write workloads) especially with multichannel when the number of credits on the secondary channels starts out low (e.g. less than about 130) or when recovering after server throttled back the number of credit. Signed-off-by: Aurelien Aptel Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/smb2pdu.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index 4bbb6126b14d..2199a9bfae8f 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -4041,8 +4041,7 @@ smb2_async_readv(struct cifs_readdata *rdata) if (rdata->credits.value > 0) { shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(rdata->bytes, SMB2_MAX_BUFFER_SIZE)); - shdr->CreditRequest = - cpu_to_le16(le16_to_cpu(shdr->CreditCharge) + 1); + shdr->CreditRequest = cpu_to_le16(le16_to_cpu(shdr->CreditCharge) + 8); rc = adjust_credits(server, &rdata->credits, rdata->bytes); if (rc) @@ -4348,8 +4347,7 @@ smb2_async_writev(struct cifs_writedata *wdata, if (wdata->credits.value > 0) { shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(wdata->bytes, SMB2_MAX_BUFFER_SIZE)); - shdr->CreditRequest = - cpu_to_le16(le16_to_cpu(shdr->CreditCharge) + 1); + shdr->CreditRequest = cpu_to_le16(le16_to_cpu(shdr->CreditCharge) + 8); rc = adjust_credits(server, &wdata->credits, wdata->bytes); if (rc) -- cgit v1.2.3 From 886d0137f104a440d9dfa1d16efc1db06c9a2c02 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 5 Mar 2021 12:59:30 -0700 Subject: io-wq: fix race in freeing 'wq' and worker access Ran into a use-after-free on the main io-wq struct, wq. It has a worker ref and completion event, but the manager itself isn't holding a reference. This can lead to a race where the manager thinks there are no workers and exits, but a worker is being added. That leads to the following trace: BUG: KASAN: use-after-free in io_wqe_worker+0x4c0/0x5e0 Read of size 8 at addr ffff888108baa8a0 by task iou-wrk-3080422/3080425 CPU: 5 PID: 3080425 Comm: iou-wrk-3080422 Not tainted 5.12.0-rc1+ #110 Hardware name: Micro-Star International Co., Ltd. MS-7C60/TRX40 PRO 10G (MS-7C60), BIOS 1.60 05/13/2020 Call Trace: dump_stack+0x90/0xbe print_address_description.constprop.0+0x67/0x28d ? io_wqe_worker+0x4c0/0x5e0 kasan_report.cold+0x7b/0xd4 ? io_wqe_worker+0x4c0/0x5e0 __asan_load8+0x6d/0xa0 io_wqe_worker+0x4c0/0x5e0 ? io_worker_handle_work+0xc00/0xc00 ? recalc_sigpending+0xe5/0x120 ? io_worker_handle_work+0xc00/0xc00 ? io_worker_handle_work+0xc00/0xc00 ret_from_fork+0x1f/0x30 Allocated by task 3080422: kasan_save_stack+0x23/0x60 __kasan_kmalloc+0x80/0xa0 kmem_cache_alloc_node_trace+0xa0/0x480 io_wq_create+0x3b5/0x600 io_uring_alloc_task_context+0x13c/0x380 io_uring_add_task_file+0x109/0x140 __x64_sys_io_uring_enter+0x45f/0x660 do_syscall_64+0x32/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xae Freed by task 3080422: kasan_save_stack+0x23/0x60 kasan_set_track+0x20/0x40 kasan_set_free_info+0x24/0x40 __kasan_slab_free+0xe8/0x120 kfree+0xa8/0x400 io_wq_put+0x14a/0x220 io_wq_put_and_exit+0x9a/0xc0 io_uring_clean_tctx+0x101/0x140 __io_uring_files_cancel+0x36e/0x3c0 do_exit+0x169/0x1340 __x64_sys_exit+0x34/0x40 do_syscall_64+0x32/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xae Have the manager itself hold a reference, and now both drop points drop and complete if we hit zero, and the manager can unconditionally do a wait_for_completion() instead of having a race between reading the ref count and waiting if it was non-zero. Fixes: fb3a1f6c745c ("io-wq: have manager wait for all workers to exit") Signed-off-by: Jens Axboe --- fs/io-wq.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 28868eb4cd09..1bfdb86336e4 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -722,9 +722,9 @@ static int io_wq_manager(void *data) io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL); rcu_read_unlock(); - /* we might not ever have created any workers */ - if (atomic_read(&wq->worker_refs)) - wait_for_completion(&wq->worker_done); + if (atomic_dec_and_test(&wq->worker_refs)) + complete(&wq->worker_done); + wait_for_completion(&wq->worker_done); spin_lock_irq(&wq->hash->wait.lock); for_each_node(node) @@ -774,7 +774,8 @@ static int io_wq_fork_manager(struct io_wq *wq) if (wq->manager) return 0; - reinit_completion(&wq->worker_done); + init_completion(&wq->worker_done); + atomic_set(&wq->worker_refs, 1); tsk = create_io_thread(io_wq_manager, wq, NUMA_NO_NODE); if (!IS_ERR(tsk)) { wq->manager = get_task_struct(tsk); @@ -782,6 +783,9 @@ static int io_wq_fork_manager(struct io_wq *wq) return 0; } + if (atomic_dec_and_test(&wq->worker_refs)) + complete(&wq->worker_done); + return PTR_ERR(tsk); } @@ -1018,13 +1022,9 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) init_completion(&wq->exited); refcount_set(&wq->refs, 1); - init_completion(&wq->worker_done); - atomic_set(&wq->worker_refs, 0); - ret = io_wq_fork_manager(wq); if (!ret) return wq; - err: io_wq_put_hash(data->hash); cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); -- cgit v1.2.3 From 003e8dccdb22712dae388e682182d5f08b32386f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 6 Mar 2021 09:22:27 -0700 Subject: io-wq: always track creds for async issue If we go async with a request, grab the creds that the task currently has assigned and make sure that the async side switches to them. This is handled in the same way that we do for registered personalities. Signed-off-by: Jens Axboe --- fs/io-wq.h | 2 +- fs/io_uring.c | 33 +++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.h b/fs/io-wq.h index 5fbf7997149e..1ac2f3248088 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -79,8 +79,8 @@ static inline void wq_list_del(struct io_wq_work_list *list, struct io_wq_work { struct io_wq_work_node list; + const struct cred *creds; unsigned flags; - unsigned short personality; }; static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) diff --git a/fs/io_uring.c b/fs/io_uring.c index 92c25b5f1349..d51c6ba9268b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1183,6 +1183,9 @@ static void io_prep_async_work(struct io_kiocb *req) const struct io_op_def *def = &io_op_defs[req->opcode]; struct io_ring_ctx *ctx = req->ctx; + if (!req->work.creds) + req->work.creds = get_current_cred(); + if (req->flags & REQ_F_FORCE_ASYNC) req->work.flags |= IO_WQ_WORK_CONCURRENT; @@ -1648,6 +1651,10 @@ static void io_dismantle_req(struct io_kiocb *req) io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE)); if (req->fixed_rsrc_refs) percpu_ref_put(req->fixed_rsrc_refs); + if (req->work.creds) { + put_cred(req->work.creds); + req->work.creds = NULL; + } if (req->flags & REQ_F_INFLIGHT) { struct io_ring_ctx *ctx = req->ctx; @@ -5916,18 +5923,8 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) const struct cred *creds = NULL; int ret; - if (req->work.personality) { - const struct cred *new_creds; - - if (!(issue_flags & IO_URING_F_NONBLOCK)) - mutex_lock(&ctx->uring_lock); - new_creds = idr_find(&ctx->personality_idr, req->work.personality); - if (!(issue_flags & IO_URING_F_NONBLOCK)) - mutex_unlock(&ctx->uring_lock); - if (!new_creds) - return -EINVAL; - creds = override_creds(new_creds); - } + if (req->work.creds && req->work.creds != current_cred()) + creds = override_creds(req->work.creds); switch (req->opcode) { case IORING_OP_NOP: @@ -6291,7 +6288,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, { struct io_submit_state *state; unsigned int sqe_flags; - int ret = 0; + int personality, ret = 0; req->opcode = READ_ONCE(sqe->opcode); /* same numerical values with corresponding REQ_F_*, safe to copy */ @@ -6324,8 +6321,16 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, return -EOPNOTSUPP; req->work.list.next = NULL; + personality = READ_ONCE(sqe->personality); + if (personality) { + req->work.creds = idr_find(&ctx->personality_idr, personality); + if (!req->work.creds) + return -EINVAL; + get_cred(req->work.creds); + } else { + req->work.creds = NULL; + } req->work.flags = 0; - req->work.personality = READ_ONCE(sqe->personality); state = &ctx->submit_state; /* -- cgit v1.2.3 From d30881f573e565ebb5dbb50b31ed6106b5c81328 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 18 Feb 2021 21:02:07 -0500 Subject: nfsd: Don't keep looking up unhashed files in the nfsd file cache If a file is unhashed, then we're going to reject it anyway and retry, so make sure we skip it when we're doing the RCU lockless lookup. This avoids a number of unnecessary nfserr_jukebox returns from nfsd_file_acquire() Fixes: 65294c1f2c5e ("nfsd: add a new struct file caching facility to nfsd") Signed-off-by: Trond Myklebust Signed-off-by: Chuck Lever --- fs/nfsd/filecache.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 53fcbf79bdca..7629248fdd53 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -898,6 +898,8 @@ nfsd_file_find_locked(struct inode *inode, unsigned int may_flags, continue; if (!nfsd_match_cred(nf->nf_cred, current_cred())) continue; + if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) + continue; if (nfsd_file_get(nf) != NULL) return nf; } -- cgit v1.2.3 From 7005227369079963d25fb2d5d736d0feb2c44cf6 Mon Sep 17 00:00:00 2001 From: Julian Braha Date: Fri, 19 Feb 2021 16:56:10 -0500 Subject: fs: nfsd: fix kconfig dependency warning for NFSD_V4 When NFSD_V4 is enabled and CRYPTO is disabled, Kbuild gives the following warning: WARNING: unmet direct dependencies detected for CRYPTO_SHA256 Depends on [n]: CRYPTO [=n] Selected by [y]: - NFSD_V4 [=y] && NETWORK_FILESYSTEMS [=y] && NFSD [=y] && PROC_FS [=y] WARNING: unmet direct dependencies detected for CRYPTO_MD5 Depends on [n]: CRYPTO [=n] Selected by [y]: - NFSD_V4 [=y] && NETWORK_FILESYSTEMS [=y] && NFSD [=y] && PROC_FS [=y] This is because NFSD_V4 selects CRYPTO_MD5 and CRYPTO_SHA256, without depending on or selecting CRYPTO, despite those config options being subordinate to CRYPTO. Signed-off-by: Julian Braha Signed-off-by: Chuck Lever --- fs/nfsd/Kconfig | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/nfsd/Kconfig b/fs/nfsd/Kconfig index 821e5913faee..d6cff5fbe705 100644 --- a/fs/nfsd/Kconfig +++ b/fs/nfsd/Kconfig @@ -73,6 +73,7 @@ config NFSD_V4 select NFSD_V3 select FS_POSIX_ACL select SUNRPC_GSS + select CRYPTO select CRYPTO_MD5 select CRYPTO_SHA256 select GRACE_PERIOD -- cgit v1.2.3 From bfdd89f232aa2de5a4b3fc985cba894148b830a8 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Wed, 24 Feb 2021 13:39:50 -0500 Subject: nfsd: don't abort copies early The typical result of the backwards comparison here is that the source server in a server-to-server copy will return BAD_STATEID within a few seconds of the copy starting, instead of giving the copy a full lease period, so the copy_file_range() call will end up unnecessarily returning a short read. Fixes: 624322f1adc5 "NFSD add COPY_NOTIFY operation" Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 423fd6683f3a..61552e89bd89 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -5389,7 +5389,7 @@ nfs4_laundromat(struct nfsd_net *nn) idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) { cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid); if (cps->cp_stateid.sc_type == NFS4_COPYNOTIFY_STID && - cps->cpntf_time > cutoff) + cps->cpntf_time < cutoff) _free_cpntf_state_locked(nn, cps); } spin_unlock(&nn->s2s_cp_lock); -- cgit v1.2.3 From eb602521f43876b3f76c4686de596c9804977228 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Thu, 4 Mar 2021 09:28:57 -0500 Subject: gfs2: make function gfs2_make_fs_ro() to void type It fixes the following warning detected by coccinelle: ./fs/gfs2/super.c:592:5-10: Unneeded variable: "error". Return "0" on line 628 Reported-by: Abaci Robot Signed-off-by: Yang Li Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/ops_fstype.c | 4 +--- fs/gfs2/super.c | 10 ++-------- fs/gfs2/super.h | 2 +- fs/gfs2/util.c | 2 +- 4 files changed, 5 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index 74c7d01723b9..aa4136055a83 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -1539,9 +1539,7 @@ static int gfs2_reconfigure(struct fs_context *fc) return -EINVAL; if (fc->sb_flags & SB_RDONLY) { - error = gfs2_make_fs_ro(sdp); - if (error) - errorfc(fc, "unable to remount read-only"); + gfs2_make_fs_ro(sdp); } else { error = gfs2_make_fs_rw(sdp); if (error) diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index 861ed5fe02a5..97076d3f562f 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -587,9 +587,8 @@ out: * Returns: errno */ -int gfs2_make_fs_ro(struct gfs2_sbd *sdp) +void gfs2_make_fs_ro(struct gfs2_sbd *sdp) { - int error = 0; int log_write_allowed = test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); gfs2_flush_delete_work(sdp); @@ -624,8 +623,6 @@ int gfs2_make_fs_ro(struct gfs2_sbd *sdp) if (!log_write_allowed) sdp->sd_vfs->s_flags |= SB_RDONLY; - - return error; } /** @@ -637,7 +634,6 @@ int gfs2_make_fs_ro(struct gfs2_sbd *sdp) static void gfs2_put_super(struct super_block *sb) { struct gfs2_sbd *sdp = sb->s_fs_info; - int error; struct gfs2_jdesc *jd; /* No more recovery requests */ @@ -658,9 +654,7 @@ restart: spin_unlock(&sdp->sd_jindex_spin); if (!sb_rdonly(sb)) { - error = gfs2_make_fs_ro(sdp); - if (error) - gfs2_io_error(sdp); + gfs2_make_fs_ro(sdp); } WARN_ON(gfs2_withdrawing(sdp)); diff --git a/fs/gfs2/super.h b/fs/gfs2/super.h index 08e502dec7ec..ec4affb33ed5 100644 --- a/fs/gfs2/super.h +++ b/fs/gfs2/super.h @@ -34,7 +34,7 @@ extern int gfs2_lookup_in_master_dir(struct gfs2_sbd *sdp, char *filename, struct gfs2_inode **ipp); extern int gfs2_make_fs_rw(struct gfs2_sbd *sdp); -extern int gfs2_make_fs_ro(struct gfs2_sbd *sdp); +extern void gfs2_make_fs_ro(struct gfs2_sbd *sdp); extern void gfs2_online_uevent(struct gfs2_sbd *sdp); extern int gfs2_statfs_init(struct gfs2_sbd *sdp); extern void gfs2_statfs_change(struct gfs2_sbd *sdp, s64 total, s64 free, diff --git a/fs/gfs2/util.c b/fs/gfs2/util.c index 8d3c670c990f..58743315cda9 100644 --- a/fs/gfs2/util.c +++ b/fs/gfs2/util.c @@ -156,7 +156,7 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) ret = 0; } if (!ret) - ret = gfs2_make_fs_ro(sdp); + gfs2_make_fs_ro(sdp); gfs2_freeze_unlock(&freeze_gh); } -- cgit v1.2.3 From 1a5a2cfd34c17db73c53ef127272c8c1ae220485 Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Thu, 25 Feb 2021 11:11:09 -0500 Subject: gfs2: fix use-after-free in trans_drain This patch adds code to function trans_drain to remove drained bd elements from the ail lists, if queued, before freeing the bd. If we don't remove the bd from the ail, function ail_drain will try to reference the bd after it has been freed by trans_drain. Thanks to Andy Price for his analysis of the problem. Reported-by: Andy Price Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 4 ++++ fs/gfs2/trans.c | 2 ++ 2 files changed, 6 insertions(+) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 16937ebb2a3e..760af666576c 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -998,12 +998,16 @@ static void trans_drain(struct gfs2_trans *tr) while (!list_empty(head)) { bd = list_first_entry(head, struct gfs2_bufdata, bd_list); list_del_init(&bd->bd_list); + if (!list_empty(&bd->bd_ail_st_list)) + gfs2_remove_from_ail(bd); kmem_cache_free(gfs2_bufdata_cachep, bd); } head = &tr->tr_databuf; while (!list_empty(head)) { bd = list_first_entry(head, struct gfs2_bufdata, bd_list); list_del_init(&bd->bd_list); + if (!list_empty(&bd->bd_ail_st_list)) + gfs2_remove_from_ail(bd); kmem_cache_free(gfs2_bufdata_cachep, bd); } } diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c index ab96cf0bf26b..63fec11ef2ce 100644 --- a/fs/gfs2/trans.c +++ b/fs/gfs2/trans.c @@ -169,6 +169,8 @@ static struct gfs2_bufdata *gfs2_alloc_bufdata(struct gfs2_glock *gl, bd->bd_bh = bh; bd->bd_gl = gl; INIT_LIST_HEAD(&bd->bd_list); + INIT_LIST_HEAD(&bd->bd_ail_st_list); + INIT_LIST_HEAD(&bd->bd_ail_gl_list); bh->b_private = bd; return bd; } -- cgit v1.2.3 From 2941267bd3dad018de1d51fe2cd996b7bc1e5a5d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 6 Mar 2021 11:02:11 +0000 Subject: io_uring: make del_task_file more forgiving Rework io_uring_del_task_file(), so it accepts an index to delete, and it's not necessarily have to be in the ->xa. Infer file from xa_erase() to maintain a single origin of truth. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index d51c6ba9268b..00a736867b76 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8785,15 +8785,18 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) /* * Remove this io_uring_file -> task mapping. */ -static void io_uring_del_task_file(struct file *file) +static void io_uring_del_task_file(unsigned long index) { struct io_uring_task *tctx = current->io_uring; + struct file *file; + + file = xa_erase(&tctx->xa, index); + if (!file) + return; if (tctx->last == file) tctx->last = NULL; - file = xa_erase(&tctx->xa, (unsigned long)file); - if (file) - fput(file); + fput(file); } static void io_uring_clean_tctx(struct io_uring_task *tctx) @@ -8802,7 +8805,7 @@ static void io_uring_clean_tctx(struct io_uring_task *tctx) unsigned long index; xa_for_each(&tctx->xa, index, file) - io_uring_del_task_file(file); + io_uring_del_task_file(index); if (tctx->io_wq) { io_wq_put_and_exit(tctx->io_wq); tctx->io_wq = NULL; -- cgit v1.2.3 From 13bf43f5f4739739751c0049a1582610c283bdde Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 6 Mar 2021 11:02:12 +0000 Subject: io_uring: introduce ctx to tctx back map For each pair tcxt-ctx create an object and chain it into ctx, so we have a way to traverse all tctx that are using current ctx. Preparation patch, will be used later. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 58 ++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 00a736867b76..9a2cff0662e0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -454,6 +454,7 @@ struct io_ring_ctx { /* Keep this last, we don't need it for the fast path */ struct work_struct exit_work; + struct list_head tctx_list; }; /* @@ -805,6 +806,13 @@ struct io_kiocb { struct io_wq_work work; }; +struct io_tctx_node { + struct list_head ctx_node; + struct task_struct *task; + struct file *file; + struct io_ring_ctx *ctx; +}; + struct io_defer_entry { struct list_head list; struct io_kiocb *req; @@ -1144,6 +1152,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->rsrc_ref_list); INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); + INIT_LIST_HEAD(&ctx->tctx_list); INIT_LIST_HEAD(&ctx->submit_state.comp.free_list); INIT_LIST_HEAD(&ctx->submit_state.comp.locked_free_list); return ctx; @@ -8748,6 +8757,7 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) { struct io_uring_task *tctx = current->io_uring; + struct io_tctx_node *node; int ret; if (unlikely(!tctx)) { @@ -8760,13 +8770,25 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) void *old = xa_load(&tctx->xa, (unsigned long)file); if (!old) { + node = kmalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + node->ctx = ctx; + node->file = file; + node->task = current; + get_file(file); ret = xa_err(xa_store(&tctx->xa, (unsigned long)file, - file, GFP_KERNEL)); + node, GFP_KERNEL)); if (ret) { fput(file); + kfree(node); return ret; } + + mutex_lock(&ctx->uring_lock); + list_add(&node->ctx_node, &ctx->tctx_list); + mutex_unlock(&ctx->uring_lock); } tctx->last = file; } @@ -8788,23 +8810,31 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) static void io_uring_del_task_file(unsigned long index) { struct io_uring_task *tctx = current->io_uring; - struct file *file; + struct io_tctx_node *node; - file = xa_erase(&tctx->xa, index); - if (!file) + node = xa_erase(&tctx->xa, index); + if (!node) return; - if (tctx->last == file) + WARN_ON_ONCE(current != node->task); + WARN_ON_ONCE(list_empty(&node->ctx_node)); + + mutex_lock(&node->ctx->uring_lock); + list_del(&node->ctx_node); + mutex_unlock(&node->ctx->uring_lock); + + if (tctx->last == node->file) tctx->last = NULL; - fput(file); + fput(node->file); + kfree(node); } static void io_uring_clean_tctx(struct io_uring_task *tctx) { - struct file *file; + struct io_tctx_node *node; unsigned long index; - xa_for_each(&tctx->xa, index, file) + xa_for_each(&tctx->xa, index, node) io_uring_del_task_file(index); if (tctx->io_wq) { io_wq_put_and_exit(tctx->io_wq); @@ -8815,13 +8845,13 @@ static void io_uring_clean_tctx(struct io_uring_task *tctx) void __io_uring_files_cancel(struct files_struct *files) { struct io_uring_task *tctx = current->io_uring; - struct file *file; + struct io_tctx_node *node; unsigned long index; /* make sure overflow events are dropped */ atomic_inc(&tctx->in_idle); - xa_for_each(&tctx->xa, index, file) - io_uring_cancel_task_requests(file->private_data, files); + xa_for_each(&tctx->xa, index, node) + io_uring_cancel_task_requests(node->ctx, files); atomic_dec(&tctx->in_idle); if (files) @@ -8884,11 +8914,11 @@ void __io_uring_task_cancel(void) atomic_inc(&tctx->in_idle); if (tctx->sqpoll) { - struct file *file; + struct io_tctx_node *node; unsigned long index; - xa_for_each(&tctx->xa, index, file) - io_uring_cancel_sqpoll(file->private_data); + xa_for_each(&tctx->xa, index, node) + io_uring_cancel_sqpoll(node->ctx); } do { -- cgit v1.2.3 From d56d938b4bef3e1421a42023cdcd6e13c1f50831 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 6 Mar 2021 11:02:13 +0000 Subject: io_uring: do ctx initiated file note removal Another preparation patch. When full quiesce is done on ctx exit, use task_work infra to remove corresponding to the ctx io_uring->xa entries. For that we use the back tctx map. Also use ->in_idle to prevent removing it while we traversing ->xa on cancellation, just ignore it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 9a2cff0662e0..8a4ab86ae64f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -987,6 +987,7 @@ static const struct io_op_def io_op_defs[] = { [IORING_OP_UNLINKAT] = {}, }; +static void io_uring_del_task_file(unsigned long index); static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, struct files_struct *files); @@ -8536,10 +8537,33 @@ static bool io_run_ctx_fallback(struct io_ring_ctx *ctx) return executed; } +struct io_tctx_exit { + struct callback_head task_work; + struct completion completion; + unsigned long index; +}; + +static void io_tctx_exit_cb(struct callback_head *cb) +{ + struct io_uring_task *tctx = current->io_uring; + struct io_tctx_exit *work; + + work = container_of(cb, struct io_tctx_exit, task_work); + /* + * When @in_idle, we're in cancellation and it's racy to remove the + * node. It'll be removed by the end of cancellation, just ignore it. + */ + if (!atomic_read(&tctx->in_idle)) + io_uring_del_task_file(work->index); + complete(&work->completion); +} + static void io_ring_exit_work(struct work_struct *work) { - struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, - exit_work); + struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work); + struct io_tctx_exit exit; + struct io_tctx_node *node; + int ret; /* * If we're doing polled IO and end up having requests being @@ -8550,6 +8574,26 @@ static void io_ring_exit_work(struct work_struct *work) do { io_uring_try_cancel_requests(ctx, NULL, NULL); } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)); + + mutex_lock(&ctx->uring_lock); + while (!list_empty(&ctx->tctx_list)) { + node = list_first_entry(&ctx->tctx_list, struct io_tctx_node, + ctx_node); + exit.index = (unsigned long)node->file; + init_completion(&exit.completion); + init_task_work(&exit.task_work, io_tctx_exit_cb); + ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL); + if (WARN_ON_ONCE(ret)) + continue; + wake_up_process(node->task); + + mutex_unlock(&ctx->uring_lock); + wait_for_completion(&exit.completion); + cond_resched(); + mutex_lock(&ctx->uring_lock); + } + mutex_unlock(&ctx->uring_lock); + io_ring_ctx_free(ctx); } -- cgit v1.2.3 From eebd2e37e662617a6b8041db75205f0a262ce870 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 6 Mar 2021 11:02:14 +0000 Subject: io_uring: don't take task ring-file notes With ->flush() gone we're now leaving all uring file notes until the task dies/execs, so the ctx will not be freed until all tasks that have ever submit a request die. It was nicer with flush but not much, we could have locked as described ctx in many cases. Now we guarantee that ctx outlives all tctx in a sense that io_ring_exit_work() waits for all tctxs to drop their corresponding enties in ->xa, and ctx won't go away until then. Hence, additional io_uring file reference (a.k.a. task file notes) are not needed anymore. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 8a4ab86ae64f..f448213267c8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8821,11 +8821,9 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) node->file = file; node->task = current; - get_file(file); ret = xa_err(xa_store(&tctx->xa, (unsigned long)file, node, GFP_KERNEL)); if (ret) { - fput(file); kfree(node); return ret; } @@ -8856,6 +8854,8 @@ static void io_uring_del_task_file(unsigned long index) struct io_uring_task *tctx = current->io_uring; struct io_tctx_node *node; + if (!tctx) + return; node = xa_erase(&tctx->xa, index); if (!node) return; @@ -8869,7 +8869,6 @@ static void io_uring_del_task_file(unsigned long index) if (tctx->last == node->file) tctx->last = NULL; - fput(node->file); kfree(node); } -- cgit v1.2.3 From baf186c4d345f5a105e63df01100936ad622f369 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 6 Mar 2021 11:02:15 +0000 Subject: io_uring: index io_uring->xa by ctx not file We don't use task file notes anymore, and no need left in indexing task->io_uring->xa by file, and replace it with ctx. It's better design-wise, especially since we keep a dangling file, and so have to keep an eye on not dereferencing it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 24 +++++++++++------------- include/linux/io_uring.h | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index f448213267c8..01a7fa4a4889 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -809,7 +809,6 @@ struct io_kiocb { struct io_tctx_node { struct list_head ctx_node; struct task_struct *task; - struct file *file; struct io_ring_ctx *ctx; }; @@ -8540,7 +8539,7 @@ static bool io_run_ctx_fallback(struct io_ring_ctx *ctx) struct io_tctx_exit { struct callback_head task_work; struct completion completion; - unsigned long index; + struct io_ring_ctx *ctx; }; static void io_tctx_exit_cb(struct callback_head *cb) @@ -8554,7 +8553,7 @@ static void io_tctx_exit_cb(struct callback_head *cb) * node. It'll be removed by the end of cancellation, just ignore it. */ if (!atomic_read(&tctx->in_idle)) - io_uring_del_task_file(work->index); + io_uring_del_task_file((unsigned long)work->ctx); complete(&work->completion); } @@ -8579,7 +8578,7 @@ static void io_ring_exit_work(struct work_struct *work) while (!list_empty(&ctx->tctx_list)) { node = list_first_entry(&ctx->tctx_list, struct io_tctx_node, ctx_node); - exit.index = (unsigned long)node->file; + exit.ctx = ctx; init_completion(&exit.completion); init_task_work(&exit.task_work, io_tctx_exit_cb); ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL); @@ -8798,7 +8797,7 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, /* * Note that this task has used io_uring. We use it for cancelation purposes. */ -static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) +static int io_uring_add_task_file(struct io_ring_ctx *ctx) { struct io_uring_task *tctx = current->io_uring; struct io_tctx_node *node; @@ -8810,18 +8809,17 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) return ret; tctx = current->io_uring; } - if (tctx->last != file) { - void *old = xa_load(&tctx->xa, (unsigned long)file); + if (tctx->last != ctx) { + void *old = xa_load(&tctx->xa, (unsigned long)ctx); if (!old) { node = kmalloc(sizeof(*node), GFP_KERNEL); if (!node) return -ENOMEM; node->ctx = ctx; - node->file = file; node->task = current; - ret = xa_err(xa_store(&tctx->xa, (unsigned long)file, + ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx, node, GFP_KERNEL)); if (ret) { kfree(node); @@ -8832,7 +8830,7 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx, struct file *file) list_add(&node->ctx_node, &ctx->tctx_list); mutex_unlock(&ctx->uring_lock); } - tctx->last = file; + tctx->last = ctx; } /* @@ -8867,7 +8865,7 @@ static void io_uring_del_task_file(unsigned long index) list_del(&node->ctx_node); mutex_unlock(&node->ctx->uring_lock); - if (tctx->last == node->file) + if (tctx->last == node->ctx) tctx->last = NULL; kfree(node); } @@ -9166,7 +9164,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, } submitted = to_submit; } else if (to_submit) { - ret = io_uring_add_task_file(ctx, f.file); + ret = io_uring_add_task_file(ctx); if (unlikely(ret)) goto out; mutex_lock(&ctx->uring_lock); @@ -9375,7 +9373,7 @@ static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file) if (fd < 0) return fd; - ret = io_uring_add_task_file(ctx, file); + ret = io_uring_add_task_file(ctx); if (ret) { put_unused_fd(fd); return ret; diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 7cb7bd0e334c..9761a0ec9f95 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -18,7 +18,7 @@ struct io_uring_task { /* submission side */ struct xarray xa; struct wait_queue_head wait; - struct file *last; + void *last; void *io_wq; struct percpu_counter inflight; atomic_t in_idle; -- cgit v1.2.3 From b5bb3a24f69da92e0ec2a301452364333e45be03 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 6 Mar 2021 11:02:16 +0000 Subject: io_uring: warn when ring exit takes too long We use system_unbound_wq to run io_ring_exit_work(), so it's hard to monitor whether removal hang or not. Add WARN_ONCE to catch hangs. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 01a7fa4a4889..945e54690b81 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8560,6 +8560,7 @@ static void io_tctx_exit_cb(struct callback_head *cb) static void io_ring_exit_work(struct work_struct *work) { struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work); + unsigned long timeout = jiffies + HZ * 60 * 5; struct io_tctx_exit exit; struct io_tctx_node *node; int ret; @@ -8572,10 +8573,14 @@ static void io_ring_exit_work(struct work_struct *work) */ do { io_uring_try_cancel_requests(ctx, NULL, NULL); + + WARN_ON_ONCE(time_after(jiffies, timeout)); } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)); mutex_lock(&ctx->uring_lock); while (!list_empty(&ctx->tctx_list)) { + WARN_ON_ONCE(time_after(jiffies, timeout)); + node = list_first_entry(&ctx->tctx_list, struct io_tctx_node, ctx_node); exit.ctx = ctx; -- cgit v1.2.3 From 1b00764f09b6912d25e188d972a7764a457926ba Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 6 Mar 2021 11:02:17 +0000 Subject: io_uring: cancel reqs of all iowq's on ring exit io_ring_exit_work() have to cancel all requests, including those staying in io-wq, however it tries only cancellation of current tctx, which is NULL. If we've got task==NULL, use the ctx-to-tctx map to go over all tctx/io-wq and try cancellations on them. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 945e54690b81..8c74c7799960 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8688,19 +8688,55 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx, } } +static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) +{ + struct io_kiocb *req = container_of(work, struct io_kiocb, work); + + return req->ctx == data; +} + +static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) +{ + struct io_tctx_node *node; + enum io_wq_cancel cret; + bool ret = false; + + mutex_lock(&ctx->uring_lock); + list_for_each_entry(node, &ctx->tctx_list, ctx_node) { + struct io_uring_task *tctx = node->task->io_uring; + + /* + * io_wq will stay alive while we hold uring_lock, because it's + * killed after ctx nodes, which requires to take the lock. + */ + if (!tctx || !tctx->io_wq) + continue; + cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true); + ret |= (cret != IO_WQ_CANCEL_NOTFOUND); + } + mutex_unlock(&ctx->uring_lock); + + return ret; +} + static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, struct files_struct *files) { struct io_task_cancel cancel = { .task = task, .files = files, }; - struct task_struct *tctx_task = task ?: current; - struct io_uring_task *tctx = tctx_task->io_uring; + struct io_uring_task *tctx = task ? task->io_uring : NULL; while (1) { enum io_wq_cancel cret; bool ret = false; - if (tctx && tctx->io_wq) { + if (!task) { + ret |= io_uring_try_cancel_iowq(ctx); + } else if (tctx && tctx->io_wq) { + /* + * Cancels requests of all rings, not only @ctx, but + * it's fine as the task is in exit/exec. + */ cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb, &cancel, true); ret |= (cret != IO_WQ_CANCEL_NOTFOUND); -- cgit v1.2.3 From 678eeba481d8c161203382832a4379d507050aed Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 6 Mar 2021 11:02:18 +0000 Subject: io-wq: warn on creating manager while exiting Add a simple warning making sure that nobody tries to create a new manager while we're under IO_WQ_BIT_EXIT. That can potentially happen due to racy work submission after final put. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io-wq.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 1bfdb86336e4..1ab9324e602f 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -774,6 +774,8 @@ static int io_wq_fork_manager(struct io_wq *wq) if (wq->manager) return 0; + WARN_ON_ONCE(test_bit(IO_WQ_BIT_EXIT, &wq->state)); + init_completion(&wq->worker_done); atomic_set(&wq->worker_refs, 1); tsk = create_io_thread(io_wq_manager, wq, NUMA_NO_NODE); -- cgit v1.2.3 From 7c30f36a98ae488741178d69662e4f2baa53e7f6 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 7 Mar 2021 11:54:28 +0100 Subject: io_uring: run __io_sq_thread() with the initial creds from io_uring_setup() With IORING_SETUP_ATTACH_WQ we should let __io_sq_thread() use the initial creds from each ctx. Signed-off-by: Stefan Metzmacher Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 8c74c7799960..4d3333ca27a3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -380,6 +380,7 @@ struct io_ring_ctx { /* Only used for accounting purposes */ struct mm_struct *mm_account; + const struct cred *sq_creds; /* cred used for __io_sq_thread() */ struct io_sq_data *sq_data; /* if using sq thread polling */ struct wait_queue_head sqo_sq_wait; @@ -6719,7 +6720,13 @@ static int io_sq_thread(void *data) sqt_spin = false; cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { + const struct cred *creds = NULL; + + if (ctx->sq_creds != current_cred()) + creds = override_creds(ctx->sq_creds); ret = __io_sq_thread(ctx, cap_entries); + if (creds) + revert_creds(creds); if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) sqt_spin = true; } @@ -7152,6 +7159,8 @@ static void io_sq_thread_finish(struct io_ring_ctx *ctx) io_put_sq_data(sqd); ctx->sq_data = NULL; + if (ctx->sq_creds) + put_cred(ctx->sq_creds); } } @@ -7890,6 +7899,7 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, goto err; } + ctx->sq_creds = get_current_cred(); ctx->sq_data = sqd; io_sq_thread_park(sqd); mutex_lock(&sqd->ctx_lock); -- cgit v1.2.3 From 041474885e9707a38fad081abe30159eb6d463f9 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 7 Mar 2021 11:54:29 +0100 Subject: io_uring: kill io_sq_thread_fork() and return -EOWNERDEAD if the sq_thread is gone This brings the behavior back in line with what 5.11 and earlier did, and this is no longer needed with the improved handling of creds not needing to do unshare(). Signed-off-by: Stefan Metzmacher Signed-off-by: Jens Axboe --- fs/io_uring.c | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 4d3333ca27a3..7cf96be691d8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -336,7 +336,6 @@ struct io_ring_ctx { unsigned int drain_next: 1; unsigned int eventfd_async: 1; unsigned int restricted: 1; - unsigned int sqo_exec: 1; /* * Ring buffer of indices into array of io_uring_sqe, which is @@ -6786,7 +6785,6 @@ static int io_sq_thread(void *data) sqd->thread = NULL; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { - ctx->sqo_exec = 1; io_ring_set_wakeup_flag(ctx); } @@ -7846,26 +7844,6 @@ void __io_uring_free(struct task_struct *tsk) tsk->io_uring = NULL; } -static int io_sq_thread_fork(struct io_sq_data *sqd, struct io_ring_ctx *ctx) -{ - struct task_struct *tsk; - int ret; - - clear_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); - reinit_completion(&sqd->parked); - ctx->sqo_exec = 0; - sqd->task_pid = current->pid; - tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE); - if (IS_ERR(tsk)) - return PTR_ERR(tsk); - ret = io_uring_alloc_task_context(tsk, ctx); - if (ret) - set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); - sqd->thread = tsk; - wake_up_new_task(tsk); - return ret; -} - static int io_sq_offload_create(struct io_ring_ctx *ctx, struct io_uring_params *p) { @@ -9199,13 +9177,10 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, if (ctx->flags & IORING_SETUP_SQPOLL) { io_cqring_overflow_flush(ctx, false, NULL, NULL); - if (unlikely(ctx->sqo_exec)) { - ret = io_sq_thread_fork(ctx->sq_data, ctx); - if (ret) - goto out; - ctx->sqo_exec = 0; - } ret = -EOWNERDEAD; + if (unlikely(ctx->sq_data->thread == NULL)) { + goto out; + } if (flags & IORING_ENTER_SQ_WAKEUP) wake_up(&ctx->sq_data->wait); if (flags & IORING_ENTER_SQ_WAIT) { -- cgit v1.2.3 From 9f377622a484de0818c49ee01e0ab4eedf6acd81 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Sat, 6 Mar 2021 12:04:38 +0800 Subject: erofs: fix bio->bi_max_vecs behavior change Martin reported an issue that directory read could be hung on the latest -rc kernel with some certain image. The root cause is that commit baa2c7c97153 ("block: set .bi_max_vecs as actual allocated vector number") changes .bi_max_vecs behavior. bio->bi_max_vecs is set as actual allocated vector number rather than the requested number now. Let's avoid using .bi_max_vecs completely instead. Link: https://lore.kernel.org/r/20210306040438.8084-1-hsiangkao@aol.com Reported-by: Martin DEVERA Reviewed-by: Chao Yu [ Gao Xiang: note that <= 5.11 kernels are not impacted. ] Signed-off-by: Gao Xiang --- fs/erofs/data.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/erofs/data.c b/fs/erofs/data.c index f88851c5c250..1249e74b3bf0 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -129,6 +129,7 @@ static inline struct bio *erofs_read_raw_page(struct bio *bio, struct page *page, erofs_off_t *last_block, unsigned int nblocks, + unsigned int *eblks, bool ra) { struct inode *const inode = mapping->host; @@ -145,8 +146,7 @@ static inline struct bio *erofs_read_raw_page(struct bio *bio, /* note that for readpage case, bio also equals to NULL */ if (bio && - /* not continuous */ - *last_block + 1 != current_block) { + (*last_block + 1 != current_block || !*eblks)) { submit_bio_retry: submit_bio(bio); bio = NULL; @@ -216,7 +216,8 @@ submit_bio_retry: if (nblocks > DIV_ROUND_UP(map.m_plen, PAGE_SIZE)) nblocks = DIV_ROUND_UP(map.m_plen, PAGE_SIZE); - bio = bio_alloc(GFP_NOIO, bio_max_segs(nblocks)); + *eblks = bio_max_segs(nblocks); + bio = bio_alloc(GFP_NOIO, *eblks); bio->bi_end_io = erofs_readendio; bio_set_dev(bio, sb->s_bdev); @@ -229,16 +230,8 @@ submit_bio_retry: /* out of the extent or bio is full */ if (err < PAGE_SIZE) goto submit_bio_retry; - + --*eblks; *last_block = current_block; - - /* shift in advance in case of it followed by too many gaps */ - if (bio->bi_iter.bi_size >= bio->bi_max_vecs * PAGE_SIZE) { - /* err should reassign to 0 after submitting */ - err = 0; - goto submit_bio_out; - } - return bio; err_out: @@ -252,7 +245,6 @@ has_updated: /* if updated manually, continuous pages has a gap */ if (bio) -submit_bio_out: submit_bio(bio); return err ? ERR_PTR(err) : NULL; } @@ -264,23 +256,26 @@ submit_bio_out: static int erofs_raw_access_readpage(struct file *file, struct page *page) { erofs_off_t last_block; + unsigned int eblks; struct bio *bio; trace_erofs_readpage(page, true); bio = erofs_read_raw_page(NULL, page->mapping, - page, &last_block, 1, false); + page, &last_block, 1, &eblks, false); if (IS_ERR(bio)) return PTR_ERR(bio); - DBG_BUGON(bio); /* since we have only one bio -- must be NULL */ + if (bio) + submit_bio(bio); return 0; } static void erofs_raw_access_readahead(struct readahead_control *rac) { erofs_off_t last_block; + unsigned int eblks; struct bio *bio = NULL; struct page *page; @@ -291,7 +286,7 @@ static void erofs_raw_access_readahead(struct readahead_control *rac) prefetchw(&page->flags); bio = erofs_read_raw_page(bio, rac->mapping, page, &last_block, - readahead_count(rac), true); + readahead_count(rac), &eblks, true); /* all the page errors are ignored when readahead */ if (IS_ERR(bio)) { @@ -305,7 +300,6 @@ static void erofs_raw_access_readahead(struct readahead_control *rac) put_page(page); } - /* the rare case (end in gaps) */ if (bio) submit_bio(bio); } -- cgit v1.2.3 From d0ed78e1780eb3738f9c106fbaff6a1181017cd3 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 1 Mar 2021 21:02:49 +0100 Subject: s390,alpha: make TMPFS_INODE64 available again Both s390 and alpha have been switched to 64-bit ino_t with commit 96c0a6a72d18 ("s390,alpha: switch to 64-bit ino_t"). Therefore enable TMPFS_INODE64 for both architectures again. Cc: Richard Henderson Cc: Ivan Kokshaysky Link: https://lore.kernel.org/linux-mm/YCV7QiyoweJwvN+m@osiris/ Signed-off-by: Heiko Carstens --- fs/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/Kconfig b/fs/Kconfig index 462253ae483a..a55bda4233bb 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -203,7 +203,7 @@ config TMPFS_XATTR config TMPFS_INODE64 bool "Use 64-bit ino_t by default in tmpfs" - depends on TMPFS && 64BIT && !(S390 || ALPHA) + depends on TMPFS && 64BIT default n help tmpfs has historically used only inode numbers as wide as an unsigned -- cgit v1.2.3 From ee2e3f50629f17b0752b55b2566c15ce8dafb557 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Sat, 6 Mar 2021 11:10:10 +0100 Subject: mount: fix mounting of detached mounts onto targets that reside on shared mounts Creating a series of detached mounts, attaching them to the filesystem, and unmounting them can be used to trigger an integer overflow in ns->mounts causing the kernel to block any new mounts in count_mounts() and returning ENOSPC because it falsely assumes that the maximum number of mounts in the mount namespace has been reached, i.e. it thinks it can't fit the new mounts into the mount namespace anymore. Depending on the number of mounts in your system, this can be reproduced on any kernel that supportes open_tree() and move_mount() by compiling and running the following program: /* SPDX-License-Identifier: LGPL-2.1+ */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include #include #include /* open_tree() */ #ifndef OPEN_TREE_CLONE #define OPEN_TREE_CLONE 1 #endif #ifndef OPEN_TREE_CLOEXEC #define OPEN_TREE_CLOEXEC O_CLOEXEC #endif #ifndef __NR_open_tree #if defined __alpha__ #define __NR_open_tree 538 #elif defined _MIPS_SIM #if _MIPS_SIM == _MIPS_SIM_ABI32 /* o32 */ #define __NR_open_tree 4428 #endif #if _MIPS_SIM == _MIPS_SIM_NABI32 /* n32 */ #define __NR_open_tree 6428 #endif #if _MIPS_SIM == _MIPS_SIM_ABI64 /* n64 */ #define __NR_open_tree 5428 #endif #elif defined __ia64__ #define __NR_open_tree (428 + 1024) #else #define __NR_open_tree 428 #endif #endif /* move_mount() */ #ifndef MOVE_MOUNT_F_EMPTY_PATH #define MOVE_MOUNT_F_EMPTY_PATH 0x00000004 /* Empty from path permitted */ #endif #ifndef __NR_move_mount #if defined __alpha__ #define __NR_move_mount 539 #elif defined _MIPS_SIM #if _MIPS_SIM == _MIPS_SIM_ABI32 /* o32 */ #define __NR_move_mount 4429 #endif #if _MIPS_SIM == _MIPS_SIM_NABI32 /* n32 */ #define __NR_move_mount 6429 #endif #if _MIPS_SIM == _MIPS_SIM_ABI64 /* n64 */ #define __NR_move_mount 5429 #endif #elif defined __ia64__ #define __NR_move_mount (428 + 1024) #else #define __NR_move_mount 429 #endif #endif static inline int sys_open_tree(int dfd, const char *filename, unsigned int flags) { return syscall(__NR_open_tree, dfd, filename, flags); } static inline int sys_move_mount(int from_dfd, const char *from_pathname, int to_dfd, const char *to_pathname, unsigned int flags) { return syscall(__NR_move_mount, from_dfd, from_pathname, to_dfd, to_pathname, flags); } static bool is_shared_mountpoint(const char *path) { bool shared = false; FILE *f = NULL; char *line = NULL; int i; size_t len = 0; f = fopen("/proc/self/mountinfo", "re"); if (!f) return 0; while (getline(&line, &len, f) > 0) { char *slider1, *slider2; for (slider1 = line, i = 0; slider1 && i < 4; i++) slider1 = strchr(slider1 + 1, ' '); if (!slider1) continue; slider2 = strchr(slider1 + 1, ' '); if (!slider2) continue; *slider2 = '\0'; if (strcmp(slider1 + 1, path) == 0) { /* This is the path. Is it shared? */ slider1 = strchr(slider2 + 1, ' '); if (slider1 && strstr(slider1, "shared:")) { shared = true; break; } } } fclose(f); free(line); return shared; } static void usage(void) { const char *text = "mount-new [--recursive] \n"; fprintf(stderr, "%s", text); _exit(EXIT_SUCCESS); } #define exit_usage(format, ...) \ ({ \ fprintf(stderr, format "\n", ##__VA_ARGS__); \ usage(); \ }) #define exit_log(format, ...) \ ({ \ fprintf(stderr, format "\n", ##__VA_ARGS__); \ exit(EXIT_FAILURE); \ }) static const struct option longopts[] = { {"help", no_argument, 0, 'a'}, { NULL, no_argument, 0, 0 }, }; int main(int argc, char *argv[]) { int exit_code = EXIT_SUCCESS, index = 0; int dfd, fd_tree, new_argc, ret; char *base_dir; char *const *new_argv; char target[PATH_MAX]; while ((ret = getopt_long_only(argc, argv, "", longopts, &index)) != -1) { switch (ret) { case 'a': /* fallthrough */ default: usage(); } } new_argv = &argv[optind]; new_argc = argc - optind; if (new_argc < 1) exit_usage("Missing base directory\n"); base_dir = new_argv[0]; if (*base_dir != '/') exit_log("Please specify an absolute path"); /* Ensure that target is a shared mountpoint. */ if (!is_shared_mountpoint(base_dir)) exit_log("Please ensure that \"%s\" is a shared mountpoint", base_dir); dfd = open(base_dir, O_RDONLY | O_DIRECTORY | O_CLOEXEC); if (dfd < 0) exit_log("%m - Failed to open base directory \"%s\"", base_dir); ret = mkdirat(dfd, "detached-move-mount", 0755); if (ret < 0) exit_log("%m - Failed to create required temporary directories"); ret = snprintf(target, sizeof(target), "%s/detached-move-mount", base_dir); if (ret < 0 || (size_t)ret >= sizeof(target)) exit_log("%m - Failed to assemble target path"); /* * Having a mount table with 10000 mounts is already quite excessive * and shoult account even for weird test systems. */ for (size_t i = 0; i < 10000; i++) { fd_tree = sys_open_tree(dfd, "detached-move-mount", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC | AT_EMPTY_PATH); if (fd_tree < 0) { fprintf(stderr, "%m - Failed to open %d(detached-move-mount)", dfd); exit_code = EXIT_FAILURE; break; } ret = sys_move_mount(fd_tree, "", dfd, "detached-move-mount", MOVE_MOUNT_F_EMPTY_PATH); if (ret < 0) { if (errno == ENOSPC) fprintf(stderr, "%m - Buggy mount counting"); else fprintf(stderr, "%m - Failed to attach mount to %d(detached-move-mount)", dfd); exit_code = EXIT_FAILURE; break; } close(fd_tree); ret = umount2(target, MNT_DETACH); if (ret < 0) { fprintf(stderr, "%m - Failed to unmount %s", target); exit_code = EXIT_FAILURE; break; } } (void)unlinkat(dfd, "detached-move-mount", AT_REMOVEDIR); close(dfd); exit(exit_code); } and wait for the kernel to refuse any new mounts by returning ENOSPC. How many iterations are needed depends on the number of mounts in your system. Assuming you have something like 50 mounts on a standard system it should be almost instantaneous. The root cause of this is that detached mounts aren't handled correctly when source and target mount are identical and reside on a shared mount causing a broken mount tree where the detached source itself is propagated which propagation prevents for regular bind-mounts and new mounts. This ultimately leads to a miscalculation of the number of mounts in the mount namespace. Detached mounts created via open_tree(fd, path, OPEN_TREE_CLONE) are essentially like an unattached new mount, or an unattached bind-mount. They can then later on be attached to the filesystem via move_mount() which calls into attach_recursive_mount(). Part of attaching it to the filesystem is making sure that mounts get correctly propagated in case the destination mountpoint is MS_SHARED, i.e. is a shared mountpoint. This is done by calling into propagate_mnt() which walks the list of peers calling propagate_one() on each mount in this list making sure it receives the propagation event. The propagate_one() functions thereby skips both new mounts and bind mounts to not propagate them "into themselves". Both are identified by checking whether the mount is already attached to any mount namespace in mnt->mnt_ns. The is what the IS_MNT_NEW() helper is responsible for. However, detached mounts have an anonymous mount namespace attached to them stashed in mnt->mnt_ns which means that IS_MNT_NEW() doesn't realize they need to be skipped causing the mount to propagate "into itself" breaking the mount table and causing a disconnect between the number of mounts recorded as being beneath or reachable from the target mountpoint and the number of mounts actually recorded/counted in ns->mounts ultimately causing an overflow which in turn prevents any new mounts via the ENOSPC issue. So teach propagation to handle detached mounts by making it aware of them. I've been tracking this issue down for the last couple of days and then verifying that the fix is correct by unmounting everything in my current mount table leaving only /proc and /sys mounted and running the reproducer above overnight verifying the number of mounts counted in ns->mounts. With this fix the counts are correct and the ENOSPC issue can't be reproduced. This change will only have an effect on mounts created with the new mount API since detached mounts cannot be created with the old mount API so regressions are extremely unlikely. Link: https://lore.kernel.org/r/20210306101010.243666-1-christian.brauner@ubuntu.com Fixes: 2db154b3ea8e ("vfs: syscall: Add move_mount(2) to move mounts around") Cc: David Howells Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Cc: Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner --- fs/pnode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/pnode.h b/fs/pnode.h index 26f74e092bd9..988f1aa9b02a 100644 --- a/fs/pnode.h +++ b/fs/pnode.h @@ -12,7 +12,7 @@ #define IS_MNT_SHARED(m) ((m)->mnt.mnt_flags & MNT_SHARED) #define IS_MNT_SLAVE(m) ((m)->mnt_master) -#define IS_MNT_NEW(m) (!(m)->mnt_ns) +#define IS_MNT_NEW(m) (!(m)->mnt_ns || is_anon_ns((m)->mnt_ns)) #define CLEAR_MNT_SHARED(m) ((m)->mnt.mnt_flags &= ~MNT_SHARED) #define IS_MNT_UNBINDABLE(m) ((m)->mnt.mnt_flags & MNT_UNBINDABLE) #define IS_MNT_MARKED(m) ((m)->mnt.mnt_flags & MNT_MARKED) -- cgit v1.2.3 From a0590473c5e6c4ef17c3132ad08fbad170f72d55 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler Date: Tue, 23 Feb 2021 15:19:01 +0100 Subject: nfs: fix PNFS_FLEXFILE_LAYOUT Kconfig default This follows what was done in 8c2fabc6542d9d0f8b16bd1045c2eda59bdcde13. With the default being m, it's impossible to build the module into the kernel. Signed-off-by: Timo Rothenpieler Signed-off-by: Anna Schumaker --- fs/nfs/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig index e2a488d403a6..14a72224b657 100644 --- a/fs/nfs/Kconfig +++ b/fs/nfs/Kconfig @@ -127,7 +127,7 @@ config PNFS_BLOCK config PNFS_FLEXFILE_LAYOUT tristate depends on NFS_V4_1 && NFS_V3 - default m + default NFS_V4 config NFS_V4_1_IMPLEMENTATION_ID_DOMAIN string "NFSv4.1 Implementation ID Domain" -- cgit v1.2.3 From ad3dbe35c833c2d4d0bbf3f04c785d32f931e7c9 Mon Sep 17 00:00:00 2001 From: Frank Sorenson Date: Mon, 8 Mar 2021 12:12:13 -0600 Subject: NFS: Correct size calculation for create reply length CREATE requests return a post_op_fh3, rather than nfs_fh3. The post_op_fh3 includes an extra word to indicate 'handle_follows'. Without that additional word, create fails when full 64-byte filehandles are in use. Add NFS3_post_op_fh_sz, and correct the size calculation for NFS3_createres_sz. Signed-off-by: Frank Sorenson Signed-off-by: Anna Schumaker --- fs/nfs/nfs3xdr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/nfs/nfs3xdr.c b/fs/nfs/nfs3xdr.c index ca10072644ff..ed1c83738c30 100644 --- a/fs/nfs/nfs3xdr.c +++ b/fs/nfs/nfs3xdr.c @@ -36,6 +36,7 @@ #define NFS3_pagepad_sz (1) /* Page padding */ #define NFS3_fhandle_sz (1+16) #define NFS3_fh_sz (NFS3_fhandle_sz) /* shorthand */ +#define NFS3_post_op_fh_sz (1+NFS3_fh_sz) #define NFS3_sattr_sz (15) #define NFS3_filename_sz (1+(NFS3_MAXNAMLEN>>2)) #define NFS3_path_sz (1+(NFS3_MAXPATHLEN>>2)) @@ -73,7 +74,7 @@ #define NFS3_readlinkres_sz (1+NFS3_post_op_attr_sz+1+NFS3_pagepad_sz) #define NFS3_readres_sz (1+NFS3_post_op_attr_sz+3+NFS3_pagepad_sz) #define NFS3_writeres_sz (1+NFS3_wcc_data_sz+4) -#define NFS3_createres_sz (1+NFS3_fh_sz+NFS3_post_op_attr_sz+NFS3_wcc_data_sz) +#define NFS3_createres_sz (1+NFS3_post_op_fh_sz+NFS3_post_op_attr_sz+NFS3_wcc_data_sz) #define NFS3_renameres_sz (1+(2 * NFS3_wcc_data_sz)) #define NFS3_linkres_sz (1+NFS3_post_op_attr_sz+NFS3_wcc_data_sz) #define NFS3_readdirres_sz (1+NFS3_post_op_attr_sz+2+NFS3_pagepad_sz) -- cgit v1.2.3 From 82e7ca1334ab16e2e04fafded1cab9dfcdc11b40 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 8 Mar 2021 14:42:51 -0500 Subject: NFS: Don't revalidate the directory permissions on a lookup failure There should be no reason to expect the directory permissions to change just because the directory contents changed or a negative lookup timed out. So let's avoid doing a full call to nfs_mark_for_revalidate() in that case. Furthermore, if this is a negative dentry, and we haven't actually done a new lookup, then we have no reason yet to believe the directory has changed at all. So let's remove the gratuitous directory inode invalidation altogether when called from nfs_lookup_revalidate_negative(). Reported-by: Geert Jansen Fixes: 5ceb9d7fdaaf ("NFS: Refactor nfs_lookup_revalidate()") Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/dir.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 19a9f434442f..08b162de627f 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -1401,6 +1401,15 @@ out_force: goto out; } +static void nfs_mark_dir_for_revalidate(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + + spin_lock(&inode->i_lock); + nfsi->cache_validity |= NFS_INO_REVAL_PAGECACHE; + spin_unlock(&inode->i_lock); +} + /* * We judge how long we want to trust negative * dentries by looking at the parent inode mtime. @@ -1435,7 +1444,6 @@ nfs_lookup_revalidate_done(struct inode *dir, struct dentry *dentry, __func__, dentry); return 1; case 0: - nfs_mark_for_revalidate(dir); if (inode && S_ISDIR(inode->i_mode)) { /* Purge readdir caches. */ nfs_zap_caches(inode); @@ -1525,6 +1533,13 @@ out: nfs_free_fattr(fattr); nfs_free_fhandle(fhandle); nfs4_label_free(label); + + /* + * If the lookup failed despite the dentry change attribute being + * a match, then we should revalidate the directory cache. + */ + if (!ret && nfs_verify_change_attribute(dir, dentry->d_time)) + nfs_mark_dir_for_revalidate(dir); return nfs_lookup_revalidate_done(dir, dentry, inode, ret); } @@ -1567,7 +1582,7 @@ nfs_do_lookup_revalidate(struct inode *dir, struct dentry *dentry, error = nfs_lookup_verify_inode(inode, flags); if (error) { if (error == -ESTALE) - nfs_zap_caches(dir); + nfs_mark_dir_for_revalidate(dir); goto out_bad; } nfs_advise_use_readdirplus(dir); @@ -2064,7 +2079,6 @@ out: dput(parent); return d; out_error: - nfs_mark_for_revalidate(dir); d = ERR_PTR(error); goto out; } -- cgit v1.2.3 From 47397915ede0192235474b145ebcd81b37b03624 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 8 Mar 2021 14:42:52 -0500 Subject: NFS: Don't gratuitously clear the inode cache when lookup failed The fact that the lookup revalidation failed, does not mean that the inode contents have changed. Fixes: 5ceb9d7fdaaf ("NFS: Refactor nfs_lookup_revalidate()") Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/dir.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 08b162de627f..a91f324cca49 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -1444,18 +1444,14 @@ nfs_lookup_revalidate_done(struct inode *dir, struct dentry *dentry, __func__, dentry); return 1; case 0: - if (inode && S_ISDIR(inode->i_mode)) { - /* Purge readdir caches. */ - nfs_zap_caches(inode); - /* - * We can't d_drop the root of a disconnected tree: - * its d_hash is on the s_anon list and d_drop() would hide - * it from shrink_dcache_for_unmount(), leading to busy - * inodes on unmount and further oopses. - */ - if (IS_ROOT(dentry)) - return 1; - } + /* + * We can't d_drop the root of a disconnected tree: + * its d_hash is on the s_anon list and d_drop() would hide + * it from shrink_dcache_for_unmount(), leading to busy + * inodes on unmount and further oopses. + */ + if (inode && IS_ROOT(dentry)) + return 1; dfprintk(LOOKUPCACHE, "NFS: %s(%pd2) is invalid\n", __func__, dentry); return 0; -- cgit v1.2.3 From fd6d3feed041e96b84680d0bfc1e7abc8f65de92 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 8 Mar 2021 14:42:53 -0500 Subject: NFS: Clean up function nfs_mark_dir_for_revalidate() Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/dir.c | 4 +--- fs/nfs/inode.c | 2 +- fs/nfs/internal.h | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index a91f324cca49..02ac982846f4 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -1403,10 +1403,8 @@ out_force: static void nfs_mark_dir_for_revalidate(struct inode *inode) { - struct nfs_inode *nfsi = NFS_I(inode); - spin_lock(&inode->i_lock); - nfsi->cache_validity |= NFS_INO_REVAL_PAGECACHE; + nfs_set_cache_invalid(inode, NFS_INO_REVAL_PAGECACHE); spin_unlock(&inode->i_lock); } diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 749bbea14d99..d21bfaac10b0 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -207,7 +207,7 @@ static bool nfs_has_xattr_cache(const struct nfs_inode *nfsi) } #endif -static void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) +void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) { struct nfs_inode *nfsi = NFS_I(inode); bool have_delegation = NFS_PROTO(inode)->have_delegation(inode, FMODE_READ); diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 25fb43b69e5a..7b644d6c09e4 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -411,7 +411,8 @@ extern int nfs_write_inode(struct inode *, struct writeback_control *); extern int nfs_drop_inode(struct inode *); extern void nfs_clear_inode(struct inode *); extern void nfs_evict_inode(struct inode *); -void nfs_zap_acl_cache(struct inode *inode); +extern void nfs_zap_acl_cache(struct inode *inode); +extern void nfs_set_cache_invalid(struct inode *inode, unsigned long flags); extern bool nfs_check_cache_invalid(struct inode *, unsigned long); extern int nfs_wait_bit_killable(struct wait_bit_key *key, int mode); extern int nfs_wait_atomic_killable(atomic_t *p, unsigned int mode); -- cgit v1.2.3 From ac46b3d768e4c2754f7b191b81e1bea582e11907 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 8 Mar 2021 14:42:54 -0500 Subject: NFS: Fix open coded versions of nfs_set_cache_invalid() nfs_set_cache_invalid() has code to handle delegations, and other optimisations, so let's use it when appropriate. Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/dir.c | 20 ++++++++++---------- fs/nfs/inode.c | 4 ++-- fs/nfs/unlink.c | 6 +++--- fs/nfs/write.c | 8 ++++---- 4 files changed, 19 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 02ac982846f4..fc4f490f2d78 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -81,8 +81,9 @@ static struct nfs_open_dir_context *alloc_nfs_open_dir_context(struct inode *dir spin_lock(&dir->i_lock); if (list_empty(&nfsi->open_files) && (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER)) - nfsi->cache_validity |= NFS_INO_INVALID_DATA | - NFS_INO_REVAL_FORCED; + nfs_set_cache_invalid(dir, + NFS_INO_INVALID_DATA | + NFS_INO_REVAL_FORCED); list_add(&ctx->list, &nfsi->open_files); spin_unlock(&dir->i_lock); return ctx; @@ -1700,10 +1701,9 @@ static void nfs_drop_nlink(struct inode *inode) if (inode->i_nlink > 0) drop_nlink(inode); NFS_I(inode)->attr_gencount = nfs_inc_attr_generation_counter(); - NFS_I(inode)->cache_validity |= NFS_INO_INVALID_CHANGE - | NFS_INO_INVALID_CTIME - | NFS_INO_INVALID_OTHER - | NFS_INO_REVAL_FORCED; + nfs_set_cache_invalid( + inode, NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_CTIME | + NFS_INO_INVALID_OTHER | NFS_INO_REVAL_FORCED); spin_unlock(&inode->i_lock); } @@ -1715,7 +1715,7 @@ static void nfs_dentry_iput(struct dentry *dentry, struct inode *inode) { if (S_ISDIR(inode->i_mode)) /* drop any readdir cache as it could easily be old */ - NFS_I(inode)->cache_validity |= NFS_INO_INVALID_DATA; + nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA); if (dentry->d_flags & DCACHE_NFSFS_RENAMED) { nfs_complete_unlink(dentry, inode); @@ -2481,9 +2481,9 @@ int nfs_rename(struct user_namespace *mnt_userns, struct inode *old_dir, if (error == 0) { spin_lock(&old_inode->i_lock); NFS_I(old_inode)->attr_gencount = nfs_inc_attr_generation_counter(); - NFS_I(old_inode)->cache_validity |= NFS_INO_INVALID_CHANGE - | NFS_INO_INVALID_CTIME - | NFS_INO_REVAL_FORCED; + nfs_set_cache_invalid(old_inode, NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME | + NFS_INO_REVAL_FORCED); spin_unlock(&old_inode->i_lock); } out: diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index d21bfaac10b0..eb1ae77f411a 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -1067,8 +1067,8 @@ void nfs_inode_attach_open_context(struct nfs_open_context *ctx) spin_lock(&inode->i_lock); if (list_empty(&nfsi->open_files) && (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER)) - nfsi->cache_validity |= NFS_INO_INVALID_DATA | - NFS_INO_REVAL_FORCED; + nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA | + NFS_INO_REVAL_FORCED); list_add_tail_rcu(&ctx->list, &nfsi->open_files); spin_unlock(&inode->i_lock); } diff --git a/fs/nfs/unlink.c b/fs/nfs/unlink.c index b27ebdccef70..5fa11e1aca4c 100644 --- a/fs/nfs/unlink.c +++ b/fs/nfs/unlink.c @@ -500,9 +500,9 @@ nfs_sillyrename(struct inode *dir, struct dentry *dentry) nfs_set_verifier(dentry, nfs_save_change_attribute(dir)); spin_lock(&inode->i_lock); NFS_I(inode)->attr_gencount = nfs_inc_attr_generation_counter(); - NFS_I(inode)->cache_validity |= NFS_INO_INVALID_CHANGE - | NFS_INO_INVALID_CTIME - | NFS_INO_REVAL_FORCED; + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME | + NFS_INO_REVAL_FORCED); spin_unlock(&inode->i_lock); d_move(dentry, sdentry); break; diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 82bdcb982186..f05a90338a76 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -303,9 +303,9 @@ static void nfs_set_pageerror(struct address_space *mapping) nfs_zap_mapping(mapping->host, mapping); /* Force file size revalidation */ spin_lock(&inode->i_lock); - NFS_I(inode)->cache_validity |= NFS_INO_REVAL_FORCED | - NFS_INO_REVAL_PAGECACHE | - NFS_INO_INVALID_SIZE; + nfs_set_cache_invalid(inode, NFS_INO_REVAL_FORCED | + NFS_INO_REVAL_PAGECACHE | + NFS_INO_INVALID_SIZE); spin_unlock(&inode->i_lock); } @@ -1604,7 +1604,7 @@ static int nfs_writeback_done(struct rpc_task *task, /* Deal with the suid/sgid bit corner case */ if (nfs_should_remove_suid(inode)) { spin_lock(&inode->i_lock); - NFS_I(inode)->cache_validity |= NFS_INO_INVALID_OTHER; + nfs_set_cache_invalid(inode, NFS_INO_INVALID_OTHER); spin_unlock(&inode->i_lock); } return 0; -- cgit v1.2.3 From b6f80a2ebb97f184c4679518ac83074598bf9bf4 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 8 Mar 2021 14:42:55 -0500 Subject: NFS: Fix open coded versions of nfs_set_cache_invalid() in NFSv4 nfs_set_cache_invalid() has code to handle delegations, and other optimisations, so let's use it when appropriate. Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/inode.c | 1 + fs/nfs/nfs42proc.c | 12 +++++++----- fs/nfs/nfs4proc.c | 28 ++++++++++++---------------- 3 files changed, 20 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index eb1ae77f411a..a7fb076a5f44 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -229,6 +229,7 @@ void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) if (flags & NFS_INO_INVALID_DATA) nfs_fscache_invalidate(inode); } +EXPORT_SYMBOL_GPL(nfs_set_cache_invalid); /* * Invalidate the local caches diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index f3fd935620fc..094024b0aca1 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -357,13 +357,15 @@ static ssize_t _nfs42_proc_copy(struct file *src, truncate_pagecache_range(dst_inode, pos_dst, pos_dst + res->write_res.count); spin_lock(&dst_inode->i_lock); - NFS_I(dst_inode)->cache_validity |= (NFS_INO_REVAL_PAGECACHE | - NFS_INO_REVAL_FORCED | NFS_INO_INVALID_SIZE | - NFS_INO_INVALID_ATTR | NFS_INO_INVALID_DATA); + nfs_set_cache_invalid( + dst_inode, NFS_INO_REVAL_PAGECACHE | NFS_INO_REVAL_FORCED | + NFS_INO_INVALID_SIZE | NFS_INO_INVALID_ATTR | + NFS_INO_INVALID_DATA); spin_unlock(&dst_inode->i_lock); spin_lock(&src_inode->i_lock); - NFS_I(src_inode)->cache_validity |= (NFS_INO_REVAL_PAGECACHE | - NFS_INO_REVAL_FORCED | NFS_INO_INVALID_ATIME); + nfs_set_cache_invalid(src_inode, NFS_INO_REVAL_PAGECACHE | + NFS_INO_REVAL_FORCED | + NFS_INO_INVALID_ATIME); spin_unlock(&src_inode->i_lock); status = res->write_res.count; out: diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 74bc5120013d..6d8fc56e5f45 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -1169,14 +1169,14 @@ int nfs4_call_sync(struct rpc_clnt *clnt, static void nfs4_inc_nlink_locked(struct inode *inode) { - NFS_I(inode)->cache_validity |= NFS_INO_INVALID_OTHER; + nfs_set_cache_invalid(inode, NFS_INO_INVALID_OTHER); inc_nlink(inode); } static void nfs4_dec_nlink_locked(struct inode *inode) { - NFS_I(inode)->cache_validity |= NFS_INO_INVALID_OTHER; + nfs_set_cache_invalid(inode, NFS_INO_INVALID_OTHER); drop_nlink(inode); } @@ -1187,35 +1187,31 @@ nfs4_update_changeattr_locked(struct inode *inode, { struct nfs_inode *nfsi = NFS_I(inode); - nfsi->cache_validity |= NFS_INO_INVALID_CTIME - | NFS_INO_INVALID_MTIME - | cache_validity; + cache_validity |= NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME; if (cinfo->atomic && cinfo->before == inode_peek_iversion_raw(inode)) { nfsi->cache_validity &= ~NFS_INO_REVAL_PAGECACHE; nfsi->attrtimeo_timestamp = jiffies; } else { if (S_ISDIR(inode->i_mode)) { - nfsi->cache_validity |= NFS_INO_INVALID_DATA; + cache_validity |= NFS_INO_INVALID_DATA; nfs_force_lookup_revalidate(inode); } else { if (!NFS_PROTO(inode)->have_delegation(inode, FMODE_READ)) - nfsi->cache_validity |= NFS_INO_REVAL_PAGECACHE; + cache_validity |= NFS_INO_REVAL_PAGECACHE; } if (cinfo->before != inode_peek_iversion_raw(inode)) - nfsi->cache_validity |= NFS_INO_INVALID_ACCESS | - NFS_INO_INVALID_ACL | - NFS_INO_INVALID_XATTR; + cache_validity |= NFS_INO_INVALID_ACCESS | + NFS_INO_INVALID_ACL | + NFS_INO_INVALID_XATTR; } inode_set_iversion_raw(inode, cinfo->after); nfsi->read_cache_jiffies = timestamp; nfsi->attr_gencount = nfs_inc_attr_generation_counter(); + nfs_set_cache_invalid(inode, cache_validity); nfsi->cache_validity &= ~NFS_INO_INVALID_CHANGE; - - if (nfsi->cache_validity & NFS_INO_INVALID_DATA) - nfs_fscache_invalidate(inode); } void @@ -5915,9 +5911,9 @@ static int __nfs4_proc_set_acl(struct inode *inode, const void *buf, size_t bufl * so mark the attribute cache invalid. */ spin_lock(&inode->i_lock); - NFS_I(inode)->cache_validity |= NFS_INO_INVALID_CHANGE - | NFS_INO_INVALID_CTIME - | NFS_INO_REVAL_FORCED; + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME | + NFS_INO_REVAL_FORCED); spin_unlock(&inode->i_lock); nfs_access_zap_cache(inode); nfs_zap_acl_cache(inode); -- cgit v1.2.3 From bf1bc694b6b0cf49756cb06f8f38501b9b2c5527 Mon Sep 17 00:00:00 2001 From: Paulo Alcantara Date: Mon, 8 Mar 2021 12:00:47 -0300 Subject: cifs: print MIDs in decimal notation The MIDs are mostly printed as decimal, so let's make it consistent. Signed-off-by: Paulo Alcantara (SUSE) Reviewed-by: Aurelien Aptel Reviewed-by: Ronnie Sahlberg Signed-off-by: Steve French --- fs/cifs/cifs_debug.c | 2 +- fs/cifs/connect.c | 4 ++-- fs/cifs/smb2misc.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c index 3aedc484e440..88a7958170ee 100644 --- a/fs/cifs/cifs_debug.c +++ b/fs/cifs/cifs_debug.c @@ -207,7 +207,7 @@ static int cifs_debug_files_proc_show(struct seq_file *m, void *v) from_kuid(&init_user_ns, cfile->uid), cfile->dentry); #ifdef CONFIG_CIFS_DEBUG2 - seq_printf(m, " 0x%llx\n", cfile->fid.mid); + seq_printf(m, " %llu\n", cfile->fid.mid); #else seq_printf(m, "\n"); #endif /* CIFS_DEBUG2 */ diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 68642e3d4270..eec8a2052da2 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -741,7 +741,7 @@ static void clean_demultiplex_info(struct TCP_Server_Info *server) spin_lock(&GlobalMid_Lock); list_for_each_safe(tmp, tmp2, &server->pending_mid_q) { mid_entry = list_entry(tmp, struct mid_q_entry, qhead); - cifs_dbg(FYI, "Clearing mid 0x%llx\n", mid_entry->mid); + cifs_dbg(FYI, "Clearing mid %llu\n", mid_entry->mid); kref_get(&mid_entry->refcount); mid_entry->mid_state = MID_SHUTDOWN; list_move(&mid_entry->qhead, &dispose_list); @@ -752,7 +752,7 @@ static void clean_demultiplex_info(struct TCP_Server_Info *server) /* now walk dispose list and issue callbacks */ list_for_each_safe(tmp, tmp2, &dispose_list) { mid_entry = list_entry(tmp, struct mid_q_entry, qhead); - cifs_dbg(FYI, "Callback mid 0x%llx\n", mid_entry->mid); + cifs_dbg(FYI, "Callback mid %llu\n", mid_entry->mid); list_del_init(&mid_entry->qhead); mid_entry->callback(mid_entry); cifs_mid_q_entry_release(mid_entry); diff --git a/fs/cifs/smb2misc.c b/fs/cifs/smb2misc.c index 60d4bd1eae2b..6e0ea19e710b 100644 --- a/fs/cifs/smb2misc.c +++ b/fs/cifs/smb2misc.c @@ -767,7 +767,7 @@ smb2_cancelled_close_fid(struct work_struct *work) int rc; if (cancelled->mid) - cifs_tcon_dbg(VFS, "Close unmatched open for MID:%llx\n", + cifs_tcon_dbg(VFS, "Close unmatched open for MID:%llu\n", cancelled->mid); else cifs_tcon_dbg(VFS, "Close interrupted close\n"); -- cgit v1.2.3 From e3d100eae44b42f309c1366efb8397368f1cf8ed Mon Sep 17 00:00:00 2001 From: Paulo Alcantara Date: Mon, 8 Mar 2021 12:00:48 -0300 Subject: cifs: change noisy error message to FYI A customer has reported that their dmesg were being flooded by CIFS: VFS: \\server Cancelling wait for mid xxx cmd: a CIFS: VFS: \\server Cancelling wait for mid yyy cmd: b CIFS: VFS: \\server Cancelling wait for mid zzz cmd: c because some processes that were performing statfs(2) on the share had been interrupted due to their automount setup when certain users logged in and out. Change it to FYI as they should be mostly informative rather than error messages. Signed-off-by: Paulo Alcantara (SUSE) Reviewed-by: Aurelien Aptel Reviewed-by: Ronnie Sahlberg Signed-off-by: Steve French --- fs/cifs/transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c index e90a1d1380b0..15b72d2e8713 100644 --- a/fs/cifs/transport.c +++ b/fs/cifs/transport.c @@ -1207,7 +1207,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, } if (rc != 0) { for (; i < num_rqst; i++) { - cifs_server_dbg(VFS, "Cancelling wait for mid %llu cmd: %d\n", + cifs_server_dbg(FYI, "Cancelling wait for mid %llu cmd: %d\n", midQ[i]->mid, le16_to_cpu(midQ[i]->command)); send_cancel(server, &rqst[i], midQ[i]); spin_lock(&GlobalMid_Lock); -- cgit v1.2.3 From 14302ee3301b3a77b331cc14efb95bf7184c73cc Mon Sep 17 00:00:00 2001 From: Paulo Alcantara Date: Mon, 8 Mar 2021 12:00:49 -0300 Subject: cifs: return proper error code in statfs(2) In cifs_statfs(), if server->ops->queryfs is not NULL, then we should use its return value rather than always returning 0. Instead, use rc variable as it is properly set to 0 in case there is no server->ops->queryfs. Signed-off-by: Paulo Alcantara (SUSE) Reviewed-by: Aurelien Aptel Reviewed-by: Ronnie Sahlberg CC: Signed-off-by: Steve French --- fs/cifs/cifsfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index d43e935d2df4..099ad9f3660b 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -290,7 +290,7 @@ cifs_statfs(struct dentry *dentry, struct kstatfs *buf) rc = server->ops->queryfs(xid, tcon, cifs_sb, buf); free_xid(xid); - return 0; + return rc; } static long cifs_fallocate(struct file *file, int mode, loff_t off, loff_t len) -- cgit v1.2.3 From 04ad69c342fc4de5bd23be9ef15ea7574fb1a87e Mon Sep 17 00:00:00 2001 From: Paulo Alcantara Date: Mon, 8 Mar 2021 12:00:50 -0300 Subject: cifs: do not send close in compound create+close requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In case of interrupted syscalls, prevent sending CLOSE commands for compound CREATE+CLOSE requests by introducing an CIFS_CP_CREATE_CLOSE_OP flag to indicate lower layers that it should not send a CLOSE command to the MIDs corresponding the compound CREATE+CLOSE request. A simple reproducer: #!/bin/bash mount //server/share /mnt -o username=foo,password=*** tc qdisc add dev eth0 root netem delay 450ms stat -f /mnt &>/dev/null & pid=$! sleep 0.01 kill $pid tc qdisc del dev eth0 root umount /mnt Before patch: ... 6 0.256893470 192.168.122.2 → 192.168.122.15 SMB2 402 Create Request File: ;GetInfo Request FS_INFO/FileFsFullSizeInformation;Close Request 7 0.257144491 192.168.122.15 → 192.168.122.2 SMB2 498 Create Response File: ;GetInfo Response;Close Response 9 0.260798209 192.168.122.2 → 192.168.122.15 SMB2 146 Close Request File: 10 0.260841089 192.168.122.15 → 192.168.122.2 SMB2 130 Close Response, Error: STATUS_FILE_CLOSED Signed-off-by: Paulo Alcantara (SUSE) Reviewed-by: Ronnie Sahlberg Reviewed-by: Aurelien Aptel CC: Signed-off-by: Steve French --- fs/cifs/cifsglob.h | 19 ++++++++++--------- fs/cifs/smb2inode.c | 1 + fs/cifs/smb2misc.c | 8 ++++---- fs/cifs/smb2ops.c | 10 +++++----- fs/cifs/smb2proto.h | 3 +-- fs/cifs/transport.c | 2 +- 6 files changed, 22 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index 3de3c5908a72..31fc8695abd6 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -257,7 +257,7 @@ struct smb_version_operations { /* verify the message */ int (*check_message)(char *, unsigned int, struct TCP_Server_Info *); bool (*is_oplock_break)(char *, struct TCP_Server_Info *); - int (*handle_cancelled_mid)(char *, struct TCP_Server_Info *); + int (*handle_cancelled_mid)(struct mid_q_entry *, struct TCP_Server_Info *); void (*downgrade_oplock)(struct TCP_Server_Info *server, struct cifsInodeInfo *cinode, __u32 oplock, unsigned int epoch, bool *purge_cache); @@ -1705,16 +1705,17 @@ static inline bool is_retryable_error(int error) #define CIFS_NO_RSP_BUF 0x040 /* no response buffer required */ /* Type of request operation */ -#define CIFS_ECHO_OP 0x080 /* echo request */ -#define CIFS_OBREAK_OP 0x0100 /* oplock break request */ -#define CIFS_NEG_OP 0x0200 /* negotiate request */ +#define CIFS_ECHO_OP 0x080 /* echo request */ +#define CIFS_OBREAK_OP 0x0100 /* oplock break request */ +#define CIFS_NEG_OP 0x0200 /* negotiate request */ +#define CIFS_CP_CREATE_CLOSE_OP 0x0400 /* compound create+close request */ /* Lower bitmask values are reserved by others below. */ -#define CIFS_SESS_OP 0x2000 /* session setup request */ -#define CIFS_OP_MASK 0x2380 /* mask request type */ +#define CIFS_SESS_OP 0x2000 /* session setup request */ +#define CIFS_OP_MASK 0x2780 /* mask request type */ -#define CIFS_HAS_CREDITS 0x0400 /* already has credits */ -#define CIFS_TRANSFORM_REQ 0x0800 /* transform request before sending */ -#define CIFS_NO_SRV_RSP 0x1000 /* there is no server response */ +#define CIFS_HAS_CREDITS 0x0400 /* already has credits */ +#define CIFS_TRANSFORM_REQ 0x0800 /* transform request before sending */ +#define CIFS_NO_SRV_RSP 0x1000 /* there is no server response */ /* Security Flags: indicate type of session setup needed */ #define CIFSSEC_MAY_SIGN 0x00001 diff --git a/fs/cifs/smb2inode.c b/fs/cifs/smb2inode.c index 1f900b81c34a..a718dc77e604 100644 --- a/fs/cifs/smb2inode.c +++ b/fs/cifs/smb2inode.c @@ -358,6 +358,7 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, if (cfile) goto after_close; /* Close */ + flags |= CIFS_CP_CREATE_CLOSE_OP; rqst[num_rqst].rq_iov = &vars->close_iov[0]; rqst[num_rqst].rq_nvec = 1; rc = SMB2_close_init(tcon, server, diff --git a/fs/cifs/smb2misc.c b/fs/cifs/smb2misc.c index 6e0ea19e710b..b50164e2c88d 100644 --- a/fs/cifs/smb2misc.c +++ b/fs/cifs/smb2misc.c @@ -844,14 +844,14 @@ smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, } int -smb2_handle_cancelled_mid(char *buffer, struct TCP_Server_Info *server) +smb2_handle_cancelled_mid(struct mid_q_entry *mid, struct TCP_Server_Info *server) { - struct smb2_sync_hdr *sync_hdr = (struct smb2_sync_hdr *)buffer; - struct smb2_create_rsp *rsp = (struct smb2_create_rsp *)buffer; + struct smb2_sync_hdr *sync_hdr = mid->resp_buf; + struct smb2_create_rsp *rsp = mid->resp_buf; struct cifs_tcon *tcon; int rc; - if (sync_hdr->Command != SMB2_CREATE || + if ((mid->optype & CIFS_CP_CREATE_CLOSE_OP) || sync_hdr->Command != SMB2_CREATE || sync_hdr->Status != STATUS_SUCCESS) return 0; diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index f5087295424c..9bae7e8deb09 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -1195,7 +1195,7 @@ smb2_set_ea(const unsigned int xid, struct cifs_tcon *tcon, struct TCP_Server_Info *server = cifs_pick_channel(ses); __le16 *utf16_path = NULL; int ea_name_len = strlen(ea_name); - int flags = 0; + int flags = CIFS_CP_CREATE_CLOSE_OP; int len; struct smb_rqst rqst[3]; int resp_buftype[3]; @@ -1573,7 +1573,7 @@ smb2_ioctl_query_info(const unsigned int xid, struct smb_query_info qi; struct smb_query_info __user *pqi; int rc = 0; - int flags = 0; + int flags = CIFS_CP_CREATE_CLOSE_OP; struct smb2_query_info_rsp *qi_rsp = NULL; struct smb2_ioctl_rsp *io_rsp = NULL; void *buffer = NULL; @@ -2577,7 +2577,7 @@ smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, { struct cifs_ses *ses = tcon->ses; struct TCP_Server_Info *server = cifs_pick_channel(ses); - int flags = 0; + int flags = CIFS_CP_CREATE_CLOSE_OP; struct smb_rqst rqst[3]; int resp_buftype[3]; struct kvec rsp_iov[3]; @@ -2975,7 +2975,7 @@ smb2_query_symlink(const unsigned int xid, struct cifs_tcon *tcon, unsigned int sub_offset; unsigned int print_len; unsigned int print_offset; - int flags = 0; + int flags = CIFS_CP_CREATE_CLOSE_OP; struct smb_rqst rqst[3]; int resp_buftype[3]; struct kvec rsp_iov[3]; @@ -3157,7 +3157,7 @@ smb2_query_reparse_tag(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_open_parms oparms; struct cifs_fid fid; struct TCP_Server_Info *server = cifs_pick_channel(tcon->ses); - int flags = 0; + int flags = CIFS_CP_CREATE_CLOSE_OP; struct smb_rqst rqst[3]; int resp_buftype[3]; struct kvec rsp_iov[3]; diff --git a/fs/cifs/smb2proto.h b/fs/cifs/smb2proto.h index 9565e27681a5..a2eb34a8d9c9 100644 --- a/fs/cifs/smb2proto.h +++ b/fs/cifs/smb2proto.h @@ -246,8 +246,7 @@ extern int SMB2_oplock_break(const unsigned int xid, struct cifs_tcon *tcon, extern int smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, __u64 volatile_fid); -extern int smb2_handle_cancelled_mid(char *buffer, - struct TCP_Server_Info *server); +extern int smb2_handle_cancelled_mid(struct mid_q_entry *mid, struct TCP_Server_Info *server); void smb2_cancelled_close_fid(struct work_struct *work); extern int SMB2_QFS_info(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_file_id, u64 volatile_file_id, diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c index 15b72d2e8713..007d99437c77 100644 --- a/fs/cifs/transport.c +++ b/fs/cifs/transport.c @@ -101,7 +101,7 @@ static void _cifs_mid_q_entry_release(struct kref *refcount) if (midEntry->resp_buf && (midEntry->mid_flags & MID_WAIT_CANCELLED) && midEntry->mid_state == MID_RESPONSE_RECEIVED && server->ops->handle_cancelled_mid) - server->ops->handle_cancelled_mid(midEntry->resp_buf, server); + server->ops->handle_cancelled_mid(midEntry, server); midEntry->mid_state = MID_FREE; atomic_dec(&midCount); -- cgit v1.2.3 From 4aa5e002034f0701c3335379fd6c22d7f3338cce Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Mon, 8 Mar 2021 10:51:51 -0500 Subject: Revert "nfsd4: remove check_conflicting_opens warning" This reverts commit 50747dd5e47b "nfsd4: remove check_conflicting_opens warning", as a prerequisite for reverting 94415b06eb8a, which has a serious bug. Cc: stable@vger.kernel.org Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 61552e89bd89..482e241c9639 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -4952,6 +4952,7 @@ static int nfsd4_check_conflicting_opens(struct nfs4_client *clp, writes--; if (fp->fi_fds[O_RDWR]) writes--; + WARN_ON_ONCE(writes < 0); if (writes > 0) return -EAGAIN; spin_lock(&fp->fi_lock); -- cgit v1.2.3 From 6ee65a773096ab3f39d9b00311ac983be5bdeb7c Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Mon, 8 Mar 2021 10:52:29 -0500 Subject: Revert "nfsd4: a client's own opens needn't prevent delegations" This reverts commit 94415b06eb8aed13481646026dc995f04a3a534a. That commit claimed to allow a client to get a read delegation when it was the only writer. Actually it allowed a client to get a read delegation when *any* client has a write open! The main problem is that it's depending on nfs4_clnt_odstate structures that are actually only maintained for pnfs exports. This causes clients to miss writes performed by other clients, even when there have been intervening closes and opens, violating close-to-open cache consistency. We can do this a different way, but first we should just revert this. I've added pynfs 4.1 test DELEG19 to test for this, as I should have done originally! Cc: stable@vger.kernel.org Reported-by: Timo Rothenpieler Signed-off-by: J. Bruce Fields Signed-off-by: Chuck Lever --- fs/locks.c | 3 --- fs/nfsd/nfs4state.c | 54 ++++++++++++++--------------------------------------- 2 files changed, 14 insertions(+), 43 deletions(-) (limited to 'fs') diff --git a/fs/locks.c b/fs/locks.c index 99ca97e81b7a..6125d2de39b8 100644 --- a/fs/locks.c +++ b/fs/locks.c @@ -1808,9 +1808,6 @@ check_conflicting_open(struct file *filp, const long arg, int flags) if (flags & FL_LAYOUT) return 0; - if (flags & FL_DELEG) - /* We leave these checks to the caller. */ - return 0; if (arg == F_RDLCK) return inode_is_open_for_write(inode) ? -EAGAIN : 0; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 482e241c9639..97447a64bad0 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -4940,32 +4940,6 @@ static struct file_lock *nfs4_alloc_init_lease(struct nfs4_delegation *dp, return fl; } -static int nfsd4_check_conflicting_opens(struct nfs4_client *clp, - struct nfs4_file *fp) -{ - struct nfs4_clnt_odstate *co; - struct file *f = fp->fi_deleg_file->nf_file; - struct inode *ino = locks_inode(f); - int writes = atomic_read(&ino->i_writecount); - - if (fp->fi_fds[O_WRONLY]) - writes--; - if (fp->fi_fds[O_RDWR]) - writes--; - WARN_ON_ONCE(writes < 0); - if (writes > 0) - return -EAGAIN; - spin_lock(&fp->fi_lock); - list_for_each_entry(co, &fp->fi_clnt_odstate, co_perfile) { - if (co->co_client != clp) { - spin_unlock(&fp->fi_lock); - return -EAGAIN; - } - } - spin_unlock(&fp->fi_lock); - return 0; -} - static struct nfs4_delegation * nfs4_set_delegation(struct nfs4_client *clp, struct svc_fh *fh, struct nfs4_file *fp, struct nfs4_clnt_odstate *odstate) @@ -4985,12 +4959,9 @@ nfs4_set_delegation(struct nfs4_client *clp, struct svc_fh *fh, nf = find_readable_file(fp); if (!nf) { - /* - * We probably could attempt another open and get a read - * delegation, but for now, don't bother until the - * client actually sends us one. - */ - return ERR_PTR(-EAGAIN); + /* We should always have a readable file here */ + WARN_ON_ONCE(1); + return ERR_PTR(-EBADF); } spin_lock(&state_lock); spin_lock(&fp->fi_lock); @@ -5020,19 +4991,11 @@ nfs4_set_delegation(struct nfs4_client *clp, struct svc_fh *fh, if (!fl) goto out_clnt_odstate; - status = nfsd4_check_conflicting_opens(clp, fp); - if (status) { - locks_free_lock(fl); - goto out_clnt_odstate; - } status = vfs_setlease(fp->fi_deleg_file->nf_file, fl->fl_type, &fl, NULL); if (fl) locks_free_lock(fl); if (status) goto out_clnt_odstate; - status = nfsd4_check_conflicting_opens(clp, fp); - if (status) - goto out_clnt_odstate; spin_lock(&state_lock); spin_lock(&fp->fi_lock); @@ -5114,6 +5077,17 @@ nfs4_open_delegation(struct svc_fh *fh, struct nfsd4_open *open, goto out_no_deleg; if (!cb_up || !(oo->oo_flags & NFS4_OO_CONFIRMED)) goto out_no_deleg; + /* + * Also, if the file was opened for write or + * create, there's a good chance the client's + * about to write to it, resulting in an + * immediate recall (since we don't support + * write delegations): + */ + if (open->op_share_access & NFS4_SHARE_ACCESS_WRITE) + goto out_no_deleg; + if (open->op_create == NFS4_OPEN_CREATE) + goto out_no_deleg; break; default: goto out_no_deleg; -- cgit v1.2.3 From 5808fecc572391867fcd929662b29c12e6d08d81 Mon Sep 17 00:00:00 2001 From: Ritesh Harjani Date: Tue, 9 Mar 2021 09:29:11 -0800 Subject: iomap: Fix negative assignment to unsigned sis->pages in iomap_swapfile_activate In case if isi.nr_pages is 0, we are making sis->pages (which is unsigned int) a huge value in iomap_swapfile_activate() by assigning -1. This could cause a kernel crash in kernel v4.18 (with below signature). Or could lead to unknown issues on latest kernel if the fake big swap gets used. Fix this issue by returning -EINVAL in case of nr_pages is 0, since it is anyway a invalid swapfile. Looks like this issue will be hit when we have pagesize < blocksize type of configuration. I was able to hit the issue in case of a tiny swap file with below test script. https://raw.githubusercontent.com/riteshharjani/LinuxStudy/master/scripts/swap-issue.sh kernel crash analysis on v4.18 ============================== On v4.18 kernel, it causes a kernel panic, since sis->pages becomes a huge value and isi.nr_extents is 0. When 0 is returned it is considered as a swapfile over NFS and SWP_FILE is set (sis->flags |= SWP_FILE). Then when swapoff was getting called it was calling a_ops->swap_deactivate() if (sis->flags & SWP_FILE) is true. Since a_ops->swap_deactivate() is NULL in case of XFS, it causes below panic. Panic signature on v4.18 kernel: ======================================= root@qemu:/home/qemu# [ 8291.723351] XFS (loop2): Unmounting Filesystem [ 8292.123104] XFS (loop2): Mounting V5 Filesystem [ 8292.132451] XFS (loop2): Ending clean mount [ 8292.263362] Adding 4294967232k swap on /mnt1/test/swapfile. Priority:-2 extents:1 across:274877906880k [ 8292.277834] Unable to handle kernel paging request for instruction fetch [ 8292.278677] Faulting instruction address: 0x00000000 cpu 0x19: Vector: 400 (Instruction Access) at [c0000009dd5b7ad0] pc: 0000000000000000 lr: c0000000003eb9dc: destroy_swap_extents+0xfc/0x120 sp: c0000009dd5b7d50 msr: 8000000040009033 current = 0xc0000009b6710080 paca = 0xc00000003ffcb280 irqmask: 0x03 irq_happened: 0x01 pid = 5604, comm = swapoff Linux version 4.18.0 (riteshh@xxxxxxx) (gcc version 8.4.0 (Ubuntu 8.4.0-1ubuntu1~18.04)) #57 SMP Wed Mar 3 01:33:04 CST 2021 enter ? for help [link register ] c0000000003eb9dc destroy_swap_extents+0xfc/0x120 [c0000009dd5b7d50] c0000000025a7058 proc_poll_event+0x0/0x4 (unreliable) [c0000009dd5b7da0] c0000000003f0498 sys_swapoff+0x3f8/0x910 [c0000009dd5b7e30] c00000000000bbe4 system_call+0x5c/0x70 Exception: c01 (System Call) at 00007ffff7d208d8 Signed-off-by: Ritesh Harjani [djwong: rework the comment to provide more details] Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/iomap/swapfile.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'fs') diff --git a/fs/iomap/swapfile.c b/fs/iomap/swapfile.c index a648dbf6991e..a5e478de1417 100644 --- a/fs/iomap/swapfile.c +++ b/fs/iomap/swapfile.c @@ -170,6 +170,16 @@ int iomap_swapfile_activate(struct swap_info_struct *sis, return ret; } + /* + * If this swapfile doesn't contain even a single page-aligned + * contiguous range of blocks, reject this useless swapfile to + * prevent confusion later on. + */ + if (isi.nr_pages == 0) { + pr_warn("swapon: Cannot find a single usable page in file.\n"); + return -EINVAL; + } + *pagespan = 1 + isi.highest_ppage - isi.lowest_ppage; sis->max = isi.nr_pages; sis->pages = isi.nr_pages - 1; -- cgit v1.2.3 From b5a08423da9da59c7f38ed8dbb6dd6cbbe9024a4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 2 Mar 2021 09:32:52 -0800 Subject: xfs: fix quota accounting when a mount is idmapped Nowadays, we indirectly use the idmap-aware helper functions in the VFS to set the initial uid and gid of a file being created. Unfortunately, we didn't convert the quota code, which means we attach the wrong dquots to files created on an idmapped mount. Fixes: f736d93d76d3 ("xfs: support idmapped mounts") Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Acked-by: Christian Brauner Reviewed-by: Dave Chinner --- fs/xfs/xfs_inode.c | 14 ++++++++------ fs/xfs/xfs_symlink.c | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 46a861d55e48..f93370bd7b1e 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -1007,9 +1007,10 @@ xfs_create( /* * Make sure that we have allocated dquot(s) on disk. */ - error = xfs_qm_vop_dqalloc(dp, current_fsuid(), current_fsgid(), prid, - XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, - &udqp, &gdqp, &pdqp); + error = xfs_qm_vop_dqalloc(dp, fsuid_into_mnt(mnt_userns), + fsgid_into_mnt(mnt_userns), prid, + XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, + &udqp, &gdqp, &pdqp); if (error) return error; @@ -1157,9 +1158,10 @@ xfs_create_tmpfile( /* * Make sure that we have allocated dquot(s) on disk. */ - error = xfs_qm_vop_dqalloc(dp, current_fsuid(), current_fsgid(), prid, - XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, - &udqp, &gdqp, &pdqp); + error = xfs_qm_vop_dqalloc(dp, fsuid_into_mnt(mnt_userns), + fsgid_into_mnt(mnt_userns), prid, + XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, + &udqp, &gdqp, &pdqp); if (error) return error; diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 1379013d74b8..7f368b10ded1 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -182,7 +182,8 @@ xfs_symlink( /* * Make sure that we have allocated dquot(s) on disk. */ - error = xfs_qm_vop_dqalloc(dp, current_fsuid(), current_fsgid(), prid, + error = xfs_qm_vop_dqalloc(dp, fsuid_into_mnt(mnt_userns), + fsgid_into_mnt(mnt_userns), prid, XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, &udqp, &gdqp, &pdqp); if (error) -- cgit v1.2.3 From 614c9750173e412663728215152cc6d12bcb3425 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Tue, 9 Mar 2021 09:41:14 -0500 Subject: NFSD: fix dest to src mount in inter-server COPY A cleanup of the inter SSC copy needs to call fput() of the source file handle to make sure that file structure is freed as well as drop the reference on the superblock to unmount the source server. Fixes: 36e1e5ba90fb ("NFSD: Fix use-after-free warning when doing inter-server copy") Signed-off-by: Olga Kornievskaia Signed-off-by: Chuck Lever Tested-by: Dai Ngo --- fs/nfsd/nfs4proc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index acdb3cd806a1..dd9f38d072dd 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1302,7 +1302,7 @@ nfsd4_cleanup_inter_ssc(struct vfsmount *ss_mnt, struct nfsd_file *src, struct nfsd_file *dst) { nfs42_ssc_close(src->nf_file); - /* 'src' is freed by nfsd4_do_async_copy */ + fput(src->nf_file); nfsd_file_put(dst); mntput(ss_mnt); } -- cgit v1.2.3 From 53cb245454df5b13d7063162afd7a785aed6ebf2 Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Fri, 15 Jan 2021 18:43:56 +0100 Subject: NFSv4.2: fix return value of _nfs4_get_security_label() An xattr 'get' handler is expected to return the length of the value on success, yet _nfs4_get_security_label() (and consequently also nfs4_xattr_get_nfs4_label(), which is used as an xattr handler) returns just 0 on success. Fix this by returning label.len instead, which contains the length of the result. Fixes: aa9c2669626c ("NFS: Client implementation of Labeled-NFS") Signed-off-by: Ondrej Mosnacek Reviewed-by: James Morris Reviewed-by: Paul Moore Signed-off-by: Anna Schumaker --- fs/nfs/nfs4proc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 6d8fc56e5f45..1002c4f66f3f 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -5965,7 +5965,7 @@ static int _nfs4_get_security_label(struct inode *inode, void *buf, return ret; if (!(fattr.valid & NFS_ATTR_FATTR_V4_SECURITY_LABEL)) return -ENOENT; - return 0; + return label.len; } static int nfs4_get_security_label(struct inode *inode, void *buf, -- cgit v1.2.3 From 05962f95f9ac7af25fea037ef51b37c0eccb5590 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 6 Mar 2021 13:58:48 -0700 Subject: io_uring: SQPOLL parking fixes We keep running into weird dependency issues between the sqd lock and the parking state. Disentangle the SQPOLL thread from the last bits of the kthread parking inheritance, and just replace the parking state, and two associated locks, with a single rw mutex. The SQPOLL thread keeps the mutex for read all the time, except if someone has marked us needing to park. Then we drop/re-acquire and try again. This greatly simplifies the parking state machine (by just getting rid of it), and makes it a lot more obvious how it works - if you need to modify the ctx list, then you simply park the thread which will grab the lock for writing. Fold in fix from Hillf Danton on not setting STOP on a fatal signal. Fixes: e54945ae947f ("io_uring: SQPOLL stop error handling fixes") Signed-off-by: Jens Axboe --- fs/io_uring.c | 133 +++++++++++++++------------------------------------------- 1 file changed, 34 insertions(+), 99 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 7cf96be691d8..2a3542b487ff 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -258,12 +258,11 @@ enum { struct io_sq_data { refcount_t refs; - struct mutex lock; + struct rw_semaphore rw_lock; /* ctx's that are using this sqd */ struct list_head ctx_list; struct list_head ctx_new_list; - struct mutex ctx_lock; struct task_struct *thread; struct wait_queue_head wait; @@ -274,7 +273,6 @@ struct io_sq_data { unsigned long state; struct completion startup; - struct completion parked; struct completion exited; }; @@ -6638,45 +6636,6 @@ static void io_sqd_init_new(struct io_sq_data *sqd) io_sqd_update_thread_idle(sqd); } -static bool io_sq_thread_should_stop(struct io_sq_data *sqd) -{ - return test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); -} - -static bool io_sq_thread_should_park(struct io_sq_data *sqd) -{ - return test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); -} - -static void io_sq_thread_parkme(struct io_sq_data *sqd) -{ - for (;;) { - /* - * TASK_PARKED is a special state; we must serialize against - * possible pending wakeups to avoid store-store collisions on - * task->state. - * - * Such a collision might possibly result in the task state - * changin from TASK_PARKED and us failing the - * wait_task_inactive() in kthread_park(). - */ - set_special_state(TASK_PARKED); - if (!test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) - break; - - /* - * Thread is going to call schedule(), do not preempt it, - * or the caller of kthread_park() may spend more time in - * wait_task_inactive(). - */ - preempt_disable(); - complete(&sqd->parked); - schedule_preempt_disabled(); - preempt_enable(); - } - __set_current_state(TASK_RUNNING); -} - static int io_sq_thread(void *data) { struct io_sq_data *sqd = data; @@ -6697,17 +6656,16 @@ static int io_sq_thread(void *data) wait_for_completion(&sqd->startup); - while (!io_sq_thread_should_stop(sqd)) { + down_read(&sqd->rw_lock); + + while (!test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) { int ret; bool cap_entries, sqt_spin, needs_sched; - /* - * Any changes to the sqd lists are synchronized through the - * thread parking. This synchronizes the thread vs users, - * the users are synchronized on the sqd->ctx_lock. - */ - if (io_sq_thread_should_park(sqd)) { - io_sq_thread_parkme(sqd); + if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) { + up_read(&sqd->rw_lock); + cond_resched(); + down_read(&sqd->rw_lock); continue; } if (unlikely(!list_empty(&sqd->ctx_new_list))) { @@ -6752,12 +6710,14 @@ static int io_sq_thread(void *data) } } - if (needs_sched && !io_sq_thread_should_park(sqd)) { + if (needs_sched && !test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) { list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); + up_read(&sqd->rw_lock); schedule(); try_to_freeze(); + down_read(&sqd->rw_lock); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_clear_wakeup_flag(ctx); } @@ -6768,28 +6728,16 @@ static int io_sq_thread(void *data) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_uring_cancel_sqpoll(ctx); + up_read(&sqd->rw_lock); io_run_task_work(); - /* - * Ensure that we park properly if racing with someone trying to park - * while we're exiting. If we fail to grab the lock, check park and - * park if necessary. The ordering with the park bit and the lock - * ensures that we catch this reliably. - */ - if (!mutex_trylock(&sqd->lock)) { - if (io_sq_thread_should_park(sqd)) - io_sq_thread_parkme(sqd); - mutex_lock(&sqd->lock); - } - + down_write(&sqd->rw_lock); sqd->thread = NULL; - list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { + list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); - } - + up_write(&sqd->rw_lock); complete(&sqd->exited); - mutex_unlock(&sqd->lock); do_exit(0); } @@ -7088,44 +7036,40 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) } static void io_sq_thread_unpark(struct io_sq_data *sqd) - __releases(&sqd->lock) + __releases(&sqd->rw_lock) { if (sqd->thread == current) return; clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); - if (sqd->thread) - wake_up_state(sqd->thread, TASK_PARKED); - mutex_unlock(&sqd->lock); + up_write(&sqd->rw_lock); } static void io_sq_thread_park(struct io_sq_data *sqd) - __acquires(&sqd->lock) + __acquires(&sqd->rw_lock) { if (sqd->thread == current) return; set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); - mutex_lock(&sqd->lock); - if (sqd->thread) { + down_write(&sqd->rw_lock); + /* set again for consistency, in case concurrent parks are happening */ + set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); + if (sqd->thread) wake_up_process(sqd->thread); - wait_for_completion(&sqd->parked); - } } static void io_sq_thread_stop(struct io_sq_data *sqd) { if (test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) return; - mutex_lock(&sqd->lock); - if (sqd->thread) { - set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); - WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)); - wake_up_process(sqd->thread); - mutex_unlock(&sqd->lock); - wait_for_completion(&sqd->exited); - WARN_ON_ONCE(sqd->thread); - } else { - mutex_unlock(&sqd->lock); + down_write(&sqd->rw_lock); + if (!sqd->thread) { + up_write(&sqd->rw_lock); + return; } + set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); + wake_up_process(sqd->thread); + up_write(&sqd->rw_lock); + wait_for_completion(&sqd->exited); } static void io_put_sq_data(struct io_sq_data *sqd) @@ -7142,18 +7086,13 @@ static void io_sq_thread_finish(struct io_ring_ctx *ctx) if (sqd) { complete(&sqd->startup); - if (sqd->thread) { + if (sqd->thread) wait_for_completion(&ctx->sq_thread_comp); - io_sq_thread_park(sqd); - } - mutex_lock(&sqd->ctx_lock); + io_sq_thread_park(sqd); list_del(&ctx->sqd_list); io_sqd_update_thread_idle(sqd); - mutex_unlock(&sqd->ctx_lock); - - if (sqd->thread) - io_sq_thread_unpark(sqd); + io_sq_thread_unpark(sqd); io_put_sq_data(sqd); ctx->sq_data = NULL; @@ -7202,11 +7141,9 @@ static struct io_sq_data *io_get_sq_data(struct io_uring_params *p) refcount_set(&sqd->refs, 1); INIT_LIST_HEAD(&sqd->ctx_list); INIT_LIST_HEAD(&sqd->ctx_new_list); - mutex_init(&sqd->ctx_lock); - mutex_init(&sqd->lock); + init_rwsem(&sqd->rw_lock); init_waitqueue_head(&sqd->wait); init_completion(&sqd->startup); - init_completion(&sqd->parked); init_completion(&sqd->exited); return sqd; } @@ -7880,9 +7817,7 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, ctx->sq_creds = get_current_cred(); ctx->sq_data = sqd; io_sq_thread_park(sqd); - mutex_lock(&sqd->ctx_lock); list_add(&ctx->sqd_list, &sqd->ctx_new_list); - mutex_unlock(&sqd->ctx_lock); io_sq_thread_unpark(sqd); ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle); -- cgit v1.2.3 From f458dd8441e56d122ddf1d8e2af0b6ee62f52af9 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 8 Mar 2021 12:14:14 +0000 Subject: io_uring: fix unrelated ctx reqs cancellation io-wq now is per-task, so cancellations now should match against request's ctx. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 2a3542b487ff..d4f018f5838d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5573,22 +5573,30 @@ add: return 0; } +struct io_cancel_data { + struct io_ring_ctx *ctx; + u64 user_data; +}; + static bool io_cancel_cb(struct io_wq_work *work, void *data) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); + struct io_cancel_data *cd = data; - return req->user_data == (unsigned long) data; + return req->ctx == cd->ctx && req->user_data == cd->user_data; } -static int io_async_cancel_one(struct io_uring_task *tctx, void *sqe_addr) +static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data, + struct io_ring_ctx *ctx) { + struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, }; enum io_wq_cancel cancel_ret; int ret = 0; - if (!tctx->io_wq) + if (!tctx || !tctx->io_wq) return -ENOENT; - cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, sqe_addr, false); + cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false); switch (cancel_ret) { case IO_WQ_CANCEL_OK: ret = 0; @@ -5611,8 +5619,7 @@ static void io_async_find_and_cancel(struct io_ring_ctx *ctx, unsigned long flags; int ret; - ret = io_async_cancel_one(req->task->io_uring, - (void *) (unsigned long) sqe_addr); + ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx); if (ret != -ENOENT) { spin_lock_irqsave(&ctx->completion_lock, flags); goto done; -- cgit v1.2.3 From 0298ef969a110ca03654f0cea9b50e3f3b331acc Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 8 Mar 2021 13:20:57 +0000 Subject: io_uring: clean R_DISABLED startup mess There are enough of problems with IORING_SETUP_R_DISABLED, including the burden of checking and kicking off the SQO task all over the codebase -- for exit/cancel/etc. Rework it, always start the thread but don't do submit unless the flag is gone, that's much easier. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index d4f018f5838d..3f6db813d670 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6606,7 +6606,8 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) if (!list_empty(&ctx->iopoll_list)) io_do_iopoll(ctx, &nr_events, 0); - if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs))) + if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) && + !(ctx->flags & IORING_SETUP_R_DISABLED)) ret = io_submit_sqes(ctx, to_submit); mutex_unlock(&ctx->uring_lock); } @@ -7861,6 +7862,7 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, wake_up_new_task(tsk); if (ret) goto err; + complete(&sqd->startup); } else if (p->flags & IORING_SETUP_SQ_AFF) { /* Can't have SQ_AFF without SQPOLL */ ret = -EINVAL; @@ -7873,15 +7875,6 @@ err: return ret; } -static void io_sq_offload_start(struct io_ring_ctx *ctx) -{ - struct io_sq_data *sqd = ctx->sq_data; - - ctx->flags &= ~IORING_SETUP_R_DISABLED; - if (ctx->flags & IORING_SETUP_SQPOLL) - complete(&sqd->startup); -} - static inline void __io_unaccount_mem(struct user_struct *user, unsigned long nr_pages) { @@ -8742,11 +8735,6 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, struct task_struct *task = current; if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { - /* never started, nothing to cancel */ - if (ctx->flags & IORING_SETUP_R_DISABLED) { - io_sq_offload_start(ctx); - return; - } io_sq_thread_park(ctx->sq_data); task = ctx->sq_data->thread; if (task) @@ -9449,9 +9437,6 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, if (ret) goto err; - if (!(p->flags & IORING_SETUP_R_DISABLED)) - io_sq_offload_start(ctx); - memset(&p->sq_off, 0, sizeof(p->sq_off)); p->sq_off.head = offsetof(struct io_rings, sq.head); p->sq_off.tail = offsetof(struct io_rings, sq.tail); @@ -9668,7 +9653,9 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx) if (ctx->restrictions.registered) ctx->restricted = 1; - io_sq_offload_start(ctx); + ctx->flags &= ~IORING_SETUP_R_DISABLED; + if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) + wake_up(&ctx->sq_data->wait); return 0; } -- cgit v1.2.3 From 61cf93700fe6359552848ed5e3becba6cd760efa Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 8 Mar 2021 14:16:16 +0000 Subject: io_uring: Convert personality_idr to XArray You can't call idr_remove() from within a idr_for_each() callback, but you can call xa_erase() from an xa_for_each() loop, so switch the entire personality_idr from the IDR to the XArray. This manifests as a use-after-free as idr_for_each() attempts to walk the rest of the node after removing the last entry from it. Fixes: 071698e13ac6 ("io_uring: allow registering credentials") Cc: stable@vger.kernel.org # 5.6+ Reported-by: yangerkun Signed-off-by: Matthew Wilcox (Oracle) [Pavel: rebased (creds load was moved into io_init_req())] Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7ccff36e1375f2b0ebf73d957f037b43becc0dde.1615212806.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 3f6db813d670..84eb499368a4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -406,7 +406,8 @@ struct io_ring_ctx { struct idr io_buffer_idr; - struct idr personality_idr; + struct xarray personalities; + u32 pers_next; struct { unsigned cached_cq_tail; @@ -1137,7 +1138,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) init_completion(&ctx->ref_comp); init_completion(&ctx->sq_thread_comp); idr_init(&ctx->io_buffer_idr); - idr_init(&ctx->personality_idr); + xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); mutex_init(&ctx->uring_lock); init_waitqueue_head(&ctx->wait); spin_lock_init(&ctx->completion_lock); @@ -6337,7 +6338,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->work.list.next = NULL; personality = READ_ONCE(sqe->personality); if (personality) { - req->work.creds = idr_find(&ctx->personality_idr, personality); + req->work.creds = xa_load(&ctx->personalities, personality); if (!req->work.creds) return -EINVAL; get_cred(req->work.creds); @@ -8355,7 +8356,6 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) mutex_unlock(&ctx->uring_lock); io_eventfd_unregister(ctx); io_destroy_buffers(ctx); - idr_destroy(&ctx->personality_idr); #if defined(CONFIG_UNIX) if (ctx->ring_sock) { @@ -8420,7 +8420,7 @@ static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) { const struct cred *creds; - creds = idr_remove(&ctx->personality_idr, id); + creds = xa_erase(&ctx->personalities, id); if (creds) { put_cred(creds); return 0; @@ -8429,14 +8429,6 @@ static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) return -EINVAL; } -static int io_remove_personalities(int id, void *p, void *data) -{ - struct io_ring_ctx *ctx = data; - - io_unregister_personality(ctx, id); - return 0; -} - static bool io_run_ctx_fallback(struct io_ring_ctx *ctx) { struct callback_head *work, *next; @@ -8526,13 +8518,17 @@ static void io_ring_exit_work(struct work_struct *work) static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) { + unsigned long index; + struct creds *creds; + mutex_lock(&ctx->uring_lock); percpu_ref_kill(&ctx->refs); /* if force is set, the ring is going away. always drop after that */ ctx->cq_overflow_flushed = 1; if (ctx->rings) __io_cqring_overflow_flush(ctx, true, NULL, NULL); - idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx); + xa_for_each(&ctx->personalities, index, creds) + io_unregister_personality(ctx, index); mutex_unlock(&ctx->uring_lock); io_kill_timeouts(ctx, NULL, NULL); @@ -9162,10 +9158,9 @@ out_fput: } #ifdef CONFIG_PROC_FS -static int io_uring_show_cred(int id, void *p, void *data) +static int io_uring_show_cred(struct seq_file *m, unsigned int id, + const struct cred *cred) { - const struct cred *cred = p; - struct seq_file *m = data; struct user_namespace *uns = seq_user_ns(m); struct group_info *gi; kernel_cap_t cap; @@ -9233,9 +9228,13 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, (unsigned int) buf->len); } - if (has_lock && !idr_is_empty(&ctx->personality_idr)) { + if (has_lock && !xa_empty(&ctx->personalities)) { + unsigned long index; + const struct cred *cred; + seq_printf(m, "Personalities:\n"); - idr_for_each(&ctx->personality_idr, io_uring_show_cred, m); + xa_for_each(&ctx->personalities, index, cred) + io_uring_show_cred(m, index, cred); } seq_printf(m, "PollList:\n"); spin_lock_irq(&ctx->completion_lock); @@ -9564,14 +9563,16 @@ out: static int io_register_personality(struct io_ring_ctx *ctx) { const struct cred *creds; + u32 id; int ret; creds = get_current_cred(); - ret = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1, - USHRT_MAX, GFP_KERNEL); - if (ret < 0) - put_cred(creds); + ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds, + XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); + if (!ret) + return id; + put_cred(creds); return ret; } -- cgit v1.2.3 From cc20e3fec682700b673fcd286e6bef8e9da947e2 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 8 Mar 2021 09:34:43 -0700 Subject: io-wq: remove unused 'user' member of io_wq Previous patches killed the last user of this, now it's just a dead member in the struct. Get rid of it. Signed-off-by: Jens Axboe --- fs/io-wq.c | 1 - 1 file changed, 1 deletion(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 1ab9324e602f..c2e7031f6d09 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -110,7 +110,6 @@ struct io_wq { io_wq_work_fn *do_work; struct task_struct *manager; - struct user_struct *user; struct io_wq_hash *hash; -- cgit v1.2.3 From 97a73a0f9fbfb2be682fd037814576dbfa0e0da8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 8 Mar 2021 17:30:54 +0000 Subject: io_uring: fix io_sq_offload_create error handling Don't set IO_SQ_THREAD_SHOULD_STOP when io_sq_offload_create() has failed on io_uring_alloc_task_context() but leave everything to io_sq_thread_finish(), because currently io_sq_thread_finish() hangs on trying to park it. That's great it stalls there, because otherwise the following io_sq_thread_stop() would be skipped on IO_SQ_THREAD_SHOULD_STOP check and the sqo would race for sqd with freeing ctx. A simple error injection gives something like this. [ 245.463955] INFO: task sqpoll-test-hang:523 blocked for more than 122 seconds. [ 245.463983] Call Trace: [ 245.463990] __schedule+0x36b/0x950 [ 245.464005] schedule+0x68/0xe0 [ 245.464013] schedule_timeout+0x209/0x2a0 [ 245.464032] wait_for_completion+0x8b/0xf0 [ 245.464043] io_sq_thread_finish+0x44/0x1a0 [ 245.464049] io_uring_setup+0x9ea/0xc80 [ 245.464058] __x64_sys_io_uring_setup+0x16/0x20 [ 245.464064] do_syscall_64+0x38/0x50 [ 245.464073] entry_SYSCALL_64_after_hwframe+0x44/0xae Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 84eb499368a4..3299807894ec 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7856,10 +7856,9 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, ret = PTR_ERR(tsk); goto err; } - ret = io_uring_alloc_task_context(tsk, ctx); - if (ret) - set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); + sqd->thread = tsk; + ret = io_uring_alloc_task_context(tsk, ctx); wake_up_new_task(tsk); if (ret) goto err; -- cgit v1.2.3 From 33cc89a9fc248a486857381584cc6b67d9405fab Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 9 Mar 2021 00:37:58 +0000 Subject: io_uring: add io_disarm_next() helper A preparation patch placing all preparations before extracting a next request into a separate helper io_disarm_next(). Also, don't spuriously do ev_posted in a rare case where REQ_F_FAIL_LINK is set but there are no requests linked (i.e. after cancelling a linked timeout or setting IOSQE_IO_LINK on a last request of a submission batch). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/44ecff68d6b47e1c4e6b891bdde1ddc08cfc3590.1615250156.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 68 ++++++++++++++++++++++++++++++----------------------------- 1 file changed, 35 insertions(+), 33 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 3299807894ec..cc9a2cc95608 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1705,15 +1705,11 @@ static inline void io_remove_next_linked(struct io_kiocb *req) nxt->link = NULL; } -static void io_kill_linked_timeout(struct io_kiocb *req) +static bool io_kill_linked_timeout(struct io_kiocb *req) + __must_hold(&req->ctx->completion_lock) { - struct io_ring_ctx *ctx = req->ctx; - struct io_kiocb *link; + struct io_kiocb *link = req->link; bool cancelled = false; - unsigned long flags; - - spin_lock_irqsave(&ctx->completion_lock, flags); - link = req->link; /* * Can happen if a linked timeout fired and link had been like @@ -1728,50 +1724,48 @@ static void io_kill_linked_timeout(struct io_kiocb *req) ret = hrtimer_try_to_cancel(&io->timer); if (ret != -1) { io_cqring_fill_event(link, -ECANCELED); - io_commit_cqring(ctx); + io_put_req_deferred(link, 1); cancelled = true; } } req->flags &= ~REQ_F_LINK_TIMEOUT; - spin_unlock_irqrestore(&ctx->completion_lock, flags); - - if (cancelled) { - io_cqring_ev_posted(ctx); - io_put_req(link); - } + return cancelled; } - static void io_fail_links(struct io_kiocb *req) + __must_hold(&req->ctx->completion_lock) { - struct io_kiocb *link, *nxt; - struct io_ring_ctx *ctx = req->ctx; - unsigned long flags; + struct io_kiocb *nxt, *link = req->link; - spin_lock_irqsave(&ctx->completion_lock, flags); - link = req->link; req->link = NULL; - while (link) { nxt = link->link; link->link = NULL; trace_io_uring_fail_link(req, link); io_cqring_fill_event(link, -ECANCELED); - io_put_req_deferred(link, 2); link = nxt; } - io_commit_cqring(ctx); - spin_unlock_irqrestore(&ctx->completion_lock, flags); +} - io_cqring_ev_posted(ctx); +static bool io_disarm_next(struct io_kiocb *req) + __must_hold(&req->ctx->completion_lock) +{ + bool posted = false; + + if (likely(req->flags & REQ_F_LINK_TIMEOUT)) + posted = io_kill_linked_timeout(req); + if (unlikely(req->flags & REQ_F_FAIL_LINK)) { + posted |= (req->link != NULL); + io_fail_links(req); + } + return posted; } static struct io_kiocb *__io_req_find_next(struct io_kiocb *req) { - if (req->flags & REQ_F_LINK_TIMEOUT) - io_kill_linked_timeout(req); + struct io_kiocb *nxt; /* * If LINK is set, we have dependent requests in this chain. If we @@ -1779,14 +1773,22 @@ static struct io_kiocb *__io_req_find_next(struct io_kiocb *req) * dependencies to the next request. In case of failure, fail the rest * of the chain. */ - if (likely(!(req->flags & REQ_F_FAIL_LINK))) { - struct io_kiocb *nxt = req->link; + if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL_LINK)) { + struct io_ring_ctx *ctx = req->ctx; + unsigned long flags; + bool posted; - req->link = NULL; - return nxt; + spin_lock_irqsave(&ctx->completion_lock, flags); + posted = io_disarm_next(req); + if (posted) + io_commit_cqring(req->ctx); + spin_unlock_irqrestore(&ctx->completion_lock, flags); + if (posted) + io_cqring_ev_posted(ctx); } - io_fail_links(req); - return NULL; + nxt = req->link; + req->link = NULL; + return nxt; } static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) -- cgit v1.2.3 From 7a612350a989866510dc5c874fd8ffe1f37555d2 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 9 Mar 2021 00:37:59 +0000 Subject: io_uring: fix complete_post races for linked req Calling io_queue_next() after spin_unlock in io_req_complete_post() races with the other side extracting and reusing this request. Hand coded parts of io_req_find_next() considering that io_disarm_next() and io_req_task_queue() have (and safe) to be called with completion_lock held. It already does io_commit_cqring() and io_cqring_ev_posted(), so just reuse it for post io_disarm_next(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/5672a62f3150ee7c55849f40c0037655c4f2840f.1615250156.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index cc9a2cc95608..f7153483a3ac 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -985,6 +985,7 @@ static const struct io_op_def io_op_defs[] = { [IORING_OP_UNLINKAT] = {}, }; +static bool io_disarm_next(struct io_kiocb *req); static void io_uring_del_task_file(unsigned long index); static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, struct task_struct *task, @@ -1525,15 +1526,14 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) __io_cqring_fill_event(req, res, 0); } -static inline void io_req_complete_post(struct io_kiocb *req, long res, - unsigned int cflags) +static void io_req_complete_post(struct io_kiocb *req, long res, + unsigned int cflags) { struct io_ring_ctx *ctx = req->ctx; unsigned long flags; spin_lock_irqsave(&ctx->completion_lock, flags); __io_cqring_fill_event(req, res, cflags); - io_commit_cqring(ctx); /* * If we're the last reference to this request, add to our locked * free_list cache. @@ -1541,19 +1541,26 @@ static inline void io_req_complete_post(struct io_kiocb *req, long res, if (refcount_dec_and_test(&req->refs)) { struct io_comp_state *cs = &ctx->submit_state.comp; + if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { + if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL_LINK)) + io_disarm_next(req); + if (req->link) { + io_req_task_queue(req->link); + req->link = NULL; + } + } io_dismantle_req(req); io_put_task(req->task, 1); list_add(&req->compl.list, &cs->locked_free_list); cs->locked_free_nr++; } else req = NULL; + io_commit_cqring(ctx); spin_unlock_irqrestore(&ctx->completion_lock, flags); - io_cqring_ev_posted(ctx); - if (req) { - io_queue_next(req); + + if (req) percpu_ref_put(&ctx->refs); - } } static void io_req_complete_state(struct io_kiocb *req, long res, -- cgit v1.2.3 From 70e35125093b05b0e607ba1f5358ddf76946756c Mon Sep 17 00:00:00 2001 From: yangerkun Date: Tue, 9 Mar 2021 11:04:10 +0800 Subject: io-wq: fix ref leak for req in case of exit cancelations do_work such as io_wq_submit_work that cancel the work may leave a ref of req as 1 if we have links. Fix it by call io_run_cancel. Fixes: 4fb6ac326204 ("io-wq: improve manager/worker handling over exec") Signed-off-by: yangerkun Link: https://lore.kernel.org/r/20210309030410.3294078-1-yangerkun@huawei.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index c2e7031f6d09..3d7060ba547a 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -799,8 +799,7 @@ static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) /* Can only happen if manager creation fails after exec */ if (io_wq_fork_manager(wqe->wq) || test_bit(IO_WQ_BIT_EXIT, &wqe->wq->state)) { - work->flags |= IO_WQ_WORK_CANCEL; - wqe->wq->do_work(work); + io_run_cancel(work, wqe); return; } -- cgit v1.2.3 From 93e68e036c2fc1ce18e784418e4e19975a5882b4 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 9 Mar 2021 07:02:21 -0700 Subject: io_uring: move all io_kiocb init early in io_init_req() If we hit an error path in the function, make sure that the io_kiocb is fully initialized at that point so that freeing the request always sees a valid state. Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index f7153483a3ac..0f18e4a7bd08 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6327,6 +6327,9 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, refcount_set(&req->refs, 2); req->task = current; req->result = 0; + req->work.list.next = NULL; + req->work.creds = NULL; + req->work.flags = 0; /* enforce forwards compatibility on users */ if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) { @@ -6344,17 +6347,13 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, !io_op_defs[req->opcode].buffer_select) return -EOPNOTSUPP; - req->work.list.next = NULL; personality = READ_ONCE(sqe->personality); if (personality) { req->work.creds = xa_load(&ctx->personalities, personality); if (!req->work.creds) return -EINVAL; get_cred(req->work.creds); - } else { - req->work.creds = NULL; } - req->work.flags = 0; state = &ctx->submit_state; /* -- cgit v1.2.3 From 5199328a0d415b3e372633096b1b92f36b8ac9e5 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Tue, 9 Mar 2021 14:30:41 +0800 Subject: io_uring: remove unneeded variable 'ret' Fix the following coccicheck warning: ./fs/io_uring.c:8984:5-8: Unneeded variable: "ret". Return "0" on line 8998 Reported-by: Abaci Robot Signed-off-by: Yang Li Link: https://lore.kernel.org/r/1615271441-33649-1-git-send-email-yang.lee@linux.alibaba.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 0f18e4a7bd08..6325f32ef6a3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9022,7 +9022,6 @@ static unsigned long io_uring_nommu_get_unmapped_area(struct file *file, static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) { - int ret = 0; DEFINE_WAIT(wait); do { @@ -9036,7 +9035,7 @@ static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) } while (!signal_pending(current)); finish_wait(&ctx->sqo_sq_wait, &wait); - return ret; + return 0; } static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz, -- cgit v1.2.3 From e8f98f24549d62cc54bf608c815904a56d4437bc Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 9 Mar 2021 16:32:13 -0700 Subject: io_uring: always wait for sqd exited when stopping SQPOLL thread We have a tiny race where io_put_sq_data() calls io_sq_thead_stop() and finds the thread gone, but the thread has indeed not fully exited or called complete() yet. Close it up by always having io_sq_thread_stop() wait on completion of the exit event. Signed-off-by: Jens Axboe --- fs/io_uring.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6325f32ef6a3..62f998bf2ce8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7079,12 +7079,9 @@ static void io_sq_thread_stop(struct io_sq_data *sqd) if (test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) return; down_write(&sqd->rw_lock); - if (!sqd->thread) { - up_write(&sqd->rw_lock); - return; - } set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); - wake_up_process(sqd->thread); + if (sqd->thread) + wake_up_process(sqd->thread); up_write(&sqd->rw_lock); wait_for_completion(&sqd->exited); } @@ -7849,9 +7846,9 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, ret = -EINVAL; if (cpu >= nr_cpu_ids) - goto err; + goto err_sqpoll; if (!cpu_online(cpu)) - goto err; + goto err_sqpoll; sqd->sq_cpu = cpu; } else { @@ -7862,7 +7859,7 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE); if (IS_ERR(tsk)) { ret = PTR_ERR(tsk); - goto err; + goto err_sqpoll; } sqd->thread = tsk; @@ -7881,6 +7878,9 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, err: io_sq_thread_finish(ctx); return ret; +err_sqpoll: + complete(&ctx->sq_data->exited); + goto err; } static inline void __io_unaccount_mem(struct user_struct *user, -- cgit v1.2.3 From e22bc9b481a90d7898984ea17621f04a653e2cd1 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 9 Mar 2021 19:49:02 -0700 Subject: kernel: make IO threads unfreezable by default The io-wq threads were already marked as no-freeze, but the manager was not. On resume, we perpetually have signal_pending() being true, and hence the manager will loop and spin 100% of the time. Just mark the tasks created by create_io_thread() as PF_NOFREEZE by default, and remove any knowledge of it in io-wq and io_uring. Reported-by: Kevin Locke Tested-by: Kevin Locke Signed-off-by: Jens Axboe --- fs/io-wq.c | 3 +-- fs/io_uring.c | 1 - kernel/fork.c | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 3d7060ba547a..0ae9ecadf295 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -591,7 +591,7 @@ static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) tsk->pf_io_worker = worker; worker->task = tsk; set_cpus_allowed_ptr(tsk, cpumask_of_node(wqe->node)); - tsk->flags |= PF_NOFREEZE | PF_NO_SETAFFINITY; + tsk->flags |= PF_NO_SETAFFINITY; raw_spin_lock_irq(&wqe->lock); hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); @@ -709,7 +709,6 @@ static int io_wq_manager(void *data) set_current_state(TASK_INTERRUPTIBLE); io_wq_check_workers(wq); schedule_timeout(HZ); - try_to_freeze(); if (fatal_signal_pending(current)) set_bit(IO_WQ_BIT_EXIT, &wq->state); } while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)); diff --git a/fs/io_uring.c b/fs/io_uring.c index 62f998bf2ce8..14165e18020c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6733,7 +6733,6 @@ static int io_sq_thread(void *data) up_read(&sqd->rw_lock); schedule(); - try_to_freeze(); down_read(&sqd->rw_lock); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_clear_wakeup_flag(ctx); diff --git a/kernel/fork.c b/kernel/fork.c index d3171e8e88e5..72e444cd0ffe 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2436,6 +2436,7 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) if (!IS_ERR(tsk)) { sigfillset(&tsk->blocked); sigdelsetmask(&tsk->blocked, sigmask(SIGKILL)); + tsk->flags |= PF_NOFREEZE; } return tsk; } -- cgit v1.2.3 From 78d7f6ba82edb7f8763390982be29051c4216772 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Mar 2021 13:13:53 +0000 Subject: io_uring: fix invalid ctx->sq_thread_idle We have to set ctx->sq_thread_idle before adding a ring to an SQ task, otherwise sqd races for seeing zero and accounting it as such. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 14165e18020c..7072c0eb22c1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7829,14 +7829,14 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, ctx->sq_creds = get_current_cred(); ctx->sq_data = sqd; - io_sq_thread_park(sqd); - list_add(&ctx->sqd_list, &sqd->ctx_new_list); - io_sq_thread_unpark(sqd); - ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle); if (!ctx->sq_thread_idle) ctx->sq_thread_idle = HZ; + io_sq_thread_park(sqd); + list_add(&ctx->sqd_list, &sqd->ctx_new_list); + io_sq_thread_unpark(sqd); + if (sqd->thread) return 0; -- cgit v1.2.3 From 7d41e8543d809c3c900d1212d6ea887eb284b69a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Mar 2021 13:13:54 +0000 Subject: io_uring: remove indirect ctx into sqo injection We use ->ctx_new_list to notify sqo about new ctx pending, then sqo should stop and splice it to its sqd->ctx_list, paired with ->sq_thread_comp. The last one is broken because nobody reinitialises it, and trying to fix it would only add more complexity and bugs. And the first isn't really needed as is done under park(), that protects from races well. Add ctx into sqd->ctx_list directly (under park()), it's much simpler and allows to kill both, ctx_new_list and sq_thread_comp. note: apparently there is no real problem at the moment, because sq_thread_comp is used only by io_sq_thread_finish() followed by parking, where list_del(&ctx->sqd_list) removes it well regardless whether it's in the new or the active list. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 7072c0eb22c1..5c045a9f7ffe 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -262,7 +262,6 @@ struct io_sq_data { /* ctx's that are using this sqd */ struct list_head ctx_list; - struct list_head ctx_new_list; struct task_struct *thread; struct wait_queue_head wait; @@ -398,7 +397,6 @@ struct io_ring_ctx { struct user_struct *user; struct completion ref_comp; - struct completion sq_thread_comp; #if defined(CONFIG_UNIX) struct socket *ring_sock; @@ -1137,7 +1135,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) init_waitqueue_head(&ctx->cq_wait); INIT_LIST_HEAD(&ctx->cq_overflow_list); init_completion(&ctx->ref_comp); - init_completion(&ctx->sq_thread_comp); idr_init(&ctx->io_buffer_idr); xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); mutex_init(&ctx->uring_lock); @@ -6640,19 +6637,6 @@ static void io_sqd_update_thread_idle(struct io_sq_data *sqd) sqd->sq_thread_idle = sq_thread_idle; } -static void io_sqd_init_new(struct io_sq_data *sqd) -{ - struct io_ring_ctx *ctx; - - while (!list_empty(&sqd->ctx_new_list)) { - ctx = list_first_entry(&sqd->ctx_new_list, struct io_ring_ctx, sqd_list); - list_move_tail(&ctx->sqd_list, &sqd->ctx_list); - complete(&ctx->sq_thread_comp); - } - - io_sqd_update_thread_idle(sqd); -} - static int io_sq_thread(void *data) { struct io_sq_data *sqd = data; @@ -6683,11 +6667,8 @@ static int io_sq_thread(void *data) up_read(&sqd->rw_lock); cond_resched(); down_read(&sqd->rw_lock); - continue; - } - if (unlikely(!list_empty(&sqd->ctx_new_list))) { - io_sqd_init_new(sqd); timeout = jiffies + sqd->sq_thread_idle; + continue; } if (fatal_signal_pending(current)) break; @@ -7099,9 +7080,6 @@ static void io_sq_thread_finish(struct io_ring_ctx *ctx) if (sqd) { complete(&sqd->startup); - if (sqd->thread) - wait_for_completion(&ctx->sq_thread_comp); - io_sq_thread_park(sqd); list_del(&ctx->sqd_list); io_sqd_update_thread_idle(sqd); @@ -7153,7 +7131,6 @@ static struct io_sq_data *io_get_sq_data(struct io_uring_params *p) refcount_set(&sqd->refs, 1); INIT_LIST_HEAD(&sqd->ctx_list); - INIT_LIST_HEAD(&sqd->ctx_new_list); init_rwsem(&sqd->rw_lock); init_waitqueue_head(&sqd->wait); init_completion(&sqd->startup); @@ -7834,7 +7811,8 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, ctx->sq_thread_idle = HZ; io_sq_thread_park(sqd); - list_add(&ctx->sqd_list, &sqd->ctx_new_list); + list_add(&ctx->sqd_list, &sqd->ctx_list); + io_sqd_update_thread_idle(sqd); io_sq_thread_unpark(sqd); if (sqd->thread) -- cgit v1.2.3 From 14fbbc8297728e880070f7b077b3301a8c698ef9 Mon Sep 17 00:00:00 2001 From: Daiyue Zhang Date: Mon, 1 Mar 2021 14:10:53 +0800 Subject: configfs: fix a use-after-free in __configfs_open_file Commit b0841eefd969 ("configfs: provide exclusion between IO and removals") uses ->frag_dead to mark the fragment state, thus no bothering with extra refcount on config_item when opening a file. The configfs_get_config_item was removed in __configfs_open_file, but not with config_item_put. So the refcount on config_item will lost its balance, causing use-after-free issues in some occasions like this: Test: 1. Mount configfs on /config with read-only items: drwxrwx--- 289 root root 0 2021-04-01 11:55 /config drwxr-xr-x 2 root root 0 2021-04-01 11:54 /config/a --w--w--w- 1 root root 4096 2021-04-01 11:53 /config/a/1.txt ...... 2. Then run: for file in /config do echo $file grep -R 'key' $file done 3. __configfs_open_file will be called in parallel, the first one got called will do: if (file->f_mode & FMODE_READ) { if (!(inode->i_mode & S_IRUGO)) goto out_put_module; config_item_put(buffer->item); kref_put() package_details_release() kfree() the other one will run into use-after-free issues like this: BUG: KASAN: use-after-free in __configfs_open_file+0x1bc/0x3b0 Read of size 8 at addr fffffff155f02480 by task grep/13096 CPU: 0 PID: 13096 Comm: grep VIP: 00 Tainted: G W 4.14.116-kasan #1 TGID: 13096 Comm: grep Call trace: dump_stack+0x118/0x160 kasan_report+0x22c/0x294 __asan_load8+0x80/0x88 __configfs_open_file+0x1bc/0x3b0 configfs_open_file+0x28/0x34 do_dentry_open+0x2cc/0x5c0 vfs_open+0x80/0xe0 path_openat+0xd8c/0x2988 do_filp_open+0x1c4/0x2fc do_sys_open+0x23c/0x404 SyS_openat+0x38/0x48 Allocated by task 2138: kasan_kmalloc+0xe0/0x1ac kmem_cache_alloc_trace+0x334/0x394 packages_make_item+0x4c/0x180 configfs_mkdir+0x358/0x740 vfs_mkdir2+0x1bc/0x2e8 SyS_mkdirat+0x154/0x23c el0_svc_naked+0x34/0x38 Freed by task 13096: kasan_slab_free+0xb8/0x194 kfree+0x13c/0x910 package_details_release+0x524/0x56c kref_put+0xc4/0x104 config_item_put+0x24/0x34 __configfs_open_file+0x35c/0x3b0 configfs_open_file+0x28/0x34 do_dentry_open+0x2cc/0x5c0 vfs_open+0x80/0xe0 path_openat+0xd8c/0x2988 do_filp_open+0x1c4/0x2fc do_sys_open+0x23c/0x404 SyS_openat+0x38/0x48 el0_svc_naked+0x34/0x38 To fix this issue, remove the config_item_put in __configfs_open_file to balance the refcount of config_item. Fixes: b0841eefd969 ("configfs: provide exclusion between IO and removals") Signed-off-by: Daiyue Zhang Signed-off-by: Yi Chen Signed-off-by: Ge Qiu Reviewed-by: Chao Yu Acked-by: Al Viro Signed-off-by: Christoph Hellwig --- fs/configfs/file.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/configfs/file.c b/fs/configfs/file.c index 1f0270229d7b..da8351d1e455 100644 --- a/fs/configfs/file.c +++ b/fs/configfs/file.c @@ -378,7 +378,7 @@ static int __configfs_open_file(struct inode *inode, struct file *file, int type attr = to_attr(dentry); if (!attr) - goto out_put_item; + goto out_free_buffer; if (type & CONFIGFS_ITEM_BIN_ATTR) { buffer->bin_attr = to_bin_attr(dentry); @@ -391,7 +391,7 @@ static int __configfs_open_file(struct inode *inode, struct file *file, int type /* Grab the module reference for this attribute if we have one */ error = -ENODEV; if (!try_module_get(buffer->owner)) - goto out_put_item; + goto out_free_buffer; error = -EACCES; if (!buffer->item->ci_type) @@ -435,8 +435,6 @@ static int __configfs_open_file(struct inode *inode, struct file *file, int type out_put_module: module_put(buffer->owner); -out_put_item: - config_item_put(buffer->item); out_free_buffer: up_read(&frag->frag_sem); kfree(buffer); -- cgit v1.2.3 From a8affc03a9b375e19bc81573de0c9108317d78c7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 11 Mar 2021 12:01:37 +0100 Subject: block: rename BIO_MAX_PAGES to BIO_MAX_VECS Ever since the addition of multipage bio_vecs BIO_MAX_PAGES has been horribly confusingly misnamed. Rename it to BIO_MAX_VECS to stop confusing users of the bio API. Signed-off-by: Christoph Hellwig Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Martin K. Petersen Link: https://lore.kernel.org/r/20210311110137.1132391-2-hch@lst.de Signed-off-by: Jens Axboe --- block/bio.c | 14 +++++++------- block/blk-crypto-fallback.c | 2 +- block/blk-lib.c | 2 +- block/blk-map.c | 2 +- block/bounce.c | 6 +++--- drivers/block/drbd/drbd_int.h | 2 +- drivers/md/bcache/super.c | 2 +- drivers/md/dm-crypt.c | 8 ++++---- drivers/md/dm-writecache.c | 4 ++-- drivers/md/raid5-cache.c | 4 ++-- drivers/md/raid5-ppl.c | 2 +- drivers/nvme/target/passthru.c | 6 +++--- fs/block_dev.c | 6 +++--- fs/btrfs/extent_io.c | 2 +- fs/btrfs/scrub.c | 2 +- fs/crypto/bio.c | 6 +++--- fs/erofs/zdata.c | 2 +- fs/ext4/page-io.c | 2 +- fs/f2fs/checkpoint.c | 2 +- fs/f2fs/data.c | 4 ++-- fs/f2fs/segment.c | 2 +- fs/f2fs/segment.h | 4 ++-- fs/f2fs/super.c | 4 ++-- fs/gfs2/lops.c | 2 +- fs/iomap/buffered-io.c | 4 ++-- fs/iomap/direct-io.c | 4 ++-- fs/mpage.c | 2 +- fs/nilfs2/segbuf.c | 2 +- fs/squashfs/block.c | 2 +- fs/zonefs/super.c | 2 +- include/linux/bio.h | 4 ++-- 31 files changed, 56 insertions(+), 56 deletions(-) (limited to 'fs') diff --git a/block/bio.c b/block/bio.c index a1c4d2900c7a..26b7f721cda8 100644 --- a/block/bio.c +++ b/block/bio.c @@ -33,7 +33,7 @@ static struct biovec_slab { { .nr_vecs = 16, .name = "biovec-16" }, { .nr_vecs = 64, .name = "biovec-64" }, { .nr_vecs = 128, .name = "biovec-128" }, - { .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" }, + { .nr_vecs = BIO_MAX_VECS, .name = "biovec-max" }, }; static struct biovec_slab *biovec_slab(unsigned short nr_vecs) @@ -46,7 +46,7 @@ static struct biovec_slab *biovec_slab(unsigned short nr_vecs) return &bvec_slabs[1]; case 65 ... 128: return &bvec_slabs[2]; - case 129 ... BIO_MAX_PAGES: + case 129 ... BIO_MAX_VECS: return &bvec_slabs[3]; default: BUG(); @@ -151,9 +151,9 @@ out: void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs) { - BIO_BUG_ON(nr_vecs > BIO_MAX_PAGES); + BIO_BUG_ON(nr_vecs > BIO_MAX_VECS); - if (nr_vecs == BIO_MAX_PAGES) + if (nr_vecs == BIO_MAX_VECS) mempool_free(bv, pool); else if (nr_vecs > BIO_INLINE_VECS) kmem_cache_free(biovec_slab(nr_vecs)->slab, bv); @@ -186,15 +186,15 @@ struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, /* * Try a slab allocation first for all smaller allocations. If that * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool. - * The mempool is sized to handle up to BIO_MAX_PAGES entries. + * The mempool is sized to handle up to BIO_MAX_VECS entries. */ - if (*nr_vecs < BIO_MAX_PAGES) { + if (*nr_vecs < BIO_MAX_VECS) { struct bio_vec *bvl; bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask)); if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) return bvl; - *nr_vecs = BIO_MAX_PAGES; + *nr_vecs = BIO_MAX_VECS; } return mempool_alloc(pool, gfp_mask); diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index c176b7af56a7..c322176a1e09 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -219,7 +219,7 @@ static bool blk_crypto_split_bio_if_needed(struct bio **bio_ptr) bio_for_each_segment(bv, bio, iter) { num_sectors += bv.bv_len >> SECTOR_SHIFT; - if (++i == BIO_MAX_PAGES) + if (++i == BIO_MAX_VECS) break; } if (num_sectors < bio_sectors(bio)) { diff --git a/block/blk-lib.c b/block/blk-lib.c index 752f9c722062..7b256131b20b 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -296,7 +296,7 @@ static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects) { sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512); - return min(pages, (sector_t)BIO_MAX_PAGES); + return min(pages, (sector_t)BIO_MAX_VECS); } static int __blkdev_issue_zero_pages(struct block_device *bdev, diff --git a/block/blk-map.c b/block/blk-map.c index 369e204d14d0..1ffef782fcf2 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -249,7 +249,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, if (!iov_iter_count(iter)) return -EINVAL; - bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES)); + bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_VECS)); if (!bio) return -ENOMEM; bio->bi_opf |= req_op(rq); diff --git a/block/bounce.c b/block/bounce.c index 87983a35079c..6c441f4f1cd4 100644 --- a/block/bounce.c +++ b/block/bounce.c @@ -229,10 +229,10 @@ static struct bio *bounce_clone_bio(struct bio *bio_src) * - The point of cloning the biovec is to produce a bio with a biovec * the caller can modify: bi_idx and bi_bvec_done should be 0. * - * - The original bio could've had more than BIO_MAX_PAGES biovecs; if + * - The original bio could've had more than BIO_MAX_VECS biovecs; if * we tried to clone the whole thing bio_alloc_bioset() would fail. * But the clone should succeed as long as the number of biovecs we - * actually need to allocate is fewer than BIO_MAX_PAGES. + * actually need to allocate is fewer than BIO_MAX_VECS. * * - Lastly, bi_vcnt should not be looked at or relied upon by code * that does not own the bio - reason being drivers don't use it for @@ -299,7 +299,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig, int sectors = 0; bio_for_each_segment(from, *bio_orig, iter) { - if (i++ < BIO_MAX_PAGES) + if (i++ < BIO_MAX_VECS) sectors += from.bv_len >> 9; if (page_to_pfn(from.bv_page) > q->limits.bounce_pfn) bounce = true; diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index 7d9cc433b758..5d9181382ce1 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -1324,7 +1324,7 @@ struct bm_extent { * A followup commit may allow even bigger BIO sizes, * once we thought that through. */ #define DRBD_MAX_BIO_SIZE (1U << 20) -#if DRBD_MAX_BIO_SIZE > (BIO_MAX_PAGES << PAGE_SHIFT) +#if DRBD_MAX_BIO_SIZE > (BIO_MAX_VECS << PAGE_SHIFT) #error Architecture not supported: DRBD_MAX_BIO_SIZE > BIO_MAX_SIZE #endif #define DRBD_MAX_BIO_SIZE_SAFE (1U << 12) /* Works always = 4k */ diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 71691f32959b..03e1fe4de53d 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -965,7 +965,7 @@ static int bcache_device_init(struct bcache_device *d, unsigned int block_size, q->limits.max_hw_sectors = UINT_MAX; q->limits.max_sectors = UINT_MAX; q->limits.max_segment_size = UINT_MAX; - q->limits.max_segments = BIO_MAX_PAGES; + q->limits.max_segments = BIO_MAX_VECS; blk_queue_max_discard_sectors(q, UINT_MAX); q->limits.discard_granularity = 512; q->limits.io_min = block_size; diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 11c105ecd165..b0ab080f2567 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -229,7 +229,7 @@ static DEFINE_SPINLOCK(dm_crypt_clients_lock); static unsigned dm_crypt_clients_n = 0; static volatile unsigned long dm_crypt_pages_per_client; #define DM_CRYPT_MEMORY_PERCENT 2 -#define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_PAGES * 16) +#define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_VECS * 16) static void clone_init(struct dm_crypt_io *, struct bio *); static void kcryptd_queue_crypt(struct dm_crypt_io *io); @@ -3246,7 +3246,7 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size, ARCH_KMALLOC_MINALIGN); - ret = mempool_init(&cc->page_pool, BIO_MAX_PAGES, crypt_page_alloc, crypt_page_free, cc); + ret = mempool_init(&cc->page_pool, BIO_MAX_VECS, crypt_page_alloc, crypt_page_free, cc); if (ret) { ti->error = "Cannot allocate page mempool"; goto bad; @@ -3373,9 +3373,9 @@ static int crypt_map(struct dm_target *ti, struct bio *bio) /* * Check if bio is too large, split as needed. */ - if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_PAGES << PAGE_SHIFT)) && + if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_VECS << PAGE_SHIFT)) && (bio_data_dir(bio) == WRITE || cc->on_disk_tag_size)) - dm_accept_partial_bio(bio, ((BIO_MAX_PAGES << PAGE_SHIFT) >> SECTOR_SHIFT)); + dm_accept_partial_bio(bio, ((BIO_MAX_VECS << PAGE_SHIFT) >> SECTOR_SHIFT)); /* * Ensure that bio is a multiple of internal sector encryption size diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index 844c4be11768..4f72b6f66c3a 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -1892,10 +1892,10 @@ restart: list_add(&g->lru, &wbl.list); wbl.size++; g->write_in_progress = true; - g->wc_list_contiguous = BIO_MAX_PAGES; + g->wc_list_contiguous = BIO_MAX_VECS; f = g; e->wc_list_contiguous++; - if (unlikely(e->wc_list_contiguous == BIO_MAX_PAGES)) { + if (unlikely(e->wc_list_contiguous == BIO_MAX_VECS)) { if (unlikely(wc->writeback_all)) { next_node = rb_next(&f->rb_node); if (likely(next_node)) diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c index 4337ae0e6af2..0b5dcaabbc15 100644 --- a/drivers/md/raid5-cache.c +++ b/drivers/md/raid5-cache.c @@ -735,7 +735,7 @@ static void r5l_submit_current_io(struct r5l_log *log) static struct bio *r5l_bio_alloc(struct r5l_log *log) { - struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, &log->bs); + struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_VECS, &log->bs); bio_set_op_attrs(bio, REQ_OP_WRITE, 0); bio_set_dev(bio, log->rdev->bdev); @@ -1634,7 +1634,7 @@ static int r5l_recovery_allocate_ra_pool(struct r5l_log *log, { struct page *page; - ctx->ra_bio = bio_alloc_bioset(GFP_KERNEL, BIO_MAX_PAGES, &log->bs); + ctx->ra_bio = bio_alloc_bioset(GFP_KERNEL, BIO_MAX_VECS, &log->bs); if (!ctx->ra_bio) return -ENOMEM; diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index e8c118e05dfd..3ddc2aa0b530 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -496,7 +496,7 @@ static void ppl_submit_iounit(struct ppl_io_unit *io) if (!bio_add_page(bio, sh->ppl_page, PAGE_SIZE, 0)) { struct bio *prev = bio; - bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, + bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_VECS, &ppl_conf->bs); bio->bi_opf = prev->bi_opf; bio->bi_write_hint = prev->bi_write_hint; diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index 26c587ccd152..2798944899b7 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -50,9 +50,9 @@ static u16 nvmet_passthru_override_id_ctrl(struct nvmet_req *req) /* * nvmet_passthru_map_sg is limitted to using a single bio so limit - * the mdts based on BIO_MAX_PAGES as well + * the mdts based on BIO_MAX_VECS as well */ - max_hw_sectors = min_not_zero(BIO_MAX_PAGES << (PAGE_SHIFT - 9), + max_hw_sectors = min_not_zero(BIO_MAX_VECS << (PAGE_SHIFT - 9), max_hw_sectors); page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12; @@ -191,7 +191,7 @@ static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq) struct bio *bio; int i; - if (req->sg_cnt > BIO_MAX_PAGES) + if (req->sg_cnt > BIO_MAX_VECS) return -EINVAL; if (req->transfer_len <= NVMET_MAX_INLINE_DATA_LEN) { diff --git a/fs/block_dev.c b/fs/block_dev.c index 03166b3dea4d..92ed7d5df677 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -432,7 +432,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, dio->size += bio->bi_iter.bi_size; pos += bio->bi_iter.bi_size; - nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_PAGES); + nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS); if (!nr_pages) { bool polled = false; @@ -500,8 +500,8 @@ blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (!iov_iter_count(iter)) return 0; - nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_PAGES + 1); - if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES) + nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS + 1); + if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_VECS) return __blkdev_direct_IO_simple(iocb, iter, nr_pages); return __blkdev_direct_IO(iocb, iter, bio_max_segs(nr_pages)); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 4dfb3ead1175..db8cb98c020c 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -3048,7 +3048,7 @@ struct bio *btrfs_bio_alloc(u64 first_byte) { struct bio *bio; - bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &btrfs_bioset); + bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_VECS, &btrfs_bioset); bio->bi_iter.bi_sector = first_byte >> 9; btrfs_io_bio_init(btrfs_io_bio(bio)); return bio; diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 582df11d298a..6daa4309c974 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -1428,7 +1428,7 @@ static void scrub_recheck_block_on_raid56(struct btrfs_fs_info *fs_info, if (!first_page->dev->bdev) goto out; - bio = btrfs_io_bio_alloc(BIO_MAX_PAGES); + bio = btrfs_io_bio_alloc(BIO_MAX_VECS); bio_set_dev(bio, first_page->dev->bdev); for (page_num = 0; page_num < sblock->page_count; page_num++) { diff --git a/fs/crypto/bio.c b/fs/crypto/bio.c index b048a0e38516..68a2de6b5a9b 100644 --- a/fs/crypto/bio.c +++ b/fs/crypto/bio.c @@ -52,7 +52,7 @@ static int fscrypt_zeroout_range_inline_crypt(const struct inode *inode, int num_pages = 0; /* This always succeeds since __GFP_DIRECT_RECLAIM is set. */ - bio = bio_alloc(GFP_NOFS, BIO_MAX_PAGES); + bio = bio_alloc(GFP_NOFS, BIO_MAX_VECS); while (len) { unsigned int blocks_this_page = min(len, blocks_per_page); @@ -74,7 +74,7 @@ static int fscrypt_zeroout_range_inline_crypt(const struct inode *inode, len -= blocks_this_page; lblk += blocks_this_page; pblk += blocks_this_page; - if (num_pages == BIO_MAX_PAGES || !len || + if (num_pages == BIO_MAX_VECS || !len || !fscrypt_mergeable_bio(bio, inode, lblk)) { err = submit_bio_wait(bio); if (err) @@ -126,7 +126,7 @@ int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, return fscrypt_zeroout_range_inline_crypt(inode, lblk, pblk, len); - BUILD_BUG_ON(ARRAY_SIZE(pages) > BIO_MAX_PAGES); + BUILD_BUG_ON(ARRAY_SIZE(pages) > BIO_MAX_VECS); nr_pages = min_t(unsigned int, ARRAY_SIZE(pages), (len + blocks_per_page - 1) >> blocks_per_page_bits); diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 6cb356c4217b..3851e1a64f73 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -1235,7 +1235,7 @@ submit_bio_retry: } if (!bio) { - bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); + bio = bio_alloc(GFP_NOIO, BIO_MAX_VECS); bio->bi_end_io = z_erofs_decompressqueue_endio; bio_set_dev(bio, sb->s_bdev); diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 03a44a0de86a..f038d578d8d8 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -398,7 +398,7 @@ static void io_submit_init_bio(struct ext4_io_submit *io, * bio_alloc will _always_ be able to allocate a bio if * __GFP_DIRECT_RECLAIM is set, see comments for bio_alloc_bioset(). */ - bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); + bio = bio_alloc(GFP_NOIO, BIO_MAX_VECS); fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio_set_dev(bio, bh->b_bdev); diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 174a0819ad96..be5415a0dbbc 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -292,7 +292,7 @@ void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index) f2fs_put_page(page, 0); if (readahead) - f2fs_ra_meta_pages(sbi, index, BIO_MAX_PAGES, META_POR, true); + f2fs_ra_meta_pages(sbi, index, BIO_MAX_VECS, META_POR, true); } static int __f2fs_write_meta_page(struct page *page, diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 7c95818639a6..4e5257c763d0 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -857,7 +857,7 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio) f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL); alloc_new: if (!bio) { - bio = __bio_alloc(fio, BIO_MAX_PAGES); + bio = __bio_alloc(fio, BIO_MAX_VECS); __attach_io_flag(fio); f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, fio->page->index, fio, GFP_NOIO); @@ -932,7 +932,7 @@ alloc_new: fio->retry = true; goto skip; } - io->bio = __bio_alloc(fio, BIO_MAX_PAGES); + io->bio = __bio_alloc(fio, BIO_MAX_VECS); f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host, bio_page->index, fio, GFP_NOIO); io->fio = *fio; diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 993004f06a77..c2866561263e 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -4381,7 +4381,7 @@ static int build_sit_entries(struct f2fs_sb_info *sbi) block_t total_node_blocks = 0; do { - readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES, + readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS, META_SIT, true); start = start_blk * sit_i->sents_per_block; diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index 229814b4f4a6..e9a7a637d688 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -851,7 +851,7 @@ static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type) else if (type == NODE) return 8 * sbi->blocks_per_seg; else if (type == META) - return 8 * BIO_MAX_PAGES; + return 8 * BIO_MAX_VECS; else return 0; } @@ -868,7 +868,7 @@ static inline long nr_pages_to_write(struct f2fs_sb_info *sbi, int type, return 0; nr_to_write = wbc->nr_to_write; - desired = BIO_MAX_PAGES; + desired = BIO_MAX_VECS; if (type == NODE) desired <<= 1; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 7069793752f1..82592b19b4e0 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -753,9 +753,9 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) case Opt_io_size_bits: if (args->from && match_int(args, &arg)) return -EINVAL; - if (arg <= 0 || arg > __ilog2_u32(BIO_MAX_PAGES)) { + if (arg <= 0 || arg > __ilog2_u32(BIO_MAX_VECS)) { f2fs_warn(sbi, "Not support %d, larger than %d", - 1 << arg, BIO_MAX_PAGES); + 1 << arg, BIO_MAX_VECS); return -EINVAL; } F2FS_OPTION(sbi).write_io_size_bits = arg; diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index dc1b93a877c6..a82f4747aa8d 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c @@ -267,7 +267,7 @@ static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno, bio_end_io_t *end_io) { struct super_block *sb = sdp->sd_vfs; - struct bio *bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); + struct bio *bio = bio_alloc(GFP_NOIO, BIO_MAX_VECS); bio->bi_iter.bi_sector = blkno << sdp->sd_fsb2bb_shift; bio_set_dev(bio, sb->s_bdev); diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 7ffcd7ef33d4..414769a6ad11 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -1221,7 +1221,7 @@ iomap_alloc_ioend(struct inode *inode, struct iomap_writepage_ctx *wpc, struct iomap_ioend *ioend; struct bio *bio; - bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &iomap_ioend_bioset); + bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_VECS, &iomap_ioend_bioset); bio_set_dev(bio, wpc->iomap.bdev); bio->bi_iter.bi_sector = sector; bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc); @@ -1252,7 +1252,7 @@ iomap_chain_bio(struct bio *prev) { struct bio *new; - new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES); + new = bio_alloc(GFP_NOFS, BIO_MAX_VECS); bio_copy_dev(new, prev);/* also copies over blkcg information */ new->bi_iter.bi_sector = bio_end_sector(prev); new->bi_opf = prev->bi_opf; diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index e2c4991833b8..bdd0d89bbf0a 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -296,7 +296,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, */ bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua); - nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_PAGES); + nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS); do { size_t n; if (dio->error) { @@ -338,7 +338,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, copied += n; nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, - BIO_MAX_PAGES); + BIO_MAX_VECS); iomap_dio_submit_bio(dio, iomap, bio, pos); pos += n; } while (nr_pages); diff --git a/fs/mpage.c b/fs/mpage.c index 961234d68779..334e7d09aa65 100644 --- a/fs/mpage.c +++ b/fs/mpage.c @@ -616,7 +616,7 @@ alloc_new: goto out; } bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9), - BIO_MAX_PAGES, GFP_NOFS|__GFP_HIGH); + BIO_MAX_VECS, GFP_NOFS|__GFP_HIGH); if (bio == NULL) goto confused; diff --git a/fs/nilfs2/segbuf.c b/fs/nilfs2/segbuf.c index 1e75417bfe6e..56872e93823d 100644 --- a/fs/nilfs2/segbuf.c +++ b/fs/nilfs2/segbuf.c @@ -399,7 +399,7 @@ static void nilfs_segbuf_prepare_write(struct nilfs_segment_buffer *segbuf, { wi->bio = NULL; wi->rest_blocks = segbuf->sb_sum.nblocks; - wi->max_pages = BIO_MAX_PAGES; + wi->max_pages = BIO_MAX_VECS; wi->nr_vecs = min(wi->max_pages, wi->rest_blocks); wi->start = wi->end = 0; wi->blocknr = segbuf->sb_pseg_start; diff --git a/fs/squashfs/block.c b/fs/squashfs/block.c index 45f44425d856..b9e87ebb1060 100644 --- a/fs/squashfs/block.c +++ b/fs/squashfs/block.c @@ -87,7 +87,7 @@ static int squashfs_bio_read(struct super_block *sb, u64 index, int length, int error, i; struct bio *bio; - if (page_count <= BIO_MAX_PAGES) + if (page_count <= BIO_MAX_VECS) bio = bio_alloc(GFP_NOIO, page_count); else bio = bio_kmalloc(GFP_NOIO, page_count); diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index b6ff4a21abac..0fe76f376dee 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -684,7 +684,7 @@ static ssize_t zonefs_file_dio_append(struct kiocb *iocb, struct iov_iter *from) max = ALIGN_DOWN(max << SECTOR_SHIFT, inode->i_sb->s_blocksize); iov_iter_truncate(from, max); - nr_pages = iov_iter_npages(from, BIO_MAX_PAGES); + nr_pages = iov_iter_npages(from, BIO_MAX_VECS); if (!nr_pages) return 0; diff --git a/include/linux/bio.h b/include/linux/bio.h index 983ed2fe7c85..d0246c92a6e8 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -20,11 +20,11 @@ #define BIO_BUG_ON #endif -#define BIO_MAX_PAGES 256U +#define BIO_MAX_VECS 256U static inline unsigned int bio_max_segs(unsigned int nr_segs) { - return min(nr_segs, BIO_MAX_PAGES); + return min(nr_segs, BIO_MAX_VECS); } #define bio_prio(bio) (bio)->bi_ioprio -- cgit v1.2.3 From f053cf7aa66cd9d592b0fc967f4d887c2abff1b7 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 2 Mar 2021 12:04:19 -0800 Subject: ext4: fix error handling in ext4_end_enable_verity() ext4 didn't properly clean up if verity failed to be enabled on a file: - It left verity metadata (pages past EOF) in the page cache, which would be exposed to userspace if the file was later extended. - It didn't truncate the verity metadata at all (either from cache or from disk) if an error occurred while setting the verity bit. Fix these bugs by adding a call to truncate_inode_pages() and ensuring that we truncate the verity metadata (both from cache and from disk) in all error paths. Also rework the code to cleanly separate the success path from the error paths, which makes it much easier to understand. Reported-by: Yunlei He Fixes: c93d8f885809 ("ext4: add basic fs-verity support") Cc: stable@vger.kernel.org # v5.4+ Signed-off-by: Eric Biggers Link: https://lore.kernel.org/r/20210302200420.137977-2-ebiggers@kernel.org Signed-off-by: Theodore Ts'o --- fs/ext4/verity.c | 89 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 34 deletions(-) (limited to 'fs') diff --git a/fs/ext4/verity.c b/fs/ext4/verity.c index 5b7ba8f71153..00e3cbde472e 100644 --- a/fs/ext4/verity.c +++ b/fs/ext4/verity.c @@ -201,55 +201,76 @@ static int ext4_end_enable_verity(struct file *filp, const void *desc, struct inode *inode = file_inode(filp); const int credits = 2; /* superblock and inode for ext4_orphan_del() */ handle_t *handle; + struct ext4_iloc iloc; int err = 0; - int err2; - if (desc != NULL) { - /* Succeeded; write the verity descriptor. */ - err = ext4_write_verity_descriptor(inode, desc, desc_size, - merkle_tree_size); - - /* Write all pages before clearing VERITY_IN_PROGRESS. */ - if (!err) - err = filemap_write_and_wait(inode->i_mapping); - } + /* + * If an error already occurred (which fs/verity/ signals by passing + * desc == NULL), then only clean-up is needed. + */ + if (desc == NULL) + goto cleanup; - /* If we failed, truncate anything we wrote past i_size. */ - if (desc == NULL || err) - ext4_truncate(inode); + /* Append the verity descriptor. */ + err = ext4_write_verity_descriptor(inode, desc, desc_size, + merkle_tree_size); + if (err) + goto cleanup; /* - * We must always clean up by clearing EXT4_STATE_VERITY_IN_PROGRESS and - * deleting the inode from the orphan list, even if something failed. - * If everything succeeded, we'll also set the verity bit in the same - * transaction. + * Write all pages (both data and verity metadata). Note that this must + * happen before clearing EXT4_STATE_VERITY_IN_PROGRESS; otherwise pages + * beyond i_size won't be written properly. For crash consistency, this + * also must happen before the verity inode flag gets persisted. */ + err = filemap_write_and_wait(inode->i_mapping); + if (err) + goto cleanup; - ext4_clear_inode_state(inode, EXT4_STATE_VERITY_IN_PROGRESS); + /* + * Finally, set the verity inode flag and remove the inode from the + * orphan list (in a single transaction). + */ handle = ext4_journal_start(inode, EXT4_HT_INODE, credits); if (IS_ERR(handle)) { - ext4_orphan_del(NULL, inode); - return PTR_ERR(handle); + err = PTR_ERR(handle); + goto cleanup; } - err2 = ext4_orphan_del(handle, inode); - if (err2) - goto out_stop; + err = ext4_orphan_del(handle, inode); + if (err) + goto stop_and_cleanup; - if (desc != NULL && !err) { - struct ext4_iloc iloc; + err = ext4_reserve_inode_write(handle, inode, &iloc); + if (err) + goto stop_and_cleanup; - err = ext4_reserve_inode_write(handle, inode, &iloc); - if (err) - goto out_stop; - ext4_set_inode_flag(inode, EXT4_INODE_VERITY); - ext4_set_inode_flags(inode, false); - err = ext4_mark_iloc_dirty(handle, inode, &iloc); - } -out_stop: + ext4_set_inode_flag(inode, EXT4_INODE_VERITY); + ext4_set_inode_flags(inode, false); + err = ext4_mark_iloc_dirty(handle, inode, &iloc); + if (err) + goto stop_and_cleanup; + + ext4_journal_stop(handle); + + ext4_clear_inode_state(inode, EXT4_STATE_VERITY_IN_PROGRESS); + return 0; + +stop_and_cleanup: ext4_journal_stop(handle); - return err ?: err2; +cleanup: + /* + * Verity failed to be enabled, so clean up by truncating any verity + * metadata that was written beyond i_size (both from cache and from + * disk), removing the inode from the orphan list (if it wasn't done + * already), and clearing EXT4_STATE_VERITY_IN_PROGRESS. + */ + truncate_inode_pages(inode->i_mapping, inode->i_size); + ext4_truncate(inode); + ext4_orphan_del(NULL, inode); + ext4_clear_inode_state(inode, EXT4_STATE_VERITY_IN_PROGRESS); + return err; } static int ext4_get_verity_descriptor_location(struct inode *inode, -- cgit v1.2.3 From b4250dd868d1b42c0a65de11ef3afbee67ba5d2f Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Thu, 11 Mar 2021 10:55:00 -0500 Subject: NFSD: fix error handling in NFSv4.0 callbacks When the server tries to do a callback and a client fails it due to authentication problems, we need the server to set callback down flag in RENEW so that client can recover. Suggested-by: Bruce Fields Signed-off-by: Olga Kornievskaia Signed-off-by: Chuck Lever Tested-by: Benjamin Coddington Link: https://lore.kernel.org/linux-nfs/FB84E90A-1A03-48B3-8BF7-D9D10AC2C9FE@oracle.com/T/#t --- fs/nfsd/nfs4callback.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 052be5bf9ef5..7325592b456e 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -1189,6 +1189,7 @@ static void nfsd4_cb_done(struct rpc_task *task, void *calldata) switch (task->tk_status) { case -EIO: case -ETIMEDOUT: + case -EACCES: nfsd4_mark_cb_down(clp, task->tk_status); } break; -- cgit v1.2.3 From 5c2469e0a22e035d52f3ba768151cc75e3d4a1cd Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 11 Mar 2021 10:17:56 -0700 Subject: io_uring: force creation of separate context for ATTACH_WQ and non-threads Earlier kernels had SQPOLL threads that could share across anything, as we grabbed the context we needed on a per-ring basis. This is no longer the case, so only allow attaching directly if we're in the same thread group. That is the common use case. For non-group tasks, just setup a new context and thread as we would've done if sharing wasn't set. This isn't 100% ideal in terms of CPU utilization for the forked and share case, but hopefully that isn't much of a concern. If it is, there are plans in motion for how to improve that. Most importantly, we want to avoid app side regressions where sharing worked before and now doesn't. With this patch, functionality is equivalent to previous kernels that supported IORING_SETUP_ATTACH_WQ with SQPOLL. Reported-by: Stefan Metzmacher Signed-off-by: Jens Axboe --- fs/io_uring.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5c045a9f7ffe..472eab7359f2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -269,6 +269,7 @@ struct io_sq_data { unsigned sq_thread_idle; int sq_cpu; pid_t task_pid; + pid_t task_tgid; unsigned long state; struct completion startup; @@ -7112,6 +7113,10 @@ static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p) fdput(f); return ERR_PTR(-EINVAL); } + if (sqd->task_tgid != current->tgid) { + fdput(f); + return ERR_PTR(-EPERM); + } refcount_inc(&sqd->refs); fdput(f); @@ -7122,8 +7127,14 @@ static struct io_sq_data *io_get_sq_data(struct io_uring_params *p) { struct io_sq_data *sqd; - if (p->flags & IORING_SETUP_ATTACH_WQ) - return io_attach_sq_data(p); + if (p->flags & IORING_SETUP_ATTACH_WQ) { + sqd = io_attach_sq_data(p); + if (!IS_ERR(sqd)) + return sqd; + /* fall through for EPERM case, setup new sqd/task */ + if (PTR_ERR(sqd) != -EPERM) + return sqd; + } sqd = kzalloc(sizeof(*sqd), GFP_KERNEL); if (!sqd) @@ -7833,6 +7844,7 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, } sqd->task_pid = current->pid; + sqd->task_tgid = current->tgid; tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE); if (IS_ERR(tsk)) { ret = PTR_ERR(tsk); -- cgit v1.2.3 From d052d1d685f5125249ab4ff887562c88ba959638 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 11 Mar 2021 10:49:20 -0700 Subject: io_uring: perform IOPOLL reaping if canceler is thread itself We bypass IOPOLL completion polling (and reaping) for the SQPOLL thread, but if it's the thread itself invoking cancelations, then we still need to perform it or no one will. Fixes: 9936c7c2bc76 ("io_uring: deduplicate core cancellations sequence") Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 472eab7359f2..49f85f49e1c3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8658,7 +8658,8 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, } /* SQPOLL thread does its own polling */ - if (!(ctx->flags & IORING_SETUP_SQPOLL) && !files) { + if ((!(ctx->flags & IORING_SETUP_SQPOLL) && !files) || + (ctx->sq_data && ctx->sq_data->thread == current)) { while (!list_empty_careful(&ctx->iopoll_list)) { io_iopoll_try_reap_events(ctx); ret = true; -- cgit v1.2.3 From 4f8be1f53bf615102d103c0509ffa9596f65b718 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 28 Jan 2021 17:36:38 -0500 Subject: nfs: we don't support removing system.nfs4_acl The NFSv4 protocol doesn't have any notion of reomoving an attribute, so removexattr(path,"system.nfs4_acl") doesn't make sense. There's no documented return value. Arguably it could be EOPNOTSUPP but I'm a little worried an application might take that to mean that we don't support ACLs or xattrs. How about EINVAL? Signed-off-by: J. Bruce Fields Signed-off-by: Anna Schumaker --- fs/nfs/nfs4proc.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 1002c4f66f3f..c65c4b41e2c1 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -5889,6 +5889,9 @@ static int __nfs4_proc_set_acl(struct inode *inode, const void *buf, size_t bufl unsigned int npages = DIV_ROUND_UP(buflen, PAGE_SIZE); int ret, i; + /* You can't remove system.nfs4_acl: */ + if (buflen == 0) + return -EINVAL; if (!nfs4_server_supports_acls(server)) return -EOPNOTSUPP; if (npages > ARRAY_SIZE(pages)) -- cgit v1.2.3 From d5bf630f355d8c532bef2347cf90e8ae60a5f1bd Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Fri, 12 Mar 2021 07:58:54 -0500 Subject: gfs2: bypass signal_our_withdraw if no journal Before this patch, function signal_our_withdraw referenced the journal inode immediately. But corrupt file systems may have some invalid journals, in which case our attempt to read it in will withdraw and the resulting signal_our_withdraw would dereference the NULL value. This patch adds a check to signal_our_withdraw so that if the journal has not yet been initialized, it simply returns and does the old-style withdraw. Thanks, Andy Price, for his analysis. Reported-by: syzbot+50a8a9cf8127f2c6f5df@syzkaller.appspotmail.com Fixes: 601ef0d52e96 ("gfs2: Force withdraw to replay journals and wait for it to finish") Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/util.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/util.c b/fs/gfs2/util.c index 58743315cda9..4f034b87b427 100644 --- a/fs/gfs2/util.c +++ b/fs/gfs2/util.c @@ -119,17 +119,22 @@ void gfs2_freeze_unlock(struct gfs2_holder *freeze_gh) static void signal_our_withdraw(struct gfs2_sbd *sdp) { struct gfs2_glock *live_gl = sdp->sd_live_gh.gh_gl; - struct inode *inode = sdp->sd_jdesc->jd_inode; - struct gfs2_inode *ip = GFS2_I(inode); - struct gfs2_glock *i_gl = ip->i_gl; - u64 no_formal_ino = ip->i_no_formal_ino; + struct inode *inode; + struct gfs2_inode *ip; + struct gfs2_glock *i_gl; + u64 no_formal_ino; int log_write_allowed = test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); int ret = 0; int tries; - if (test_bit(SDF_NORECOVERY, &sdp->sd_flags)) + if (test_bit(SDF_NORECOVERY, &sdp->sd_flags) || !sdp->sd_jdesc) return; + inode = sdp->sd_jdesc->jd_inode; + ip = GFS2_I(inode); + i_gl = ip->i_gl; + no_formal_ino = ip->i_no_formal_ino; + /* Prevent any glock dq until withdraw recovery is complete */ set_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags); /* -- cgit v1.2.3 From e1915f76a8981f0a750cf56515df42582a37c4b0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Mar 2021 23:29:35 +0000 Subject: io_uring: cancel deferred requests in try_cancel As io_uring_cancel_files() and others let SQO to run between io_uring_try_cancel_requests(), SQO may generate new deferred requests, so it's safer to try to cancel them in it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 49f85f49e1c3..56f3d8f408c9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8577,11 +8577,11 @@ static bool io_cancel_task_cb(struct io_wq_work *work, void *data) return ret; } -static void io_cancel_defer_files(struct io_ring_ctx *ctx, +static bool io_cancel_defer_files(struct io_ring_ctx *ctx, struct task_struct *task, struct files_struct *files) { - struct io_defer_entry *de = NULL; + struct io_defer_entry *de; LIST_HEAD(list); spin_lock_irq(&ctx->completion_lock); @@ -8592,6 +8592,8 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx, } } spin_unlock_irq(&ctx->completion_lock); + if (list_empty(&list)) + return false; while (!list_empty(&list)) { de = list_first_entry(&list, struct io_defer_entry, list); @@ -8601,6 +8603,7 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx, io_req_complete(de->req, -ECANCELED); kfree(de); } + return true; } static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) @@ -8666,6 +8669,7 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, } } + ret |= io_cancel_defer_files(ctx, task, files); ret |= io_poll_remove_all(ctx, task, files); ret |= io_kill_timeouts(ctx, task, files); ret |= io_run_task_work(); @@ -8734,8 +8738,6 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, atomic_inc(&task->io_uring->in_idle); } - io_cancel_defer_files(ctx, task, files); - io_uring_cancel_files(ctx, task, files); if (!files) io_uring_try_cancel_requests(ctx, task, NULL); -- cgit v1.2.3 From 0df8ea602b3fe80819a34361027ad40485e78909 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Mar 2021 23:29:36 +0000 Subject: io_uring: remove useless ->startup completion We always do complete(&sqd->startup) almost right after sqd->thread creation, either in the success path or in io_sq_thread_finish(). It's specifically created not started for us to be able to set some stuff like sqd->thread and io_uring_alloc_task_context() before following right after wake_up_new_task(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 56f3d8f408c9..6349374d715d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -272,7 +272,6 @@ struct io_sq_data { pid_t task_tgid; unsigned long state; - struct completion startup; struct completion exited; }; @@ -6656,8 +6655,6 @@ static int io_sq_thread(void *data) set_cpus_allowed_ptr(current, cpu_online_mask); current->flags |= PF_NO_SETAFFINITY; - wait_for_completion(&sqd->startup); - down_read(&sqd->rw_lock); while (!test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) { @@ -7080,7 +7077,6 @@ static void io_sq_thread_finish(struct io_ring_ctx *ctx) struct io_sq_data *sqd = ctx->sq_data; if (sqd) { - complete(&sqd->startup); io_sq_thread_park(sqd); list_del(&ctx->sqd_list); io_sqd_update_thread_idle(sqd); @@ -7144,7 +7140,6 @@ static struct io_sq_data *io_get_sq_data(struct io_uring_params *p) INIT_LIST_HEAD(&sqd->ctx_list); init_rwsem(&sqd->rw_lock); init_waitqueue_head(&sqd->wait); - init_completion(&sqd->startup); init_completion(&sqd->exited); return sqd; } @@ -7856,7 +7851,6 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, wake_up_new_task(tsk); if (ret) goto err; - complete(&sqd->startup); } else if (p->flags & IORING_SETUP_SQ_AFF) { /* Can't have SQ_AFF without SQPOLL */ ret = -EINVAL; -- cgit v1.2.3 From 0efc4976e3da40b09c592b21f722022d8f12a16b Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Fri, 12 Mar 2021 08:47:47 -0500 Subject: gfs2: bypass log flush if the journal is not live Patch fe3e397668775 ("gfs2: Rework the log space allocation logic") changed gfs2_log_flush to reserve a set of journal blocks in case no transaction is active. However, gfs2_log_flush also gets called in cases where we don't have an active journal, for example, for spectator mounts. In that case, trying to reserve blocks would sleep forever, but we want gfs2_log_flush to be a no-op instead. Fixes: fe3e397668775 ("gfs2: Rework the log space allocation logic") Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 760af666576c..6410281546f9 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -1036,7 +1036,7 @@ repeat: * Do this check while holding the log_flush_lock to prevent new * buffers from being added to the ail via gfs2_pin() */ - if (gfs2_withdrawn(sdp)) + if (gfs2_withdrawn(sdp) || !test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) goto out; /* Log might have been flushed while we waited for the flush lock */ -- cgit v1.2.3 From 26984fbf3ad9d1c1fb56a0c1e0cdf9fa3b806f0c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Mar 2021 23:29:37 +0000 Subject: io_uring: prevent racy sqd->thread checks SQPOLL thread to which we're trying to attach may be going away, it's not nice but a more serious problem is if io_sq_offload_create() sees sqd->thread==NULL, and tries to init it with a new thread. There are tons of ways it can be exploited or fail. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6349374d715d..cdec59510433 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7119,14 +7119,18 @@ static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p) return sqd; } -static struct io_sq_data *io_get_sq_data(struct io_uring_params *p) +static struct io_sq_data *io_get_sq_data(struct io_uring_params *p, + bool *attached) { struct io_sq_data *sqd; + *attached = false; if (p->flags & IORING_SETUP_ATTACH_WQ) { sqd = io_attach_sq_data(p); - if (!IS_ERR(sqd)) + if (!IS_ERR(sqd)) { + *attached = true; return sqd; + } /* fall through for EPERM case, setup new sqd/task */ if (PTR_ERR(sqd) != -EPERM) return sqd; @@ -7799,12 +7803,13 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, if (ctx->flags & IORING_SETUP_SQPOLL) { struct task_struct *tsk; struct io_sq_data *sqd; + bool attached; ret = -EPERM; if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_NICE)) goto err; - sqd = io_get_sq_data(p); + sqd = io_get_sq_data(p, &attached); if (IS_ERR(sqd)) { ret = PTR_ERR(sqd); goto err; @@ -7816,13 +7821,24 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, if (!ctx->sq_thread_idle) ctx->sq_thread_idle = HZ; + ret = 0; io_sq_thread_park(sqd); - list_add(&ctx->sqd_list, &sqd->ctx_list); - io_sqd_update_thread_idle(sqd); + /* don't attach to a dying SQPOLL thread, would be racy */ + if (attached && !sqd->thread) { + ret = -ENXIO; + } else { + list_add(&ctx->sqd_list, &sqd->ctx_list); + io_sqd_update_thread_idle(sqd); + } io_sq_thread_unpark(sqd); - if (sqd->thread) + if (ret < 0) { + io_put_sq_data(sqd); + ctx->sq_data = NULL; + return ret; + } else if (attached) { return 0; + } if (p->flags & IORING_SETUP_SQ_AFF) { int cpu = p->sq_thread_cpu; -- cgit v1.2.3 From 521d6a737a31c08dbab204a95cd4fb5bee725f0f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 11 Mar 2021 23:29:38 +0000 Subject: io_uring: cancel sqpoll via task_work 1) The first problem is io_uring_cancel_sqpoll() -> io_uring_cancel_task_requests() basically doing park(); park(); and so hanging. 2) Another one is more subtle, when the master task is doing cancellations, but SQPOLL task submits in-between the end of the cancellation but before finish() requests taking a ref to the ctx, and so eternally locking it up. 3) Yet another is a dying SQPOLL task doing io_uring_cancel_sqpoll() and same io_uring_cancel_sqpoll() from the owner task, they race for tctx->wait events. And there probably more of them. Instead do SQPOLL cancellations from within SQPOLL task context via task_work, see io_sqpoll_cancel_sync(). With that we don't need temporal park()/unpark() during cancellation, which is ugly, subtle and anyway doesn't allow to do io_run_task_work() properly. io_uring_cancel_sqpoll() is called only from SQPOLL task context and under sqd locking, so all parking is removed from there. And so, io_sq_thread_[un]park() and io_sq_thread_stop() are not used now by SQPOLL task, and that spare us from some headache. Also remove ctx->sqd_list early to avoid 2). And kill tctx->sqpoll, which is not used anymore. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 151 +++++++++++++++++++++++++++------------------------------- 1 file changed, 71 insertions(+), 80 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index cdec59510433..70286b393c0e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6665,6 +6665,7 @@ static int io_sq_thread(void *data) up_read(&sqd->rw_lock); cond_resched(); down_read(&sqd->rw_lock); + io_run_task_work(); timeout = jiffies + sqd->sq_thread_idle; continue; } @@ -6720,18 +6721,22 @@ static int io_sq_thread(void *data) finish_wait(&sqd->wait, &wait); timeout = jiffies + sqd->sq_thread_idle; } - - list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) - io_uring_cancel_sqpoll(ctx); up_read(&sqd->rw_lock); - + down_write(&sqd->rw_lock); + /* + * someone may have parked and added a cancellation task_work, run + * it first because we don't want it in io_uring_cancel_sqpoll() + */ io_run_task_work(); - down_write(&sqd->rw_lock); + list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) + io_uring_cancel_sqpoll(ctx); sqd->thread = NULL; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); up_write(&sqd->rw_lock); + + io_run_task_work(); complete(&sqd->exited); do_exit(0); } @@ -7033,8 +7038,8 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) static void io_sq_thread_unpark(struct io_sq_data *sqd) __releases(&sqd->rw_lock) { - if (sqd->thread == current) - return; + WARN_ON_ONCE(sqd->thread == current); + clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); up_write(&sqd->rw_lock); } @@ -7042,8 +7047,8 @@ static void io_sq_thread_unpark(struct io_sq_data *sqd) static void io_sq_thread_park(struct io_sq_data *sqd) __acquires(&sqd->rw_lock) { - if (sqd->thread == current) - return; + WARN_ON_ONCE(sqd->thread == current); + set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); down_write(&sqd->rw_lock); /* set again for consistency, in case concurrent parks are happening */ @@ -7054,8 +7059,8 @@ static void io_sq_thread_park(struct io_sq_data *sqd) static void io_sq_thread_stop(struct io_sq_data *sqd) { - if (test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) - return; + WARN_ON_ONCE(sqd->thread == current); + down_write(&sqd->rw_lock); set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); if (sqd->thread) @@ -7078,7 +7083,7 @@ static void io_sq_thread_finish(struct io_ring_ctx *ctx) if (sqd) { io_sq_thread_park(sqd); - list_del(&ctx->sqd_list); + list_del_init(&ctx->sqd_list); io_sqd_update_thread_idle(sqd); io_sq_thread_unpark(sqd); @@ -7760,7 +7765,6 @@ static int io_uring_alloc_task_context(struct task_struct *task, init_waitqueue_head(&tctx->wait); tctx->last = NULL; atomic_set(&tctx->in_idle, 0); - tctx->sqpoll = false; task->io_uring = tctx; spin_lock_init(&tctx->task_lock); INIT_WQ_LIST(&tctx->task_list); @@ -8719,43 +8723,12 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, io_uring_try_cancel_requests(ctx, task, files); - if (ctx->sq_data) - io_sq_thread_unpark(ctx->sq_data); prepare_to_wait(&task->io_uring->wait, &wait, TASK_UNINTERRUPTIBLE); if (inflight == io_uring_count_inflight(ctx, task, files)) schedule(); finish_wait(&task->io_uring->wait, &wait); - if (ctx->sq_data) - io_sq_thread_park(ctx->sq_data); - } -} - -/* - * We need to iteratively cancel requests, in case a request has dependent - * hard links. These persist even for failure of cancelations, hence keep - * looping until none are found. - */ -static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx, - struct files_struct *files) -{ - struct task_struct *task = current; - - if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) { - io_sq_thread_park(ctx->sq_data); - task = ctx->sq_data->thread; - if (task) - atomic_inc(&task->io_uring->in_idle); } - - io_uring_cancel_files(ctx, task, files); - if (!files) - io_uring_try_cancel_requests(ctx, task, NULL); - - if (task) - atomic_dec(&task->io_uring->in_idle); - if (ctx->sq_data) - io_sq_thread_unpark(ctx->sq_data); } /* @@ -8796,15 +8769,6 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx) } tctx->last = ctx; } - - /* - * This is race safe in that the task itself is doing this, hence it - * cannot be going through the exit/cancel paths at the same time. - * This cannot be modified while exit/cancel is running. - */ - if (!tctx->sqpoll && (ctx->flags & IORING_SETUP_SQPOLL)) - tctx->sqpoll = true; - return 0; } @@ -8847,6 +8811,44 @@ static void io_uring_clean_tctx(struct io_uring_task *tctx) } } +static s64 tctx_inflight(struct io_uring_task *tctx) +{ + return percpu_counter_sum(&tctx->inflight); +} + +static void io_sqpoll_cancel_cb(struct callback_head *cb) +{ + struct io_tctx_exit *work = container_of(cb, struct io_tctx_exit, task_work); + struct io_ring_ctx *ctx = work->ctx; + struct io_sq_data *sqd = ctx->sq_data; + + if (sqd->thread) + io_uring_cancel_sqpoll(ctx); + complete(&work->completion); +} + +static void io_sqpoll_cancel_sync(struct io_ring_ctx *ctx) +{ + struct io_sq_data *sqd = ctx->sq_data; + struct io_tctx_exit work = { .ctx = ctx, }; + struct task_struct *task; + + io_sq_thread_park(sqd); + list_del_init(&ctx->sqd_list); + io_sqd_update_thread_idle(sqd); + task = sqd->thread; + if (task) { + init_completion(&work.completion); + init_task_work(&work.task_work, io_sqpoll_cancel_cb); + WARN_ON_ONCE(task_work_add(task, &work.task_work, TWA_SIGNAL)); + wake_up_process(task); + } + io_sq_thread_unpark(sqd); + + if (task) + wait_for_completion(&work.completion); +} + void __io_uring_files_cancel(struct files_struct *files) { struct io_uring_task *tctx = current->io_uring; @@ -8855,41 +8857,40 @@ void __io_uring_files_cancel(struct files_struct *files) /* make sure overflow events are dropped */ atomic_inc(&tctx->in_idle); - xa_for_each(&tctx->xa, index, node) - io_uring_cancel_task_requests(node->ctx, files); + xa_for_each(&tctx->xa, index, node) { + struct io_ring_ctx *ctx = node->ctx; + + if (ctx->sq_data) { + io_sqpoll_cancel_sync(ctx); + continue; + } + io_uring_cancel_files(ctx, current, files); + if (!files) + io_uring_try_cancel_requests(ctx, current, NULL); + } atomic_dec(&tctx->in_idle); if (files) io_uring_clean_tctx(tctx); } -static s64 tctx_inflight(struct io_uring_task *tctx) -{ - return percpu_counter_sum(&tctx->inflight); -} - +/* should only be called by SQPOLL task */ static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) { struct io_sq_data *sqd = ctx->sq_data; - struct io_uring_task *tctx; + struct io_uring_task *tctx = current->io_uring; s64 inflight; DEFINE_WAIT(wait); - if (!sqd) - return; - io_sq_thread_park(sqd); - if (!sqd->thread || !sqd->thread->io_uring) { - io_sq_thread_unpark(sqd); - return; - } - tctx = ctx->sq_data->thread->io_uring; + WARN_ON_ONCE(!sqd || ctx->sq_data->thread != current); + atomic_inc(&tctx->in_idle); do { /* read completions before cancelations */ inflight = tctx_inflight(tctx); if (!inflight) break; - io_uring_cancel_task_requests(ctx, NULL); + io_uring_try_cancel_requests(ctx, current, NULL); prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE); /* @@ -8902,7 +8903,6 @@ static void io_uring_cancel_sqpoll(struct io_ring_ctx *ctx) finish_wait(&tctx->wait, &wait); } while (1); atomic_dec(&tctx->in_idle); - io_sq_thread_unpark(sqd); } /* @@ -8917,15 +8917,6 @@ void __io_uring_task_cancel(void) /* make sure overflow events are dropped */ atomic_inc(&tctx->in_idle); - - if (tctx->sqpoll) { - struct io_tctx_node *node; - unsigned long index; - - xa_for_each(&tctx->xa, index, node) - io_uring_cancel_sqpoll(node->ctx); - } - do { /* read completions before cancelations */ inflight = tctx_inflight(tctx); -- cgit v1.2.3 From 58f99373834151e1ca7edc49bc5578d9d40db099 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 12 Mar 2021 16:25:55 +0000 Subject: io_uring: fix OP_ASYNC_CANCEL across tasks IORING_OP_ASYNC_CANCEL tries io-wq cancellation only for current task. If it fails go over tctx_list and try it out for every single tctx. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 70286b393c0e..a4bce17af506 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5667,8 +5667,47 @@ static int io_async_cancel_prep(struct io_kiocb *req, static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; + u64 sqe_addr = req->cancel.addr; + struct io_tctx_node *node; + int ret; - io_async_find_and_cancel(ctx, req, req->cancel.addr, 0); + /* tasks should wait for their io-wq threads, so safe w/o sync */ + ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx); + spin_lock_irq(&ctx->completion_lock); + if (ret != -ENOENT) + goto done; + ret = io_timeout_cancel(ctx, sqe_addr); + if (ret != -ENOENT) + goto done; + ret = io_poll_cancel(ctx, sqe_addr); + if (ret != -ENOENT) + goto done; + spin_unlock_irq(&ctx->completion_lock); + + /* slow path, try all io-wq's */ + io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); + ret = -ENOENT; + list_for_each_entry(node, &ctx->tctx_list, ctx_node) { + struct io_uring_task *tctx = node->task->io_uring; + + if (!tctx || !tctx->io_wq) + continue; + ret = io_async_cancel_one(tctx, req->cancel.addr, ctx); + if (ret != -ENOENT) + break; + } + io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); + + spin_lock_irq(&ctx->completion_lock); +done: + io_cqring_fill_event(req, ret); + io_commit_cqring(ctx); + spin_unlock_irq(&ctx->completion_lock); + io_cqring_ev_posted(ctx); + + if (ret < 0) + req_set_fail_links(req); + io_put_req(req); return 0; } -- cgit v1.2.3 From 16efa4fce3b7af17bb45d635c3e89992d721e0f3 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 12 Mar 2021 20:26:13 -0700 Subject: io_uring: allow IO worker threads to be frozen With the freezer using the proper signaling to notify us of when it's time to freeze a thread, we can re-enable normal freezer usage for the IO threads. Ensure that SQPOLL, io-wq, and the io-wq manager call try_to_freeze() appropriately, and remove the default setting of PF_NOFREEZE from create_io_thread(). Signed-off-by: Jens Axboe --- fs/io-wq.c | 6 +++++- fs/io_uring.c | 1 + kernel/fork.c | 1 - 3 files changed, 6 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 0ae9ecadf295..e05f996d088f 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -488,6 +488,8 @@ static int io_wqe_worker(void *data) set_task_comm(current, buf); while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { + long ret; + set_current_state(TASK_INTERRUPTIBLE); loop: raw_spin_lock_irq(&wqe->lock); @@ -498,7 +500,8 @@ loop: __io_worker_idle(wqe, worker); raw_spin_unlock_irq(&wqe->lock); io_flush_signals(); - if (schedule_timeout(WORKER_IDLE_TIMEOUT)) + ret = schedule_timeout(WORKER_IDLE_TIMEOUT); + if (try_to_freeze() || ret) continue; if (fatal_signal_pending(current)) break; @@ -709,6 +712,7 @@ static int io_wq_manager(void *data) set_current_state(TASK_INTERRUPTIBLE); io_wq_check_workers(wq); schedule_timeout(HZ); + try_to_freeze(); if (fatal_signal_pending(current)) set_bit(IO_WQ_BIT_EXIT, &wq->state); } while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)); diff --git a/fs/io_uring.c b/fs/io_uring.c index a4bce17af506..05adc4887ef3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6752,6 +6752,7 @@ static int io_sq_thread(void *data) up_read(&sqd->rw_lock); schedule(); + try_to_freeze(); down_read(&sqd->rw_lock); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_clear_wakeup_flag(ctx); diff --git a/kernel/fork.c b/kernel/fork.c index 72e444cd0ffe..d3171e8e88e5 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2436,7 +2436,6 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) if (!IS_ERR(tsk)) { sigfillset(&tsk->blocked); sigdelsetmask(&tsk->blocked, sigmask(SIGKILL)); - tsk->flags |= PF_NOFREEZE; } return tsk; } -- cgit v1.2.3 From ca6eb14d6453bea85ac66fa4c6ab75dfe93eaf45 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Fri, 12 Mar 2021 21:07:30 -0800 Subject: mm: use is_cow_mapping() across tree where proper After is_cow_mapping() is exported in mm.h, replace some manual checks elsewhere throughout the tree but start to use the new helper. Link: https://lkml.kernel.org/r/20210217233547.93892-5-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Jason Gunthorpe Cc: VMware Graphics Cc: Roland Scheidegger Cc: David Airlie Cc: Daniel Vetter Cc: Mike Kravetz Cc: Alexey Dobriyan Cc: Andrea Arcangeli Cc: Christoph Hellwig Cc: David Gibson Cc: Gal Pressman Cc: Jan Kara Cc: Jann Horn Cc: Kirill Shutemov Cc: Kirill Tkhai Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Mike Rapoport Cc: Wei Zhang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c | 4 +--- drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c | 2 +- fs/proc/task_mmu.c | 2 -- mm/hugetlb.c | 4 +--- 4 files changed, 3 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c index 0a900afc66ff..45c9c6a7f1d6 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c @@ -500,8 +500,6 @@ vm_fault_t vmw_bo_vm_huge_fault(struct vm_fault *vmf, vm_fault_t ret; pgoff_t fault_page_size; bool write = vmf->flags & FAULT_FLAG_WRITE; - bool is_cow_mapping = - (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; switch (pe_size) { case PE_SIZE_PMD: @@ -518,7 +516,7 @@ vm_fault_t vmw_bo_vm_huge_fault(struct vm_fault *vmf, } /* Always do write dirty-tracking and COW on PTE level. */ - if (write && (READ_ONCE(vbo->dirty) || is_cow_mapping)) + if (write && (READ_ONCE(vbo->dirty) || is_cow_mapping(vma->vm_flags))) return VM_FAULT_FALLBACK; ret = ttm_bo_vm_reserve(bo, vmf); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c index 3c03b1746661..cb9975889e2f 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c @@ -49,7 +49,7 @@ int vmw_mmap(struct file *filp, struct vm_area_struct *vma) vma->vm_ops = &vmw_vm_ops; /* Use VM_PFNMAP rather than VM_MIXEDMAP if not a COW mapping */ - if ((vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) != VM_MAYWRITE) + if (!is_cow_mapping(vma->vm_flags)) vma->vm_flags = (vma->vm_flags & ~VM_MIXEDMAP) | VM_PFNMAP; return 0; diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 3cec6fbef725..e862cab69583 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1036,8 +1036,6 @@ struct clear_refs_private { #ifdef CONFIG_MEM_SOFT_DIRTY -#define is_cow_mapping(flags) (((flags) & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE) - static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { struct page *page; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index b905143a523e..7786267da2fe 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3734,15 +3734,13 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, pte_t *src_pte, *dst_pte, entry, dst_entry; struct page *ptepage; unsigned long addr; - int cow; + bool cow = is_cow_mapping(vma->vm_flags); struct hstate *h = hstate_vma(vma); unsigned long sz = huge_page_size(h); struct address_space *mapping = vma->vm_file->f_mapping; struct mmu_notifier_range range; int ret = 0; - cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; - if (cow) { mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, src, vma->vm_start, -- cgit v1.2.3 From e7850f4d844e0acfac7e570af611d89deade3146 Mon Sep 17 00:00:00 2001 From: Lior Ribak Date: Fri, 12 Mar 2021 21:07:41 -0800 Subject: binfmt_misc: fix possible deadlock in bm_register_write There is a deadlock in bm_register_write: First, in the begining of the function, a lock is taken on the binfmt_misc root inode with inode_lock(d_inode(root)). Then, if the user used the MISC_FMT_OPEN_FILE flag, the function will call open_exec on the user-provided interpreter. open_exec will call a path lookup, and if the path lookup process includes the root of binfmt_misc, it will try to take a shared lock on its inode again, but it is already locked, and the code will get stuck in a deadlock To reproduce the bug: $ echo ":iiiii:E::ii::/proc/sys/fs/binfmt_misc/bla:F" > /proc/sys/fs/binfmt_misc/register backtrace of where the lock occurs (#5): 0 schedule () at ./arch/x86/include/asm/current.h:15 1 0xffffffff81b51237 in rwsem_down_read_slowpath (sem=0xffff888003b202e0, count=, state=state@entry=2) at kernel/locking/rwsem.c:992 2 0xffffffff81b5150a in __down_read_common (state=2, sem=) at kernel/locking/rwsem.c:1213 3 __down_read (sem=) at kernel/locking/rwsem.c:1222 4 down_read (sem=) at kernel/locking/rwsem.c:1355 5 0xffffffff811ee22a in inode_lock_shared (inode=) at ./include/linux/fs.h:783 6 open_last_lookups (op=0xffffc9000022fe34, file=0xffff888004098600, nd=0xffffc9000022fd10) at fs/namei.c:3177 7 path_openat (nd=nd@entry=0xffffc9000022fd10, op=op@entry=0xffffc9000022fe34, flags=flags@entry=65) at fs/namei.c:3366 8 0xffffffff811efe1c in do_filp_open (dfd=, pathname=pathname@entry=0xffff8880031b9000, op=op@entry=0xffffc9000022fe34) at fs/namei.c:3396 9 0xffffffff811e493f in do_open_execat (fd=fd@entry=-100, name=name@entry=0xffff8880031b9000, flags=, flags@entry=0) at fs/exec.c:913 10 0xffffffff811e4a92 in open_exec (name=) at fs/exec.c:948 11 0xffffffff8124aa84 in bm_register_write (file=, buffer=, count=19, ppos=) at fs/binfmt_misc.c:682 12 0xffffffff811decd2 in vfs_write (file=file@entry=0xffff888004098500, buf=buf@entry=0xa758d0 ":iiiii:E::ii::i:CF ", count=count@entry=19, pos=pos@entry=0xffffc9000022ff10) at fs/read_write.c:603 13 0xffffffff811defda in ksys_write (fd=, buf=0xa758d0 ":iiiii:E::ii::i:CF ", count=19) at fs/read_write.c:658 14 0xffffffff81b49813 in do_syscall_64 (nr=, regs=0xffffc9000022ff58) at arch/x86/entry/common.c:46 15 0xffffffff81c0007c in entry_SYSCALL_64 () at arch/x86/entry/entry_64.S:120 To solve the issue, the open_exec call is moved to before the write lock is taken by bm_register_write Link: https://lkml.kernel.org/r/20210228224414.95962-1-liorribak@gmail.com Fixes: 948b701a607f1 ("binfmt_misc: add persistent opened binary handler for containers") Signed-off-by: Lior Ribak Acked-by: Helge Deller Cc: Al Viro Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/binfmt_misc.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/binfmt_misc.c b/fs/binfmt_misc.c index c457334de43f..e1eae7ea823a 100644 --- a/fs/binfmt_misc.c +++ b/fs/binfmt_misc.c @@ -649,12 +649,24 @@ static ssize_t bm_register_write(struct file *file, const char __user *buffer, struct super_block *sb = file_inode(file)->i_sb; struct dentry *root = sb->s_root, *dentry; int err = 0; + struct file *f = NULL; e = create_entry(buffer, count); if (IS_ERR(e)) return PTR_ERR(e); + if (e->flags & MISC_FMT_OPEN_FILE) { + f = open_exec(e->interpreter); + if (IS_ERR(f)) { + pr_notice("register: failed to install interpreter file %s\n", + e->interpreter); + kfree(e); + return PTR_ERR(f); + } + e->interp_file = f; + } + inode_lock(d_inode(root)); dentry = lookup_one_len(e->name, root, strlen(e->name)); err = PTR_ERR(dentry); @@ -678,21 +690,6 @@ static ssize_t bm_register_write(struct file *file, const char __user *buffer, goto out2; } - if (e->flags & MISC_FMT_OPEN_FILE) { - struct file *f; - - f = open_exec(e->interpreter); - if (IS_ERR(f)) { - err = PTR_ERR(f); - pr_notice("register: failed to install interpreter file %s\n", e->interpreter); - simple_release_fs(&bm_mnt, &entry_count); - iput(inode); - inode = NULL; - goto out2; - } - e->interp_file = f; - } - e->dentry = dget(dentry); inode->i_private = e; inode->i_fop = &bm_entry_operations; @@ -709,6 +706,8 @@ out: inode_unlock(d_inode(root)); if (err) { + if (f) + filp_close(f, NULL); kfree(e); return err; } -- cgit v1.2.3 From 9e15c3a0ced5a61f320b989072c24983cb1620c1 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 13 Mar 2021 12:29:43 -0700 Subject: io_uring: convert io_buffer_idr to XArray Like we did for the personality idr, convert the IO buffer idr to use XArray. This avoids a use-after-free on removal of entries, since idr doesn't like doing so from inside an iterator, and it nicely reduces the amount of code we need to support this feature. Fixes: 5a2e745d4d43 ("io_uring: buffer registration infrastructure") Cc: stable@vger.kernel.org Cc: Matthew Wilcox Cc: yangerkun Reported-by: Hulk Robot Signed-off-by: Jens Axboe --- fs/io_uring.c | 43 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 28 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 05adc4887ef3..58d62dd9f8e4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -402,7 +402,7 @@ struct io_ring_ctx { struct socket *ring_sock; #endif - struct idr io_buffer_idr; + struct xarray io_buffers; struct xarray personalities; u32 pers_next; @@ -1135,7 +1135,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) init_waitqueue_head(&ctx->cq_wait); INIT_LIST_HEAD(&ctx->cq_overflow_list); init_completion(&ctx->ref_comp); - idr_init(&ctx->io_buffer_idr); + xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1); xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); mutex_init(&ctx->uring_lock); init_waitqueue_head(&ctx->wait); @@ -2843,7 +2843,7 @@ static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, lockdep_assert_held(&req->ctx->uring_lock); - head = idr_find(&req->ctx->io_buffer_idr, bgid); + head = xa_load(&req->ctx->io_buffers, bgid); if (head) { if (!list_empty(&head->list)) { kbuf = list_last_entry(&head->list, struct io_buffer, @@ -2851,7 +2851,7 @@ static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, list_del(&kbuf->list); } else { kbuf = head; - idr_remove(&req->ctx->io_buffer_idr, bgid); + xa_erase(&req->ctx->io_buffers, bgid); } if (*len > kbuf->len) *len = kbuf->len; @@ -3892,7 +3892,7 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, } i++; kfree(buf); - idr_remove(&ctx->io_buffer_idr, bgid); + xa_erase(&ctx->io_buffers, bgid); return i; } @@ -3910,7 +3910,7 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) lockdep_assert_held(&ctx->uring_lock); ret = -ENOENT; - head = idr_find(&ctx->io_buffer_idr, p->bgid); + head = xa_load(&ctx->io_buffers, p->bgid); if (head) ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs); if (ret < 0) @@ -3993,21 +3993,14 @@ static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) lockdep_assert_held(&ctx->uring_lock); - list = head = idr_find(&ctx->io_buffer_idr, p->bgid); + list = head = xa_load(&ctx->io_buffers, p->bgid); ret = io_add_buffers(p, &head); - if (ret < 0) - goto out; - - if (!list) { - ret = idr_alloc(&ctx->io_buffer_idr, head, p->bgid, p->bgid + 1, - GFP_KERNEL); - if (ret < 0) { + if (ret >= 0 && !list) { + ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL); + if (ret < 0) __io_remove_buffers(ctx, head, p->bgid, -1U); - goto out; - } } -out: if (ret < 0) req_set_fail_links(req); @@ -8333,19 +8326,13 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) return -ENXIO; } -static int __io_destroy_buffers(int id, void *p, void *data) -{ - struct io_ring_ctx *ctx = data; - struct io_buffer *buf = p; - - __io_remove_buffers(ctx, buf, id, -1U); - return 0; -} - static void io_destroy_buffers(struct io_ring_ctx *ctx) { - idr_for_each(&ctx->io_buffer_idr, __io_destroy_buffers, ctx); - idr_destroy(&ctx->io_buffer_idr); + struct io_buffer *buf; + unsigned long index; + + xa_for_each(&ctx->io_buffers, index, buf) + __io_remove_buffers(ctx, buf, index, -1U); } static void io_req_cache_free(struct list_head *list, struct task_struct *tsk) -- cgit v1.2.3 From 5171317dfd9afcf729799d31fffdbb9e71e45402 Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Wed, 10 Mar 2021 10:22:27 +0000 Subject: cifs: update new ACE pointer after populate_new_aces. After the fix for retaining externally set ACEs with cifsacl and modefromsid,idsfromsid, there was an issue in populating the inherited ACEs after setting the ACEs introduced by these two modes. Fixed this by updating the ACE pointer again after the call to populate_new_aces. Signed-off-by: Shyam Prasad N Reviewed-by: Rohith Surabattula Signed-off-by: Steve French --- fs/cifs/cifsacl.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsacl.c b/fs/cifs/cifsacl.c index 9d29eb9660c2..2be22a5c690f 100644 --- a/fs/cifs/cifsacl.c +++ b/fs/cifs/cifsacl.c @@ -1118,7 +1118,6 @@ static int set_chmod_dacl(struct cifs_acl *pdacl, struct cifs_acl *pndacl, /* Retain old ACEs which we can retain */ for (i = 0; i < src_num_aces; ++i) { pntace = (struct cifs_ace *) (acl_base + size); - pnntace = (struct cifs_ace *) (nacl_base + nsize); if (!new_aces_set && (pntace->flags & INHERITED_ACE)) { /* Place the new ACEs in between existing explicit and inherited */ @@ -1131,14 +1130,18 @@ static int set_chmod_dacl(struct cifs_acl *pdacl, struct cifs_acl *pndacl, } /* If it's any one of the ACE we're replacing, skip! */ - if ((compare_sids(&pntace->sid, &sid_unix_NFS_mode) == 0) || + if (!mode_from_sid && + ((compare_sids(&pntace->sid, &sid_unix_NFS_mode) == 0) || (compare_sids(&pntace->sid, pownersid) == 0) || (compare_sids(&pntace->sid, pgrpsid) == 0) || (compare_sids(&pntace->sid, &sid_everyone) == 0) || - (compare_sids(&pntace->sid, &sid_authusers) == 0)) { + (compare_sids(&pntace->sid, &sid_authusers) == 0))) { goto next_ace; } + /* update the pointer to the next ACE to populate*/ + pnntace = (struct cifs_ace *) (nacl_base + nsize); + nsize += cifs_copy_ace(pnntace, pntace, NULL); num_aces++; -- cgit v1.2.3 From 05946d4b7a7349ae58bfa2d51ae832e64a394c2d Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Wed, 10 Mar 2021 13:20:40 +0100 Subject: cifs: Fix preauth hash corruption smb311_update_preauth_hash() uses the shash in server->secmech without appropriate locking, and this can lead to sessions corrupting each other's preauth hashes. The following script can easily trigger the problem: #!/bin/sh -e NMOUNTS=10 for i in $(seq $NMOUNTS); mkdir -p /tmp/mnt$i umount /tmp/mnt$i 2>/dev/null || : done while :; do for i in $(seq $NMOUNTS); do mount -t cifs //192.168.0.1/test /tmp/mnt$i -o ... & done wait for i in $(seq $NMOUNTS); do umount /tmp/mnt$i done done Usually within seconds this leads to one or more of the mounts failing with the following errors, and a "Bad SMB2 signature for message" is seen in the server logs: CIFS: VFS: \\192.168.0.1 failed to connect to IPC (rc=-13) CIFS: VFS: cifs_mount failed w/return code = -13 Fix it by holding the server mutex just like in the other places where the shashes are used. Fixes: 8bd68c6e47abff34e4 ("CIFS: implement v3.11 preauth integrity") Signed-off-by: Vincent Whitchurch CC: Reviewed-by: Aurelien Aptel Signed-off-by: Steve French --- fs/cifs/transport.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c index 007d99437c77..c1725b55f364 100644 --- a/fs/cifs/transport.c +++ b/fs/cifs/transport.c @@ -1196,9 +1196,12 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, /* * Compounding is never used during session establish. */ - if ((ses->status == CifsNew) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) + if ((ses->status == CifsNew) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) { + mutex_lock(&server->srv_mutex); smb311_update_preauth_hash(ses, rqst[0].rq_iov, rqst[0].rq_nvec); + mutex_unlock(&server->srv_mutex); + } for (i = 0; i < num_rqst; i++) { rc = wait_for_response(server, midQ[i]); @@ -1266,7 +1269,9 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, .iov_base = resp_iov[0].iov_base, .iov_len = resp_iov[0].iov_len }; + mutex_lock(&server->srv_mutex); smb311_update_preauth_hash(ses, &iov, 1); + mutex_unlock(&server->srv_mutex); } out: -- cgit v1.2.3 From 62dd0f98a0e5668424270b47a0c2e973795faba7 Mon Sep 17 00:00:00 2001 From: Andrew Price Date: Mon, 15 Mar 2021 12:24:00 +0000 Subject: gfs2: Flag a withdraw if init_threads() fails Interrupting mount with ^C quickly enough can cause the kthread_run() calls in gfs2's init_threads() to fail and the error path leads to a deadlock on the s_umount rwsem. The abridged chain of events is: [mount path] get_tree_bdev() sget_fc() alloc_super() down_write_nested(&s->s_umount, SINGLE_DEPTH_NESTING); [acquired] gfs2_fill_super() gfs2_make_fs_rw() init_threads() kthread_run() ( Interrupted ) [Error path] gfs2_gl_hash_clear() flush_workqueue(glock_workqueue) wait_for_completion() [workqueue context] glock_work_func() run_queue() do_xmote() freeze_go_sync() freeze_super() down_write(&sb->s_umount) [deadlock] In freeze_go_sync() there is a gfs2_withdrawn() check that we can use to make sure freeze_super() is not called in the error path, so add a gfs2_withdraw_delayed() call when init_threads() fails. Ref: https://bugzilla.kernel.org/show_bug.cgi?id=212231 Reported-by: Alexander Aring Signed-off-by: Andrew Price Signed-off-by: Andreas Gruenbacher --- fs/gfs2/super.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index 97076d3f562f..9e91c9d92bd6 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -162,8 +162,10 @@ int gfs2_make_fs_rw(struct gfs2_sbd *sdp) int error; error = init_threads(sdp); - if (error) + if (error) { + gfs2_withdraw_delayed(sdp); return error; + } j_gl->gl_ops->go_inval(j_gl, DIO_METADATA); if (gfs2_withdrawn(sdp)) { -- cgit v1.2.3 From efe814a471e0e58f28f1efaf430c8784a4f36626 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 14 Mar 2021 20:57:08 +0000 Subject: io_uring: fix ->flags races by linked timeouts It's racy to modify req->flags from a not owning context, e.g. linked timeout calling req_set_fail_links() for the master request might race with that request setting/clearing flags while being executed concurrently. Just remove req_set_fail_links(prev) from io_link_timeout_fn(), io_async_find_and_cancel() and functions down the line take care of setting the fail bit. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 1 - 1 file changed, 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 58d62dd9f8e4..217f72d50ff5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6197,7 +6197,6 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) spin_unlock_irqrestore(&ctx->completion_lock, flags); if (prev) { - req_set_fail_links(prev); io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME); io_put_req_deferred(prev, 1); } else { -- cgit v1.2.3 From 180f829fe4026bd192447d261e712b6cb84f6202 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 14 Mar 2021 20:57:09 +0000 Subject: io_uring: fix complete_post use ctx after free If io_req_complete_post() put not a final ref, we can't rely on the request's ctx ref, and so ctx may potentially be freed while complete_post() is in io_cqring_ev_posted()/etc. In that case get an additional ctx reference, and put it in the end, so protecting following io_cqring_ev_posted(). And also prolong ctx lifetime until spin_unlock happens, as we do with mutexes, so added percpu_ref_get() doesn't race with ctx free. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 217f72d50ff5..f8a683607b25 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1550,14 +1550,17 @@ static void io_req_complete_post(struct io_kiocb *req, long res, io_put_task(req->task, 1); list_add(&req->compl.list, &cs->locked_free_list); cs->locked_free_nr++; - } else - req = NULL; + } else { + if (!percpu_ref_tryget(&ctx->refs)) + req = NULL; + } io_commit_cqring(ctx); spin_unlock_irqrestore(&ctx->completion_lock, flags); - io_cqring_ev_posted(ctx); - if (req) + if (req) { + io_cqring_ev_posted(ctx); percpu_ref_put(&ctx->refs); + } } static void io_req_complete_state(struct io_kiocb *req, long res, @@ -8373,11 +8376,13 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) { /* * Some may use context even when all refs and requests have been put, - * and they are free to do so while still holding uring_lock, see - * __io_req_task_submit(). Wait for them to finish. + * and they are free to do so while still holding uring_lock or + * completion_lock, see __io_req_task_submit(). Wait for them to finish. */ mutex_lock(&ctx->uring_lock); mutex_unlock(&ctx->uring_lock); + spin_lock_irq(&ctx->completion_lock); + spin_unlock_irq(&ctx->completion_lock); io_sq_thread_finish(ctx); io_sqe_buffers_unregister(ctx); -- cgit v1.2.3 From 09a6f4efaa6536e760385f949e24078fd78305ad Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 14 Mar 2021 20:57:10 +0000 Subject: io_uring: replace sqd rw_semaphore with mutex The only user of read-locking of sqd->rw_lock is sq_thread itself, which is by definition alone, so we don't really need rw_semaphore, but mutex will do. Replace it with a mutex, and kill read-to-write upgrading and extra task_work handling in io_sq_thread(). Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index f8a683607b25..6de779aafd33 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -258,7 +258,7 @@ enum { struct io_sq_data { refcount_t refs; - struct rw_semaphore rw_lock; + struct mutex lock; /* ctx's that are using this sqd */ struct list_head ctx_list; @@ -6689,16 +6689,15 @@ static int io_sq_thread(void *data) set_cpus_allowed_ptr(current, cpu_online_mask); current->flags |= PF_NO_SETAFFINITY; - down_read(&sqd->rw_lock); - + mutex_lock(&sqd->lock); while (!test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state)) { int ret; bool cap_entries, sqt_spin, needs_sched; if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) { - up_read(&sqd->rw_lock); + mutex_unlock(&sqd->lock); cond_resched(); - down_read(&sqd->rw_lock); + mutex_lock(&sqd->lock); io_run_task_work(); timeout = jiffies + sqd->sq_thread_idle; continue; @@ -6745,10 +6744,10 @@ static int io_sq_thread(void *data) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); - up_read(&sqd->rw_lock); + mutex_unlock(&sqd->lock); schedule(); try_to_freeze(); - down_read(&sqd->rw_lock); + mutex_lock(&sqd->lock); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_clear_wakeup_flag(ctx); } @@ -6756,20 +6755,13 @@ static int io_sq_thread(void *data) finish_wait(&sqd->wait, &wait); timeout = jiffies + sqd->sq_thread_idle; } - up_read(&sqd->rw_lock); - down_write(&sqd->rw_lock); - /* - * someone may have parked and added a cancellation task_work, run - * it first because we don't want it in io_uring_cancel_sqpoll() - */ - io_run_task_work(); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_uring_cancel_sqpoll(ctx); sqd->thread = NULL; list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_set_wakeup_flag(ctx); - up_write(&sqd->rw_lock); + mutex_unlock(&sqd->lock); io_run_task_work(); complete(&sqd->exited); @@ -7071,21 +7063,21 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) } static void io_sq_thread_unpark(struct io_sq_data *sqd) - __releases(&sqd->rw_lock) + __releases(&sqd->lock) { WARN_ON_ONCE(sqd->thread == current); clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); - up_write(&sqd->rw_lock); + mutex_unlock(&sqd->lock); } static void io_sq_thread_park(struct io_sq_data *sqd) - __acquires(&sqd->rw_lock) + __acquires(&sqd->lock) { WARN_ON_ONCE(sqd->thread == current); set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); - down_write(&sqd->rw_lock); + mutex_lock(&sqd->lock); /* set again for consistency, in case concurrent parks are happening */ set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); if (sqd->thread) @@ -7096,11 +7088,11 @@ static void io_sq_thread_stop(struct io_sq_data *sqd) { WARN_ON_ONCE(sqd->thread == current); - down_write(&sqd->rw_lock); + mutex_lock(&sqd->lock); set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state); if (sqd->thread) wake_up_process(sqd->thread); - up_write(&sqd->rw_lock); + mutex_unlock(&sqd->lock); wait_for_completion(&sqd->exited); } @@ -7182,7 +7174,7 @@ static struct io_sq_data *io_get_sq_data(struct io_uring_params *p, refcount_set(&sqd->refs, 1); INIT_LIST_HEAD(&sqd->ctx_list); - init_rwsem(&sqd->rw_lock); + mutex_init(&sqd->lock); init_waitqueue_head(&sqd->wait); init_completion(&sqd->exited); return sqd; -- cgit v1.2.3 From f6d54255f4235448d4bbe442362d4caa62da97d5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 14 Mar 2021 20:57:11 +0000 Subject: io_uring: halt SQO submission on ctx exit io_sq_thread_finish() is called in io_ring_ctx_free(), so SQPOLL task is potentially running submitting new requests. It's not a disaster because of using a "try" variant of percpu_ref_get, but is far from nice. Remove ctx from the sqd ctx list earlier, before cancellation loop, so SQPOLL can't find it and so won't submit new requests. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6de779aafd33..b87012a21775 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8564,6 +8564,14 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) io_unregister_personality(ctx, index); mutex_unlock(&ctx->uring_lock); + /* prevent SQPOLL from submitting new requests */ + if (ctx->sq_data) { + io_sq_thread_park(ctx->sq_data); + list_del_init(&ctx->sqd_list); + io_sqd_update_thread_idle(ctx->sq_data); + io_sq_thread_unpark(ctx->sq_data); + } + io_kill_timeouts(ctx, NULL, NULL); io_poll_remove_all(ctx, NULL, NULL); -- cgit v1.2.3 From 9e138a48345427fa42f6076396ea069cebf3c08f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 14 Mar 2021 20:57:12 +0000 Subject: io_uring: fix concurrent parking If io_sq_thread_park() of one task got rescheduled right after set_bit(), before it gets back to mutex_lock() there can happen park()/unpark() by another task with SQPOLL locking again and continuing running never seeing that first set_bit(SHOULD_PARK), so won't even try to put the mutex down for parking. It will get parked eventually when SQPOLL drops the lock for reschedule, but may be problematic and will get in the way of further fixes. Account number of tasks waiting for parking with a new atomic variable park_pending and adjust SHOULD_PARK accordingly. It doesn't entirely replaces SHOULD_PARK bit with this atomic var because it's convenient to have it as a bit in the state and will help to do optimisations later. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index b87012a21775..70ceb8ed5950 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -258,6 +258,7 @@ enum { struct io_sq_data { refcount_t refs; + atomic_t park_pending; struct mutex lock; /* ctx's that are using this sqd */ @@ -7067,7 +7068,13 @@ static void io_sq_thread_unpark(struct io_sq_data *sqd) { WARN_ON_ONCE(sqd->thread == current); + /* + * Do the dance but not conditional clear_bit() because it'd race with + * other threads incrementing park_pending and setting the bit. + */ clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); + if (atomic_dec_return(&sqd->park_pending)) + set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); mutex_unlock(&sqd->lock); } @@ -7076,10 +7083,9 @@ static void io_sq_thread_park(struct io_sq_data *sqd) { WARN_ON_ONCE(sqd->thread == current); + atomic_inc(&sqd->park_pending); set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); mutex_lock(&sqd->lock); - /* set again for consistency, in case concurrent parks are happening */ - set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state); if (sqd->thread) wake_up_process(sqd->thread); } @@ -7099,6 +7105,8 @@ static void io_sq_thread_stop(struct io_sq_data *sqd) static void io_put_sq_data(struct io_sq_data *sqd) { if (refcount_dec_and_test(&sqd->refs)) { + WARN_ON_ONCE(atomic_read(&sqd->park_pending)); + io_sq_thread_stop(sqd); kfree(sqd); } @@ -7172,6 +7180,7 @@ static struct io_sq_data *io_get_sq_data(struct io_uring_params *p, if (!sqd) return ERR_PTR(-ENOMEM); + atomic_set(&sqd->park_pending, 0); refcount_set(&sqd->refs, 1); INIT_LIST_HEAD(&sqd->ctx_list); mutex_init(&sqd->lock); -- cgit v1.2.3 From 9b46571142e47503ed4f3ae3be5ed3968d8cb9cc Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 15 Mar 2021 14:23:07 +0000 Subject: io_uring: add generic callback_head helpers We already have helpers to run/add callback_head but taking ctx and working with ctx->exit_task_work. Extract generic versions of them implemented in terms of struct callback_head, it will be used later. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 62 ++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 26 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 70ceb8ed5950..5c2de43b99f5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1929,17 +1929,44 @@ static int io_req_task_work_add(struct io_kiocb *req) return ret; } -static void io_req_task_work_add_fallback(struct io_kiocb *req, - task_work_func_t cb) +static bool io_run_task_work_head(struct callback_head **work_head) +{ + struct callback_head *work, *next; + bool executed = false; + + do { + work = xchg(work_head, NULL); + if (!work) + break; + + do { + next = work->next; + work->func(work); + work = next; + cond_resched(); + } while (work); + executed = true; + } while (1); + + return executed; +} + +static void io_task_work_add_head(struct callback_head **work_head, + struct callback_head *task_work) { - struct io_ring_ctx *ctx = req->ctx; struct callback_head *head; - init_task_work(&req->task_work, cb); do { - head = READ_ONCE(ctx->exit_task_work); - req->task_work.next = head; - } while (cmpxchg(&ctx->exit_task_work, head, &req->task_work) != head); + head = READ_ONCE(*work_head); + task_work->next = head; + } while (cmpxchg(work_head, head, task_work) != head); +} + +static void io_req_task_work_add_fallback(struct io_kiocb *req, + task_work_func_t cb) +{ + init_task_work(&req->task_work, cb); + io_task_work_add_head(&req->ctx->exit_task_work, &req->task_work); } static void __io_req_task_cancel(struct io_kiocb *req, int error) @@ -8471,26 +8498,9 @@ static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) return -EINVAL; } -static bool io_run_ctx_fallback(struct io_ring_ctx *ctx) +static inline bool io_run_ctx_fallback(struct io_ring_ctx *ctx) { - struct callback_head *work, *next; - bool executed = false; - - do { - work = xchg(&ctx->exit_task_work, NULL); - if (!work) - break; - - do { - next = work->next; - work->func(work); - work = next; - cond_resched(); - } while (work); - executed = true; - } while (1); - - return executed; + return io_run_task_work_head(&ctx->exit_task_work); } struct io_tctx_exit { -- cgit v1.2.3 From b7f5a0bfe2061b2c7b2164de06fa4072d7373a45 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 15 Mar 2021 14:23:08 +0000 Subject: io_uring: fix sqpoll cancellation via task_work Running sqpoll cancellations via task_work_run() is a bad idea because it depends on other task works to be run, but those may be locked in currently running task_work_run() because of how it's (splicing the list in batches). Enqueue and run them through a separate callback head, namely struct io_sq_data::park_task_work. As a nice bonus we now precisely control where it's run, that's much safer than guessing where it can happen as it was before. Reported-by: Jens Axboe Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5c2de43b99f5..5f4e312111ea 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -274,6 +274,7 @@ struct io_sq_data { unsigned long state; struct completion exited; + struct callback_head *park_task_work; }; #define IO_IOPOLL_BATCH 8 @@ -6727,6 +6728,7 @@ static int io_sq_thread(void *data) cond_resched(); mutex_lock(&sqd->lock); io_run_task_work(); + io_run_task_work_head(&sqd->park_task_work); timeout = jiffies + sqd->sq_thread_idle; continue; } @@ -6781,6 +6783,7 @@ static int io_sq_thread(void *data) } finish_wait(&sqd->wait, &wait); + io_run_task_work_head(&sqd->park_task_work); timeout = jiffies + sqd->sq_thread_idle; } @@ -6792,6 +6795,7 @@ static int io_sq_thread(void *data) mutex_unlock(&sqd->lock); io_run_task_work(); + io_run_task_work_head(&sqd->park_task_work); complete(&sqd->exited); do_exit(0); } @@ -8890,7 +8894,7 @@ static void io_sqpoll_cancel_sync(struct io_ring_ctx *ctx) if (task) { init_completion(&work.completion); init_task_work(&work.task_work, io_sqpoll_cancel_cb); - WARN_ON_ONCE(task_work_add(task, &work.task_work, TWA_SIGNAL)); + io_task_work_add_head(&sqd->park_task_work, &work.task_work); wake_up_process(task); } io_sq_thread_unpark(sqd); -- cgit v1.2.3 From d336f7ebc65007f5831e2297e6f3383ae8dbf8ed Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 2 Mar 2021 09:32:53 -0800 Subject: xfs: force log and push AIL to clear pinned inodes when aborting mount If we allocate quota inodes in the process of mounting a filesystem but then decide to abort the mount, it's possible that the quota inodes are sitting around pinned by the log. Now that inode reclaim relies on the AIL to flush inodes, we have to force the log and push the AIL in between releasing the quota inodes and kicking off reclaim to tear down all the incore inodes. Do this by extracting the bits we need from the unmount path and reusing them. As an added bonus, failed writes during a failed mount will not retry forever now. This was originally found during a fuzz test of metadata directories (xfs/1546), but the actual symptom was that reclaim hung up on the quota inodes. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Dave Chinner --- fs/xfs/xfs_mount.c | 90 ++++++++++++++++++++++++++---------------------------- 1 file changed, 44 insertions(+), 46 deletions(-) (limited to 'fs') diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index 52370d0a3f43..1c97b155a8ee 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -634,6 +634,47 @@ xfs_check_summary_counts( return xfs_initialize_perag_data(mp, mp->m_sb.sb_agcount); } +/* + * Flush and reclaim dirty inodes in preparation for unmount. Inodes and + * internal inode structures can be sitting in the CIL and AIL at this point, + * so we need to unpin them, write them back and/or reclaim them before unmount + * can proceed. + * + * An inode cluster that has been freed can have its buffer still pinned in + * memory because the transaction is still sitting in a iclog. The stale inodes + * on that buffer will be pinned to the buffer until the transaction hits the + * disk and the callbacks run. Pushing the AIL will skip the stale inodes and + * may never see the pinned buffer, so nothing will push out the iclog and + * unpin the buffer. + * + * Hence we need to force the log to unpin everything first. However, log + * forces don't wait for the discards they issue to complete, so we have to + * explicitly wait for them to complete here as well. + * + * Then we can tell the world we are unmounting so that error handling knows + * that the filesystem is going away and we should error out anything that we + * have been retrying in the background. This will prevent never-ending + * retries in AIL pushing from hanging the unmount. + * + * Finally, we can push the AIL to clean all the remaining dirty objects, then + * reclaim the remaining inodes that are still in memory at this point in time. + */ +static void +xfs_unmount_flush_inodes( + struct xfs_mount *mp) +{ + xfs_log_force(mp, XFS_LOG_SYNC); + xfs_extent_busy_wait_all(mp); + flush_workqueue(xfs_discard_wq); + + mp->m_flags |= XFS_MOUNT_UNMOUNTING; + + xfs_ail_push_all_sync(mp->m_ail); + cancel_delayed_work_sync(&mp->m_reclaim_work); + xfs_reclaim_inodes(mp); + xfs_health_unmount(mp); +} + /* * This function does the following on an initial mount of a file system: * - reads the superblock from disk and init the mount struct @@ -1008,7 +1049,7 @@ xfs_mountfs( /* Clean out dquots that might be in memory after quotacheck. */ xfs_qm_unmount(mp); /* - * Cancel all delayed reclaim work and reclaim the inodes directly. + * Flush all inode reclamation work and flush the log. * We have to do this /after/ rtunmount and qm_unmount because those * two will have scheduled delayed reclaim for the rt/quota inodes. * @@ -1018,11 +1059,8 @@ xfs_mountfs( * qm_unmount_quotas and therefore rely on qm_unmount to release the * quota inodes. */ - cancel_delayed_work_sync(&mp->m_reclaim_work); - xfs_reclaim_inodes(mp); - xfs_health_unmount(mp); + xfs_unmount_flush_inodes(mp); out_log_dealloc: - mp->m_flags |= XFS_MOUNT_UNMOUNTING; xfs_log_mount_cancel(mp); out_fail_wait: if (mp->m_logdev_targp && mp->m_logdev_targp != mp->m_ddev_targp) @@ -1063,47 +1101,7 @@ xfs_unmountfs( xfs_rtunmount_inodes(mp); xfs_irele(mp->m_rootip); - /* - * We can potentially deadlock here if we have an inode cluster - * that has been freed has its buffer still pinned in memory because - * the transaction is still sitting in a iclog. The stale inodes - * on that buffer will be pinned to the buffer until the - * transaction hits the disk and the callbacks run. Pushing the AIL will - * skip the stale inodes and may never see the pinned buffer, so - * nothing will push out the iclog and unpin the buffer. Hence we - * need to force the log here to ensure all items are flushed into the - * AIL before we go any further. - */ - xfs_log_force(mp, XFS_LOG_SYNC); - - /* - * Wait for all busy extents to be freed, including completion of - * any discard operation. - */ - xfs_extent_busy_wait_all(mp); - flush_workqueue(xfs_discard_wq); - - /* - * We now need to tell the world we are unmounting. This will allow - * us to detect that the filesystem is going away and we should error - * out anything that we have been retrying in the background. This will - * prevent neverending retries in AIL pushing from hanging the unmount. - */ - mp->m_flags |= XFS_MOUNT_UNMOUNTING; - - /* - * Flush all pending changes from the AIL. - */ - xfs_ail_push_all_sync(mp->m_ail); - - /* - * Reclaim all inodes. At this point there should be no dirty inodes and - * none should be pinned or locked. Stop background inode reclaim here - * if it is still running. - */ - cancel_delayed_work_sync(&mp->m_reclaim_work); - xfs_reclaim_inodes(mp); - xfs_health_unmount(mp); + xfs_unmount_flush_inodes(mp); xfs_qm_unmount(mp); -- cgit v1.2.3 From 8723d5ba8bdae1c41be7a6fc8469dc9aa551e7d0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 14 Mar 2021 10:59:39 -0700 Subject: xfs: also reject BULKSTAT_SINGLE in a mount user namespace BULKSTAT_SINGLE exposed the ondisk uids/gids just like bulkstat, and can be called on any inode, including ones not visible in the current mount. Fixes: f736d93d76d3 ("xfs: support idmapped mounts") Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_itable.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'fs') diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c index ca310a125d1e..3498b97fb06d 100644 --- a/fs/xfs/xfs_itable.c +++ b/fs/xfs/xfs_itable.c @@ -168,6 +168,12 @@ xfs_bulkstat_one( }; int error; + if (breq->mnt_userns != &init_user_ns) { + xfs_warn_ratelimited(breq->mp, + "bulkstat not supported inside of idmapped mounts."); + return -EINVAL; + } + ASSERT(breq->icount == 1); bc.buf = kmem_zalloc(sizeof(struct xfs_bulkstat), -- cgit v1.2.3 From d2dcc8ed8ec650a793e81d8b2222146eb6ddd84f Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 8 Mar 2021 17:20:17 +0800 Subject: btrfs: fix wrong offset to zero out range beyond i_size [BUG] The test generic/091 fails , with the following output: fsx -N 10000 -o 128000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -W mapped writes DISABLED Seed set to 1 main: filesystem does not support fallocate mode FALLOC_FL_COLLAPSE_RANGE, disabling! main: filesystem does not support fallocate mode FALLOC_FL_INSERT_RANGE, disabling! skipping zero size read truncating to largest ever: 0xe400 copying to largest ever: 0x1f400 cloning to largest ever: 0x70000 cloning to largest ever: 0x77000 fallocating to largest ever: 0x7a120 Mapped Read: non-zero data past EOF (0x3a7ff) page offset 0x800 is 0xf2e1 <<< ... [CAUSE] In commit c28ea613fafa ("btrfs: subpage: fix the false data csum mismatch error") end_bio_extent_readpage() changes to only zero the range inside the bvec for incoming subpage support. But that commit is using incorrect offset to calculate the start. For subpage, we can have a case that the whole bvec is beyond isize, thus we need to calculate the correct offset. But the offending commit is using @end (bvec end), other than @start (bvec start) to calculate the start offset. This means, we only zero the last byte of the bvec, not from the isize. This stupid bug makes the range beyond isize is not properly zeroed, and failed above test. [FIX] Use correct @start to calculate the range start. Reported-by: kernel test robot Fixes: c28ea613fafa ("btrfs: subpage: fix the false data csum mismatch error") Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 4671c99d468d..f3d7be975c3a 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -3020,7 +3020,7 @@ readpage_ok: */ if (page->index == end_index && i_size <= end) { u32 zero_start = max(offset_in_page(i_size), - offset_in_page(end)); + offset_in_page(start)); zero_user_segment(page, zero_start, offset_in_page(end) + 1); -- cgit v1.2.3 From fbf48bb0b197e6894a04c714728c952af7153bf3 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Wed, 3 Mar 2021 18:41:51 +0800 Subject: btrfs: track qgroup released data in own variable in insert_prealloc_file_extent There is a piece of weird code in insert_prealloc_file_extent(), which looks like: ret = btrfs_qgroup_release_data(inode, file_offset, len); if (ret < 0) return ERR_PTR(ret); if (trans) { ret = insert_reserved_file_extent(trans, inode, file_offset, &stack_fi, true, ret); ... } extent_info.is_new_extent = true; extent_info.qgroup_reserved = ret; ... Note how the variable @ret is abused here, and if anyone is adding code just after btrfs_qgroup_release_data() call, it's super easy to overwrite the @ret and cause tons of qgroup related bugs. Fix such abuse by introducing new variable @qgroup_released, so that we won't reuse the existing variable @ret. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index c35b724a5611..77182be403c5 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -9873,6 +9873,7 @@ static struct btrfs_trans_handle *insert_prealloc_file_extent( struct btrfs_path *path; u64 start = ins->objectid; u64 len = ins->offset; + int qgroup_released; int ret; memset(&stack_fi, 0, sizeof(stack_fi)); @@ -9885,14 +9886,14 @@ static struct btrfs_trans_handle *insert_prealloc_file_extent( btrfs_set_stack_file_extent_compression(&stack_fi, BTRFS_COMPRESS_NONE); /* Encryption and other encoding is reserved and all 0 */ - ret = btrfs_qgroup_release_data(inode, file_offset, len); - if (ret < 0) - return ERR_PTR(ret); + qgroup_released = btrfs_qgroup_release_data(inode, file_offset, len); + if (qgroup_released < 0) + return ERR_PTR(qgroup_released); if (trans) { ret = insert_reserved_file_extent(trans, inode, file_offset, &stack_fi, - true, ret); + true, qgroup_released); if (ret) return ERR_PTR(ret); return trans; @@ -9905,7 +9906,7 @@ static struct btrfs_trans_handle *insert_prealloc_file_extent( extent_info.file_offset = file_offset; extent_info.extent_buf = (char *)&stack_fi; extent_info.is_new_extent = true; - extent_info.qgroup_reserved = ret; + extent_info.qgroup_reserved = qgroup_released; extent_info.insertions = 0; path = btrfs_alloc_path(); -- cgit v1.2.3 From a3ee79bd8fe17812d2305ccc4bf81bfeab395576 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Wed, 3 Mar 2021 18:41:52 +0800 Subject: btrfs: fix qgroup data rsv leak caused by falloc failure [BUG] When running fsstress with only falloc workload, and a very low qgroup limit set, we can get qgroup data rsv leak at unmount time. BTRFS warning (device dm-0): qgroup 0/5 has unreleased space, type 0 rsv 20480 BTRFS error (device dm-0): qgroup reserved space leaked The minimal reproducer looks like: #!/bin/bash dev=/dev/test/test mnt="/mnt/btrfs" fsstress=~/xfstests-dev/ltp/fsstress runtime=8 workload() { umount $dev &> /dev/null umount $mnt &> /dev/null mkfs.btrfs -f $dev > /dev/null mount $dev $mnt btrfs quota en $mnt btrfs quota rescan -w $mnt btrfs qgroup limit 16m 0/5 $mnt $fsstress -w -z -f creat=10 -f fallocate=10 -p 2 -n 100 \ -d $mnt -v > /tmp/fsstress umount $mnt if dmesg | grep leak ; then echo "!!! FAILED !!!" exit 1 fi } for (( i=0; i < $runtime; i++)); do echo "=== $i/$runtime===" workload done Normally it would fail before round 4. [CAUSE] In function insert_prealloc_file_extent(), we first call btrfs_qgroup_release_data() to know how many bytes are reserved for qgroup data rsv. Then use that @qgroup_released number to continue our work. But after we call btrfs_qgroup_release_data(), we should either queue @qgroup_released to delayed ref or free them manually in error path. Unfortunately, we lack the error handling to free the released bytes, leaking qgroup data rsv. All the error handling function outside won't help at all, as we have released the range, meaning in inode io tree, the EXTENT_QGROUP_RESERVED bit is already cleared, thus all btrfs_qgroup_free_data() call won't free any data rsv. [FIX] Add free_qgroup tag to manually free the released qgroup data rsv. Reported-by: Nikolay Borisov Reported-by: David Sterba Fixes: 9729f10a608f ("btrfs: inode: move qgroup reserved space release to the callers of insert_reserved_file_extent()") CC: stable@vger.kernel.org # 5.10+ Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/inode.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 77182be403c5..ea5ede619220 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -9895,7 +9895,7 @@ static struct btrfs_trans_handle *insert_prealloc_file_extent( file_offset, &stack_fi, true, qgroup_released); if (ret) - return ERR_PTR(ret); + goto free_qgroup; return trans; } @@ -9910,17 +9910,31 @@ static struct btrfs_trans_handle *insert_prealloc_file_extent( extent_info.insertions = 0; path = btrfs_alloc_path(); - if (!path) - return ERR_PTR(-ENOMEM); + if (!path) { + ret = -ENOMEM; + goto free_qgroup; + } ret = btrfs_replace_file_extents(&inode->vfs_inode, path, file_offset, file_offset + len - 1, &extent_info, &trans); btrfs_free_path(path); if (ret) - return ERR_PTR(ret); - + goto free_qgroup; return trans; + +free_qgroup: + /* + * We have released qgroup data range at the beginning of the function, + * and normally qgroup_released bytes will be freed when committing + * transaction. + * But if we error out early, we have to free what we have released + * or we leak qgroup data reservation. + */ + btrfs_qgroup_free_refroot(inode->root->fs_info, + inode->root->root_key.objectid, qgroup_released, + BTRFS_QGROUP_RSV_DATA); + return ERR_PTR(ret); } static int __btrfs_prealloc_file_range(struct inode *inode, int mode, -- cgit v1.2.3 From e3d3b4157610164b0ec43d968b0dfedfe7c68992 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Thu, 11 Mar 2021 15:13:30 +0000 Subject: btrfs: zoned: fix linked list corruption after log root tree allocation failure When using a zoned filesystem, while syncing the log, if we fail to allocate the root node for the log root tree, we are not removing the log context we allocated on stack from the list of log contexts of the log root tree. This means after the return from btrfs_sync_log() we get a corrupted linked list. Fix this by allocating the node before adding our stack allocated context to the list of log contexts of the log root tree. Fixes: 3ddebf27fcd3a9 ("btrfs: zoned: reorder log node allocation on zoned filesystem") Reviewed-by: Johannes Thumshirn Reviewed-by: Naohiro Aota Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 2f1acc9aea9e..92a368627791 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3169,10 +3169,6 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, mutex_lock(&log_root_tree->log_mutex); - index2 = log_root_tree->log_transid % 2; - list_add_tail(&root_log_ctx.list, &log_root_tree->log_ctxs[index2]); - root_log_ctx.log_transid = log_root_tree->log_transid; - if (btrfs_is_zoned(fs_info)) { if (!log_root_tree->node) { ret = btrfs_alloc_log_tree_node(trans, log_root_tree); @@ -3183,6 +3179,10 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, } } + index2 = log_root_tree->log_transid % 2; + list_add_tail(&root_log_ctx.list, &log_root_tree->log_ctxs[index2]); + root_log_ctx.log_transid = log_root_tree->log_transid; + /* * Now we are safe to update the log_root_tree because we're under the * log_mutex, and we're a current writer so we're holding the commit -- cgit v1.2.3 From 64fcbb6158ecc684d84c64424830a9c37c77c5b9 Mon Sep 17 00:00:00 2001 From: David Howells Date: Tue, 2 Mar 2021 10:26:45 +0000 Subject: afs: Fix accessing YFS xattrs on a non-YFS server If someone attempts to access YFS-related xattrs (e.g. afs.yfs.acl) on a file on a non-YFS AFS server (such as OpenAFS), then the kernel will jump to a NULL function pointer because the afs_fetch_acl_operation descriptor doesn't point to a function for issuing an operation on a non-YFS server[1]. Fix this by making afs_wait_for_operation() check that the issue_afs_rpc method is set before jumping to it and setting -ENOTSUPP if not. This fix also covers other potential operations that also only exist on YFS servers. afs_xattr_get/set_yfs() then need to translate -ENOTSUPP to -ENODATA as the former error is internal to the kernel. The bug shows up as an oops like the following: BUG: kernel NULL pointer dereference, address: 0000000000000000 [...] Code: Unable to access opcode bytes at RIP 0xffffffffffffffd6. [...] Call Trace: afs_wait_for_operation+0x83/0x1b0 [kafs] afs_xattr_get_yfs+0xe6/0x270 [kafs] __vfs_getxattr+0x59/0x80 vfs_getxattr+0x11c/0x140 getxattr+0x181/0x250 ? __check_object_size+0x13f/0x150 ? __fput+0x16d/0x250 __x64_sys_fgetxattr+0x64/0xb0 do_syscall_64+0x49/0xc0 entry_SYSCALL_64_after_hwframe+0x44/0xa9 RIP: 0033:0x7fb120a9defe This was triggered with "cp -a" which attempts to copy xattrs, including afs ones, but is easier to reproduce with getfattr, e.g.: getfattr -d -m ".*" /afs/openafs.org/ Fixes: e49c7b2f6de7 ("afs: Build an abstraction around an "operation" concept") Reported-by: Gaja Sophie Peters Signed-off-by: David Howells Tested-by: Gaja Sophie Peters Reviewed-by: Marc Dionne Reviewed-by: Jeffrey Altman cc: linux-afs@lists.infradead.org Link: http://lists.infradead.org/pipermail/linux-afs/2021-March/003498.html [1] Link: http://lists.infradead.org/pipermail/linux-afs/2021-March/003566.html # v1 Link: http://lists.infradead.org/pipermail/linux-afs/2021-March/003572.html # v2 --- fs/afs/fs_operation.c | 7 +++++-- fs/afs/xattr.c | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/afs/fs_operation.c b/fs/afs/fs_operation.c index 97cab12b0a6c..71c58723763d 100644 --- a/fs/afs/fs_operation.c +++ b/fs/afs/fs_operation.c @@ -181,10 +181,13 @@ void afs_wait_for_operation(struct afs_operation *op) if (test_bit(AFS_SERVER_FL_IS_YFS, &op->server->flags) && op->ops->issue_yfs_rpc) op->ops->issue_yfs_rpc(op); - else + else if (op->ops->issue_afs_rpc) op->ops->issue_afs_rpc(op); + else + op->ac.error = -ENOTSUPP; - op->error = afs_wait_for_call_to_complete(op->call, &op->ac); + if (op->call) + op->error = afs_wait_for_call_to_complete(op->call, &op->ac); } switch (op->error) { diff --git a/fs/afs/xattr.c b/fs/afs/xattr.c index c629caae5002..4934e325a14a 100644 --- a/fs/afs/xattr.c +++ b/fs/afs/xattr.c @@ -231,6 +231,8 @@ static int afs_xattr_get_yfs(const struct xattr_handler *handler, else ret = -ERANGE; } + } else if (ret == -ENOTSUPP) { + ret = -ENODATA; } error_yacl: @@ -256,6 +258,7 @@ static int afs_xattr_set_yfs(const struct xattr_handler *handler, { struct afs_operation *op; struct afs_vnode *vnode = AFS_FS_I(inode); + int ret; if (flags == XATTR_CREATE || strcmp(name, "acl") != 0) @@ -270,7 +273,10 @@ static int afs_xattr_set_yfs(const struct xattr_handler *handler, return afs_put_operation(op); op->ops = &yfs_store_opaque_acl2_operation; - return afs_do_sync_operation(op); + ret = afs_do_sync_operation(op); + if (ret == -ENOTSUPP) + ret = -ENODATA; + return ret; } static const struct xattr_handler afs_xattr_yfs_handler = { -- cgit v1.2.3 From a7889c6320b9200e3fe415238f546db677310fa9 Mon Sep 17 00:00:00 2001 From: David Howells Date: Tue, 9 Mar 2021 08:27:39 +0000 Subject: afs: Stop listxattr() from listing "afs.*" attributes afs_listxattr() lists all the available special afs xattrs (i.e. those in the "afs.*" space), no matter what type of server we're dealing with. But OpenAFS servers, for example, cannot deal with some of the extra-capable attributes that AuriStor (YFS) servers provide. Unfortunately, the presence of the afs.yfs.* attributes causes errors[1] for anything that tries to read them if the server is of the wrong type. Fix the problem by removing afs_listxattr() so that none of the special xattrs are listed (AFS doesn't support xattrs). It does mean, however, that getfattr won't list them, though they can still be accessed with getxattr() and setxattr(). This can be tested with something like: getfattr -d -m ".*" /afs/example.com/path/to/file With this change, none of the afs.* attributes should be visible. Changes: ver #2: - Hide all of the afs.* xattrs, not just the ACL ones. Fixes: ae46578b963f ("afs: Get YFS ACLs and information through xattrs") Reported-by: Gaja Sophie Peters Signed-off-by: David Howells Tested-by: Gaja Sophie Peters Reviewed-by: Jeffrey Altman Reviewed-by: Marc Dionne cc: linux-afs@lists.infradead.org Link: http://lists.infradead.org/pipermail/linux-afs/2021-March/003502.html [1] Link: http://lists.infradead.org/pipermail/linux-afs/2021-March/003567.html # v1 Link: http://lists.infradead.org/pipermail/linux-afs/2021-March/003573.html # v2 --- fs/afs/dir.c | 1 - fs/afs/file.c | 1 - fs/afs/inode.c | 1 - fs/afs/internal.h | 1 - fs/afs/mntpt.c | 1 - fs/afs/xattr.c | 23 ----------------------- 6 files changed, 28 deletions(-) (limited to 'fs') diff --git a/fs/afs/dir.c b/fs/afs/dir.c index 714fcca9af99..17548c1faf02 100644 --- a/fs/afs/dir.c +++ b/fs/afs/dir.c @@ -70,7 +70,6 @@ const struct inode_operations afs_dir_inode_operations = { .permission = afs_permission, .getattr = afs_getattr, .setattr = afs_setattr, - .listxattr = afs_listxattr, }; const struct address_space_operations afs_dir_aops = { diff --git a/fs/afs/file.c b/fs/afs/file.c index 85f5adf21aa0..960b64268623 100644 --- a/fs/afs/file.c +++ b/fs/afs/file.c @@ -43,7 +43,6 @@ const struct inode_operations afs_file_inode_operations = { .getattr = afs_getattr, .setattr = afs_setattr, .permission = afs_permission, - .listxattr = afs_listxattr, }; const struct address_space_operations afs_fs_aops = { diff --git a/fs/afs/inode.c b/fs/afs/inode.c index 1156b2df28d3..12be88716e4c 100644 --- a/fs/afs/inode.c +++ b/fs/afs/inode.c @@ -27,7 +27,6 @@ static const struct inode_operations afs_symlink_inode_operations = { .get_link = page_get_link, - .listxattr = afs_listxattr, }; static noinline void dump_vnode(struct afs_vnode *vnode, struct afs_vnode *parent_vnode) diff --git a/fs/afs/internal.h b/fs/afs/internal.h index b626e38e9ab5..1627b1872812 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -1509,7 +1509,6 @@ extern int afs_launder_page(struct page *); * xattr.c */ extern const struct xattr_handler *afs_xattr_handlers[]; -extern ssize_t afs_listxattr(struct dentry *, char *, size_t); /* * yfsclient.c diff --git a/fs/afs/mntpt.c b/fs/afs/mntpt.c index 052dab2f5c03..bbb2c210d139 100644 --- a/fs/afs/mntpt.c +++ b/fs/afs/mntpt.c @@ -32,7 +32,6 @@ const struct inode_operations afs_mntpt_inode_operations = { .lookup = afs_mntpt_lookup, .readlink = page_readlink, .getattr = afs_getattr, - .listxattr = afs_listxattr, }; const struct inode_operations afs_autocell_inode_operations = { diff --git a/fs/afs/xattr.c b/fs/afs/xattr.c index 4934e325a14a..7751b0b3f81d 100644 --- a/fs/afs/xattr.c +++ b/fs/afs/xattr.c @@ -11,29 +11,6 @@ #include #include "internal.h" -static const char afs_xattr_list[] = - "afs.acl\0" - "afs.cell\0" - "afs.fid\0" - "afs.volume\0" - "afs.yfs.acl\0" - "afs.yfs.acl_inherited\0" - "afs.yfs.acl_num_cleaned\0" - "afs.yfs.vol_acl"; - -/* - * Retrieve a list of the supported xattrs. - */ -ssize_t afs_listxattr(struct dentry *dentry, char *buffer, size_t size) -{ - if (size == 0) - return sizeof(afs_xattr_list); - if (size < sizeof(afs_xattr_list)) - return -ERANGE; - memcpy(buffer, afs_xattr_list, sizeof(afs_xattr_list)); - return sizeof(afs_xattr_list); -} - /* * Deal with the result of a successful fetch ACL operation. */ -- cgit v1.2.3 From 1601ea068b886da1f8f8d4e18b9403e9e24adef6 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Mon, 15 Mar 2021 12:43:55 +0900 Subject: zonefs: prevent use of seq files as swap file The sequential write constraint of sequential zone file prevent their use as swap files. Only allow conventional zone files to be used as swap files. Fixes: 8dcc1a9d90c1 ("fs: New zonefs file system") Cc: Reviewed-by: Johannes Thumshirn Signed-off-by: Damien Le Moal --- fs/zonefs/super.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'fs') diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 0fe76f376dee..a3d074f98660 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -165,6 +165,21 @@ static int zonefs_writepages(struct address_space *mapping, return iomap_writepages(mapping, wbc, &wpc, &zonefs_writeback_ops); } +static int zonefs_swap_activate(struct swap_info_struct *sis, + struct file *swap_file, sector_t *span) +{ + struct inode *inode = file_inode(swap_file); + struct zonefs_inode_info *zi = ZONEFS_I(inode); + + if (zi->i_ztype != ZONEFS_ZTYPE_CNV) { + zonefs_err(inode->i_sb, + "swap file: not a conventional zone file\n"); + return -EINVAL; + } + + return iomap_swapfile_activate(sis, swap_file, span, &zonefs_iomap_ops); +} + static const struct address_space_operations zonefs_file_aops = { .readpage = zonefs_readpage, .readahead = zonefs_readahead, @@ -177,6 +192,7 @@ static const struct address_space_operations zonefs_file_aops = { .is_partially_uptodate = iomap_is_partially_uptodate, .error_remove_page = generic_error_remove_page, .direct_IO = noop_direct_IO, + .swap_activate = zonefs_swap_activate, }; static void zonefs_update_stats(struct inode *inode, loff_t new_isize) -- cgit v1.2.3 From ebfd68cd0c1e81267c757332385cb96df30dacce Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Wed, 10 Mar 2021 15:20:28 +0900 Subject: zonefs: Fix O_APPEND async write handling zonefs updates the size of a sequential zone file inode only on completion of direct writes. When executing asynchronous append writes (with a file open with O_APPEND or using RWF_APPEND), the use of the current inode size in generic_write_checks() to set an iocb offset thus leads to unaligned write if an application issues an append write operation with another write already being executed. Fix this problem by introducing zonefs_write_checks() as a modified version of generic_write_checks() using the file inode wp_offset for an append write iocb offset. Also introduce zonefs_write_check_limits() to replace generic_write_check_limits() call. This zonefs special helper makes sure that the maximum file limit used is the maximum size of the file being accessed. Since zonefs_write_checks() already truncates the iov_iter, the calls to iov_iter_truncate() in zonefs_file_dio_write() and zonefs_file_buffered_write() are removed. Fixes: 8dcc1a9d90c1 ("fs: New zonefs file system") Cc: Reviewed-by: Johannes Thumshirn Signed-off-by: Damien Le Moal --- fs/zonefs/super.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index a3d074f98660..3427c99abb4d 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -743,6 +743,68 @@ out_release: return ret; } +/* + * Do not exceed the LFS limits nor the file zone size. If pos is under the + * limit it becomes a short access. If it exceeds the limit, return -EFBIG. + */ +static loff_t zonefs_write_check_limits(struct file *file, loff_t pos, + loff_t count) +{ + struct inode *inode = file_inode(file); + struct zonefs_inode_info *zi = ZONEFS_I(inode); + loff_t limit = rlimit(RLIMIT_FSIZE); + loff_t max_size = zi->i_max_size; + + if (limit != RLIM_INFINITY) { + if (pos >= limit) { + send_sig(SIGXFSZ, current, 0); + return -EFBIG; + } + count = min(count, limit - pos); + } + + if (!(file->f_flags & O_LARGEFILE)) + max_size = min_t(loff_t, MAX_NON_LFS, max_size); + + if (unlikely(pos >= max_size)) + return -EFBIG; + + return min(count, max_size - pos); +} + +static ssize_t zonefs_write_checks(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct zonefs_inode_info *zi = ZONEFS_I(inode); + loff_t count; + + if (IS_SWAPFILE(inode)) + return -ETXTBSY; + + if (!iov_iter_count(from)) + return 0; + + if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT)) + return -EINVAL; + + if (iocb->ki_flags & IOCB_APPEND) { + if (zi->i_ztype != ZONEFS_ZTYPE_SEQ) + return -EINVAL; + mutex_lock(&zi->i_truncate_mutex); + iocb->ki_pos = zi->i_wpoffset; + mutex_unlock(&zi->i_truncate_mutex); + } + + count = zonefs_write_check_limits(file, iocb->ki_pos, + iov_iter_count(from)); + if (count < 0) + return count; + + iov_iter_truncate(from, count); + return iov_iter_count(from); +} + /* * Handle direct writes. For sequential zone files, this is the only possible * write path. For these files, check that the user is issuing writes @@ -760,8 +822,7 @@ static ssize_t zonefs_file_dio_write(struct kiocb *iocb, struct iov_iter *from) struct super_block *sb = inode->i_sb; bool sync = is_sync_kiocb(iocb); bool append = false; - size_t count; - ssize_t ret; + ssize_t ret, count; /* * For async direct IOs to sequential zone files, refuse IOCB_NOWAIT @@ -779,12 +840,11 @@ static ssize_t zonefs_file_dio_write(struct kiocb *iocb, struct iov_iter *from) inode_lock(inode); } - ret = generic_write_checks(iocb, from); - if (ret <= 0) + count = zonefs_write_checks(iocb, from); + if (count <= 0) { + ret = count; goto inode_unlock; - - iov_iter_truncate(from, zi->i_max_size - iocb->ki_pos); - count = iov_iter_count(from); + } if ((iocb->ki_pos | count) & (sb->s_blocksize - 1)) { ret = -EINVAL; @@ -844,12 +904,10 @@ static ssize_t zonefs_file_buffered_write(struct kiocb *iocb, inode_lock(inode); } - ret = generic_write_checks(iocb, from); + ret = zonefs_write_checks(iocb, from); if (ret <= 0) goto inode_unlock; - iov_iter_truncate(from, zi->i_max_size - iocb->ki_pos); - ret = iomap_file_buffered_write(iocb, from, &zonefs_iomap_ops); if (ret > 0) iocb->ki_pos += ret; -- cgit v1.2.3 From d9bb77d51e668a1a6d4530c1ea471574d0ce465f Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 15 Mar 2021 13:39:14 +0800 Subject: btrfs: subpage: fix wild pointer access during metadata read failure [BUG] When running fstests for btrfs subpage read-write test, it has a very high chance to crash at generic/475 with the following stack: BTRFS warning (device dm-8): direct IO failed ino 510 rw 1,34817 sector 0xcdf0 len 94208 err no 10 Unable to handle kernel paging request at virtual address ffff80001157e7c0 CPU: 2 PID: 687125 Comm: kworker/u12:4 Tainted: G WC 5.12.0-rc2-custom+ #5 Hardware name: Khadas VIM3 (DT) Workqueue: btrfs-endio-meta btrfs_work_helper [btrfs] pc : queued_spin_lock_slowpath+0x1a0/0x390 lr : do_raw_spin_lock+0xc4/0x11c Call trace: queued_spin_lock_slowpath+0x1a0/0x390 _raw_spin_lock+0x68/0x84 btree_readahead_hook+0x38/0xc0 [btrfs] end_bio_extent_readpage+0x504/0x5f4 [btrfs] bio_endio+0x170/0x1a4 end_workqueue_fn+0x3c/0x60 [btrfs] btrfs_work_helper+0x1b0/0x1b4 [btrfs] process_one_work+0x22c/0x430 worker_thread+0x70/0x3a0 kthread+0x13c/0x140 ret_from_fork+0x10/0x30 Code: 910020e0 8b0200c2 f861d884 aa0203e1 (f8246827) [CAUSE] In end_bio_extent_readpage(), if we hit an error during read, we will handle the error differently for data and metadata. For data we queue a repair, while for metadata, we record the error and let the caller choose what to do. But the code is still using page->private to grab extent buffer, which no longer points to extent buffer for subpage metadata pages. Thus this wild pointer access leads to above crash. [FIX] Introduce a helper, find_extent_buffer_readpage(), to grab extent buffer. The difference against find_extent_buffer_nospinlock() is: - Also handles regular sectorsize == PAGE_SIZE case - No extent buffer refs increase/decrease As extent buffer under IO must have non-zero refs, so this is safe Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index f3d7be975c3a..0d00a9cd123a 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2885,6 +2885,35 @@ static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len) btrfs_subpage_end_reader(fs_info, page, start, len); } +/* + * Find extent buffer for a givne bytenr. + * + * This is for end_bio_extent_readpage(), thus we can't do any unsafe locking + * in endio context. + */ +static struct extent_buffer *find_extent_buffer_readpage( + struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr) +{ + struct extent_buffer *eb; + + /* + * For regular sectorsize, we can use page->private to grab extent + * buffer + */ + if (fs_info->sectorsize == PAGE_SIZE) { + ASSERT(PagePrivate(page) && page->private); + return (struct extent_buffer *)page->private; + } + + /* For subpage case, we need to lookup buffer radix tree */ + rcu_read_lock(); + eb = radix_tree_lookup(&fs_info->buffer_radix, + bytenr >> fs_info->sectorsize_bits); + rcu_read_unlock(); + ASSERT(eb); + return eb; +} + /* * after a readpage IO is done, we need to: * clear the uptodate bits on error @@ -2996,7 +3025,7 @@ static void end_bio_extent_readpage(struct bio *bio) } else { struct extent_buffer *eb; - eb = (struct extent_buffer *)page->private; + eb = find_extent_buffer_readpage(fs_info, page, start); set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags); eb->read_mirror = mirror; atomic_dec(&eb->io_pages); -- cgit v1.2.3 From 60484cd9d50117017cf53d5310c6cd629600dc69 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 15 Mar 2021 13:39:15 +0800 Subject: btrfs: subpage: make readahead work properly In readahead infrastructure, we are using a lot of hard coded PAGE_SHIFT while we're not doing anything specific to PAGE_SIZE. One of the most affected part is the radix tree operation of btrfs_fs_info::reada_tree. If using PAGE_SHIFT, subpage metadata readahead is broken and does no help reading metadata ahead. Fix the problem by using btrfs_fs_info::sectorsize_bits so that readahead could work for subpage. Reviewed-by: Johannes Thumshirn Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/reada.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/reada.c b/fs/btrfs/reada.c index 20fd4aa48a8c..06713a8fe26b 100644 --- a/fs/btrfs/reada.c +++ b/fs/btrfs/reada.c @@ -209,7 +209,7 @@ int btree_readahead_hook(struct extent_buffer *eb, int err) /* find extent */ spin_lock(&fs_info->reada_lock); re = radix_tree_lookup(&fs_info->reada_tree, - eb->start >> PAGE_SHIFT); + eb->start >> fs_info->sectorsize_bits); if (re) re->refcnt++; spin_unlock(&fs_info->reada_lock); @@ -240,7 +240,7 @@ static struct reada_zone *reada_find_zone(struct btrfs_device *dev, u64 logical, zone = NULL; spin_lock(&fs_info->reada_lock); ret = radix_tree_gang_lookup(&dev->reada_zones, (void **)&zone, - logical >> PAGE_SHIFT, 1); + logical >> fs_info->sectorsize_bits, 1); if (ret == 1 && logical >= zone->start && logical <= zone->end) { kref_get(&zone->refcnt); spin_unlock(&fs_info->reada_lock); @@ -283,13 +283,13 @@ static struct reada_zone *reada_find_zone(struct btrfs_device *dev, u64 logical, spin_lock(&fs_info->reada_lock); ret = radix_tree_insert(&dev->reada_zones, - (unsigned long)(zone->end >> PAGE_SHIFT), - zone); + (unsigned long)(zone->end >> fs_info->sectorsize_bits), + zone); if (ret == -EEXIST) { kfree(zone); ret = radix_tree_gang_lookup(&dev->reada_zones, (void **)&zone, - logical >> PAGE_SHIFT, 1); + logical >> fs_info->sectorsize_bits, 1); if (ret == 1 && logical >= zone->start && logical <= zone->end) kref_get(&zone->refcnt); else @@ -315,7 +315,7 @@ static struct reada_extent *reada_find_extent(struct btrfs_fs_info *fs_info, u64 length; int real_stripes; int nzones = 0; - unsigned long index = logical >> PAGE_SHIFT; + unsigned long index = logical >> fs_info->sectorsize_bits; int dev_replace_is_ongoing; int have_zone = 0; @@ -497,7 +497,7 @@ static void reada_extent_put(struct btrfs_fs_info *fs_info, struct reada_extent *re) { int i; - unsigned long index = re->logical >> PAGE_SHIFT; + unsigned long index = re->logical >> fs_info->sectorsize_bits; spin_lock(&fs_info->reada_lock); if (--re->refcnt) { @@ -538,11 +538,12 @@ static void reada_extent_put(struct btrfs_fs_info *fs_info, static void reada_zone_release(struct kref *kref) { struct reada_zone *zone = container_of(kref, struct reada_zone, refcnt); + struct btrfs_fs_info *fs_info = zone->device->fs_info; - lockdep_assert_held(&zone->device->fs_info->reada_lock); + lockdep_assert_held(&fs_info->reada_lock); radix_tree_delete(&zone->device->reada_zones, - zone->end >> PAGE_SHIFT); + zone->end >> fs_info->sectorsize_bits); kfree(zone); } @@ -593,7 +594,7 @@ static int reada_add_block(struct reada_control *rc, u64 logical, static void reada_peer_zones_set_lock(struct reada_zone *zone, int lock) { int i; - unsigned long index = zone->end >> PAGE_SHIFT; + unsigned long index = zone->end >> zone->device->fs_info->sectorsize_bits; for (i = 0; i < zone->ndevs; ++i) { struct reada_zone *peer; @@ -628,7 +629,7 @@ static int reada_pick_zone(struct btrfs_device *dev) (void **)&zone, index, 1); if (ret == 0) break; - index = (zone->end >> PAGE_SHIFT) + 1; + index = (zone->end >> dev->fs_info->sectorsize_bits) + 1; if (zone->locked) { if (zone->elems > top_locked_elems) { top_locked_elems = zone->elems; @@ -709,7 +710,7 @@ static int reada_start_machine_dev(struct btrfs_device *dev) * plugging to speed things up */ ret = radix_tree_gang_lookup(&dev->reada_extents, (void **)&re, - dev->reada_next >> PAGE_SHIFT, 1); + dev->reada_next >> fs_info->sectorsize_bits, 1); if (ret == 0 || re->logical > dev->reada_curr_zone->end) { ret = reada_pick_zone(dev); if (!ret) { @@ -718,7 +719,7 @@ static int reada_start_machine_dev(struct btrfs_device *dev) } re = NULL; ret = radix_tree_gang_lookup(&dev->reada_extents, (void **)&re, - dev->reada_next >> PAGE_SHIFT, 1); + dev->reada_next >> fs_info->sectorsize_bits, 1); } if (ret == 0) { spin_unlock(&fs_info->reada_lock); @@ -885,7 +886,7 @@ static void dump_devs(struct btrfs_fs_info *fs_info, int all) pr_cont(" curr off %llu", device->reada_next - zone->start); pr_cont("\n"); - index = (zone->end >> PAGE_SHIFT) + 1; + index = (zone->end >> fs_info->sectorsize_bits) + 1; } cnt = 0; index = 0; @@ -910,7 +911,7 @@ static void dump_devs(struct btrfs_fs_info *fs_info, int all) } } pr_cont("\n"); - index = (re->logical >> PAGE_SHIFT) + 1; + index = (re->logical >> fs_info->sectorsize_bits) + 1; if (++cnt > 15) break; } @@ -926,7 +927,7 @@ static void dump_devs(struct btrfs_fs_info *fs_info, int all) if (ret == 0) break; if (!re->scheduled) { - index = (re->logical >> PAGE_SHIFT) + 1; + index = (re->logical >> fs_info->sectorsize_bits) + 1; continue; } pr_debug("re: logical %llu size %u list empty %d scheduled %d", @@ -942,7 +943,7 @@ static void dump_devs(struct btrfs_fs_info *fs_info, int all) } } pr_cont("\n"); - index = (re->logical >> PAGE_SHIFT) + 1; + index = (re->logical >> fs_info->sectorsize_bits) + 1; } spin_unlock(&fs_info->reada_lock); } -- cgit v1.2.3 From f8425c9396639cc462bcce44b1051f8b4e62fddb Mon Sep 17 00:00:00 2001 From: Alessio Balsini Date: Mon, 25 Jan 2021 15:30:51 +0000 Subject: fuse: 32-bit user space ioctl compat for fuse device With a 64-bit kernel build the FUSE device cannot handle ioctl requests coming from 32-bit user space. This is due to the ioctl command translation that generates different command identifiers that thus cannot be used for direct comparisons without proper manipulation. Explicitly extract type and number from the ioctl command to enable 32-bit user space compatibility on 64-bit kernel builds. Signed-off-by: Alessio Balsini Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 26 ++++++++++++++++---------- include/uapi/linux/fuse.h | 3 ++- 2 files changed, 18 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index c6636b4c4ccf..c0fee830a34e 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -2229,19 +2229,21 @@ static int fuse_device_clone(struct fuse_conn *fc, struct file *new) static long fuse_dev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { - int err = -ENOTTY; + int res; + int oldfd; + struct fuse_dev *fud = NULL; - if (cmd == FUSE_DEV_IOC_CLONE) { - int oldfd; + if (_IOC_TYPE(cmd) != FUSE_DEV_IOC_MAGIC) + return -ENOTTY; - err = -EFAULT; - if (!get_user(oldfd, (__u32 __user *) arg)) { + switch (_IOC_NR(cmd)) { + case _IOC_NR(FUSE_DEV_IOC_CLONE): + res = -EFAULT; + if (!get_user(oldfd, (__u32 __user *)arg)) { struct file *old = fget(oldfd); - err = -EINVAL; + res = -EINVAL; if (old) { - struct fuse_dev *fud = NULL; - /* * Check against file->f_op because CUSE * uses the same ioctl handler. @@ -2252,14 +2254,18 @@ static long fuse_dev_ioctl(struct file *file, unsigned int cmd, if (fud) { mutex_lock(&fuse_mutex); - err = fuse_device_clone(fud->fc, file); + res = fuse_device_clone(fud->fc, file); mutex_unlock(&fuse_mutex); } fput(old); } } + break; + default: + res = -ENOTTY; + break; } - return err; + return res; } const struct file_operations fuse_dev_operations = { diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 98ca64d1beb6..54442612c48b 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -903,7 +903,8 @@ struct fuse_notify_retrieve_in { }; /* Device ioctls: */ -#define FUSE_DEV_IOC_CLONE _IOR(229, 0, uint32_t) +#define FUSE_DEV_IOC_MAGIC 229 +#define FUSE_DEV_IOC_CLONE _IOR(FUSE_DEV_IOC_MAGIC, 0, uint32_t) struct fuse_lseek_in { uint64_t fh; -- cgit v1.2.3 From 34e49994d0dcdb2d31d4d2908d04f4e9ce57e4d7 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 15 Mar 2021 15:18:24 +0100 Subject: btrfs: fix slab cache flags for free space tree bitmap The free space tree bitmap slab cache is created with SLAB_RED_ZONE but that's a debugging flag and not always enabled. Also the other slabs are created with at least SLAB_MEM_SPREAD that we want as well to average the memory placement cost. Reported-by: Vlastimil Babka Fixes: 3acd48507dc4 ("btrfs: fix allocation of free space cache v1 bitmap pages") CC: stable@vger.kernel.org # 5.4+ Signed-off-by: David Sterba --- fs/btrfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index ea5ede619220..5092dea21448 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -9005,7 +9005,7 @@ int __init btrfs_init_cachep(void) btrfs_free_space_bitmap_cachep = kmem_cache_create("btrfs_free_space_bitmap", PAGE_SIZE, PAGE_SIZE, - SLAB_RED_ZONE, NULL); + SLAB_MEM_SPREAD, NULL); if (!btrfs_free_space_bitmap_cachep) goto fail; -- cgit v1.2.3 From dbcc7d57bffc0c8cac9dac11bec548597d59a6a5 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Thu, 11 Mar 2021 14:31:05 +0000 Subject: btrfs: fix race when cloning extent buffer during rewind of an old root While resolving backreferences, as part of a logical ino ioctl call or fiemap, we can end up hitting a BUG_ON() when replaying tree mod log operations of a root, triggering a stack trace like the following: ------------[ cut here ]------------ kernel BUG at fs/btrfs/ctree.c:1210! invalid opcode: 0000 [#1] SMP KASAN PTI CPU: 1 PID: 19054 Comm: crawl_335 Tainted: G W 5.11.0-2d11c0084b02-misc-next+ #89 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 RIP: 0010:__tree_mod_log_rewind+0x3b1/0x3c0 Code: 05 48 8d 74 10 (...) RSP: 0018:ffffc90001eb70b8 EFLAGS: 00010297 RAX: 0000000000000000 RBX: ffff88812344e400 RCX: ffffffffb28933b6 RDX: 0000000000000007 RSI: dffffc0000000000 RDI: ffff88812344e42c RBP: ffffc90001eb7108 R08: 1ffff11020b60a20 R09: ffffed1020b60a20 R10: ffff888105b050f9 R11: ffffed1020b60a1f R12: 00000000000000ee R13: ffff8880195520c0 R14: ffff8881bc958500 R15: ffff88812344e42c FS: 00007fd1955e8700(0000) GS:ffff8881f5600000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007efdb7928718 CR3: 000000010103a006 CR4: 0000000000170ee0 Call Trace: btrfs_search_old_slot+0x265/0x10d0 ? lock_acquired+0xbb/0x600 ? btrfs_search_slot+0x1090/0x1090 ? free_extent_buffer.part.61+0xd7/0x140 ? free_extent_buffer+0x13/0x20 resolve_indirect_refs+0x3e9/0xfc0 ? lock_downgrade+0x3d0/0x3d0 ? __kasan_check_read+0x11/0x20 ? add_prelim_ref.part.11+0x150/0x150 ? lock_downgrade+0x3d0/0x3d0 ? __kasan_check_read+0x11/0x20 ? lock_acquired+0xbb/0x600 ? __kasan_check_write+0x14/0x20 ? do_raw_spin_unlock+0xa8/0x140 ? rb_insert_color+0x30/0x360 ? prelim_ref_insert+0x12d/0x430 find_parent_nodes+0x5c3/0x1830 ? resolve_indirect_refs+0xfc0/0xfc0 ? lock_release+0xc8/0x620 ? fs_reclaim_acquire+0x67/0xf0 ? lock_acquire+0xc7/0x510 ? lock_downgrade+0x3d0/0x3d0 ? lockdep_hardirqs_on_prepare+0x160/0x210 ? lock_release+0xc8/0x620 ? fs_reclaim_acquire+0x67/0xf0 ? lock_acquire+0xc7/0x510 ? poison_range+0x38/0x40 ? unpoison_range+0x14/0x40 ? trace_hardirqs_on+0x55/0x120 btrfs_find_all_roots_safe+0x142/0x1e0 ? find_parent_nodes+0x1830/0x1830 ? btrfs_inode_flags_to_xflags+0x50/0x50 iterate_extent_inodes+0x20e/0x580 ? tree_backref_for_extent+0x230/0x230 ? lock_downgrade+0x3d0/0x3d0 ? read_extent_buffer+0xdd/0x110 ? lock_downgrade+0x3d0/0x3d0 ? __kasan_check_read+0x11/0x20 ? lock_acquired+0xbb/0x600 ? __kasan_check_write+0x14/0x20 ? _raw_spin_unlock+0x22/0x30 ? __kasan_check_write+0x14/0x20 iterate_inodes_from_logical+0x129/0x170 ? iterate_inodes_from_logical+0x129/0x170 ? btrfs_inode_flags_to_xflags+0x50/0x50 ? iterate_extent_inodes+0x580/0x580 ? __vmalloc_node+0x92/0xb0 ? init_data_container+0x34/0xb0 ? init_data_container+0x34/0xb0 ? kvmalloc_node+0x60/0x80 btrfs_ioctl_logical_to_ino+0x158/0x230 btrfs_ioctl+0x205e/0x4040 ? __might_sleep+0x71/0xe0 ? btrfs_ioctl_get_supported_features+0x30/0x30 ? getrusage+0x4b6/0x9c0 ? __kasan_check_read+0x11/0x20 ? lock_release+0xc8/0x620 ? __might_fault+0x64/0xd0 ? lock_acquire+0xc7/0x510 ? lock_downgrade+0x3d0/0x3d0 ? lockdep_hardirqs_on_prepare+0x210/0x210 ? lockdep_hardirqs_on_prepare+0x210/0x210 ? __kasan_check_read+0x11/0x20 ? do_vfs_ioctl+0xfc/0x9d0 ? ioctl_file_clone+0xe0/0xe0 ? lock_downgrade+0x3d0/0x3d0 ? lockdep_hardirqs_on_prepare+0x210/0x210 ? __kasan_check_read+0x11/0x20 ? lock_release+0xc8/0x620 ? __task_pid_nr_ns+0xd3/0x250 ? lock_acquire+0xc7/0x510 ? __fget_files+0x160/0x230 ? __fget_light+0xf2/0x110 __x64_sys_ioctl+0xc3/0x100 do_syscall_64+0x37/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xa9 RIP: 0033:0x7fd1976e2427 Code: 00 00 90 48 8b 05 (...) RSP: 002b:00007fd1955e5cf8 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 RAX: ffffffffffffffda RBX: 00007fd1955e5f40 RCX: 00007fd1976e2427 RDX: 00007fd1955e5f48 RSI: 00000000c038943b RDI: 0000000000000004 RBP: 0000000001000000 R08: 0000000000000000 R09: 00007fd1955e6120 R10: 0000557835366b00 R11: 0000000000000246 R12: 0000000000000004 R13: 00007fd1955e5f48 R14: 00007fd1955e5f40 R15: 00007fd1955e5ef8 Modules linked in: ---[ end trace ec8931a1c36e57be ]--- (gdb) l *(__tree_mod_log_rewind+0x3b1) 0xffffffff81893521 is in __tree_mod_log_rewind (fs/btrfs/ctree.c:1210). 1205 * the modification. as we're going backwards, we do the 1206 * opposite of each operation here. 1207 */ 1208 switch (tm->op) { 1209 case MOD_LOG_KEY_REMOVE_WHILE_FREEING: 1210 BUG_ON(tm->slot < n); 1211 fallthrough; 1212 case MOD_LOG_KEY_REMOVE_WHILE_MOVING: 1213 case MOD_LOG_KEY_REMOVE: 1214 btrfs_set_node_key(eb, &tm->key, tm->slot); Here's what happens to hit that BUG_ON(): 1) We have one tree mod log user (through fiemap or the logical ino ioctl), with a sequence number of 1, so we have fs_info->tree_mod_seq == 1; 2) Another task is at ctree.c:balance_level() and we have eb X currently as the root of the tree, and we promote its single child, eb Y, as the new root. Then, at ctree.c:balance_level(), we call: tree_mod_log_insert_root(eb X, eb Y, 1); 3) At tree_mod_log_insert_root() we create tree mod log elements for each slot of eb X, of operation type MOD_LOG_KEY_REMOVE_WHILE_FREEING each with a ->logical pointing to ebX->start. These are placed in an array named tm_list. Lets assume there are N elements (N pointers in eb X); 4) Then, still at tree_mod_log_insert_root(), we create a tree mod log element of operation type MOD_LOG_ROOT_REPLACE, ->logical set to ebY->start, ->old_root.logical set to ebX->start, ->old_root.level set to the level of eb X and ->generation set to the generation of eb X; 5) Then tree_mod_log_insert_root() calls tree_mod_log_free_eb() with tm_list as argument. After that, tree_mod_log_free_eb() calls __tree_mod_log_insert() for each member of tm_list in reverse order, from highest slot in eb X, slot N - 1, to slot 0 of eb X; 6) __tree_mod_log_insert() sets the sequence number of each given tree mod log operation - it increments fs_info->tree_mod_seq and sets fs_info->tree_mod_seq as the sequence number of the given tree mod log operation. This means that for the tm_list created at tree_mod_log_insert_root(), the element corresponding to slot 0 of eb X has the highest sequence number (1 + N), and the element corresponding to the last slot has the lowest sequence number (2); 7) Then, after inserting tm_list's elements into the tree mod log rbtree, the MOD_LOG_ROOT_REPLACE element is inserted, which gets the highest sequence number, which is N + 2; 8) Back to ctree.c:balance_level(), we free eb X by calling btrfs_free_tree_block() on it. Because eb X was created in the current transaction, has no other references and writeback did not happen for it, we add it back to the free space cache/tree; 9) Later some other task T allocates the metadata extent from eb X, since it is marked as free space in the space cache/tree, and uses it as a node for some other btree; 10) The tree mod log user task calls btrfs_search_old_slot(), which calls get_old_root(), and finally that calls __tree_mod_log_oldest_root() with time_seq == 1 and eb_root == eb Y; 11) First iteration of the while loop finds the tree mod log element with sequence number N + 2, for the logical address of eb Y and of type MOD_LOG_ROOT_REPLACE; 12) Because the operation type is MOD_LOG_ROOT_REPLACE, we don't break out of the loop, and set root_logical to point to tm->old_root.logical which corresponds to the logical address of eb X; 13) On the next iteration of the while loop, the call to tree_mod_log_search_oldest() returns the smallest tree mod log element for the logical address of eb X, which has a sequence number of 2, an operation type of MOD_LOG_KEY_REMOVE_WHILE_FREEING and corresponds to the old slot N - 1 of eb X (eb X had N items in it before being freed); 14) We then break out of the while loop and return the tree mod log operation of type MOD_LOG_ROOT_REPLACE (eb Y), and not the one for slot N - 1 of eb X, to get_old_root(); 15) At get_old_root(), we process the MOD_LOG_ROOT_REPLACE operation and set "logical" to the logical address of eb X, which was the old root. We then call tree_mod_log_search() passing it the logical address of eb X and time_seq == 1; 16) Then before calling tree_mod_log_search(), task T adds a key to eb X, which results in adding a tree mod log operation of type MOD_LOG_KEY_ADD to the tree mod log - this is done at ctree.c:insert_ptr() - but after adding the tree mod log operation and before updating the number of items in eb X from 0 to 1... 17) The task at get_old_root() calls tree_mod_log_search() and gets the tree mod log operation of type MOD_LOG_KEY_ADD just added by task T. Then it enters the following if branch: if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) { (...) } (...) Calls read_tree_block() for eb X, which gets a reference on eb X but does not lock it - task T has it locked. Then it clones eb X while it has nritems set to 0 in its header, before task T sets nritems to 1 in eb X's header. From hereupon we use the clone of eb X which no other task has access to; 18) Then we call __tree_mod_log_rewind(), passing it the MOD_LOG_KEY_ADD mod log operation we just got from tree_mod_log_search() in the previous step and the cloned version of eb X; 19) At __tree_mod_log_rewind(), we set the local variable "n" to the number of items set in eb X's clone, which is 0. Then we enter the while loop, and in its first iteration we process the MOD_LOG_KEY_ADD operation, which just decrements "n" from 0 to (u32)-1, since "n" is declared with a type of u32. At the end of this iteration we call rb_next() to find the next tree mod log operation for eb X, that gives us the mod log operation of type MOD_LOG_KEY_REMOVE_WHILE_FREEING, for slot 0, with a sequence number of N + 1 (steps 3 to 6); 20) Then we go back to the top of the while loop and trigger the following BUG_ON(): (...) switch (tm->op) { case MOD_LOG_KEY_REMOVE_WHILE_FREEING: BUG_ON(tm->slot < n); fallthrough; (...) Because "n" has a value of (u32)-1 (4294967295) and tm->slot is 0. Fix this by taking a read lock on the extent buffer before cloning it at ctree.c:get_old_root(). This should be done regardless of the extent buffer having been freed and reused, as a concurrent task might be modifying it (while holding a write lock on it). Reported-by: Zygo Blaxell Link: https://lore.kernel.org/linux-btrfs/20210227155037.GN28049@hungrycats.org/ Fixes: 834328a8493079 ("Btrfs: tree mod log's old roots could still be part of the tree") CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index d56730a67885..34b929bd5c1a 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -1365,7 +1365,9 @@ get_old_root(struct btrfs_root *root, u64 time_seq) "failed to read tree block %llu from get_old_root", logical); } else { + btrfs_tree_read_lock(old); eb = btrfs_clone_extent_buffer(old); + btrfs_tree_read_unlock(old); free_extent_buffer(old); } } else if (old_root) { -- cgit v1.2.3 From 485df75554257e883d0ce39bb886e8212349748e Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Thu, 11 Mar 2021 14:31:06 +0000 Subject: btrfs: always pin deleted leaves when there are active tree mod log users When freeing a tree block we may end up adding its extent back to the free space cache/tree, as long as there are no more references for it, it was created in the current transaction and writeback for it never happened. This is generally fine, however when we have tree mod log operations it can result in inconsistent versions of a btree after unwinding extent buffers with the recorded tree mod log operations. This is because: * We only log operations for nodes (adding and removing key/pointers), for leaves we don't do anything; * This means that we can log a MOD_LOG_KEY_REMOVE_WHILE_FREEING operation for a node that points to a leaf that was deleted; * Before we apply the logged operation to unwind a node, we can have that leaf's extent allocated again, either as a node or as a leaf, and possibly for another btree. This is possible if the leaf was created in the current transaction and writeback for it never started, in which case btrfs_free_tree_block() returns its extent back to the free space cache/tree; * Then, before applying the tree mod log operation, some task allocates the metadata extent just freed before, and uses it either as a leaf or as a node for some btree (can be the same or another one, it does not matter); * After applying the MOD_LOG_KEY_REMOVE_WHILE_FREEING operation we now get the target node with an item pointing to the metadata extent that now has content different from what it had before the leaf was deleted. It might now belong to a different btree and be a node and not a leaf anymore. As a consequence, the results of searches after the unwinding can be unpredictable and produce unexpected results. So make sure we pin extent buffers corresponding to leaves when there are tree mod log users. CC: stable@vger.kernel.org # 4.14+ Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 78ad31a59e59..36a3c973fda1 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -3323,6 +3323,7 @@ void btrfs_free_tree_block(struct btrfs_trans_handle *trans, if (last_ref && btrfs_header_generation(buf) == trans->transid) { struct btrfs_block_group *cache; + bool must_pin = false; if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { ret = check_ref_cleanup(trans, buf->start); @@ -3340,7 +3341,27 @@ void btrfs_free_tree_block(struct btrfs_trans_handle *trans, goto out; } - if (btrfs_is_zoned(fs_info)) { + /* + * If this is a leaf and there are tree mod log users, we may + * have recorded mod log operations that point to this leaf. + * So we must make sure no one reuses this leaf's extent before + * mod log operations are applied to a node, otherwise after + * rewinding a node using the mod log operations we get an + * inconsistent btree, as the leaf's extent may now be used as + * a node or leaf for another different btree. + * We are safe from races here because at this point no other + * node or root points to this extent buffer, so if after this + * check a new tree mod log user joins, it will not be able to + * find a node pointing to this leaf and record operations that + * point to this leaf. + */ + if (btrfs_header_level(buf) == 0) { + read_lock(&fs_info->tree_mod_log_lock); + must_pin = !list_empty(&fs_info->tree_mod_seq_list); + read_unlock(&fs_info->tree_mod_log_lock); + } + + if (must_pin || btrfs_is_zoned(fs_info)) { btrfs_redirty_list_add(trans->transaction, buf); pin_down_extent(trans, cache, buf->start, buf->len, 1); btrfs_put_block_group(cache); -- cgit v1.2.3 From 5abbe51a526253b9f003e9a0a195638dc882d660 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Mon, 1 Feb 2021 18:46:41 +0100 Subject: kernel, fs: Introduce and use set_restart_fn() and arch_set_restart_data() Preparation for fixing get_nr_restart_syscall() on X86 for COMPAT. Add a new helper which sets restart_block->fn and calls a dummy arch_set_restart_data() helper. Fixes: 609c19a385c8 ("x86/ptrace: Stop setting TS_COMPAT in ptrace code") Signed-off-by: Oleg Nesterov Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20210201174641.GA17871@redhat.com --- fs/select.c | 10 ++++------ include/linux/thread_info.h | 13 +++++++++++++ kernel/futex.c | 3 +-- kernel/time/alarmtimer.c | 2 +- kernel/time/hrtimer.c | 2 +- kernel/time/posix-cpu-timers.c | 2 +- 6 files changed, 21 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/select.c b/fs/select.c index 37aaa8317f3a..945896d0ac9e 100644 --- a/fs/select.c +++ b/fs/select.c @@ -1055,10 +1055,9 @@ static long do_restart_poll(struct restart_block *restart_block) ret = do_sys_poll(ufds, nfds, to); - if (ret == -ERESTARTNOHAND) { - restart_block->fn = do_restart_poll; - ret = -ERESTART_RESTARTBLOCK; - } + if (ret == -ERESTARTNOHAND) + ret = set_restart_fn(restart_block, do_restart_poll); + return ret; } @@ -1080,7 +1079,6 @@ SYSCALL_DEFINE3(poll, struct pollfd __user *, ufds, unsigned int, nfds, struct restart_block *restart_block; restart_block = ¤t->restart_block; - restart_block->fn = do_restart_poll; restart_block->poll.ufds = ufds; restart_block->poll.nfds = nfds; @@ -1091,7 +1089,7 @@ SYSCALL_DEFINE3(poll, struct pollfd __user *, ufds, unsigned int, nfds, } else restart_block->poll.has_timeout = 0; - ret = -ERESTART_RESTARTBLOCK; + ret = set_restart_fn(restart_block, do_restart_poll); } return ret; } diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h index 9b2158c69275..157762db9d4b 100644 --- a/include/linux/thread_info.h +++ b/include/linux/thread_info.h @@ -11,6 +11,7 @@ #include #include #include +#include #ifdef CONFIG_THREAD_INFO_IN_TASK /* @@ -59,6 +60,18 @@ enum syscall_work_bit { #ifdef __KERNEL__ +#ifndef arch_set_restart_data +#define arch_set_restart_data(restart) do { } while (0) +#endif + +static inline long set_restart_fn(struct restart_block *restart, + long (*fn)(struct restart_block *)) +{ + restart->fn = fn; + arch_set_restart_data(restart); + return -ERESTART_RESTARTBLOCK; +} + #ifndef THREAD_ALIGN #define THREAD_ALIGN THREAD_SIZE #endif diff --git a/kernel/futex.c b/kernel/futex.c index e68db7745039..00febd6dea9c 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -2728,14 +2728,13 @@ retry: goto out; restart = ¤t->restart_block; - restart->fn = futex_wait_restart; restart->futex.uaddr = uaddr; restart->futex.val = val; restart->futex.time = *abs_time; restart->futex.bitset = bitset; restart->futex.flags = flags | FLAGS_HAS_TIMEOUT; - ret = -ERESTART_RESTARTBLOCK; + ret = set_restart_fn(restart, futex_wait_restart); out: if (to) { diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 98d7a15e8cf6..4d94e2b5499d 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -854,9 +854,9 @@ static int alarm_timer_nsleep(const clockid_t which_clock, int flags, if (flags == TIMER_ABSTIME) return -ERESTARTNOHAND; - restart->fn = alarm_timer_nsleep_restart; restart->nanosleep.clockid = type; restart->nanosleep.expires = exp; + set_restart_fn(restart, alarm_timer_nsleep_restart); return ret; } diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 788b9d137de4..5c9d968187ae 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1957,9 +1957,9 @@ long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode, } restart = ¤t->restart_block; - restart->fn = hrtimer_nanosleep_restart; restart->nanosleep.clockid = t.timer.base->clockid; restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer); + set_restart_fn(restart, hrtimer_nanosleep_restart); out: destroy_hrtimer_on_stack(&t.timer); return ret; diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index a71758e34e45..9abe15255bc4 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -1480,8 +1480,8 @@ static int posix_cpu_nsleep(const clockid_t which_clock, int flags, if (flags & TIMER_ABSTIME) return -ERESTARTNOHAND; - restart_block->fn = posix_cpu_nsleep_restart; restart_block->nanosleep.clockid = which_clock; + set_restart_fn(restart_block, posix_cpu_nsleep_restart); } return error; } -- cgit v1.2.3 From 6980d29ce4da223ad7f0751c7f1d61d3c6b54ab3 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 16 Mar 2021 20:30:26 +0800 Subject: zonefs: fix to update .i_wr_refcnt correctly in zonefs_open_zone() In zonefs_open_zone(), if opened zone count is larger than .s_max_open_zones threshold, we missed to recover .i_wr_refcnt, fix this. Fixes: b5c00e975779 ("zonefs: open/close zone on file open/close") Cc: Signed-off-by: Chao Yu Signed-off-by: Damien Le Moal --- fs/zonefs/super.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 3427c99abb4d..049e36c69ed7 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -1040,9 +1040,7 @@ static int zonefs_open_zone(struct inode *inode) mutex_lock(&zi->i_truncate_mutex); - zi->i_wr_refcnt++; - if (zi->i_wr_refcnt == 1) { - + if (!zi->i_wr_refcnt) { if (atomic_inc_return(&sbi->s_open_zones) > sbi->s_max_open_zones) { atomic_dec(&sbi->s_open_zones); ret = -EBUSY; @@ -1052,7 +1050,6 @@ static int zonefs_open_zone(struct inode *inode) if (i_size_read(inode) < zi->i_max_size) { ret = zonefs_zone_mgmt(inode, REQ_OP_ZONE_OPEN); if (ret) { - zi->i_wr_refcnt--; atomic_dec(&sbi->s_open_zones); goto unlock; } @@ -1060,6 +1057,8 @@ static int zonefs_open_zone(struct inode *inode) } } + zi->i_wr_refcnt++; + unlock: mutex_unlock(&zi->i_truncate_mutex); -- cgit v1.2.3 From f3da882eae2d6ba6c72062a46344ba096a7c2b3d Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Wed, 17 Mar 2021 17:57:31 +0900 Subject: btrfs: zoned: remove outdated WARN_ON in direct IO In btrfs_submit_direct() there's a WAN_ON_ONCE() that will trigger if we're submitting a DIO write on a zoned filesystem but are not using REQ_OP_ZONE_APPEND to submit the IO to the block device. This is a left over from a previous version where btrfs_dio_iomap_begin() didn't use btrfs_use_zone_append() to check for sequential write only zones. It is an oversight from the development phase. In v11 (I think) I've added 08f455593fff ("btrfs: zoned: cache if block group is on a sequential zone") and forgot to remove the WARN_ON_ONCE() for 544d24f9de73 ("btrfs: zoned: enable zone append writing for direct IO"). When developing auto relocation I got hit by the WARN as a block groups where relocated to conventional zone and the dio code calls btrfs_use_zone_append() introduced by 08f455593fff to check if it can use zone append (a.k.a. if it's a sequential zone) or not and sets the appropriate flags for iomap. I've never hit it in testing before, as I was relying on emulation to test the conventional zones code but this one case wasn't hit, because on emulation fs_info->max_zone_append_size is 0 and the WARN doesn't trigger either. Fixes: 544d24f9de73 ("btrfs: zoned: enable zone append writing for direct IO") Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 5092dea21448..4120fd96a134 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8166,10 +8166,6 @@ static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap, bio->bi_end_io = btrfs_end_dio_bio; btrfs_io_bio(bio)->logical = file_offset; - WARN_ON_ONCE(write && btrfs_is_zoned(fs_info) && - fs_info->max_zone_append_size && - bio_op(bio) != REQ_OP_ZONE_APPEND); - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { status = extract_ordered_extent(BTRFS_I(inode), bio, file_offset); -- cgit v1.2.3 From 82d62d06db404d03836cdabbca41d38646d97cbb Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 11 Mar 2021 11:23:15 -0500 Subject: btrfs: do not initialize dev stats if we have no dev_root Neal reported a panic trying to use -o rescue=all BUG: kernel NULL pointer dereference, address: 0000000000000030 PGD 0 P4D 0 Oops: 0000 [#1] SMP PTI CPU: 0 PID: 4095 Comm: mount Not tainted 5.11.0-0.rc7.149.fc34.x86_64 #1 RIP: 0010:btrfs_device_init_dev_stats+0x4c/0x1f0 RSP: 0018:ffffa60285fbfb68 EFLAGS: 00010246 RAX: 0000000000000000 RBX: ffff88b88f806498 RCX: ffff88b82e7a2a10 RDX: ffffa60285fbfb97 RSI: ffff88b82e7a2a10 RDI: 0000000000000000 RBP: ffff88b88f806b3c R08: 0000000000000000 R09: 0000000000000000 R10: ffff88b82e7a2a10 R11: 0000000000000000 R12: ffff88b88f806a00 R13: ffff88b88f806478 R14: ffff88b88f806a00 R15: ffff88b82e7a2a10 FS: 00007f698be1ec40(0000) GS:ffff88b937e00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000000030 CR3: 0000000092c9c006 CR4: 00000000003706f0 Call Trace: ? btrfs_init_dev_stats+0x1f/0xf0 btrfs_init_dev_stats+0x62/0xf0 open_ctree+0x1019/0x15ff btrfs_mount_root.cold+0x13/0xfa legacy_get_tree+0x27/0x40 vfs_get_tree+0x25/0xb0 vfs_kern_mount.part.0+0x71/0xb0 btrfs_mount+0x131/0x3d0 ? legacy_get_tree+0x27/0x40 ? btrfs_show_options+0x640/0x640 legacy_get_tree+0x27/0x40 vfs_get_tree+0x25/0xb0 path_mount+0x441/0xa80 __x64_sys_mount+0xf4/0x130 do_syscall_64+0x33/0x40 entry_SYSCALL_64_after_hwframe+0x44/0xa9 RIP: 0033:0x7f698c04e52e This happens because we unconditionally attempt to initialize device stats on mount, but we may not have been able to read the device root. Fix this by skipping initializing the device stats if we do not have a device root. Reported-by: Neal Gompa CC: stable@vger.kernel.org # 5.11+ Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index b8fab44394f5..e2df6e6e2c18 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -7448,6 +7448,9 @@ static int btrfs_device_init_dev_stats(struct btrfs_device *device, int item_size; int i, ret, slot; + if (!device->fs_info->dev_root) + return 0; + key.objectid = BTRFS_DEV_STATS_OBJECTID; key.type = BTRFS_PERSISTENT_ITEM_KEY; key.offset = device->devid; -- cgit v1.2.3 From 820a49dafc3304de06f296c35c9ff1ebc1666343 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 11 Mar 2021 11:23:14 -0500 Subject: btrfs: initialize device::fs_info always Neal reported a panic trying to use -o rescue=all BUG: kernel NULL pointer dereference, address: 0000000000000030 PGD 0 P4D 0 Oops: 0000 [#1] SMP NOPTI CPU: 0 PID: 696 Comm: mount Tainted: G W 5.12.0-rc2+ #296 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.13.0-2.fc32 04/01/2014 RIP: 0010:btrfs_device_init_dev_stats+0x1d/0x200 RSP: 0018:ffffafaec1483bb8 EFLAGS: 00010286 RAX: 0000000000000000 RBX: ffff9a5715bcb298 RCX: 0000000000000070 RDX: ffff9a5703248000 RSI: ffff9a57052ea150 RDI: ffff9a5715bca400 RBP: ffff9a57052ea150 R08: 0000000000000070 R09: ffff9a57052ea150 R10: 000130faf0741c10 R11: 0000000000000000 R12: ffff9a5703700000 R13: 0000000000000000 R14: ffff9a5715bcb278 R15: ffff9a57052ea150 FS: 00007f600d122c40(0000) GS:ffff9a577bc00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000000030 CR3: 0000000112a46005 CR4: 0000000000370ef0 Call Trace: ? btrfs_init_dev_stats+0x1f/0xf0 ? kmem_cache_alloc+0xef/0x1f0 btrfs_init_dev_stats+0x5f/0xf0 open_ctree+0x10cb/0x1720 btrfs_mount_root.cold+0x12/0xea legacy_get_tree+0x27/0x40 vfs_get_tree+0x25/0xb0 vfs_kern_mount.part.0+0x71/0xb0 btrfs_mount+0x10d/0x380 legacy_get_tree+0x27/0x40 vfs_get_tree+0x25/0xb0 path_mount+0x433/0xa00 __x64_sys_mount+0xe3/0x120 do_syscall_64+0x33/0x40 entry_SYSCALL_64_after_hwframe+0x44/0xae This happens because when we call btrfs_init_dev_stats we do device->fs_info->dev_root. However device->fs_info isn't initialized because we were only calling btrfs_init_devices_late() if we properly read the device root. However we don't actually need the device root to init the devices, this function simply assigns the devices their ->fs_info pointer properly, so this needs to be done unconditionally always so that we can properly dereference device->fs_info in rescue cases. Reported-by: Neal Gompa CC: stable@vger.kernel.org # 5.11+ Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index c2576c5fe62e..98c46cde7d79 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -2387,8 +2387,9 @@ static int btrfs_read_roots(struct btrfs_fs_info *fs_info) } else { set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); fs_info->dev_root = root; - btrfs_init_devices_late(fs_info); } + /* Initialize fs_info for all devices in any case */ + btrfs_init_devices_late(fs_info); /* If IGNOREDATACSUMS is set don't bother reading the csum root. */ if (!btrfs_test_opt(fs_info, IGNOREDATACSUMS)) { -- cgit v1.2.3 From 3cb894972f1809aa8d087c42e5e8b26c64b7d508 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 11 Mar 2021 11:23:16 -0500 Subject: btrfs: do not initialize dev replace for bad dev root While helping Neal fix his broken file system I added a debug patch to catch if we were calling btrfs_search_slot with a NULL root, and this stack trace popped: we tried to search with a NULL root CPU: 0 PID: 1760 Comm: mount Not tainted 5.11.0-155.nealbtrfstest.1.fc34.x86_64 #1 Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020 Call Trace: dump_stack+0x6b/0x83 btrfs_search_slot.cold+0x11/0x1b ? btrfs_init_dev_replace+0x36/0x450 btrfs_init_dev_replace+0x71/0x450 open_ctree+0x1054/0x1610 btrfs_mount_root.cold+0x13/0xfa legacy_get_tree+0x27/0x40 vfs_get_tree+0x25/0xb0 vfs_kern_mount.part.0+0x71/0xb0 btrfs_mount+0x131/0x3d0 ? legacy_get_tree+0x27/0x40 ? btrfs_show_options+0x640/0x640 legacy_get_tree+0x27/0x40 vfs_get_tree+0x25/0xb0 path_mount+0x441/0xa80 __x64_sys_mount+0xf4/0x130 do_syscall_64+0x33/0x40 entry_SYSCALL_64_after_hwframe+0x44/0xa9 RIP: 0033:0x7f644730352e Fix this by not starting the device replace stuff if we do not have a NULL dev root. Reported-by: Neal Gompa CC: stable@vger.kernel.org # 5.11+ Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/dev-replace.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index 3a9c1e046ebe..d05f73530af7 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -81,6 +81,9 @@ int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info) struct btrfs_dev_replace_item *ptr; u64 src_devid; + if (!dev_root) + return 0; + path = btrfs_alloc_path(); if (!path) { ret = -ENOMEM; -- cgit v1.2.3 From ebd99a6b34fbcccf21067b66d1718000feb80ce8 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 17 Mar 2021 11:43:13 +0100 Subject: btrfs: fix build when using M=fs/btrfs There are people building the module with M= that's supposed to be used for external modules. This got broken in e9aa7c285d20 ("btrfs: enable W=1 checks for btrfs"). $ make M=fs/btrfs scripts/Makefile.lib:10: *** Recursive variable 'KBUILD_CFLAGS' references itself (eventually). Stop. make: *** [Makefile:1755: modules] Error 2 There's a difference compared to 'make fs/btrfs/btrfs.ko' which needs to rebuild a few more things and also the dependency modules need to be available. It could fail with eg. WARNING: Symbol version dump "Module.symvers" is missing. Modules may not have dependencies or modversions. In some environments it's more convenient to rebuild just the btrfs module by M= so let's make it work. The problem is with recursive variable evaluation in += so the conditional C options are stored in a temporary variable to avoid the recursion. Signed-off-by: David Sterba --- fs/btrfs/Makefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile index b634c42115ea..b4fb997eda16 100644 --- a/fs/btrfs/Makefile +++ b/fs/btrfs/Makefile @@ -7,10 +7,12 @@ subdir-ccflags-y += -Wmissing-format-attribute subdir-ccflags-y += -Wmissing-prototypes subdir-ccflags-y += -Wold-style-definition subdir-ccflags-y += -Wmissing-include-dirs -subdir-ccflags-y += $(call cc-option, -Wunused-but-set-variable) -subdir-ccflags-y += $(call cc-option, -Wunused-const-variable) -subdir-ccflags-y += $(call cc-option, -Wpacked-not-aligned) -subdir-ccflags-y += $(call cc-option, -Wstringop-truncation) +condflags := \ + $(call cc-option, -Wunused-but-set-variable) \ + $(call cc-option, -Wunused-const-variable) \ + $(call cc-option, -Wpacked-not-aligned) \ + $(call cc-option, -Wstringop-truncation) +subdir-ccflags-y += $(condflags) # The following turn off the warnings enabled by -Wextra subdir-ccflags-y += -Wno-missing-field-initializers subdir-ccflags-y += -Wno-sign-compare -- cgit v1.2.3 From 8d488a8c7ba22d7112fbf6b0a82beb1cdea1c0d5 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 16 Mar 2021 16:53:46 +0000 Subject: btrfs: fix subvolume/snapshot deletion not triggered on mount During the mount procedure we are calling btrfs_orphan_cleanup() against the root tree, which will find all orphans items in this tree. When an orphan item corresponds to a deleted subvolume/snapshot (instead of an inode space cache), it must not delete the orphan item, because that will cause btrfs_find_orphan_roots() to not find the orphan item and therefore not add the corresponding subvolume root to the list of dead roots, which results in the subvolume's tree never being deleted by the cleanup thread. The same applies to the remount from RO to RW path. Fix this by making btrfs_find_orphan_roots() run before calling btrfs_orphan_cleanup() against the root tree. A test case for fstests will follow soon. Reported-by: Robbie Ko Link: https://lore.kernel.org/linux-btrfs/b19f4310-35e0-606e-1eea-2dd84d28c5da@synology.com/ Fixes: 638331fa56caea ("btrfs: fix transaction leak and crash after cleaning up orphans on RO mount") CC: stable@vger.kernel.org # 5.11+ Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 98c46cde7d79..48e9cf01abc5 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -3010,6 +3010,21 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) } } + /* + * btrfs_find_orphan_roots() is responsible for finding all the dead + * roots (with 0 refs), flag them with BTRFS_ROOT_DEAD_TREE and load + * them into the fs_info->fs_roots_radix tree. This must be done before + * calling btrfs_orphan_cleanup() on the tree root. If we don't do it + * first, then btrfs_orphan_cleanup() will delete a dead root's orphan + * item before the root's tree is deleted - this means that if we unmount + * or crash before the deletion completes, on the next mount we will not + * delete what remains of the tree because the orphan item does not + * exists anymore, which is what tells us we have a pending deletion. + */ + ret = btrfs_find_orphan_roots(fs_info); + if (ret) + goto out; + ret = btrfs_cleanup_fs_roots(fs_info); if (ret) goto out; @@ -3069,7 +3084,6 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) } } - ret = btrfs_find_orphan_roots(fs_info); out: return ret; } -- cgit v1.2.3 From 76cd979f4f38a27df22efb5773a0d567181a9392 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 16 Mar 2021 16:33:27 +0100 Subject: io_uring: imply MSG_NOSIGNAL for send[msg]()/recv[msg]() calls We never want to generate any SIGPIPE, -EPIPE only is much better. Signed-off-by: Stefan Metzmacher Link: https://lore.kernel.org/r/38961085c3ec49fd21550c7788f214d1ff02d2d4.1615908477.git.metze@samba.org Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 5f4e312111ea..a81f7a30ea70 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4384,7 +4384,7 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) kmsg = &iomsg; } - flags = req->sr_msg.msg_flags; + flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; if (flags & MSG_DONTWAIT) req->flags |= REQ_F_NOWAIT; else if (issue_flags & IO_URING_F_NONBLOCK) @@ -4428,7 +4428,7 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) msg.msg_controllen = 0; msg.msg_namelen = 0; - flags = req->sr_msg.msg_flags; + flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; if (flags & MSG_DONTWAIT) req->flags |= REQ_F_NOWAIT; else if (issue_flags & IO_URING_F_NONBLOCK) @@ -4618,7 +4618,7 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 1, req->sr_msg.len); } - flags = req->sr_msg.msg_flags; + flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; if (flags & MSG_DONTWAIT) req->flags |= REQ_F_NOWAIT; else if (force_nonblock) @@ -4677,7 +4677,7 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags) msg.msg_iocb = NULL; msg.msg_flags = 0; - flags = req->sr_msg.msg_flags; + flags = req->sr_msg.msg_flags | MSG_NOSIGNAL; if (flags & MSG_DONTWAIT) req->flags |= REQ_F_NOWAIT; else if (force_nonblock) -- cgit v1.2.3 From 53e043b2b432ef2294efec04dd8a88d96c024624 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 15 Mar 2021 12:56:56 +0100 Subject: io_uring: remove structures from include/linux/io_uring.h Link: https://lore.kernel.org/r/8c1d14f3748105f4caeda01716d47af2fa41d11c.1615809009.git.metze@samba.org Signed-off-by: Stefan Metzmacher Signed-off-by: Jens Axboe --- fs/io-wq.h | 10 +++++++++- fs/io_uring.c | 16 ++++++++++++++++ include/linux/io_uring.h | 25 ------------------------- 3 files changed, 25 insertions(+), 26 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.h b/fs/io-wq.h index 1ac2f3248088..80d590564ff9 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -2,7 +2,6 @@ #define INTERNAL_IO_WQ_H #include -#include struct io_wq; @@ -21,6 +20,15 @@ enum io_wq_cancel { IO_WQ_CANCEL_NOTFOUND, /* work not found */ }; +struct io_wq_work_node { + struct io_wq_work_node *next; +}; + +struct io_wq_work_list { + struct io_wq_work_node *first; + struct io_wq_work_node *last; +}; + static inline void wq_list_add_after(struct io_wq_work_node *node, struct io_wq_work_node *pos, struct io_wq_work_list *list) diff --git a/fs/io_uring.c b/fs/io_uring.c index a81f7a30ea70..52ba8d7f3eb8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -456,6 +456,22 @@ struct io_ring_ctx { struct list_head tctx_list; }; +struct io_uring_task { + /* submission side */ + struct xarray xa; + struct wait_queue_head wait; + void *last; + void *io_wq; + struct percpu_counter inflight; + atomic_t in_idle; + bool sqpoll; + + spinlock_t task_lock; + struct io_wq_work_list task_list; + unsigned long task_state; + struct callback_head task_work; +}; + /* * First field must be the file pointer in all the * iocb unions! See also 'struct kiocb' in diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 9761a0ec9f95..79cde9906be0 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -5,31 +5,6 @@ #include #include -struct io_wq_work_node { - struct io_wq_work_node *next; -}; - -struct io_wq_work_list { - struct io_wq_work_node *first; - struct io_wq_work_node *last; -}; - -struct io_uring_task { - /* submission side */ - struct xarray xa; - struct wait_queue_head wait; - void *last; - void *io_wq; - struct percpu_counter inflight; - atomic_t in_idle; - bool sqpoll; - - spinlock_t task_lock; - struct io_wq_work_list task_list; - unsigned long task_state; - struct callback_head task_work; -}; - #if defined(CONFIG_IO_URING) struct sock *io_uring_get_socket(struct file *file); void __io_uring_task_cancel(void); -- cgit v1.2.3 From ee53fb2b197b72b126ca0387ae636da75d969428 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 15 Mar 2021 12:56:57 +0100 Subject: io_uring: use typesafe pointers in io_uring_task Signed-off-by: Stefan Metzmacher Link: https://lore.kernel.org/r/ce2a598e66e48347bb04afbaf2acc67c0cc7971a.1615809009.git.metze@samba.org Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 52ba8d7f3eb8..675073966760 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -460,8 +460,8 @@ struct io_uring_task { /* submission side */ struct xarray xa; struct wait_queue_head wait; - void *last; - void *io_wq; + const struct io_ring_ctx *last; + struct io_wq *io_wq; struct percpu_counter inflight; atomic_t in_idle; bool sqpoll; -- cgit v1.2.3 From de75a3d3f5a14c9ab3c4883de3471d3c92a8ee78 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 18 Mar 2021 11:54:35 +0000 Subject: io_uring: don't leak creds on SQO attach error Attaching to already dead/dying SQPOLL task is disallowed in io_sq_offload_create(), but cleanup is hand coded by calling io_put_sq_data()/etc., that miss to put ctx->sq_creds. Defer everything to error-path io_sq_thread_finish(), adding ctx->sqd_list in the error case as well as finish will handle it. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 675073966760..ea2d3e120555 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7910,22 +7910,17 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, ret = 0; io_sq_thread_park(sqd); + list_add(&ctx->sqd_list, &sqd->ctx_list); + io_sqd_update_thread_idle(sqd); /* don't attach to a dying SQPOLL thread, would be racy */ - if (attached && !sqd->thread) { + if (attached && !sqd->thread) ret = -ENXIO; - } else { - list_add(&ctx->sqd_list, &sqd->ctx_list); - io_sqd_update_thread_idle(sqd); - } io_sq_thread_unpark(sqd); - if (ret < 0) { - io_put_sq_data(sqd); - ctx->sq_data = NULL; - return ret; - } else if (attached) { + if (ret < 0) + goto err; + if (attached) return 0; - } if (p->flags & IORING_SETUP_SQ_AFF) { int cpu = p->sq_thread_cpu; -- cgit v1.2.3 From 0bb788300990d3eb5582d3301a720f846c78925c Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Thu, 18 Mar 2021 11:22:05 +0000 Subject: btrfs: fix sleep while in non-sleep context during qgroup removal While removing a qgroup's sysfs entry we end up taking the kernfs_mutex, through kobject_del(), while holding the fs_info->qgroup_lock spinlock, producing the following trace: [821.843637] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:281 [821.843641] in_atomic(): 1, irqs_disabled(): 0, non_block: 0, pid: 28214, name: podman [821.843644] CPU: 3 PID: 28214 Comm: podman Tainted: G W 5.11.6 #15 [821.843646] Hardware name: Dell Inc. PowerEdge R330/084XW4, BIOS 2.11.0 12/08/2020 [821.843647] Call Trace: [821.843650] dump_stack+0xa1/0xfb [821.843656] ___might_sleep+0x144/0x160 [821.843659] mutex_lock+0x17/0x40 [821.843662] kernfs_remove_by_name_ns+0x1f/0x80 [821.843666] sysfs_remove_group+0x7d/0xe0 [821.843668] sysfs_remove_groups+0x28/0x40 [821.843670] kobject_del+0x2a/0x80 [821.843672] btrfs_sysfs_del_one_qgroup+0x2b/0x40 [btrfs] [821.843685] __del_qgroup_rb+0x12/0x150 [btrfs] [821.843696] btrfs_remove_qgroup+0x288/0x2a0 [btrfs] [821.843707] btrfs_ioctl+0x3129/0x36a0 [btrfs] [821.843717] ? __mod_lruvec_page_state+0x5e/0xb0 [821.843719] ? page_add_new_anon_rmap+0xbc/0x150 [821.843723] ? kfree+0x1b4/0x300 [821.843725] ? mntput_no_expire+0x55/0x330 [821.843728] __x64_sys_ioctl+0x5a/0xa0 [821.843731] do_syscall_64+0x33/0x70 [821.843733] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [821.843736] RIP: 0033:0x4cd3fb [821.843741] RSP: 002b:000000c000906b20 EFLAGS: 00000206 ORIG_RAX: 0000000000000010 [821.843744] RAX: ffffffffffffffda RBX: 000000c000050000 RCX: 00000000004cd3fb [821.843745] RDX: 000000c000906b98 RSI: 000000004010942a RDI: 000000000000000f [821.843747] RBP: 000000c000907cd0 R08: 000000c000622901 R09: 0000000000000000 [821.843748] R10: 000000c000d992c0 R11: 0000000000000206 R12: 000000000000012d [821.843749] R13: 000000000000012c R14: 0000000000000200 R15: 0000000000000049 Fix this by removing the qgroup sysfs entry while not holding the spinlock, since the spinlock is only meant for protection of the qgroup rbtree. Reported-by: Stuart Shelton Link: https://lore.kernel.org/linux-btrfs/7A5485BB-0628-419D-A4D3-27B1AF47E25A@gmail.com/ Fixes: 49e5fb46211de0 ("btrfs: qgroup: export qgroups in sysfs") CC: stable@vger.kernel.org # 5.10+ Reviewed-by: Qu Wenruo Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/qgroup.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 14ff388fd3bd..f0b9ef13153a 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -226,7 +226,6 @@ static void __del_qgroup_rb(struct btrfs_fs_info *fs_info, { struct btrfs_qgroup_list *list; - btrfs_sysfs_del_one_qgroup(fs_info, qgroup); list_del(&qgroup->dirty); while (!list_empty(&qgroup->groups)) { list = list_first_entry(&qgroup->groups, @@ -243,7 +242,6 @@ static void __del_qgroup_rb(struct btrfs_fs_info *fs_info, list_del(&list->next_member); kfree(list); } - kfree(qgroup); } /* must be called with qgroup_lock held */ @@ -569,6 +567,8 @@ void btrfs_free_qgroup_config(struct btrfs_fs_info *fs_info) qgroup = rb_entry(n, struct btrfs_qgroup, node); rb_erase(n, &fs_info->qgroup_tree); __del_qgroup_rb(fs_info, qgroup); + btrfs_sysfs_del_one_qgroup(fs_info, qgroup); + kfree(qgroup); } /* * We call btrfs_free_qgroup_config() when unmounting @@ -1578,6 +1578,14 @@ int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid) spin_lock(&fs_info->qgroup_lock); del_qgroup_rb(fs_info, qgroupid); spin_unlock(&fs_info->qgroup_lock); + + /* + * Remove the qgroup from sysfs now without holding the qgroup_lock + * spinlock, since the sysfs_remove_group() function needs to take + * the mutex kernfs_mutex through kernfs_remove_by_name_ns(). + */ + btrfs_sysfs_del_one_qgroup(fs_info, qgroup); + kfree(qgroup); out: mutex_unlock(&fs_info->qgroup_ioctl_lock); return ret; -- cgit v1.2.3 From c1d6abdac46ca8127274bea195d804e3f2cec7ee Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Tue, 16 Mar 2021 12:43:00 -0700 Subject: btrfs: fix check_data_csum() error message for direct I/O Commit 1dae796aabf6 ("btrfs: inode: sink parameter start and len to check_data_csum()") replaced the start parameter to check_data_csum() with page_offset(), but page_offset() is not meaningful for direct I/O pages. Bring back the start parameter. Fixes: 265d4ac03fdf ("btrfs: sink parameter start and len to check_data_csum") CC: stable@vger.kernel.org # 5.11+ Reviewed-by: Qu Wenruo Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/inode.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 4120fd96a134..a7be4c9ec835 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3099,11 +3099,13 @@ void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start, * @bio_offset: offset to the beginning of the bio (in bytes) * @page: page where is the data to be verified * @pgoff: offset inside the page + * @start: logical offset in the file * * The length of such check is always one sector size. */ static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio, - u32 bio_offset, struct page *page, u32 pgoff) + u32 bio_offset, struct page *page, u32 pgoff, + u64 start) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); @@ -3130,8 +3132,8 @@ static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio, kunmap_atomic(kaddr); return 0; zeroit: - btrfs_print_data_csum_error(BTRFS_I(inode), page_offset(page) + pgoff, - csum, csum_expected, io_bio->mirror_num); + btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected, + io_bio->mirror_num); if (io_bio->device) btrfs_dev_stat_inc_and_print(io_bio->device, BTRFS_DEV_STAT_CORRUPTION_ERRS); @@ -3184,7 +3186,8 @@ int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u32 bio_offset, pg_off += sectorsize, bio_offset += sectorsize) { int ret; - ret = check_data_csum(inode, io_bio, bio_offset, page, pg_off); + ret = check_data_csum(inode, io_bio, bio_offset, page, pg_off, + page_offset(page) + pg_off); if (ret < 0) return -EIO; } @@ -7907,7 +7910,8 @@ static blk_status_t btrfs_check_read_dio_bio(struct inode *inode, ASSERT(pgoff < PAGE_SIZE); if (uptodate && (!csum || !check_data_csum(inode, io_bio, - bio_offset, bvec.bv_page, pgoff))) { + bio_offset, bvec.bv_page, + pgoff, start))) { clean_io_failure(fs_info, failure_tree, io_tree, start, bvec.bv_page, btrfs_ino(BTRFS_I(inode)), -- cgit v1.2.3 From 403dba003d17b3f0c1627b355cec2d74041cf648 Mon Sep 17 00:00:00 2001 From: Liu xuzhi Date: Thu, 18 Mar 2021 17:46:57 -0700 Subject: fs/cifs/: fix misspellings using codespell tool A typo is found out by codespell tool in 251th lines of cifs_swn.c: $ codespell ./fs/cifs/ ./cifs_swn.c:251: funciton ==> function Fix a typo found by codespell. Signed-off-by: Liu xuzhi Signed-off-by: Steve French --- fs/cifs/cifs_swn.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/cifs/cifs_swn.c b/fs/cifs/cifs_swn.c index f2d730fffccb..d829b8bf833e 100644 --- a/fs/cifs/cifs_swn.c +++ b/fs/cifs/cifs_swn.c @@ -248,7 +248,7 @@ nlmsg_fail: /* * Try to find a matching registration for the tcon's server name and share name. - * Calls to this funciton must be protected by cifs_swnreg_idr_mutex. + * Calls to this function must be protected by cifs_swnreg_idr_mutex. * TODO Try to avoid memory allocations */ static struct cifs_swn_reg *cifs_find_swn_reg(struct cifs_tcon *tcon) -- cgit v1.2.3 From af3ef3b1031634724a3763606695ebcd113d782b Mon Sep 17 00:00:00 2001 From: Aurelien Aptel Date: Thu, 18 Mar 2021 19:17:10 +0100 Subject: cifs: warn and fail if trying to use rootfs without the config option If CONFIG_CIFS_ROOT is not set, rootfs mount option is invalid Signed-off-by: Aurelien Aptel CC: # v5.11 Signed-off-by: Steve French --- fs/cifs/fs_context.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 892f51a21278..78889024a7ed 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -1196,9 +1196,11 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, pr_warn_once("Witness protocol support is experimental\n"); break; case Opt_rootfs: -#ifdef CONFIG_CIFS_ROOT - ctx->rootfs = true; +#ifndef CONFIG_CIFS_ROOT + cifs_dbg(VFS, "rootfs support requires CONFIG_CIFS_ROOT config option\n"); + goto cifs_parse_mount_err; #endif + ctx->rootfs = true; break; case Opt_posixpaths: if (result.negated) -- cgit v1.2.3 From 65af8f0166f4d15e61c63db498ec7981acdd897f Mon Sep 17 00:00:00 2001 From: Steve French Date: Fri, 19 Mar 2021 00:05:48 -0500 Subject: cifs: fix allocation size on newly created files Applications that create and extend and write to a file do not expect to see 0 allocation size. When file is extended, set its allocation size to a plausible value until we have a chance to query the server for it. When the file is cached this will prevent showing an impossible number of allocated blocks (like 0). This fixes e.g. xfstests 614 which does 1) create a file and set its size to 64K 2) mmap write 64K to the file 3) stat -c %b for the file (to query the number of allocated blocks) It was failing because we returned 0 blocks. Even though we would return the correct cached file size, we returned an impossible allocation size. Signed-off-by: Steve French CC: Reviewed-by: Aurelien Aptel --- fs/cifs/inode.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index 7c61bc9573c0..f2df4422e54a 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -2395,7 +2395,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, * We need to be sure that all dirty pages are written and the server * has actual ctime, mtime and file length. */ - if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE)) && + if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE | STATX_BLOCKS)) && !CIFS_CACHE_READ(CIFS_I(inode)) && inode->i_mapping && inode->i_mapping->nrpages != 0) { rc = filemap_fdatawait(inode->i_mapping); @@ -2585,6 +2585,14 @@ set_size_out: if (rc == 0) { cifsInode->server_eof = attrs->ia_size; cifs_setsize(inode, attrs->ia_size); + /* + * i_blocks is not related to (i_size / i_blksize), but instead + * 512 byte (2**9) size is required for calculating num blocks. + * Until we can query the server for actual allocation size, + * this is best estimate we have for blocks allocated for a file + * Number of blocks must be rounded up so size 1 is not 0 blocks + */ + inode->i_blocks = (512 - 1 + attrs->ia_size) >> 9; /* * The man page of truncate says if the size changed, -- cgit v1.2.3 From b7ff91fd030dc9d72ed91b1aab36e445a003af4f Mon Sep 17 00:00:00 2001 From: "zhangyi (F)" Date: Wed, 3 Mar 2021 21:17:02 +0800 Subject: ext4: find old entry again if failed to rename whiteout If we failed to add new entry on rename whiteout, we cannot reset the old->de entry directly, because the old->de could have moved from under us during make indexed dir. So find the old entry again before reset is needed, otherwise it may corrupt the filesystem as below. /dev/sda: Entry '00000001' in ??? (12) has deleted/unused inode 15. CLEARED. /dev/sda: Unattached inode 75 /dev/sda: UNEXPECTED INCONSISTENCY; RUN fsck MANUALLY. Fixes: 6b4b8e6b4ad ("ext4: fix bug for rename with RENAME_WHITEOUT") Cc: stable@vger.kernel.org Signed-off-by: zhangyi (F) Link: https://lore.kernel.org/r/20210303131703.330415-1-yi.zhang@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/namei.c | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 686bf982c84e..4aae59d55288 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -3613,6 +3613,31 @@ static int ext4_setent(handle_t *handle, struct ext4_renament *ent, return retval; } +static void ext4_resetent(handle_t *handle, struct ext4_renament *ent, + unsigned ino, unsigned file_type) +{ + struct ext4_renament old = *ent; + int retval = 0; + + /* + * old->de could have moved from under us during make indexed dir, + * so the old->de may no longer valid and need to find it again + * before reset old inode info. + */ + old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de, NULL); + if (IS_ERR(old.bh)) + retval = PTR_ERR(old.bh); + if (!old.bh) + retval = -ENOENT; + if (retval) { + ext4_std_error(old.dir->i_sb, retval); + return; + } + + ext4_setent(handle, &old, ino, file_type); + brelse(old.bh); +} + static int ext4_find_delete_entry(handle_t *handle, struct inode *dir, const struct qstr *d_name) { @@ -3937,8 +3962,8 @@ static int ext4_rename(struct user_namespace *mnt_userns, struct inode *old_dir, end_rename: if (whiteout) { if (retval) { - ext4_setent(handle, &old, - old.inode->i_ino, old_file_type); + ext4_resetent(handle, &old, + old.inode->i_ino, old_file_type); drop_nlink(whiteout); } unlock_new_inode(whiteout); -- cgit v1.2.3 From 5dccdc5a1916d4266edd251f20bbbb113a5c495f Mon Sep 17 00:00:00 2001 From: "zhangyi (F)" Date: Wed, 3 Mar 2021 21:17:03 +0800 Subject: ext4: do not iput inode under running transaction in ext4_rename() In ext4_rename(), when RENAME_WHITEOUT failed to add new entry into directory, it ends up dropping new created whiteout inode under the running transaction. After commit <9b88f9fb0d2> ("ext4: Do not iput inode under running transaction"), we follow the assumptions that evict() does not get called from a transaction context but in ext4_rename() it breaks this suggestion. Although it's not a real problem, better to obey it, so this patch add inode to orphan list and stop transaction before final iput(). Signed-off-by: zhangyi (F) Link: https://lore.kernel.org/r/20210303131703.330415-2-yi.zhang@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/namei.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 4aae59d55288..1fe8370dbf0b 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -3799,14 +3799,14 @@ static int ext4_rename(struct user_namespace *mnt_userns, struct inode *old_dir, */ retval = -ENOENT; if (!old.bh || le32_to_cpu(old.de->inode) != old.inode->i_ino) - goto end_rename; + goto release_bh; new.bh = ext4_find_entry(new.dir, &new.dentry->d_name, &new.de, &new.inlined); if (IS_ERR(new.bh)) { retval = PTR_ERR(new.bh); new.bh = NULL; - goto end_rename; + goto release_bh; } if (new.bh) { if (!new.inode) { @@ -3823,15 +3823,13 @@ static int ext4_rename(struct user_namespace *mnt_userns, struct inode *old_dir, handle = ext4_journal_start(old.dir, EXT4_HT_DIR, credits); if (IS_ERR(handle)) { retval = PTR_ERR(handle); - handle = NULL; - goto end_rename; + goto release_bh; } } else { whiteout = ext4_whiteout_for_rename(mnt_userns, &old, credits, &handle); if (IS_ERR(whiteout)) { retval = PTR_ERR(whiteout); - whiteout = NULL; - goto end_rename; + goto release_bh; } } @@ -3965,16 +3963,18 @@ end_rename: ext4_resetent(handle, &old, old.inode->i_ino, old_file_type); drop_nlink(whiteout); + ext4_orphan_add(handle, whiteout); } unlock_new_inode(whiteout); + ext4_journal_stop(handle); iput(whiteout); - + } else { + ext4_journal_stop(handle); } +release_bh: brelse(old.dir_bh); brelse(old.bh); brelse(new.bh); - if (handle) - ext4_journal_stop(handle); return retval; } -- cgit v1.2.3 From 6b22489911b726eebbf169caee52fea52013fbdd Mon Sep 17 00:00:00 2001 From: "zhangyi (F)" Date: Fri, 5 Mar 2021 20:05:08 +0800 Subject: ext4: do not try to set xattr into ea_inode if value is empty Syzbot report a warning that ext4 may create an empty ea_inode if set an empty extent attribute to a file on the file system which is no free blocks left. WARNING: CPU: 6 PID: 10667 at fs/ext4/xattr.c:1640 ext4_xattr_set_entry+0x10f8/0x1114 fs/ext4/xattr.c:1640 ... Call trace: ext4_xattr_set_entry+0x10f8/0x1114 fs/ext4/xattr.c:1640 ext4_xattr_block_set+0x1d0/0x1b1c fs/ext4/xattr.c:1942 ext4_xattr_set_handle+0x8a0/0xf1c fs/ext4/xattr.c:2390 ext4_xattr_set+0x120/0x1f0 fs/ext4/xattr.c:2491 ext4_xattr_trusted_set+0x48/0x5c fs/ext4/xattr_trusted.c:37 __vfs_setxattr+0x208/0x23c fs/xattr.c:177 ... Now, ext4 try to store extent attribute into an external inode if ext4_xattr_block_set() return -ENOSPC, but for the case of store an empty extent attribute, store the extent entry into the extent attribute block is enough. A simple reproduce below. fallocate test.img -l 1M mkfs.ext4 -F -b 2048 -O ea_inode test.img mount test.img /mnt dd if=/dev/zero of=/mnt/foo bs=2048 count=500 setfattr -n "user.test" /mnt/foo Reported-by: syzbot+98b881fdd8ebf45ab4ae@syzkaller.appspotmail.com Fixes: 9c6e7853c531 ("ext4: reserve space for xattr entries/names") Cc: stable@kernel.org Signed-off-by: zhangyi (F) Link: https://lore.kernel.org/r/20210305120508.298465-1-yi.zhang@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/xattr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 083c95126781..6c1018223c54 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -2404,7 +2404,7 @@ retry_inode: * external inode if possible. */ if (ext4_has_feature_ea_inode(inode->i_sb) && - !i.in_inode) { + i.value_len && !i.in_inode) { i.in_inode = 1; goto retry_inode; } -- cgit v1.2.3 From 7d8bd3c76da1d94b85e6c9b7007e20e980bfcfe6 Mon Sep 17 00:00:00 2001 From: Shijie Luo Date: Fri, 12 Mar 2021 01:50:51 -0500 Subject: ext4: fix potential error in ext4_do_update_inode If set_large_file = 1 and errors occur in ext4_handle_dirty_metadata(), the error code will be overridden, go to out_brelse to avoid this situation. Signed-off-by: Shijie Luo Link: https://lore.kernel.org/r/20210312065051.36314-1-luoshijie1@huawei.com Cc: stable@kernel.org Reviewed-by: Jan Kara Signed-off-by: Theodore Ts'o --- fs/ext4/inode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index a79a9ea58c56..927e47db7f00 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -5026,7 +5026,7 @@ static int ext4_do_update_inode(handle_t *handle, struct ext4_inode_info *ei = EXT4_I(inode); struct buffer_head *bh = iloc->bh; struct super_block *sb = inode->i_sb; - int err = 0, rc, block; + int err = 0, block; int need_datasync = 0, set_large_file = 0; uid_t i_uid; gid_t i_gid; @@ -5138,9 +5138,9 @@ static int ext4_do_update_inode(handle_t *handle, bh->b_data); BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); - rc = ext4_handle_dirty_metadata(handle, NULL, bh); - if (!err) - err = rc; + err = ext4_handle_dirty_metadata(handle, NULL, bh); + if (err) + goto out_brelse; ext4_clear_inode_state(inode, EXT4_STATE_NEW); if (set_large_file) { BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access"); -- cgit v1.2.3 From 2a4ae3bcdf05b8639406eaa09a2939f3c6dd8e75 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 15 Mar 2021 17:59:06 +0100 Subject: ext4: fix timer use-after-free on failed mount When filesystem mount fails because of corrupted filesystem we first cancel the s_err_report timer reminding fs errors every day and only then we flush s_error_work. However s_error_work may report another fs error and re-arm timer thus resulting in timer use-after-free. Fix the problem by first flushing the work and only after that canceling the s_err_report timer. Reported-by: syzbot+628472a2aac693ab0fcd@syzkaller.appspotmail.com Fixes: 2d01ddc86606 ("ext4: save error info to sb through journal if available") CC: stable@vger.kernel.org Signed-off-by: Jan Kara Link: https://lore.kernel.org/r/20210315165906.2175-1-jack@suse.cz Signed-off-by: Theodore Ts'o --- fs/ext4/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ext4/super.c b/fs/ext4/super.c index a0a256859662..b9693680463a 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -5154,8 +5154,8 @@ failed_mount_wq: failed_mount3a: ext4_es_unregister_shrinker(sbi); failed_mount3: - del_timer_sync(&sbi->s_err_report); flush_work(&sbi->s_error_work); + del_timer_sync(&sbi->s_err_report); if (sbi->s_mmp_tsk) kthread_stop(sbi->s_mmp_tsk); failed_mount2: -- cgit v1.2.3 From 8210bb29c1b66200cff7b25febcf6e39baf49fbf Mon Sep 17 00:00:00 2001 From: Harshad Shirwadkar Date: Tue, 16 Mar 2021 15:19:21 -0700 Subject: ext4: fix rename whiteout with fast commit This patch adds rename whiteout support in fast commits. Note that the whiteout object that gets created is actually char device. Which imples, the function ext4_inode_journal_mode(struct inode *inode) would return "JOURNAL_DATA" for this inode. This has a consequence in fast commit code that it will make creation of the whiteout object a fast-commit ineligible behavior and thus will fall back to full commits. With this patch, this can be observed by running fast commits with rename whiteout and seeing the stats generated by ext4_fc_stats tracepoint as follows: ext4_fc_stats: dev 254:32 fc ineligible reasons: XATTR:0, CROSS_RENAME:0, JOURNAL_FLAG_CHANGE:0, NO_MEM:0, SWAP_BOOT:0, RESIZE:0, RENAME_DIR:0, FALLOC_RANGE:0, INODE_JOURNAL_DATA:16; num_commits:6, ineligible: 6, numblks: 3 So in short, this patch guarantees that in case of rename whiteout, we fall back to full commits. Amir mentioned that instead of creating a new whiteout object for every rename, we can create a static whiteout object with irrelevant nlink. That will make fast commits to not fall back to full commit. But until this happens, this patch will ensure correctness by falling back to full commits. Fixes: 8016e29f4362 ("ext4: fast commit recovery path") Cc: stable@kernel.org Signed-off-by: Harshad Shirwadkar Link: https://lore.kernel.org/r/20210316221921.1124955-1-harshadshirwadkar@gmail.com Signed-off-by: Theodore Ts'o --- fs/ext4/ext4.h | 2 ++ fs/ext4/fast_commit.c | 9 +++++++-- fs/ext4/namei.c | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index ea3b41579f38..826a56e3bbd2 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -2794,6 +2794,8 @@ void __ext4_fc_track_link(handle_t *handle, struct inode *inode, struct dentry *dentry); void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry); void ext4_fc_track_link(handle_t *handle, struct dentry *dentry); +void __ext4_fc_track_create(handle_t *handle, struct inode *inode, + struct dentry *dentry); void ext4_fc_track_create(handle_t *handle, struct dentry *dentry); void ext4_fc_track_inode(handle_t *handle, struct inode *inode); void ext4_fc_mark_ineligible(struct super_block *sb, int reason); diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 6c4f19b0a556..7541d0b5d706 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -513,10 +513,10 @@ void ext4_fc_track_link(handle_t *handle, struct dentry *dentry) __ext4_fc_track_link(handle, d_inode(dentry), dentry); } -void ext4_fc_track_create(handle_t *handle, struct dentry *dentry) +void __ext4_fc_track_create(handle_t *handle, struct inode *inode, + struct dentry *dentry) { struct __track_dentry_update_args args; - struct inode *inode = d_inode(dentry); int ret; args.dentry = dentry; @@ -527,6 +527,11 @@ void ext4_fc_track_create(handle_t *handle, struct dentry *dentry) trace_ext4_fc_track_create(inode, dentry, ret); } +void ext4_fc_track_create(handle_t *handle, struct dentry *dentry) +{ + __ext4_fc_track_create(handle, d_inode(dentry), dentry); +} + /* __track_fn for inode tracking */ static int __track_inode(struct inode *inode, void *arg, bool update) { diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 1fe8370dbf0b..883e2a7cd4ab 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -3873,6 +3873,7 @@ static int ext4_rename(struct user_namespace *mnt_userns, struct inode *old_dir, retval = ext4_mark_inode_dirty(handle, whiteout); if (unlikely(retval)) goto end_rename; + } if (!new.bh) { retval = ext4_add_entry(handle, new.dentry, old.inode); @@ -3946,6 +3947,8 @@ static int ext4_rename(struct user_namespace *mnt_userns, struct inode *old_dir, ext4_fc_track_unlink(handle, new.dentry); __ext4_fc_track_link(handle, old.inode, new.dentry); __ext4_fc_track_unlink(handle, old.inode, old.dentry); + if (whiteout) + __ext4_fc_track_create(handle, whiteout, old.dentry); } if (new.inode) { -- cgit v1.2.3 From 512c15ef05d73a04f1aef18a3bc61a8bb516f323 Mon Sep 17 00:00:00 2001 From: Pan Bian Date: Sun, 17 Jan 2021 00:57:32 -0800 Subject: ext4: stop inode update before return The inode update should be stopped before returing the error code. Signed-off-by: Pan Bian Link: https://lore.kernel.org/r/20210117085732.93788-1-bianpan2016@163.com Fixes: 8016e29f4362 ("ext4: fast commit recovery path") Cc: stable@kernel.org Reviewed-by: Harshad Shirwadkar Signed-off-by: Theodore Ts'o --- fs/ext4/inode.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 927e47db7f00..0948a43f1b3d 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -5387,8 +5387,10 @@ int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, inode->i_gid = attr->ia_gid; error = ext4_mark_inode_dirty(handle, inode); ext4_journal_stop(handle); - if (unlikely(error)) + if (unlikely(error)) { + ext4_fc_stop_update(inode); return error; + } } if (attr->ia_valid & ATTR_SIZE) { -- cgit v1.2.3 From 64395d950bc476106b39341e42ebfd4d2eb71d2c Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Sun, 21 Mar 2021 00:45:37 -0400 Subject: ext4: initialize ret to suppress smatch warning Signed-off-by: Theodore Ts'o --- fs/ext4/extents.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 77c7c8a54da7..77c84d6f1af6 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -4382,7 +4382,7 @@ static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset, { struct inode *inode = file_inode(file); handle_t *handle; - int ret, ret2 = 0, ret3 = 0; + int ret = 0, ret2 = 0, ret3 = 0; int retries = 0; int depth = 0; struct ext4_map_blocks map; -- cgit v1.2.3 From 00ddff431a458bbf143ea7c4c42d022676da1b17 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 21 Mar 2021 07:06:56 -0600 Subject: io-wq: ensure task is running before processing task_work Mark the current task as running if we need to run task_work from the io-wq threads as part of work handling. If that is the case, then return as such so that the caller can appropriately loop back and reset if it was part of a going-to-sleep flush. Fixes: 3bfe6106693b ("io-wq: fork worker threads from original task") Signed-off-by: Jens Axboe --- fs/io-wq.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index e05f996d088f..3dc10bfd8c3b 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -386,13 +386,16 @@ static struct io_wq_work *io_get_next_work(struct io_wqe *wqe) return NULL; } -static void io_flush_signals(void) +static bool io_flush_signals(void) { if (unlikely(test_tsk_thread_flag(current, TIF_NOTIFY_SIGNAL))) { + __set_current_state(TASK_RUNNING); if (current->task_works) task_work_run(); clear_tsk_thread_flag(current, TIF_NOTIFY_SIGNAL); + return true; } + return false; } static void io_assign_current_work(struct io_worker *worker, @@ -499,7 +502,8 @@ loop: } __io_worker_idle(wqe, worker); raw_spin_unlock_irq(&wqe->lock); - io_flush_signals(); + if (io_flush_signals()) + continue; ret = schedule_timeout(WORKER_IDLE_TIMEOUT); if (try_to_freeze() || ret) continue; -- cgit v1.2.3 From 0031275d119efe16711cd93519b595e6f9b4b330 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sat, 20 Mar 2021 20:33:36 +0100 Subject: io_uring: call req_set_fail_links() on short send[msg]()/recv[msg]() with MSG_WAITALL Without that it's not safe to use them in a linked combination with others. Now combinations like IORING_OP_SENDMSG followed by IORING_OP_SPLICE should be possible. We already handle short reads and writes for the following opcodes: - IORING_OP_READV - IORING_OP_READ_FIXED - IORING_OP_READ - IORING_OP_WRITEV - IORING_OP_WRITE_FIXED - IORING_OP_WRITE - IORING_OP_SPLICE - IORING_OP_TEE Now we have it for these as well: - IORING_OP_SENDMSG - IORING_OP_SEND - IORING_OP_RECVMSG - IORING_OP_RECV For IORING_OP_RECVMSG we also check for the MSG_TRUNC and MSG_CTRUNC flags in order to call req_set_fail_links(). There might be applications arround depending on the behavior that even short send[msg]()/recv[msg]() retuns continue an IOSQE_IO_LINK chain. It's very unlikely that such applications pass in MSG_WAITALL, which is only defined in 'man 2 recvmsg', but not in 'man 2 sendmsg'. It's expected that the low level sock_sendmsg() call just ignores MSG_WAITALL, as MSG_ZEROCOPY is also ignored without explicitly set SO_ZEROCOPY. We also expect the caller to know about the implicit truncation to MAX_RW_COUNT, which we don't detect. cc: netdev@vger.kernel.org Link: https://lore.kernel.org/r/c4e1a4cc0d905314f4d5dc567e65a7b09621aab3.1615908477.git.metze@samba.org Signed-off-by: Stefan Metzmacher Signed-off-by: Jens Axboe --- fs/io_uring.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index ea2d3e120555..543551d70327 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4386,6 +4386,7 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) struct io_async_msghdr iomsg, *kmsg; struct socket *sock; unsigned flags; + int min_ret = 0; int ret; sock = sock_from_file(req->file); @@ -4406,6 +4407,9 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) else if (issue_flags & IO_URING_F_NONBLOCK) flags |= MSG_DONTWAIT; + if (flags & MSG_WAITALL) + min_ret = iov_iter_count(&kmsg->msg.msg_iter); + ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN) return io_setup_async_msg(req, kmsg); @@ -4416,7 +4420,7 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) if (kmsg->free_iov) kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; - if (ret < 0) + if (ret < min_ret) req_set_fail_links(req); __io_req_complete(req, issue_flags, ret, 0); return 0; @@ -4429,6 +4433,7 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) struct iovec iov; struct socket *sock; unsigned flags; + int min_ret = 0; int ret; sock = sock_from_file(req->file); @@ -4450,6 +4455,9 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) else if (issue_flags & IO_URING_F_NONBLOCK) flags |= MSG_DONTWAIT; + if (flags & MSG_WAITALL) + min_ret = iov_iter_count(&msg.msg_iter); + msg.msg_flags = flags; ret = sock_sendmsg(sock, &msg); if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN) @@ -4457,7 +4465,7 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) if (ret == -ERESTARTSYS) ret = -EINTR; - if (ret < 0) + if (ret < min_ret) req_set_fail_links(req); __io_req_complete(req, issue_flags, ret, 0); return 0; @@ -4609,6 +4617,7 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) struct socket *sock; struct io_buffer *kbuf; unsigned flags; + int min_ret = 0; int ret, cflags = 0; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; @@ -4640,6 +4649,9 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) else if (force_nonblock) flags |= MSG_DONTWAIT; + if (flags & MSG_WAITALL) + min_ret = iov_iter_count(&kmsg->msg.msg_iter); + ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg, kmsg->uaddr, flags); if (force_nonblock && ret == -EAGAIN) @@ -4653,7 +4665,7 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) if (kmsg->free_iov) kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; - if (ret < 0) + if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) req_set_fail_links(req); __io_req_complete(req, issue_flags, ret, cflags); return 0; @@ -4668,6 +4680,7 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags) struct socket *sock; struct iovec iov; unsigned flags; + int min_ret = 0; int ret, cflags = 0; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; @@ -4699,6 +4712,9 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags) else if (force_nonblock) flags |= MSG_DONTWAIT; + if (flags & MSG_WAITALL) + min_ret = iov_iter_count(&msg.msg_iter); + ret = sock_recvmsg(sock, &msg, flags); if (force_nonblock && ret == -EAGAIN) return -EAGAIN; @@ -4707,7 +4723,7 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags) out_free: if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_recv_kbuf(req); - if (ret < 0) + if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)))) req_set_fail_links(req); __io_req_complete(req, issue_flags, ret, cflags); return 0; -- cgit v1.2.3 From 0b8cfa974dfc964e6382c9e25fa6c1bdac6ef499 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 21 Mar 2021 14:16:08 -0600 Subject: io_uring: don't use {test,clear}_tsk_thread_flag() for current Linus correctly points out that this is both unnecessary and generates much worse code on some archs as going from current to thread_info is actually backwards - and obviously just wasteful, since the thread_info is what we care about. Since io_uring only operates on current for these operations, just use test_thread_flag() instead. For io-wq, we can further simplify and use tracehook_notify_signal() to handle the TIF_NOTIFY_SIGNAL work and clear the flag. The latter isn't an actual bug right now, but it may very well be in the future if we place other work items under TIF_NOTIFY_SIGNAL. Reported-by: Linus Torvalds Link: https://lore.kernel.org/io-uring/CAHk-=wgYhNck33YHKZ14mFB5MzTTk8gqXHcfj=RWTAXKwgQJgg@mail.gmail.com/ Signed-off-by: Jens Axboe --- fs/io-wq.c | 6 ++---- fs/io_uring.c | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 3dc10bfd8c3b..2dd43bdb9c7b 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -388,11 +388,9 @@ static struct io_wq_work *io_get_next_work(struct io_wqe *wqe) static bool io_flush_signals(void) { - if (unlikely(test_tsk_thread_flag(current, TIF_NOTIFY_SIGNAL))) { + if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL))) { __set_current_state(TASK_RUNNING); - if (current->task_works) - task_work_run(); - clear_tsk_thread_flag(current, TIF_NOTIFY_SIGNAL); + tracehook_notify_signal(); return true; } return false; diff --git a/fs/io_uring.c b/fs/io_uring.c index 543551d70327..be04bc6b5b99 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6873,7 +6873,7 @@ static int io_run_task_work_sig(void) return 1; if (!signal_pending(current)) return 0; - if (test_tsk_thread_flag(current, TIF_NOTIFY_SIGNAL)) + if (test_thread_flag(TIF_NOTIFY_SIGNAL)) return -ERESTARTSYS; return -EINTR; } -- cgit v1.2.3 From d07f1e8a42614cc938c9c88866d4474a5a7fee31 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 22 Mar 2021 01:45:58 +0000 Subject: io_uring: correct io_queue_async_work() traces Request's io-wq work is hashed in io_prep_async_link(), so as trace_io_uring_queue_async_work() looks at it should follow after prep has been done. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/709c9f872f4d2e198c7aed9c49019ca7095dd24d.1616366969.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index be04bc6b5b99..50acd6dfd966 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1239,10 +1239,10 @@ static void io_queue_async_work(struct io_kiocb *req) BUG_ON(!tctx); BUG_ON(!tctx->io_wq); - trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req, - &req->work, req->flags); /* init ->work of the whole link before punting */ io_prep_async_link(req); + trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req, + &req->work, req->flags); io_wq_enqueue(tctx->io_wq, &req->work); if (link) io_queue_linked_timeout(link); -- cgit v1.2.3 From b65c128f963df367a8adcfb08f5ecf8721052723 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 22 Mar 2021 01:45:59 +0000 Subject: io_uring: don't skip file_end_write() on reissue Don't miss to call kiocb_end_write() from __io_complete_rw() on reissue. Shouldn't be much of a problem as the function actually does some work only for ISREG, and NONBLOCK won't be reissued. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/32af9b77c5b874e1bee1a3c46396094bd969e577.1616366969.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 50acd6dfd966..a664b44a5a94 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2524,13 +2524,12 @@ static void __io_complete_rw(struct io_kiocb *req, long res, long res2, { int cflags = 0; + if (req->rw.kiocb.ki_flags & IOCB_WRITE) + kiocb_end_write(req); if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_reissue(req)) return; if (res != req->result) req_set_fail_links(req); - - if (req->rw.kiocb.ki_flags & IOCB_WRITE) - kiocb_end_write(req); if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_rw_kbuf(req); __io_req_complete(req, issue_flags, res, cflags); -- cgit v1.2.3 From d81269fecb8ce16eb07efafc9ff5520b2a31c486 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 19 Mar 2021 10:21:19 +0000 Subject: io_uring: fix provide_buffers sign extension io_provide_buffers_prep()'s "p->len * p->nbufs" to sign extension problems. Not a huge problem as it's only used for access_ok() and increases the checked length, but better to keep typing right. Reported-by: Colin Ian King Fixes: efe68c1ca8f49 ("io_uring: validate the full range of provided buffers for access") Signed-off-by: Pavel Begunkov Reviewed-by: Colin Ian King Link: https://lore.kernel.org/r/562376a39509e260d8532186a06226e56eb1f594.1616149233.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index a664b44a5a94..f3ae83a2d7bc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3977,6 +3977,7 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) static int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { + unsigned long size; struct io_provide_buf *p = &req->pbuf; u64 tmp; @@ -3990,7 +3991,8 @@ static int io_provide_buffers_prep(struct io_kiocb *req, p->addr = READ_ONCE(sqe->addr); p->len = READ_ONCE(sqe->len); - if (!access_ok(u64_to_user_ptr(p->addr), (p->len * p->nbufs))) + size = (unsigned long)p->len * p->nbufs; + if (!access_ok(u64_to_user_ptr(p->addr), size)) return -EFAULT; p->bgid = READ_ONCE(sqe->buf_group); -- cgit v1.2.3 From 5116784039f0421e9a619023cfba3e302c3d9adc Mon Sep 17 00:00:00 2001 From: Chris Chiu Date: Tue, 23 Mar 2021 16:52:19 +0800 Subject: block: clear GD_NEED_PART_SCAN later in bdev_disk_changed The GD_NEED_PART_SCAN is set by bdev_check_media_change to initiate a partition scan while removing a block device. It should be cleared after blk_drop_paritions because blk_drop_paritions could return -EBUSY and then the consequence __blkdev_get has no chance to do delete_partition if GD_NEED_PART_SCAN already cleared. It causes some problems on some card readers. Ex. Realtek card reader 0bda:0328 and 0bda:0158. The device node of the partition will not disappear after the memory card removed. Thus the user applications can not update the device mapping correctly. BugLink: https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1920874 Signed-off-by: Chris Chiu Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210323085219.24428-1-chris.chiu@canonical.com Signed-off-by: Jens Axboe --- fs/block_dev.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/block_dev.c b/fs/block_dev.c index 92ed7d5df677..28d583fcdc2c 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -1240,13 +1240,13 @@ int bdev_disk_changed(struct block_device *bdev, bool invalidate) lockdep_assert_held(&bdev->bd_mutex); - clear_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state); - rescan: ret = blk_drop_partitions(bdev); if (ret) return ret; + clear_bit(GD_NEED_PART_SCAN, &disk->state); + /* * Historically we only set the capacity to zero for devices that * support partitions (independ of actually having partitions created). -- cgit v1.2.3 From 39f985c8f667c80a3d1eb19d31138032fa36b09e Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 20 Mar 2021 05:40:38 +0000 Subject: fs/cachefiles: Remove wait_bit_key layout dependency Cachefiles was relying on wait_page_key and wait_bit_key being the same layout, which is fragile. Now that wait_page_key is exposed in the pagemap.h header, we can remove that fragility A comment on the need to maintain structure layout equivalence was added by Linus[1] and that is no longer applicable. Fixes: 62906027091f ("mm: add PageWaiters indicating tasks are waiting for a page bit") Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Signed-off-by: David Howells Tested-by: kafs-testing@auristor.com cc: linux-cachefs@redhat.com cc: linux-mm@kvack.org Link: https://lore.kernel.org/r/20210320054104.1300774-2-willy@infradead.org/ Link: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=3510ca20ece0150af6b10c77a74ff1b5c198e3e2 [1] --- fs/cachefiles/rdwr.c | 7 +++---- include/linux/pagemap.h | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/cachefiles/rdwr.c b/fs/cachefiles/rdwr.c index e027c718ca01..8ffc40e84a59 100644 --- a/fs/cachefiles/rdwr.c +++ b/fs/cachefiles/rdwr.c @@ -24,17 +24,16 @@ static int cachefiles_read_waiter(wait_queue_entry_t *wait, unsigned mode, container_of(wait, struct cachefiles_one_read, monitor); struct cachefiles_object *object; struct fscache_retrieval *op = monitor->op; - struct wait_bit_key *key = _key; + struct wait_page_key *key = _key; struct page *page = wait->private; ASSERT(key); _enter("{%lu},%u,%d,{%p,%u}", monitor->netfs_page->index, mode, sync, - key->flags, key->bit_nr); + key->page, key->bit_nr); - if (key->flags != &page->flags || - key->bit_nr != PG_locked) + if (key->page != page || key->bit_nr != PG_locked) return 0; _debug("--- monitor %p %lx ---", page, page->flags); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 20225b067583..8f4daac6eb4b 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -559,7 +559,6 @@ static inline pgoff_t linear_page_index(struct vm_area_struct *vma, return pgoff; } -/* This has the same layout as wait_bit_key - see fs/cachefiles/rdwr.c */ struct wait_page_key { struct page *page; int bit_nr; -- cgit v1.2.3 From 75b69799610c2b909a18e709c402923ea61aedc0 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 20 Mar 2021 05:40:40 +0000 Subject: afs: Use wait_on_page_writeback_killable Open-coding this function meant it missed out on the recent bugfix for waiters being woken by a delayed wake event from a previous instantiation of the page[1]. [DH: Changed the patch to use vmf->page rather than variable page which doesn't exist yet upstream] Fixes: 1cf7a1518aef ("afs: Implement shared-writeable mmap") Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Signed-off-by: David Howells Tested-by: kafs-testing@auristor.com cc: linux-afs@lists.infradead.org cc: linux-mm@kvack.org Link: https://lore.kernel.org/r/20210320054104.1300774-4-willy@infradead.org Link: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=c2407cf7d22d0c0d94cf20342b3b8f06f1d904e7 [1] --- fs/afs/write.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/afs/write.c b/fs/afs/write.c index c9195fc67fd8..eb737ed63afb 100644 --- a/fs/afs/write.c +++ b/fs/afs/write.c @@ -851,8 +851,7 @@ vm_fault_t afs_page_mkwrite(struct vm_fault *vmf) fscache_wait_on_page_write(vnode->cache, vmf->page); #endif - if (PageWriteback(vmf->page) && - wait_on_page_bit_killable(vmf->page, PG_writeback) < 0) + if (wait_on_page_writeback_killable(vmf->page)) return VM_FAULT_RETRY; if (lock_page_killable(vmf->page) < 0) -- cgit v1.2.3 From a185f1db59f13de73aa470559030e90e50b34d93 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 23 Mar 2021 10:52:38 +0000 Subject: io_uring: do ctx sqd ejection in a clear context WARNING: CPU: 1 PID: 27907 at fs/io_uring.c:7147 io_sq_thread_park+0xb5/0xd0 fs/io_uring.c:7147 CPU: 1 PID: 27907 Comm: iou-sqp-27905 Not tainted 5.12.0-rc4-syzkaller #0 RIP: 0010:io_sq_thread_park+0xb5/0xd0 fs/io_uring.c:7147 Call Trace: io_ring_ctx_wait_and_kill+0x214/0x700 fs/io_uring.c:8619 io_uring_release+0x3e/0x50 fs/io_uring.c:8646 __fput+0x288/0x920 fs/file_table.c:280 task_work_run+0xdd/0x1a0 kernel/task_work.c:140 io_run_task_work fs/io_uring.c:2238 [inline] io_run_task_work fs/io_uring.c:2228 [inline] io_uring_try_cancel_requests+0x8ec/0xc60 fs/io_uring.c:8770 io_uring_cancel_sqpoll+0x1cf/0x290 fs/io_uring.c:8974 io_sqpoll_cancel_cb+0x87/0xb0 fs/io_uring.c:8907 io_run_task_work_head+0x58/0xb0 fs/io_uring.c:1961 io_sq_thread+0x3e2/0x18d0 fs/io_uring.c:6763 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:294 May happen that last ctx ref is killed in io_uring_cancel_sqpoll(), so fput callback (i.e. io_uring_release()) is enqueued through task_work, and run by same cancellation. As it's deeply nested we can't do parking or taking sqd->lock there, because its state is unclear. So avoid ctx ejection from sqd list from io_ring_ctx_wait_and_kill() and do it in a clear context in io_ring_exit_work(). Fixes: f6d54255f423 ("io_uring: halt SQO submission on ctx exit") Reported-by: syzbot+e3a3f84f5cecf61f0583@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e90df88b8ff2cabb14a7534601d35d62ab4cb8c7.1616496707.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index f3ae83a2d7bc..8c5789b96dbb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8564,6 +8564,14 @@ static void io_ring_exit_work(struct work_struct *work) struct io_tctx_node *node; int ret; + /* prevent SQPOLL from submitting new requests */ + if (ctx->sq_data) { + io_sq_thread_park(ctx->sq_data); + list_del_init(&ctx->sqd_list); + io_sqd_update_thread_idle(ctx->sq_data); + io_sq_thread_unpark(ctx->sq_data); + } + /* * If we're doing polled IO and end up having requests being * submitted async (out-of-line), then completions can come in while @@ -8615,14 +8623,6 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) io_unregister_personality(ctx, index); mutex_unlock(&ctx->uring_lock); - /* prevent SQPOLL from submitting new requests */ - if (ctx->sq_data) { - io_sq_thread_park(ctx->sq_data); - list_del_init(&ctx->sqd_list); - io_sqd_update_thread_idle(ctx->sq_data); - io_sq_thread_unpark(ctx->sq_data); - } - io_kill_timeouts(ctx, NULL, NULL); io_poll_remove_all(ctx, NULL, NULL); -- cgit v1.2.3 From bf1c82a5389061d989f5e07f1c958db4efaf2141 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Wed, 24 Mar 2021 08:51:10 +0000 Subject: cachefiles: do not yet allow on idmapped mounts Based on discussions (e.g. in [1]) my understanding of cachefiles and the cachefiles userspace daemon is that it creates a cache on a local filesystem (e.g. ext4, xfs etc.) for a network filesystem. The way this is done is by writing "bind" to /dev/cachefiles and pointing it to the directory to use as the cache. Currently this directory can technically also be an idmapped mount but cachefiles aren't yet fully aware of such mounts and thus don't take the idmapping into account when creating cache entries. This could leave users confused as the ownership of the files wouldn't match to what they expressed in the idmapping. Block cache files on idmapped mounts until the fscache rework is done and we have ported it to support idmapped mounts. Signed-off-by: Christian Brauner Signed-off-by: David Howells Cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/lkml/20210303161528.n3jzg66ou2wa43qb@wittgenstein [1] Link: https://lore.kernel.org/r/20210316112257.2974212-1-christian.brauner@ubuntu.com/ # v1 Link: https://listman.redhat.com/archives/linux-cachefs/2021-March/msg00044.html # v2 Link: https://lore.kernel.org/r/20210319114146.410329-1-christian.brauner@ubuntu.com/ # v3 Signed-off-by: Linus Torvalds --- fs/cachefiles/bind.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'fs') diff --git a/fs/cachefiles/bind.c b/fs/cachefiles/bind.c index dfb14dbddf51..38bb7764b454 100644 --- a/fs/cachefiles/bind.c +++ b/fs/cachefiles/bind.c @@ -118,6 +118,12 @@ static int cachefiles_daemon_add_cache(struct cachefiles_cache *cache) cache->mnt = path.mnt; root = path.dentry; + ret = -EINVAL; + if (mnt_user_ns(path.mnt) != &init_user_ns) { + pr_warn("File cache on idmapped mounts not supported"); + goto error_unsupported; + } + /* check parameters */ ret = -EOPNOTSUPP; if (d_is_negative(root) || -- cgit v1.2.3 From f5d2d23bf0d948ce0b9307b7bacae7ff0bc03c71 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 25 Mar 2021 10:16:12 -0600 Subject: io-wq: fix race around pending work on teardown syzbot reports that it's triggering the warning condition on having pending work on shutdown: WARNING: CPU: 1 PID: 12346 at fs/io-wq.c:1061 io_wq_destroy fs/io-wq.c:1061 [inline] WARNING: CPU: 1 PID: 12346 at fs/io-wq.c:1061 io_wq_put+0x153/0x260 fs/io-wq.c:1072 Modules linked in: CPU: 1 PID: 12346 Comm: syz-executor.5 Not tainted 5.12.0-rc2-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 RIP: 0010:io_wq_destroy fs/io-wq.c:1061 [inline] RIP: 0010:io_wq_put+0x153/0x260 fs/io-wq.c:1072 Code: 8d e8 71 90 ea 01 49 89 c4 41 83 fc 40 7d 4f e8 33 4d 97 ff 42 80 7c 2d 00 00 0f 85 77 ff ff ff e9 7a ff ff ff e8 1d 4d 97 ff <0f> 0b eb b9 8d 6b ff 89 ee 09 de bf ff ff ff ff e8 18 51 97 ff 09 RSP: 0018:ffffc90001ebfb08 EFLAGS: 00010293 RAX: ffffffff81e16083 RBX: ffff888019038040 RCX: ffff88801e86b780 RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000040 RBP: 1ffff1100b2f8a80 R08: ffffffff81e15fce R09: ffffed100b2f8a82 R10: ffffed100b2f8a82 R11: 0000000000000000 R12: 0000000000000000 R13: dffffc0000000000 R14: ffff8880597c5400 R15: ffff888019038000 FS: 00007f8dcd89c700(0000) GS:ffff8880b9c00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 000055e9a054e160 CR3: 000000001dfb8000 CR4: 00000000001506f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: io_uring_clean_tctx+0x1b7/0x210 fs/io_uring.c:8802 __io_uring_files_cancel+0x13c/0x170 fs/io_uring.c:8820 io_uring_files_cancel include/linux/io_uring.h:47 [inline] do_exit+0x258/0x2340 kernel/exit.c:780 do_group_exit+0x168/0x2d0 kernel/exit.c:922 get_signal+0x1734/0x1ef0 kernel/signal.c:2773 arch_do_signal_or_restart+0x3c/0x610 arch/x86/kernel/signal.c:811 handle_signal_work kernel/entry/common.c:147 [inline] exit_to_user_mode_loop kernel/entry/common.c:171 [inline] exit_to_user_mode_prepare+0xac/0x1e0 kernel/entry/common.c:208 __syscall_exit_to_user_mode_work kernel/entry/common.c:290 [inline] syscall_exit_to_user_mode+0x48/0x180 kernel/entry/common.c:301 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x465f69 which shouldn't happen, but seems to be possible due to a race on whether or not the io-wq manager sees a fatal signal first, or whether the io-wq workers do. If we race with queueing work and then send a fatal signal to the owning task, and the io-wq worker sees that before the manager sets IO_WQ_BIT_EXIT, then it's possible to have the worker exit and leave work behind. Just turn the WARN_ON_ONCE() into a cancelation condition instead. Reported-by: syzbot+77a738a6bc947bf639ca@syzkaller.appspotmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 2dd43bdb9c7b..b7c1fa932cb3 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -1063,7 +1063,11 @@ static void io_wq_destroy(struct io_wq *wq) for_each_node(node) { struct io_wqe *wqe = wq->wqes[node]; - WARN_ON_ONCE(!wq_list_empty(&wqe->work_list)); + struct io_cb_cancel_data match = { + .fn = io_wq_work_match_all, + .cancel_all = true, + }; + io_wqe_cancel_pending_work(wqe, &match); kfree(wqe); } io_wq_put_hash(wq->hash); -- cgit v1.2.3 From c1b2028315c6b15e8d6725e0d5884b15887d3daa Mon Sep 17 00:00:00 2001 From: Sean Nyekjaer Date: Wed, 24 Mar 2021 21:37:32 -0700 Subject: squashfs: fix inode lookup sanity checks When mouting a squashfs image created without inode compression it fails with: "unable to read inode lookup table" It turns out that the BLOCK_OFFSET is missing when checking the SQUASHFS_METADATA_SIZE agaist the actual size. Link: https://lkml.kernel.org/r/20210226092903.1473545-1-sean@geanix.com Fixes: eabac19e40c0 ("squashfs: add more sanity checks in inode lookup") Signed-off-by: Sean Nyekjaer Acked-by: Phillip Lougher Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/squashfs/export.c | 8 ++++++-- fs/squashfs/squashfs_fs.h | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/squashfs/export.c b/fs/squashfs/export.c index eb02072d28dd..723763746238 100644 --- a/fs/squashfs/export.c +++ b/fs/squashfs/export.c @@ -152,14 +152,18 @@ __le64 *squashfs_read_inode_lookup_table(struct super_block *sb, start = le64_to_cpu(table[n]); end = le64_to_cpu(table[n + 1]); - if (start >= end || (end - start) > SQUASHFS_METADATA_SIZE) { + if (start >= end + || (end - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { kfree(table); return ERR_PTR(-EINVAL); } } start = le64_to_cpu(table[indexes - 1]); - if (start >= lookup_table_start || (lookup_table_start - start) > SQUASHFS_METADATA_SIZE) { + if (start >= lookup_table_start || + (lookup_table_start - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { kfree(table); return ERR_PTR(-EINVAL); } diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h index 8d64edb80ebf..b3fdc8212c5f 100644 --- a/fs/squashfs/squashfs_fs.h +++ b/fs/squashfs/squashfs_fs.h @@ -17,6 +17,7 @@ /* size of metadata (inode and directory) blocks */ #define SQUASHFS_METADATA_SIZE 8192 +#define SQUASHFS_BLOCK_OFFSET 2 /* default size of block device I/O */ #ifdef CONFIG_SQUASHFS_4K_DEVBLK_SIZE -- cgit v1.2.3 From 8b44ca2b634527151af07447a8090a5f3a043321 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Wed, 24 Mar 2021 21:37:35 -0700 Subject: squashfs: fix xattr id and id lookup sanity checks The checks for maximum metadata block size is missing SQUASHFS_BLOCK_OFFSET (the two byte length count). Link: https://lkml.kernel.org/r/2069685113.2081245.1614583677427@webmail.123-reg.co.uk Fixes: f37aa4c7366e23f ("squashfs: add more sanity checks in id lookup") Signed-off-by: Phillip Lougher Cc: Sean Nyekjaer Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/squashfs/id.c | 6 ++++-- fs/squashfs/xattr_id.c | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/squashfs/id.c b/fs/squashfs/id.c index 11581bf31af4..ea5387679723 100644 --- a/fs/squashfs/id.c +++ b/fs/squashfs/id.c @@ -97,14 +97,16 @@ __le64 *squashfs_read_id_index_table(struct super_block *sb, start = le64_to_cpu(table[n]); end = le64_to_cpu(table[n + 1]); - if (start >= end || (end - start) > SQUASHFS_METADATA_SIZE) { + if (start >= end || (end - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { kfree(table); return ERR_PTR(-EINVAL); } } start = le64_to_cpu(table[indexes - 1]); - if (start >= id_table_start || (id_table_start - start) > SQUASHFS_METADATA_SIZE) { + if (start >= id_table_start || (id_table_start - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { kfree(table); return ERR_PTR(-EINVAL); } diff --git a/fs/squashfs/xattr_id.c b/fs/squashfs/xattr_id.c index ead66670b41a..087cab8c78f4 100644 --- a/fs/squashfs/xattr_id.c +++ b/fs/squashfs/xattr_id.c @@ -109,14 +109,16 @@ __le64 *squashfs_read_xattr_id_table(struct super_block *sb, u64 table_start, start = le64_to_cpu(table[n]); end = le64_to_cpu(table[n + 1]); - if (start >= end || (end - start) > SQUASHFS_METADATA_SIZE) { + if (start >= end || (end - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { kfree(table); return ERR_PTR(-EINVAL); } } start = le64_to_cpu(table[indexes - 1]); - if (start >= table_start || (table_start - start) > SQUASHFS_METADATA_SIZE) { + if (start >= table_start || (table_start - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { kfree(table); return ERR_PTR(-EINVAL); } -- cgit v1.2.3 From ff132c5f93c06bd4432bbab5c369e468653bdec4 Mon Sep 17 00:00:00 2001 From: Bob Peterson Date: Thu, 25 Mar 2021 08:51:13 -0400 Subject: gfs2: report "already frozen/thawed" errors Before this patch, gfs2's freeze function failed to report an error when the target file system was already frozen as it should (and as generic vfs function freeze_super does. Similarly, gfs2's thaw function failed to report an error when trying to thaw a file system that is not frozen, as vfs function thaw_super does. The errors were checked, but it always returned a 0 return code. This patch adds the missing error return codes to gfs2 freeze and thaw. Signed-off-by: Bob Peterson Signed-off-by: Andreas Gruenbacher --- fs/gfs2/super.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index 9e91c9d92bd6..8fb9602d79b4 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -752,11 +752,13 @@ void gfs2_freeze_func(struct work_struct *work) static int gfs2_freeze(struct super_block *sb) { struct gfs2_sbd *sdp = sb->s_fs_info; - int error = 0; + int error; mutex_lock(&sdp->sd_freeze_mutex); - if (atomic_read(&sdp->sd_freeze_state) != SFS_UNFROZEN) + if (atomic_read(&sdp->sd_freeze_state) != SFS_UNFROZEN) { + error = -EBUSY; goto out; + } for (;;) { if (gfs2_withdrawn(sdp)) { @@ -797,10 +799,10 @@ static int gfs2_unfreeze(struct super_block *sb) struct gfs2_sbd *sdp = sb->s_fs_info; mutex_lock(&sdp->sd_freeze_mutex); - if (atomic_read(&sdp->sd_freeze_state) != SFS_FROZEN || + if (atomic_read(&sdp->sd_freeze_state) != SFS_FROZEN || !gfs2_holder_initialized(&sdp->sd_freeze_gh)) { mutex_unlock(&sdp->sd_freeze_mutex); - return 0; + return -EINVAL; } gfs2_freeze_unlock(&sdp->sd_freeze_gh); -- cgit v1.2.3 From 90b8749022bbdd0c94a13182a78f4903b98fd0d7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 25 Mar 2021 19:05:14 +0000 Subject: io_uring: maintain CQE order of a failed link Arguably we want CQEs of linked requests be in a strict order of submission as it always was. Now if init of a request fails its CQE may be posted before all prior linked requests including the head of the link. Fix it by failing it last. Fixes: de59bc104c24f ("io_uring: fail links more in io_submit_sqe()") Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b7a96b05832e7ab23ad55f84092a2548c4a888b0.1616699075.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 8c5789b96dbb..54ea561db4a5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6480,8 +6480,6 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, ret = io_init_req(ctx, req, sqe); if (unlikely(ret)) { fail_req: - io_put_req(req); - io_req_complete(req, ret); if (link->head) { /* fail even hard links since we don't submit */ link->head->flags |= REQ_F_FAIL_LINK; @@ -6489,6 +6487,8 @@ fail_req: io_req_complete(link->head, -ECANCELED); link->head = NULL; } + io_put_req(req); + io_req_complete(req, ret); return ret; } ret = io_req_prep(req, sqe); -- cgit v1.2.3 From 7f6c411c9b50cfab41cc798e003eff27608c7016 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Thu, 25 Mar 2021 14:12:34 -0400 Subject: hostfs: fix memory handling in follow_link() 1) argument should not be freed in any case - the caller already has it as ->s_fs_info (and uses it a lot afterwards) 2) allocate readlink buffer with kmalloc() - the caller has no way to tell if it's got that (on absolute symlink) or a result of kasprintf(). Sure, for SLAB and SLUB kfree() works on results of kmem_cache_alloc(), but that's not documented anywhere, might change in the future *and* is already not true for SLOB. Fixes: 52b209f7b848 ("get rid of hostfs_read_inode()") Signed-off-by: Al Viro --- fs/hostfs/hostfs_kern.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c index 29e407762626..743a005a5c64 100644 --- a/fs/hostfs/hostfs_kern.c +++ b/fs/hostfs/hostfs_kern.c @@ -144,7 +144,7 @@ static char *follow_link(char *link) char *name, *resolved, *end; int n; - name = __getname(); + name = kmalloc(PATH_MAX, GFP_KERNEL); if (!name) { n = -ENOMEM; goto out_free; @@ -173,12 +173,11 @@ static char *follow_link(char *link) goto out_free; } - __putname(name); - kfree(link); + kfree(name); return resolved; out_free: - __putname(name); + kfree(name); return ERR_PTR(n); } -- cgit v1.2.3 From 45a4546c6167a2da348a31ca439d8a8ff773b6ea Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Thu, 25 Mar 2021 12:34:54 +0000 Subject: cifs: Adjust key sizes and key generation routines for AES256 encryption For AES256 encryption (GCM and CCM), we need to adjust the size of a few fields to 32 bytes instead of 16 to accommodate the larger keys. Also, the L value supplied to the key generator needs to be changed from to 256 when these algorithms are used. Keeping the ioctl struct for dumping keys of the same size for now. Will send out a different patch for that one. Signed-off-by: Shyam Prasad N Reviewed-by: Ronnie Sahlberg CC: # v5.10+ Signed-off-by: Steve French --- fs/cifs/cifsglob.h | 4 ++-- fs/cifs/cifspdu.h | 5 +++++ fs/cifs/smb2glob.h | 1 + fs/cifs/smb2ops.c | 9 +++++---- fs/cifs/smb2transport.c | 37 ++++++++++++++++++++++++++++--------- 5 files changed, 41 insertions(+), 15 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index 31fc8695abd6..67c056a9a519 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -919,8 +919,8 @@ struct cifs_ses { bool binding:1; /* are we binding the session? */ __u16 session_flags; __u8 smb3signingkey[SMB3_SIGN_KEY_SIZE]; - __u8 smb3encryptionkey[SMB3_SIGN_KEY_SIZE]; - __u8 smb3decryptionkey[SMB3_SIGN_KEY_SIZE]; + __u8 smb3encryptionkey[SMB3_ENC_DEC_KEY_SIZE]; + __u8 smb3decryptionkey[SMB3_ENC_DEC_KEY_SIZE]; __u8 preauth_sha_hash[SMB2_PREAUTH_HASH_SIZE]; __u8 binding_preauth_sha_hash[SMB2_PREAUTH_HASH_SIZE]; diff --git a/fs/cifs/cifspdu.h b/fs/cifs/cifspdu.h index 64fe5a47b5e8..9adc74bd9f8f 100644 --- a/fs/cifs/cifspdu.h +++ b/fs/cifs/cifspdu.h @@ -147,6 +147,11 @@ */ #define SMB3_SIGN_KEY_SIZE (16) +/* + * Size of the smb3 encryption/decryption keys + */ +#define SMB3_ENC_DEC_KEY_SIZE (32) + #define CIFS_CLIENT_CHALLENGE_SIZE (8) #define CIFS_SERVER_CHALLENGE_SIZE (8) #define CIFS_HMAC_MD5_HASH_SIZE (16) diff --git a/fs/cifs/smb2glob.h b/fs/cifs/smb2glob.h index 99a1951a01ec..d9a990c99121 100644 --- a/fs/cifs/smb2glob.h +++ b/fs/cifs/smb2glob.h @@ -58,6 +58,7 @@ #define SMB2_HMACSHA256_SIZE (32) #define SMB2_CMACAES_SIZE (16) #define SMB3_SIGNKEY_SIZE (16) +#define SMB3_GCM128_CRYPTKEY_SIZE (16) #define SMB3_GCM256_CRYPTKEY_SIZE (32) /* Maximum buffer size value we can send with 1 credit */ diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index 9bae7e8deb09..3e94f83897e9 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -4158,7 +4158,7 @@ smb2_get_enc_key(struct TCP_Server_Info *server, __u64 ses_id, int enc, u8 *key) if (ses->Suid == ses_id) { ses_enc_key = enc ? ses->smb3encryptionkey : ses->smb3decryptionkey; - memcpy(key, ses_enc_key, SMB3_SIGN_KEY_SIZE); + memcpy(key, ses_enc_key, SMB3_ENC_DEC_KEY_SIZE); spin_unlock(&cifs_tcp_ses_lock); return 0; } @@ -4185,7 +4185,7 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst, int rc = 0; struct scatterlist *sg; u8 sign[SMB2_SIGNATURE_SIZE] = {}; - u8 key[SMB3_SIGN_KEY_SIZE]; + u8 key[SMB3_ENC_DEC_KEY_SIZE]; struct aead_request *req; char *iv; unsigned int iv_len; @@ -4209,10 +4209,11 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst, tfm = enc ? server->secmech.ccmaesencrypt : server->secmech.ccmaesdecrypt; - if (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM) + if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) || + (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) rc = crypto_aead_setkey(tfm, key, SMB3_GCM256_CRYPTKEY_SIZE); else - rc = crypto_aead_setkey(tfm, key, SMB3_SIGN_KEY_SIZE); + rc = crypto_aead_setkey(tfm, key, SMB3_GCM128_CRYPTKEY_SIZE); if (rc) { cifs_server_dbg(VFS, "%s: Failed to set aead key %d\n", __func__, rc); diff --git a/fs/cifs/smb2transport.c b/fs/cifs/smb2transport.c index ebccd71cc60a..e6fa76ab70be 100644 --- a/fs/cifs/smb2transport.c +++ b/fs/cifs/smb2transport.c @@ -298,7 +298,8 @@ static int generate_key(struct cifs_ses *ses, struct kvec label, { unsigned char zero = 0x0; __u8 i[4] = {0, 0, 0, 1}; - __u8 L[4] = {0, 0, 0, 128}; + __u8 L128[4] = {0, 0, 0, 128}; + __u8 L256[4] = {0, 0, 1, 0}; int rc = 0; unsigned char prfhash[SMB2_HMACSHA256_SIZE]; unsigned char *hashptr = prfhash; @@ -354,8 +355,14 @@ static int generate_key(struct cifs_ses *ses, struct kvec label, goto smb3signkey_ret; } - rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash, - L, 4); + if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) || + (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) { + rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash, + L256, 4); + } else { + rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash, + L128, 4); + } if (rc) { cifs_server_dbg(VFS, "%s: Could not update with L\n", __func__); goto smb3signkey_ret; @@ -390,6 +397,9 @@ generate_smb3signingkey(struct cifs_ses *ses, const struct derivation_triplet *ptriplet) { int rc; +#ifdef CONFIG_CIFS_DEBUG_DUMP_KEYS + struct TCP_Server_Info *server = ses->server; +#endif /* * All channels use the same encryption/decryption keys but @@ -422,11 +432,11 @@ generate_smb3signingkey(struct cifs_ses *ses, rc = generate_key(ses, ptriplet->encryption.label, ptriplet->encryption.context, ses->smb3encryptionkey, - SMB3_SIGN_KEY_SIZE); + SMB3_ENC_DEC_KEY_SIZE); rc = generate_key(ses, ptriplet->decryption.label, ptriplet->decryption.context, ses->smb3decryptionkey, - SMB3_SIGN_KEY_SIZE); + SMB3_ENC_DEC_KEY_SIZE); if (rc) return rc; } @@ -442,14 +452,23 @@ generate_smb3signingkey(struct cifs_ses *ses, */ cifs_dbg(VFS, "Session Id %*ph\n", (int)sizeof(ses->Suid), &ses->Suid); + cifs_dbg(VFS, "Cipher type %d\n", server->cipher_type); cifs_dbg(VFS, "Session Key %*ph\n", SMB2_NTLMV2_SESSKEY_SIZE, ses->auth_key.response); cifs_dbg(VFS, "Signing Key %*ph\n", SMB3_SIGN_KEY_SIZE, ses->smb3signingkey); - cifs_dbg(VFS, "ServerIn Key %*ph\n", - SMB3_SIGN_KEY_SIZE, ses->smb3encryptionkey); - cifs_dbg(VFS, "ServerOut Key %*ph\n", - SMB3_SIGN_KEY_SIZE, ses->smb3decryptionkey); + if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) || + (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) { + cifs_dbg(VFS, "ServerIn Key %*ph\n", + SMB3_GCM256_CRYPTKEY_SIZE, ses->smb3encryptionkey); + cifs_dbg(VFS, "ServerOut Key %*ph\n", + SMB3_GCM256_CRYPTKEY_SIZE, ses->smb3decryptionkey); + } else { + cifs_dbg(VFS, "ServerIn Key %*ph\n", + SMB3_GCM128_CRYPTKEY_SIZE, ses->smb3encryptionkey); + cifs_dbg(VFS, "ServerOut Key %*ph\n", + SMB3_GCM128_CRYPTKEY_SIZE, ses->smb3decryptionkey); + } #endif return rc; } -- cgit v1.2.3 From 3bffbe9e0b2721bb62d226a4d4211bddae52b00a Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Fri, 26 Mar 2021 10:28:16 +0000 Subject: cifs: Fix chmod with modefromsid when an older ACE already exists. My recent fixes to cifsacl to maintain inherited ACEs had regressed modefromsid when an older ACL already exists. Found testing xfstest 495 with modefromsid mount option Fixes: f5065508897a ("cifs: Retain old ACEs when converting between mode bits and ACL") Signed-off-by: Shyam Prasad N Reviewed-by: Paulo Alcantara (SUSE) Signed-off-by: Steve French --- fs/cifs/cifsacl.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsacl.c b/fs/cifs/cifsacl.c index 2be22a5c690f..d178cf85e926 100644 --- a/fs/cifs/cifsacl.c +++ b/fs/cifs/cifsacl.c @@ -1130,8 +1130,7 @@ static int set_chmod_dacl(struct cifs_acl *pdacl, struct cifs_acl *pndacl, } /* If it's any one of the ACE we're replacing, skip! */ - if (!mode_from_sid && - ((compare_sids(&pntace->sid, &sid_unix_NFS_mode) == 0) || + if (((compare_sids(&pntace->sid, &sid_unix_NFS_mode) == 0) || (compare_sids(&pntace->sid, pownersid) == 0) || (compare_sids(&pntace->sid, pgrpsid) == 0) || (compare_sids(&pntace->sid, &sid_everyone) == 0) || -- cgit v1.2.3 From cee8f4f6fcabfdf229542926128e9874d19016d5 Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Thu, 25 Mar 2021 16:26:35 +1000 Subject: cifs: revalidate mapping when we open files for SMB1 POSIX RHBZ: 1933527 Under SMB1 + POSIX, if an inode is reused on a server after we have read and cached a part of a file, when we then open the new file with the re-cycled inode there is a chance that we may serve the old data out of cache to the application. This only happens for SMB1 (deprecated) and when posix are used. The simplest solution to avoid this race is to force a revalidate on smb1-posix open. Signed-off-by: Ronnie Sahlberg Reviewed-by: Paulo Alcantara (SUSE) Signed-off-by: Steve French --- fs/cifs/file.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/cifs/file.c b/fs/cifs/file.c index 26de4329d161..042e24aad410 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c @@ -165,6 +165,7 @@ int cifs_posix_open(char *full_path, struct inode **pinode, goto posix_open_ret; } } else { + cifs_revalidate_mapping(*pinode); cifs_fattr_to_inode(*pinode, &fattr); } -- cgit v1.2.3 From 219481a8f90ec3a5eed9638fb35609e4b1aeece7 Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Fri, 19 Mar 2021 14:57:11 +0100 Subject: cifs: Silently ignore unknown oplock break handle Make SMB2 not print out an error when an oplock break is received for an unknown handle, similar to SMB1. The debug message which is printed for these unknown handles may also be misleading, so fix that too. The SMB2 lease break path is not affected by this patch. Without this, a program which writes to a file from one thread, and opens, reads, and writes the same file from another thread triggers the below errors several times a minute when run against a Samba server configured with "smb2 leases = no". CIFS: VFS: \\192.168.0.1 No task to wake, unknown frame received! NumMids 2 00000000: 424d53fe 00000040 00000000 00000012 .SMB@........... 00000010: 00000001 00000000 ffffffff ffffffff ................ 00000020: 00000000 00000000 00000000 00000000 ................ 00000030: 00000000 00000000 00000000 00000000 ................ Signed-off-by: Vincent Whitchurch Reviewed-by: Tom Talpey Reviewed-by: Paulo Alcantara (SUSE) Signed-off-by: Steve French --- fs/cifs/smb2misc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/smb2misc.c b/fs/cifs/smb2misc.c index b50164e2c88d..aac384f69f74 100644 --- a/fs/cifs/smb2misc.c +++ b/fs/cifs/smb2misc.c @@ -754,8 +754,8 @@ smb2_is_valid_oplock_break(char *buffer, struct TCP_Server_Info *server) } } spin_unlock(&cifs_tcp_ses_lock); - cifs_dbg(FYI, "Can not process oplock break for non-existent connection\n"); - return false; + cifs_dbg(FYI, "No file id matched, oplock break ignored\n"); + return true; } void -- cgit v1.2.3 From cfc63fc8126a93cbf95379bc4cad79a7b15b6ece Mon Sep 17 00:00:00 2001 From: Steve French Date: Fri, 26 Mar 2021 18:41:55 -0500 Subject: smb3: fix cached file size problems in duplicate extents (reflink) There were two problems (one of which could cause data corruption) that were noticed with duplicate extents (ie reflink) when debugging why various xfstests were being incorrectly skipped (e.g. generic/138, generic/140, generic/142). First, we were not updating the file size locally in the cache when extending a file due to reflink (it would refresh after actimeo expires) but xfstest was checking the size immediately which was still 0 so caused the test to be skipped. Second, we were setting the target file size (which could shrink the file) in all cases to the end of the reflinked range rather than only setting the target file size when reflink would extend the file. CC: Signed-off-by: Steve French --- fs/cifs/smb2ops.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index 3e94f83897e9..f703204fb185 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -2038,6 +2038,7 @@ smb2_duplicate_extents(const unsigned int xid, { int rc; unsigned int ret_data_len; + struct inode *inode; struct duplicate_extents_to_file dup_ext_buf; struct cifs_tcon *tcon = tlink_tcon(trgtfile->tlink); @@ -2054,10 +2055,21 @@ smb2_duplicate_extents(const unsigned int xid, cifs_dbg(FYI, "Duplicate extents: src off %lld dst off %lld len %lld\n", src_off, dest_off, len); - rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false); - if (rc) - goto duplicate_extents_out; + inode = d_inode(trgtfile->dentry); + if (inode->i_size < dest_off + len) { + rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false); + if (rc) + goto duplicate_extents_out; + /* + * Although also could set plausible allocation size (i_blocks) + * here in addition to setting the file size, in reflink + * it is likely that the target file is sparse. Its allocation + * size will be queried on next revalidate, but it is important + * to make sure that file's cached size is updated immediately + */ + cifs_setsize(inode, dest_off + len); + } rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid, trgtfile->fid.volatile_fid, FSCTL_DUPLICATE_EXTENTS_TO_FILE, -- cgit v1.2.3 From dbe1bdbb39db7dfe80a903f0d267f62cf3f093d2 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 25 Mar 2021 18:16:06 -0600 Subject: io_uring: handle signals for IO threads like a normal thread We go through various hoops to disallow signals for the IO threads, but there's really no reason why we cannot just allow them. The IO threads never return to userspace like a normal thread, and hence don't go through normal signal processing. Instead, just check for a pending signal as part of the work loop, and call get_signal() to handle it for us if anything is pending. With that, we can support receiving signals, including special ones like SIGSTOP. Acked-by: "Eric W. Biederman" Signed-off-by: Jens Axboe --- fs/io-wq.c | 20 ++++++++++++++------ fs/io_uring.c | 9 ++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index b7c1fa932cb3..7434eb40ca8c 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -16,7 +16,6 @@ #include #include #include -#include #include "../kernel/sched/sched.h" #include "io-wq.h" @@ -503,10 +502,15 @@ loop: if (io_flush_signals()) continue; ret = schedule_timeout(WORKER_IDLE_TIMEOUT); - if (try_to_freeze() || ret) - continue; - if (fatal_signal_pending(current)) + if (signal_pending(current)) { + struct ksignal ksig; + + if (!get_signal(&ksig)) + continue; break; + } + if (ret) + continue; /* timed out, exit unless we're the fixed worker */ if (test_bit(IO_WQ_BIT_EXIT, &wq->state) || !(worker->flags & IO_WORKER_F_FIXED)) @@ -714,9 +718,13 @@ static int io_wq_manager(void *data) set_current_state(TASK_INTERRUPTIBLE); io_wq_check_workers(wq); schedule_timeout(HZ); - try_to_freeze(); - if (fatal_signal_pending(current)) + if (signal_pending(current)) { + struct ksignal ksig; + + if (!get_signal(&ksig)) + continue; set_bit(IO_WQ_BIT_EXIT, &wq->state); + } } while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)); io_wq_check_workers(wq); diff --git a/fs/io_uring.c b/fs/io_uring.c index 54ea561db4a5..2d43f7b87083 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -78,7 +78,6 @@ #include #include #include -#include #define CREATE_TRACE_POINTS #include @@ -6765,8 +6764,13 @@ static int io_sq_thread(void *data) timeout = jiffies + sqd->sq_thread_idle; continue; } - if (fatal_signal_pending(current)) + if (signal_pending(current)) { + struct ksignal ksig; + + if (!get_signal(&ksig)) + continue; break; + } sqt_spin = false; cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { @@ -6809,7 +6813,6 @@ static int io_sq_thread(void *data) mutex_unlock(&sqd->lock); schedule(); - try_to_freeze(); mutex_lock(&sqd->lock); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) io_ring_clear_wakeup_flag(ctx); -- cgit v1.2.3 From 1ee4160c73b2102a52bc97a4128a89c34821414f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 25 Mar 2021 18:32:42 +0000 Subject: io_uring: fix timeout cancel return code When we cancel a timeout we should emit a sensible return code, like -ECANCELED but not 0, otherwise it may trick users. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7b0ad1065e3bd1994722702bd0ba9e7bc9b0683b.1616696997.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 2d43f7b87083..229ab9bfb45b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1247,7 +1247,7 @@ static void io_queue_async_work(struct io_kiocb *req) io_queue_linked_timeout(link); } -static void io_kill_timeout(struct io_kiocb *req) +static void io_kill_timeout(struct io_kiocb *req, int status) { struct io_timeout_data *io = req->async_data; int ret; @@ -1257,7 +1257,7 @@ static void io_kill_timeout(struct io_kiocb *req) atomic_set(&req->ctx->cq_timeouts, atomic_read(&req->ctx->cq_timeouts) + 1); list_del_init(&req->timeout.list); - io_cqring_fill_event(req, 0); + io_cqring_fill_event(req, status); io_put_req_deferred(req, 1); } } @@ -1274,7 +1274,7 @@ static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, spin_lock_irq(&ctx->completion_lock); list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { if (io_match_task(req, tsk, files)) { - io_kill_timeout(req); + io_kill_timeout(req, -ECANCELED); canceled++; } } @@ -1326,7 +1326,7 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) break; list_del_init(&req->timeout.list); - io_kill_timeout(req); + io_kill_timeout(req, 0); } while (!list_empty(&ctx->timeout_list)); ctx->cq_last_tm_flush = seq; -- cgit v1.2.3 From 80c4cbdb5ee604712e59fe304d7bf084b562f705 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 25 Mar 2021 18:32:43 +0000 Subject: io_uring: do post-completion chore on t-out cancel Don't forget about io_commit_cqring() + io_cqring_ev_posted() after exit/exec cancelling timeouts. Both functions declared only after io_kill_timeouts(), so to avoid tons of forward declarations move it down. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/72ace588772c0f14834a6a4185d56c445a366fb4.1616696997.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 229ab9bfb45b..8498f74595f3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1262,26 +1262,6 @@ static void io_kill_timeout(struct io_kiocb *req, int status) } } -/* - * Returns true if we found and killed one or more timeouts - */ -static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, - struct files_struct *files) -{ - struct io_kiocb *req, *tmp; - int canceled = 0; - - spin_lock_irq(&ctx->completion_lock); - list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { - if (io_match_task(req, tsk, files)) { - io_kill_timeout(req, -ECANCELED); - canceled++; - } - } - spin_unlock_irq(&ctx->completion_lock); - return canceled != 0; -} - static void __io_queue_deferred(struct io_ring_ctx *ctx) { do { @@ -8611,6 +8591,28 @@ static void io_ring_exit_work(struct work_struct *work) io_ring_ctx_free(ctx); } +/* Returns true if we found and killed one or more timeouts */ +static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, + struct files_struct *files) +{ + struct io_kiocb *req, *tmp; + int canceled = 0; + + spin_lock_irq(&ctx->completion_lock); + list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) { + if (io_match_task(req, tsk, files)) { + io_kill_timeout(req, -ECANCELED); + canceled++; + } + } + io_commit_cqring(ctx); + spin_unlock_irq(&ctx->completion_lock); + + if (canceled != 0) + io_cqring_ev_posted(ctx); + return canceled != 0; +} + static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) { unsigned long index; -- cgit v1.2.3 From 2482b58ffbdc80cfaae969ad19cb32803056505b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 25 Mar 2021 18:32:44 +0000 Subject: io_uring: don't cancel-track common timeouts Don't account usual timeouts (i.e. not linked) as REQ_F_INFLIGHT but keep behaviour prior to dd59a3d595cc1 ("io_uring: reliably cancel linked timeouts"). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/104441ef5d97e3932113d44501fda0df88656b83.1616696997.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 8498f74595f3..87d198d223a2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5563,7 +5563,8 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, data->mode = io_translate_timeout_mode(flags); hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode); - io_req_track_inflight(req); + if (is_timeout_link) + io_req_track_inflight(req); return 0; } -- cgit v1.2.3 From 78d9d7c2a331fb7a68a86e53ef7e12966459e0c5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 25 Mar 2021 18:32:45 +0000 Subject: io_uring: don't cancel extra on files match As tasks always wait and kill their io-wq on exec/exit, files are of no more concern to us, so we don't need to specifically cancel them by hand in those cases. Moreover we should not, because io_match_task() looks at req->task->files now, which is always true and so leads to extra cancellations, that wasn't a case before per-task io-wq. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/0566c1de9b9dd417f5de345c817ca953580e0e2e.1616696997.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 87d198d223a2..880abd8b6d31 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1094,8 +1094,6 @@ static bool io_match_task(struct io_kiocb *head, io_for_each_link(req, head) { if (req->flags & REQ_F_INFLIGHT) return true; - if (req->task->files == files) - return true; } return false; } -- cgit v1.2.3 From 2b8ed1c94182dbbd0163d0eb443a934cbf6b0d85 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 26 Mar 2021 19:52:51 +0000 Subject: io_uring: remove unsued assignment to pointer io There is an assignment to io that is never read after the assignment, the assignment is redundant and can be removed. Signed-off-by: Colin Ian King Signed-off-by: Jens Axboe --- fs/io_uring.c | 1 - 1 file changed, 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 880abd8b6d31..1949b80677e7 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4798,7 +4798,6 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags) ret = -ENOMEM; goto out; } - io = req->async_data; memcpy(req->async_data, &__io, sizeof(__io)); return -EAGAIN; } -- cgit v1.2.3 From 5a978dcfc0f054e4f6983a0a26355a65e34708cb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 27 Mar 2021 09:59:30 +0000 Subject: io_uring: always go for cancellation spin on exec Always try to do cancellation in __io_uring_task_cancel() at least once, so it actually goes and cleans its sqpoll tasks (i.e. via io_sqpoll_cancel_sync()), otherwise sqpoll task may submit new requests after cancellation and it's racy for many reasons. Fixes: 521d6a737a31c ("io_uring: cancel sqpoll via task_work") Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/0a21bd6d794bb1629bc906dd57a57b2c2985a8ac.1616839147.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 1949b80677e7..a4a944da95a0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9002,6 +9002,8 @@ void __io_uring_task_cancel(void) /* make sure overflow events are dropped */ atomic_inc(&tctx->in_idle); + __io_uring_files_cancel(NULL); + do { /* read completions before cancelations */ inflight = tctx_inflight(tctx); -- cgit v1.2.3 From 51520426f4bc3e61cbbf7a39ccf4e411b665002d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 29 Mar 2021 11:39:29 +0100 Subject: io_uring: handle setup-failed ctx in kill_timeouts general protection fault, probably for non-canonical address 0xdffffc0000000018: 0000 [#1] KASAN: null-ptr-deref in range [0x00000000000000c0-0x00000000000000c7] RIP: 0010:io_commit_cqring+0x37f/0xc10 fs/io_uring.c:1318 Call Trace: io_kill_timeouts+0x2b5/0x320 fs/io_uring.c:8606 io_ring_ctx_wait_and_kill+0x1da/0x400 fs/io_uring.c:8629 io_uring_create fs/io_uring.c:9572 [inline] io_uring_setup+0x10da/0x2ae0 fs/io_uring.c:9599 do_syscall_64+0x2d/0x70 arch/x86/entry/common.c:46 entry_SYSCALL_64_after_hwframe+0x44/0xae It can get into wait_and_kill() before setting up ctx->rings, and hence io_commit_cqring() fails. Mimic poll cancel and do it only when we completed events, there can't be any requests if it failed before initialising rings. Fixes: 80c4cbdb5ee60 ("io_uring: do post-completion chore on t-out cancel") Reported-by: syzbot+0e905eb8228070c457a0@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/660261a48f0e7abf260c8e43c87edab3c16736fa.1617014345.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index a4a944da95a0..088a9d3c420a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8603,9 +8603,9 @@ static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, canceled++; } } - io_commit_cqring(ctx); + if (canceled != 0) + io_commit_cqring(ctx); spin_unlock_irq(&ctx->completion_lock); - if (canceled != 0) io_cqring_ev_posted(ctx); return canceled != 0; -- cgit v1.2.3 From 82734c5b1b24c020d701cf90ccb075e43a5ccb07 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 29 Mar 2021 06:52:44 -0600 Subject: io_uring: drop sqd lock before handling signals for SQPOLL Don't call into get_signal() with the sqd mutex held, it'll fail if we're freezing the task and we'll get complaints on locks still being held: ==================================== WARNING: iou-sqp-8386/8387 still has locks held! 5.12.0-rc4-syzkaller #0 Not tainted ------------------------------------ 1 lock held by iou-sqp-8386/8387: #0: ffff88801e1d2470 (&sqd->lock){+.+.}-{3:3}, at: io_sq_thread+0x24c/0x13a0 fs/io_uring.c:6731 stack backtrace: CPU: 1 PID: 8387 Comm: iou-sqp-8386 Not tainted 5.12.0-rc4-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:79 [inline] dump_stack+0x141/0x1d7 lib/dump_stack.c:120 try_to_freeze include/linux/freezer.h:66 [inline] get_signal+0x171a/0x2150 kernel/signal.c:2576 io_sq_thread+0x8d2/0x13a0 fs/io_uring.c:6748 Fold the get_signal() case in with the parking checks, as we need to drop the lock in both cases, and since we need to be checking for parking when juggling the lock anyway. Reported-by: syzbot+796d767eb376810256f5@syzkaller.appspotmail.com Fixes: dbe1bdbb39db ("io_uring: handle signals for IO threads like a normal thread") Signed-off-by: Jens Axboe --- fs/io_uring.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 088a9d3c420a..6d7a1b69712b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6733,22 +6733,25 @@ static int io_sq_thread(void *data) int ret; bool cap_entries, sqt_spin, needs_sched; - if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state)) { + if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) || + signal_pending(current)) { + bool did_sig = false; + mutex_unlock(&sqd->lock); + if (signal_pending(current)) { + struct ksignal ksig; + + did_sig = get_signal(&ksig); + } cond_resched(); mutex_lock(&sqd->lock); + if (did_sig) + break; io_run_task_work(); io_run_task_work_head(&sqd->park_task_work); timeout = jiffies + sqd->sq_thread_idle; continue; } - if (signal_pending(current)) { - struct ksignal ksig; - - if (!get_signal(&ksig)) - continue; - break; - } sqt_spin = false; cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { -- cgit v1.2.3 From 5e46d1b78a03d52306f21f77a4e4a144b6d31486 Mon Sep 17 00:00:00 2001 From: Tetsuo Handa Date: Sun, 21 Mar 2021 23:37:49 +0900 Subject: reiserfs: update reiserfs_xattrs_initialized() condition syzbot is reporting NULL pointer dereference at reiserfs_security_init() [1], for commit ab17c4f02156c4f7 ("reiserfs: fixup xattr_root caching") is assuming that REISERFS_SB(s)->xattr_root != NULL in reiserfs_xattr_jcreate_nblocks() despite that commit made REISERFS_SB(sb)->priv_root != NULL && REISERFS_SB(s)->xattr_root == NULL case possible. I guess that commit 6cb4aff0a77cc0e6 ("reiserfs: fix oops while creating privroot with selinux enabled") wanted to check xattr_root != NULL before reiserfs_xattr_jcreate_nblocks(), for the changelog is talking about the xattr root. The issue is that while creating the privroot during mount reiserfs_security_init calls reiserfs_xattr_jcreate_nblocks which dereferences the xattr root. The xattr root doesn't exist, so we get an oops. Therefore, update reiserfs_xattrs_initialized() to check both the privroot and the xattr root. Link: https://syzkaller.appspot.com/bug?id=8abaedbdeb32c861dc5340544284167dd0e46cde # [1] Reported-and-tested-by: syzbot Signed-off-by: Tetsuo Handa Fixes: 6cb4aff0a77c ("reiserfs: fix oops while creating privroot with selinux enabled") Acked-by: Jeff Mahoney Acked-by: Jan Kara Signed-off-by: Linus Torvalds --- fs/reiserfs/xattr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/reiserfs/xattr.h b/fs/reiserfs/xattr.h index 9b3b06da568c..e47fde1182de 100644 --- a/fs/reiserfs/xattr.h +++ b/fs/reiserfs/xattr.h @@ -44,7 +44,7 @@ void reiserfs_security_free(struct reiserfs_security_handle *sec); static inline int reiserfs_xattrs_initialized(struct super_block *sb) { - return REISERFS_SB(sb)->priv_root != NULL; + return REISERFS_SB(sb)->priv_root && REISERFS_SB(sb)->xattr_root; } #define xattr_size(size) ((size) + sizeof(struct reiserfs_xattr_header)) -- cgit v1.2.3 From 4b982bd0f383db9132e892c0c5144117359a6289 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 1 Apr 2021 08:38:34 -0600 Subject: io_uring: don't mark S_ISBLK async work as unbounded S_ISBLK is marked as unbounded work for async preparation, because it doesn't match S_ISREG. That is incorrect, as any read/write to a block device is also a bounded operation. Fix it up and ensure that S_ISBLK isn't marked unbounded. Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 6d7a1b69712b..a16b7df934d1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1213,7 +1213,7 @@ static void io_prep_async_work(struct io_kiocb *req) if (req->flags & REQ_F_ISREG) { if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL)) io_wq_hash_work(&req->work, file_inode(req->file)); - } else { + } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) { if (def->unbound_nonreg_file) req->work.flags |= IO_WQ_WORK_UNBOUND; } -- cgit v1.2.3 From 696ee88a7c50f96573f98aa76cc74286033140c1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 1 Apr 2021 09:55:04 +0100 Subject: io_uring/io-wq: protect against sprintf overflow task_pid may be large enough to not fit into the left space of TASK_COMM_LEN-sized buffers and overflow in sprintf. We not so care about uniqueness, so replace it with safer snprintf(). Reported-by: Alexey Dobriyan Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1702c6145d7e1c46fbc382f28334c02e1a3d3994.1617267273.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 4 ++-- fs/io_uring.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 7434eb40ca8c..433c4d3c3c1c 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -484,7 +484,7 @@ static int io_wqe_worker(void *data) worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING); io_wqe_inc_running(worker); - sprintf(buf, "iou-wrk-%d", wq->task_pid); + snprintf(buf, sizeof(buf), "iou-wrk-%d", wq->task_pid); set_task_comm(current, buf); while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { @@ -711,7 +711,7 @@ static int io_wq_manager(void *data) char buf[TASK_COMM_LEN]; int node; - sprintf(buf, "iou-mgr-%d", wq->task_pid); + snprintf(buf, sizeof(buf), "iou-mgr-%d", wq->task_pid); set_task_comm(current, buf); do { diff --git a/fs/io_uring.c b/fs/io_uring.c index a16b7df934d1..4a6701b5065e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6718,7 +6718,7 @@ static int io_sq_thread(void *data) char buf[TASK_COMM_LEN]; DEFINE_WAIT(wait); - sprintf(buf, "iou-sqp-%d", sqd->task_pid); + snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid); set_task_comm(current, buf); current->pf_io_worker = NULL; -- cgit v1.2.3 From 07204f21577a1d882f0259590c3553fe6a476381 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 1 Apr 2021 12:18:48 +0100 Subject: io_uring: fix EIOCBQUEUED iter revert iov_iter_revert() is done in completion handlers that happensf before read/write returns -EIOCBQUEUED, no need to repeat reverting afterwards. Moreover, even though it may appear being just a no-op, it's actually races with 1) user forging a new iovec of a different size 2) reissue, that is done via io-wq continues completely asynchronously. Fixes: 3e6a0d3c7571c ("io_uring: fix -EAGAIN retry with IOPOLL") Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 4a6701b5065e..717942474fa9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3284,8 +3284,6 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) ret = io_iter_do_read(req, iter); if (ret == -EIOCBQUEUED) { - if (req->async_data) - iov_iter_revert(iter, io_size - iov_iter_count(iter)); goto out_free; } else if (ret == -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ @@ -3418,8 +3416,6 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) /* no retry on NONBLOCK nor RWF_NOWAIT */ if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) goto done; - if (ret2 == -EIOCBQUEUED && req->async_data) - iov_iter_revert(iter, io_size - iov_iter_count(iter)); if (!force_nonblock || ret2 != -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) -- cgit v1.2.3 From 9b5b872215fe6d1ca6a1ef411f130bd58e269012 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 2 Apr 2021 10:29:36 +0200 Subject: file: fix close_range() for unshare+cloexec syzbot reported a bug when putting the last reference to a tasks file descriptor table. Debugging this showed we didn't recalculate the current maximum fd number for CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC after we unshared the file descriptors table. So max_fd could exceed the current fdtable maximum causing us to set excessive bits. As a concrete example, let's say the user requested everything from fd 4 to ~0UL to be closed and their current fdtable size is 256 with their highest open fd being 4. With CLOSE_RANGE_UNSHARE the caller will end up with a new fdtable which has room for 64 file descriptors since that is the lowest fdtable size we accept. But now max_fd will still point to 255 and needs to be adjusted. Fix this by retrieving the correct maximum fd value in __range_cloexec(). Reported-by: syzbot+283ce5a46486d6acdbaf@syzkaller.appspotmail.com Fixes: 582f1fb6b721 ("fs, close_range: add flag CLOSE_RANGE_CLOEXEC") Fixes: fec8a6a69103 ("close_range: unshare all fds for CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC") Cc: Christoph Hellwig Cc: Giuseppe Scrivano Cc: Al Viro Cc: linux-fsdevel@vger.kernel.org Cc: stable@vger.kernel.org Signed-off-by: Christian Brauner --- fs/file.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/file.c b/fs/file.c index f3a4bac2cbe9..f633348029a5 100644 --- a/fs/file.c +++ b/fs/file.c @@ -629,17 +629,30 @@ int close_fd(unsigned fd) } EXPORT_SYMBOL(close_fd); /* for ksys_close() */ +/** + * last_fd - return last valid index into fd table + * @cur_fds: files struct + * + * Context: Either rcu read lock or files_lock must be held. + * + * Returns: Last valid index into fdtable. + */ +static inline unsigned last_fd(struct fdtable *fdt) +{ + return fdt->max_fds - 1; +} + static inline void __range_cloexec(struct files_struct *cur_fds, unsigned int fd, unsigned int max_fd) { struct fdtable *fdt; - if (fd > max_fd) - return; - + /* make sure we're using the correct maximum value */ spin_lock(&cur_fds->file_lock); fdt = files_fdtable(cur_fds); - bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1); + max_fd = min(last_fd(fdt), max_fd); + if (fd <= max_fd) + bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1); spin_unlock(&cur_fds->file_lock); } -- cgit v1.2.3 From f8b78caf21d5bc3fcfc40c18898f9d52ed1451a5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 20 Nov 2020 17:10:28 +0000 Subject: block: don't ignore REQ_NOWAIT for direct IO If IOCB_NOWAIT is set on submission, then that needs to get propagated to REQ_NOWAIT on the block side. Otherwise we completely lose this information, and any issuer of IOCB_NOWAIT IO will potentially end up blocking on eg request allocation on the storage side. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/block_dev.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'fs') diff --git a/fs/block_dev.c b/fs/block_dev.c index 28d583fcdc2c..09d6f7229db9 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -275,6 +275,8 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter, bio.bi_opf = dio_bio_write_op(iocb); task_io_account_write(ret); } + if (iocb->ki_flags & IOCB_NOWAIT) + bio.bi_opf |= REQ_NOWAIT; if (iocb->ki_flags & IOCB_HIPRI) bio_set_polled(&bio, iocb); @@ -428,6 +430,8 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, bio->bi_opf = dio_bio_write_op(iocb); task_io_account_write(bio->bi_iter.bi_size); } + if (iocb->ki_flags & IOCB_NOWAIT) + bio->bi_opf |= REQ_NOWAIT; dio->size += bio->bi_iter.bi_size; pos += bio->bi_iter.bi_size; -- cgit v1.2.3 From 230d50d448acb6639991440913299e50cacf1daf Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 1 Apr 2021 20:41:15 -0600 Subject: io_uring: move reissue into regular IO path It's non-obvious how retry is done for block backed files, when it happens off the kiocb done path. It also makes it tricky to deal with the iov_iter handling. Just mark the req as needing a reissue, and handling it from the submission path instead. This makes it directly obvious that we're not re-importing the iovec from userspace past the submit point, and it means that we can just reuse our usual -EAGAIN retry path from the read/write handling. At some point in the future, we'll gain the ability to always reliably return -EAGAIN through the stack. A previous attempt on the block side didn't pan out and got reverted, hence the need to check for this information out-of-band right now. Signed-off-by: Jens Axboe --- fs/io_uring.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 717942474fa9..8be542050648 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -697,6 +697,7 @@ enum { REQ_F_NO_FILE_TABLE_BIT, REQ_F_LTIMEOUT_ACTIVE_BIT, REQ_F_COMPLETE_INLINE_BIT, + REQ_F_REISSUE_BIT, /* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT, @@ -740,6 +741,8 @@ enum { REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT), /* completion is deferred through io_comp_state */ REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT), + /* caller should reissue async */ + REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT), }; struct async_poll { @@ -2503,8 +2506,10 @@ static void __io_complete_rw(struct io_kiocb *req, long res, long res2, if (req->rw.kiocb.ki_flags & IOCB_WRITE) kiocb_end_write(req); - if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_reissue(req)) + if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_should_reissue(req)) { + req->flags |= REQ_F_REISSUE; return; + } if (res != req->result) req_set_fail_links(req); if (req->flags & REQ_F_BUFFER_SELECTED) @@ -3283,9 +3288,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) ret = io_iter_do_read(req, iter); - if (ret == -EIOCBQUEUED) { - goto out_free; - } else if (ret == -EAGAIN) { + if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { /* IOPOLL retry should happen for io-wq threads */ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) goto done; @@ -3295,6 +3298,8 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) /* some cases will consume bytes even on error returns */ iov_iter_revert(iter, io_size - iov_iter_count(iter)); ret = 0; + } else if (ret == -EIOCBQUEUED) { + goto out_free; } else if (ret <= 0 || ret == io_size || !force_nonblock || (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) { /* read all, failed, already did sync or don't want to retry */ @@ -3407,6 +3412,9 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) else ret2 = -EINVAL; + if (req->flags & REQ_F_REISSUE) + ret2 = -EAGAIN; + /* * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just * retry them without IOCB_NOWAIT. @@ -6160,6 +6168,7 @@ static void io_wq_submit_work(struct io_wq_work *work) ret = -ECANCELED; if (!ret) { + req->flags &= ~REQ_F_REISSUE; do { ret = io_issue_sqe(req, 0); /* -- cgit v1.2.3 From e82ad4853948382d37ac512b27a3e70b6f01c103 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 2 Apr 2021 19:45:34 -0600 Subject: io_uring: fix !CONFIG_BLOCK compilation failure kernel test robot correctly pinpoints a compilation failure if CONFIG_BLOCK isn't set: fs/io_uring.c: In function '__io_complete_rw': >> fs/io_uring.c:2509:48: error: implicit declaration of function 'io_rw_should_reissue'; did you mean 'io_rw_reissue'? [-Werror=implicit-function-declaration] 2509 | if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_should_reissue(req)) { | ^~~~~~~~~~~~~~~~~~~~ | io_rw_reissue cc1: some warnings being treated as errors Ensure that we have a stub declaration of io_rw_should_reissue() for !CONFIG_BLOCK. Fixes: 230d50d448ac ("io_uring: move reissue into regular IO path") Reported-by: kernel test robot Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 8be542050648..65a17d560a73 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2479,6 +2479,11 @@ static bool io_rw_should_reissue(struct io_kiocb *req) return false; return true; } +#else +static bool io_rw_should_reissue(struct io_kiocb *req) +{ + return false; +} #endif static bool io_rw_reissue(struct io_kiocb *req) -- cgit v1.2.3 From 7d01ef7585c07afaf487759a48486228cd065726 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 6 Apr 2021 12:33:07 -0400 Subject: Make sure nd->path.mnt and nd->path.dentry are always valid pointers Initialize them in set_nameidata() and make sure that terminate_walk() clears them once the pointers become potentially invalid (i.e. we leave RCU mode or drop them in non-RCU one). Currently we have "path_init() always initializes them and nobody accesses them outside of path_init()/terminate_walk() segments", which is asking for trouble. With that change we would have nd->path.{mnt,dentry} 1) always valid - NULL or pointing to currently allocated objects. 2) non-NULL while we are successfully walking 3) NULL when we are not walking at all 4) contributing to refcounts whenever non-NULL outside of RCU mode. Fixes: 6c6ec2b0a3e0 ("fs: add support for LOOKUP_CACHED") Reported-by: syzbot+c88a7030da47945a3cc3@syzkaller.appspotmail.com Tested-by: Christian Brauner Signed-off-by: Al Viro --- fs/namei.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/namei.c b/fs/namei.c index 216f16e74351..fc8760d4314e 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -579,6 +579,8 @@ static void set_nameidata(struct nameidata *p, int dfd, struct filename *name) p->stack = p->internal; p->dfd = dfd; p->name = name; + p->path.mnt = NULL; + p->path.dentry = NULL; p->total_link_count = old ? old->total_link_count : 0; p->saved = old; current->nameidata = p; @@ -652,6 +654,8 @@ static void terminate_walk(struct nameidata *nd) rcu_read_unlock(); } nd->depth = 0; + nd->path.mnt = NULL; + nd->path.dentry = NULL; } /* path_put is needed afterwards regardless of success or failure */ @@ -2322,8 +2326,6 @@ static const char *path_init(struct nameidata *nd, unsigned flags) } nd->root.mnt = NULL; - nd->path.mnt = NULL; - nd->path.dentry = NULL; /* Absolute pathname -- fetch the root (LOOKUP_IN_ROOT uses nd->dfd). */ if (*s == '/' && !(flags & LOOKUP_IN_ROOT)) { -- cgit v1.2.3 From 4f0ed93fb92d3528c73c80317509df3f800a222b Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 6 Apr 2021 19:46:51 -0400 Subject: LOOKUP_MOUNTPOINT: we are cleaning "jumped" flag too late That (and traversals in case of umount .) should be done before complete_walk(). Either a braino or mismerge damage on queue reorders - either way, I should've spotted that much earlier. Fucked-up-by: Al Viro X-Paperbag: Brown Fixes: 161aff1d93ab "LOOKUP_MOUNTPOINT: fold path_mountpointat() into path_lookupat()" Cc: stable@vger.kernel.org # v5.7+ Signed-off-by: Al Viro --- fs/namei.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/namei.c b/fs/namei.c index fc8760d4314e..48a2f288e802 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -2421,16 +2421,16 @@ static int path_lookupat(struct nameidata *nd, unsigned flags, struct path *path while (!(err = link_path_walk(s, nd)) && (s = lookup_last(nd)) != NULL) ; + if (!err && unlikely(nd->flags & LOOKUP_MOUNTPOINT)) { + err = handle_lookup_down(nd); + nd->flags &= ~LOOKUP_JUMPED; // no d_weak_revalidate(), please... + } if (!err) err = complete_walk(nd); if (!err && nd->flags & LOOKUP_DIRECTORY) if (!d_can_lookup(nd->path.dentry)) err = -ENOTDIR; - if (!err && unlikely(nd->flags & LOOKUP_MOUNTPOINT)) { - err = handle_lookup_down(nd); - nd->flags &= ~LOOKUP_JUMPED; // no d_weak_revalidate(), please... - } if (!err) { *path = nd->path; nd->path.mnt = NULL; -- cgit v1.2.3 From 4e456b30f78c429b183db420e23b26cde7e03a78 Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Wed, 31 Mar 2021 14:35:24 +0000 Subject: cifs: On cifs_reconnect, resolve the hostname again. On cifs_reconnect, make sure that DNS resolution happens again. It could be the cause of connection to go dead in the first place. This also contains the fix for a build issue identified by Intel bot. Reported-by: kernel test robot Signed-off-by: Shyam Prasad N Reviewed-by: Paulo Alcantara (SUSE) Reviewed-by: Pavel Shilovsky CC: # 5.11+ Signed-off-by: Steve French --- fs/cifs/Kconfig | 3 +-- fs/cifs/Makefile | 5 +++-- fs/cifs/connect.c | 17 ++++++++++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/cifs/Kconfig b/fs/cifs/Kconfig index fe03cbdae959..bf52e9326ebe 100644 --- a/fs/cifs/Kconfig +++ b/fs/cifs/Kconfig @@ -18,6 +18,7 @@ config CIFS select CRYPTO_AES select CRYPTO_LIB_DES select KEYS + select DNS_RESOLVER help This is the client VFS module for the SMB3 family of NAS protocols, (including support for the most recent, most secure dialect SMB3.1.1) @@ -112,7 +113,6 @@ config CIFS_WEAK_PW_HASH config CIFS_UPCALL bool "Kerberos/SPNEGO advanced session setup" depends on CIFS - select DNS_RESOLVER help Enables an upcall mechanism for CIFS which accesses userspace helper utilities to provide SPNEGO packaged (RFC 4178) Kerberos tickets @@ -179,7 +179,6 @@ config CIFS_DEBUG_DUMP_KEYS config CIFS_DFS_UPCALL bool "DFS feature support" depends on CIFS - select DNS_RESOLVER help Distributed File System (DFS) support is used to access shares transparently in an enterprise name space, even if the share diff --git a/fs/cifs/Makefile b/fs/cifs/Makefile index 5213b20843b5..3ee3b7de4ded 100644 --- a/fs/cifs/Makefile +++ b/fs/cifs/Makefile @@ -10,13 +10,14 @@ cifs-y := trace.o cifsfs.o cifssmb.o cifs_debug.o connect.o dir.o file.o \ cifs_unicode.o nterr.o cifsencrypt.o \ readdir.o ioctl.o sess.o export.o smb1ops.o unc.o winucase.o \ smb2ops.o smb2maperror.o smb2transport.o \ - smb2misc.o smb2pdu.o smb2inode.o smb2file.o cifsacl.o fs_context.o + smb2misc.o smb2pdu.o smb2inode.o smb2file.o cifsacl.o fs_context.o \ + dns_resolve.o cifs-$(CONFIG_CIFS_XATTR) += xattr.o cifs-$(CONFIG_CIFS_UPCALL) += cifs_spnego.o -cifs-$(CONFIG_CIFS_DFS_UPCALL) += dns_resolve.o cifs_dfs_ref.o dfs_cache.o +cifs-$(CONFIG_CIFS_DFS_UPCALL) += cifs_dfs_ref.o dfs_cache.o cifs-$(CONFIG_CIFS_SWN_UPCALL) += netlink.o cifs_swn.o diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index eec8a2052da2..24668eb006c6 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -87,7 +87,6 @@ static void cifs_prune_tlinks(struct work_struct *work); * * This should be called with server->srv_mutex held. */ -#ifdef CONFIG_CIFS_DFS_UPCALL static int reconn_set_ipaddr_from_hostname(struct TCP_Server_Info *server) { int rc; @@ -124,6 +123,7 @@ static int reconn_set_ipaddr_from_hostname(struct TCP_Server_Info *server) return !rc ? -1 : 0; } +#ifdef CONFIG_CIFS_DFS_UPCALL /* These functions must be called with server->srv_mutex held */ static void reconn_set_next_dfs_target(struct TCP_Server_Info *server, struct cifs_sb_info *cifs_sb, @@ -321,14 +321,29 @@ cifs_reconnect(struct TCP_Server_Info *server) #endif #ifdef CONFIG_CIFS_DFS_UPCALL + if (cifs_sb && cifs_sb->origin_fullpath) /* * Set up next DFS target server (if any) for reconnect. If DFS * feature is disabled, then we will retry last server we * connected to before. */ reconn_set_next_dfs_target(server, cifs_sb, &tgt_list, &tgt_it); + else { +#endif + /* + * Resolve the hostname again to make sure that IP address is up-to-date. + */ + rc = reconn_set_ipaddr_from_hostname(server); + if (rc) { + cifs_dbg(FYI, "%s: failed to resolve hostname: %d\n", + __func__, rc); + } + +#ifdef CONFIG_CIFS_DFS_UPCALL + } #endif + #ifdef CONFIG_CIFS_SWN_UPCALL } #endif -- cgit v1.2.3 From d135be0a7fb83f4dd68721b3355fec6de686834c Mon Sep 17 00:00:00 2001 From: Wan Jiabing Date: Thu, 1 Apr 2021 15:51:17 +0800 Subject: fs: cifs: Remove unnecessary struct declaration struct cifs_readdata is declared twice. One is declared at 208th line. And struct cifs_readdata is defined blew. The declaration here is not needed. Remove the duplicate. Signed-off-by: Wan Jiabing Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/cifsglob.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index 67c056a9a519..ec824ab8c5ca 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -1283,8 +1283,6 @@ struct cifs_aio_ctx { bool direct_io; }; -struct cifs_readdata; - /* asynchronous read support */ struct cifs_readdata { struct kref refcount; -- cgit v1.2.3 From 0fc9322ab5e1fe6910c9673e1a7ff29f7dd72611 Mon Sep 17 00:00:00 2001 From: Maciek Borzecki Date: Tue, 6 Apr 2021 17:02:29 +0200 Subject: cifs: escape spaces in share names Commit 653a5efb849a ("cifs: update super_operations to show_devname") introduced the display of devname for cifs mounts. However, when mounting a share which has a whitespace in the name, that exact share name is also displayed in mountinfo. Make sure that all whitespace is escaped. Signed-off-by: Maciek Borzecki CC: # 5.11+ Reviewed-by: Shyam Prasad N Signed-off-by: Steve French --- fs/cifs/cifsfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index 099ad9f3660b..5ddd20b62484 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -476,7 +476,8 @@ static int cifs_show_devname(struct seq_file *m, struct dentry *root) seq_puts(m, "none"); else { convert_delimiter(devname, '/'); - seq_puts(m, devname); + /* escape all spaces in share names */ + seq_escape(m, devname, " \t"); kfree(devname); } return 0; -- cgit v1.2.3 From 6ad7f2332e84c46f0c94e73e05b5b7c2bc1a6b74 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 8 Apr 2021 01:54:39 +0100 Subject: io_uring: clear F_REISSUE right after getting it There are lots of ways r/w request may continue its path after getting REQ_F_REISSUE, it's not necessarily io-wq and can be, e.g. apoll, and submitted via io_async_task_func() -> __io_req_task_submit() Clear the flag right after getting it, so the next attempt is well prepared regardless how the request will be executed. Fixes: 230d50d448ac ("io_uring: move reissue into regular IO path") Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/11dcead939343f4e27cab0074d34afcab771bfa4.1617842918.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index 65a17d560a73..f1881ac0744b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3294,6 +3294,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) ret = io_iter_do_read(req, iter); if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { + req->flags &= ~REQ_F_REISSUE; /* IOPOLL retry should happen for io-wq threads */ if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) goto done; @@ -3417,8 +3418,10 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) else ret2 = -EINVAL; - if (req->flags & REQ_F_REISSUE) + if (req->flags & REQ_F_REISSUE) { + req->flags &= ~REQ_F_REISSUE; ret2 = -EAGAIN; + } /* * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just @@ -6173,7 +6176,6 @@ static void io_wq_submit_work(struct io_wq_work *work) ret = -ECANCELED; if (!ret) { - req->flags &= ~REQ_F_REISSUE; do { ret = io_issue_sqe(req, 0); /* -- cgit v1.2.3 From 9728463737db027557e8ba315cbbca6b81122c04 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 8 Apr 2021 19:28:03 +0100 Subject: io_uring: fix rw req completion WARNING: at fs/io_uring.c:8578 io_ring_exit_work.cold+0x0/0x18 As reissuing is now passed back by REQ_F_REISSUE and kiocb_done() internally uses __io_complete_rw(), it may stop after setting the flag so leaving a dangling request. There are tricky edge cases, e.g. reading beyound file, boundary, so the easiest way is to hand code reissue in kiocb_done() as __io_complete_rw() was doing for us before. Fixes: 230d50d448ac ("io_uring: move reissue into regular IO path") Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f602250d292f8a84cca9a01d747744d1e797be26.1617842918.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'fs') diff --git a/fs/io_uring.c b/fs/io_uring.c index f1881ac0744b..bd14327c8e7e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2762,6 +2762,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); struct io_async_rw *io = req->async_data; + bool check_reissue = kiocb->ki_complete == io_complete_rw; /* add previously done IO, if any */ if (io && io->bytes_done > 0) { @@ -2777,6 +2778,18 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, __io_complete_rw(req, ret, 0, issue_flags); else io_rw_done(kiocb, ret); + + if (check_reissue && req->flags & REQ_F_REISSUE) { + req->flags &= ~REQ_F_REISSUE; + if (!io_rw_reissue(req)) { + int cflags = 0; + + req_set_fail_links(req); + if (req->flags & REQ_F_BUFFER_SELECTED) + cflags = io_put_rw_kbuf(req); + __io_req_complete(req, issue_flags, ret, cflags); + } + } } static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter) -- cgit v1.2.3 From c60eb049f4a19ddddcd3ee97a9c79ab8066a6a03 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 8 Apr 2021 01:54:42 +0100 Subject: io-wq: cancel unbounded works on io-wq destroy WARNING: CPU: 5 PID: 227 at fs/io_uring.c:8578 io_ring_exit_work+0xe6/0x470 RIP: 0010:io_ring_exit_work+0xe6/0x470 Call Trace: process_one_work+0x206/0x400 worker_thread+0x4a/0x3d0 kthread+0x129/0x170 ret_from_fork+0x22/0x30 INFO: task lfs-openat:2359 blocked for more than 245 seconds. task:lfs-openat state:D stack: 0 pid: 2359 ppid: 1 flags:0x00000004 Call Trace: ... wait_for_completion+0x8b/0xf0 io_wq_destroy_manager+0x24/0x60 io_wq_put_and_exit+0x18/0x30 io_uring_clean_tctx+0x76/0xa0 __io_uring_files_cancel+0x1b9/0x2e0 do_exit+0xc0/0xb40 ... Even after io-wq destroy has been issued io-wq worker threads will continue executing all left work items as usual, and may hang waiting for I/O that won't ever complete (aka unbounded). [<0>] pipe_read+0x306/0x450 [<0>] io_iter_do_read+0x1e/0x40 [<0>] io_read+0xd5/0x330 [<0>] io_issue_sqe+0xd21/0x18a0 [<0>] io_wq_submit_work+0x6c/0x140 [<0>] io_worker_handle_work+0x17d/0x400 [<0>] io_wqe_worker+0x2c0/0x330 [<0>] ret_from_fork+0x22/0x30 Cancel all unbounded I/O instead of executing them. This changes the user visible behaviour, but that's inevitable as io-wq is not per task. Suggested-by: Jens Axboe Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/cd4b543154154cba055cf86f351441c2174d7f71.1617842918.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'fs') diff --git a/fs/io-wq.c b/fs/io-wq.c index 433c4d3c3c1c..4eba531bea5a 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -415,6 +415,7 @@ static void io_worker_handle_work(struct io_worker *worker) { struct io_wqe *wqe = worker->wqe; struct io_wq *wq = wqe->wq; + bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); do { struct io_wq_work *work; @@ -444,6 +445,9 @@ get_next: unsigned int hash = io_get_work_hash(work); next_hashed = wq_next_work(work); + + if (unlikely(do_kill) && (work->flags & IO_WQ_WORK_UNBOUND)) + work->flags |= IO_WQ_WORK_CANCEL; wq->do_work(work); io_assign_current_work(worker, NULL); -- cgit v1.2.3 From 90bd070aae6c4fb5d302f9c4b9c88be60c8197ec Mon Sep 17 00:00:00 2001 From: Wengang Wang Date: Fri, 9 Apr 2021 13:27:29 -0700 Subject: ocfs2: fix deadlock between setattr and dio_end_io_write The following deadlock is detected: truncate -> setattr path is waiting for pending direct IO to be done (inode->i_dio_count become zero) with inode->i_rwsem held (down_write). PID: 14827 TASK: ffff881686a9af80 CPU: 20 COMMAND: "ora_p005_hrltd9" #0 __schedule at ffffffff818667cc #1 schedule at ffffffff81866de6 #2 inode_dio_wait at ffffffff812a2d04 #3 ocfs2_setattr at ffffffffc05f322e [ocfs2] #4 notify_change at ffffffff812a5a09 #5 do_truncate at ffffffff812808f5 #6 do_sys_ftruncate.constprop.18 at ffffffff81280cf2 #7 sys_ftruncate at ffffffff81280d8e #8 do_syscall_64 at ffffffff81003949 #9 entry_SYSCALL_64_after_hwframe at ffffffff81a001ad dio completion path is going to complete one direct IO (decrement inode->i_dio_count), but before that it hung at locking inode->i_rwsem: #0 __schedule+700 at ffffffff818667cc #1 schedule+54 at ffffffff81866de6 #2 rwsem_down_write_failed+536 at ffffffff8186aa28 #3 call_rwsem_down_write_failed+23 at ffffffff8185a1b7 #4 down_write+45 at ffffffff81869c9d #5 ocfs2_dio_end_io_write+180 at ffffffffc05d5444 [ocfs2] #6 ocfs2_dio_end_io+85 at ffffffffc05d5a85 [ocfs2] #7 dio_complete+140 at ffffffff812c873c #8 dio_aio_complete_work+25 at ffffffff812c89f9 #9 process_one_work+361 at ffffffff810b1889 #10 worker_thread+77 at ffffffff810b233d #11 kthread+261 at ffffffff810b7fd5 #12 ret_from_fork+62 at ffffffff81a0035e Thus above forms ABBA deadlock. The same deadlock was mentioned in upstream commit 28f5a8a7c033 ("ocfs2: should wait dio before inode lock in ocfs2_setattr()"). It seems that that commit only removed the cluster lock (the victim of above dead lock) from the ABBA deadlock party. End-user visible effects: Process hang in truncate -> ocfs2_setattr path and other processes hang at ocfs2_dio_end_io_write path. This is to fix the deadlock itself. It removes inode_lock() call from dio completion path to remove the deadlock and add ip_alloc_sem lock in setattr path to synchronize the inode modifications. [wen.gang.wang@oracle.com: remove the "had_alloc_lock" as suggested] Link: https://lkml.kernel.org/r/20210402171344.1605-1-wen.gang.wang@oracle.com Link: https://lkml.kernel.org/r/20210331203654.3911-1-wen.gang.wang@oracle.com Signed-off-by: Wengang Wang Reviewed-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ocfs2/aops.c | 11 +---------- fs/ocfs2/file.c | 8 ++++++-- 2 files changed, 7 insertions(+), 12 deletions(-) (limited to 'fs') diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index 3bfb4147895a..ad20403b383f 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c @@ -2295,7 +2295,7 @@ static int ocfs2_dio_end_io_write(struct inode *inode, struct ocfs2_alloc_context *meta_ac = NULL; handle_t *handle = NULL; loff_t end = offset + bytes; - int ret = 0, credits = 0, locked = 0; + int ret = 0, credits = 0; ocfs2_init_dealloc_ctxt(&dealloc); @@ -2306,13 +2306,6 @@ static int ocfs2_dio_end_io_write(struct inode *inode, !dwc->dw_orphaned) goto out; - /* ocfs2_file_write_iter will get i_mutex, so we need not lock if we - * are in that context. */ - if (dwc->dw_writer_pid != task_pid_nr(current)) { - inode_lock(inode); - locked = 1; - } - ret = ocfs2_inode_lock(inode, &di_bh, 1); if (ret < 0) { mlog_errno(ret); @@ -2393,8 +2386,6 @@ out: if (meta_ac) ocfs2_free_alloc_context(meta_ac); ocfs2_run_deallocs(osb, &dealloc); - if (locked) - inode_unlock(inode); ocfs2_dio_free_write_ctx(inode, dwc); return ret; diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 6611c64ca0be..5edc1d0cf115 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -1245,22 +1245,24 @@ int ocfs2_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, goto bail_unlock; } } + down_write(&OCFS2_I(inode)->ip_alloc_sem); handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS + 2 * ocfs2_quota_trans_credits(sb)); if (IS_ERR(handle)) { status = PTR_ERR(handle); mlog_errno(status); - goto bail_unlock; + goto bail_unlock_alloc; } status = __dquot_transfer(inode, transfer_to); if (status < 0) goto bail_commit; } else { + down_write(&OCFS2_I(inode)->ip_alloc_sem); handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); if (IS_ERR(handle)) { status = PTR_ERR(handle); mlog_errno(status); - goto bail_unlock; + goto bail_unlock_alloc; } } @@ -1273,6 +1275,8 @@ int ocfs2_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, bail_commit: ocfs2_commit_trans(osb, handle); +bail_unlock_alloc: + up_write(&OCFS2_I(inode)->ip_alloc_sem); bail_unlock: if (status && inode_locked) { ocfs2_inode_unlock_tracker(inode, 1, &oh, had_lock); -- cgit v1.2.3 From df41872b68601059dd4a84858952dcae58acd331 Mon Sep 17 00:00:00 2001 From: Jack Qiu Date: Fri, 9 Apr 2021 13:27:35 -0700 Subject: fs: direct-io: fix missing sdio->boundary I encountered a hung task issue, but not a performance one. I run DIO on a device (need lba continuous, for example open channel ssd), maybe hungtask in below case: DIO: Checkpoint: get addr A(at boundary), merge into BIO, no submit because boundary missing flush dirty data(get addr A+1), wait IO(A+1) writeback timeout, because DIO(A) didn't submit get addr A+2 fail, because checkpoint is doing dio_send_cur_page() may clear sdio->boundary, so prevent it from missing a boundary. Link: https://lkml.kernel.org/r/20210322042253.38312-1-jack.qiu@huawei.com Fixes: b1058b981272 ("direct-io: submit bio after boundary buffer is added to it") Signed-off-by: Jack Qiu Reviewed-by: Jan Kara Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/direct-io.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/direct-io.c b/fs/direct-io.c index b61491bf3166..b2e86e739d7a 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -812,6 +812,7 @@ submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, struct buffer_head *map_bh) { int ret = 0; + int boundary = sdio->boundary; /* dio_send_cur_page may clear it */ if (dio->op == REQ_OP_WRITE) { /* @@ -850,10 +851,10 @@ submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits; out: /* - * If sdio->boundary then we want to schedule the IO now to + * If boundary then we want to schedule the IO now to * avoid metadata seeks. */ - if (sdio->boundary) { + if (boundary) { ret = dio_send_cur_page(dio, sdio, map_bh); if (sdio->bio) dio_bio_submit(dio, sdio); -- cgit v1.2.3 From 53b74fa990bf76f290aa5930abfcf37424a1a865 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 8 Apr 2021 17:25:28 +0900 Subject: btrfs: zoned: move superblock logging zone location Moves the location of the superblock logging zones. The new locations of the logging zones are now determined based on fixed block addresses instead of on fixed zone numbers. The old placement method based on fixed zone numbers causes problems when one needs to inspect a file system image without access to the drive zone information. In such case, the super block locations cannot be reliably determined as the zone size is unknown. By locating the superblock logging zones using fixed addresses, we can scan a dumped file system image without the zone information since a super block copy will always be present at or after the fixed known locations. Introduce the following three pairs of zones containing fixed offset locations, regardless of the device zone size. - primary superblock: offset 0B (and the following zone) - first copy: offset 512G (and the following zone) - Second copy: offset 4T (4096G, and the following zone) If a logging zone is outside of the disk capacity, we do not record the superblock copy. The first copy position is much larger than for a non-zoned filesystem, which is at 64M. This is to avoid overlapping with the log zones for the primary superblock. This higher location is arbitrary but allows supporting devices with very large zone sizes, plus some space around in between. Such large zone size is unrealistic and very unlikely to ever be seen in real devices. Currently, SMR disks have a zone size of 256MB, and we are expecting ZNS drives to be in the 1-4GB range, so this limit gives us room to breathe. For now, we only allow zone sizes up to 8GB. The maximum zone size that would still fit in the space is 256G. The fixed location addresses are somewhat arbitrary, with the intent of maintaining superblock reliability for smaller and larger devices, with the preference for the latter. For this reason, there are two superblocks under the first 1T. This should cover use cases for physical devices and for emulated/device-mapper devices. The superblock logging zones are reserved for superblock logging and never used for data or metadata blocks. Note that we only reserve the two zones per primary/copy actually used for superblock logging. We do not reserve the ranges of zones possibly containing superblocks with the largest supported zone size (0-16GB, 512G-528GB, 4096G-4112G). The zones containing the fixed location offsets used to store superblocks on a non-zoned volume are also reserved to avoid confusion. Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/btrfs/zoned.c | 53 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 11 deletions(-) (limited to 'fs') diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 43948bd40e02..ba7a303300a3 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -21,9 +21,30 @@ /* Pseudo write pointer value for conventional zone */ #define WP_CONVENTIONAL ((u64)-2) +/* + * Location of the first zone of superblock logging zone pairs. + * + * - primary superblock: 0B (zone 0) + * - first copy: 512G (zone starting at that offset) + * - second copy: 4T (zone starting at that offset) + */ +#define BTRFS_SB_LOG_PRIMARY_OFFSET (0ULL) +#define BTRFS_SB_LOG_FIRST_OFFSET (512ULL * SZ_1G) +#define BTRFS_SB_LOG_SECOND_OFFSET (4096ULL * SZ_1G) + +#define BTRFS_SB_LOG_FIRST_SHIFT const_ilog2(BTRFS_SB_LOG_FIRST_OFFSET) +#define BTRFS_SB_LOG_SECOND_SHIFT const_ilog2(BTRFS_SB_LOG_SECOND_OFFSET) + /* Number of superblock log zones */ #define BTRFS_NR_SB_LOG_ZONES 2 +/* + * Maximum supported zone size. Currently, SMR disks have a zone size of + * 256MiB, and we are expecting ZNS drives to be in the 1-4GiB range. We do not + * expect the zone size to become larger than 8GiB in the near future. + */ +#define BTRFS_MAX_ZONE_SIZE SZ_8G + static int copy_zone_info_cb(struct blk_zone *zone, unsigned int idx, void *data) { struct blk_zone *zones = data; @@ -111,23 +132,22 @@ static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones, } /* - * The following zones are reserved as the circular buffer on ZONED btrfs. - * - The primary superblock: zones 0 and 1 - * - The first copy: zones 16 and 17 - * - The second copy: zones 1024 or zone at 256GB which is minimum, and - * the following one + * Get the first zone number of the superblock mirror */ static inline u32 sb_zone_number(int shift, int mirror) { - ASSERT(mirror < BTRFS_SUPER_MIRROR_MAX); + u64 zone; + ASSERT(mirror < BTRFS_SUPER_MIRROR_MAX); switch (mirror) { - case 0: return 0; - case 1: return 16; - case 2: return min_t(u64, btrfs_sb_offset(mirror) >> shift, 1024); + case 0: zone = 0; break; + case 1: zone = 1ULL << (BTRFS_SB_LOG_FIRST_SHIFT - shift); break; + case 2: zone = 1ULL << (BTRFS_SB_LOG_SECOND_SHIFT - shift); break; } - return 0; + ASSERT(zone <= U32_MAX); + + return (u32)zone; } /* @@ -300,10 +320,21 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device) zone_sectors = bdev_zone_sectors(bdev); } - nr_sectors = bdev_nr_sectors(bdev); /* Check if it's power of 2 (see is_power_of_2) */ ASSERT(zone_sectors != 0 && (zone_sectors & (zone_sectors - 1)) == 0); zone_info->zone_size = zone_sectors << SECTOR_SHIFT; + + /* We reject devices with a zone size larger than 8GB */ + if (zone_info->zone_size > BTRFS_MAX_ZONE_SIZE) { + btrfs_err_in_rcu(fs_info, + "zoned: %s: zone size %llu larger than supported maximum %llu", + rcu_str_deref(device->name), + zone_info->zone_size, BTRFS_MAX_ZONE_SIZE); + ret = -EINVAL; + goto out; + } + + nr_sectors = bdev_nr_sectors(bdev); zone_info->zone_size_shift = ilog2(zone_info->zone_size); zone_info->max_zone_append_size = (u64)queue_max_zone_append_sectors(queue) << SECTOR_SHIFT; -- cgit v1.2.3