bcachefs: Don't hold inode lock longer than necessary in dio write path

In theory we should be able to do (non appending/extending) dio writes
without taking the inode lock at all - but this gets us most of the way
there.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

1 files changed, 54 insertions, 35 deletions

diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c
index 436676f4fa2a..37c81a664430 100644
--- a/fs/bcachefs/fs-io.c
+++ b/fs/bcachefs/fs-io.c
@@ -1774,9 +1774,6 @@ static long bch2_dio_write_loop(struct dio_write *dio)
 	if (dio->loop)
 		goto loop;
 
-	inode_dio_begin(&inode->v);
-	bch2_pagecache_block_get(&inode->ei_pagecache_lock);
-
 	/* Write and invalidate pagecache range that we're writing to: */
 	offset = req->ki_pos + (dio->op.written << 9);
 	ret = write_invalidate_inode_pages_range(mapping,
@@ -1904,15 +1901,39 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 	struct bch_io_opts opts = io_opts(c, &inode->ei_inode);
 	struct dio_write *dio;
 	struct bio *bio;
+	bool locked = true, extending;
 	ssize_t ret;
 
-	lockdep_assert_held(&inode->v.i_rwsem);
+	prefetch(&c->opts);
+	prefetch((void *) &c->opts + 64);
+	prefetch(&inode->ei_inode);
+	prefetch((void *) &inode->ei_inode + 64);
 
-	if (unlikely(!iter->count))
-		return 0;
+	inode_lock(&inode->v);
+
+	ret = generic_write_checks(req, iter);
+	if (unlikely(ret <= 0))
+		goto err;
+
+	ret = file_remove_privs(file);
+	if (unlikely(ret))
+		goto err;
+
+	ret = file_update_time(file);
+	if (unlikely(ret))
+		goto err;
 
 	if (unlikely((req->ki_pos|iter->count) & (block_bytes(c) - 1)))
-		return -EINVAL;
+		goto err;
+
+	inode_dio_begin(&inode->v);
+	bch2_pagecache_block_get(&inode->ei_pagecache_lock);
+
+	extending = req->ki_pos + iter->count > inode->v.i_size;
+	if (!extending) {
+		inode_unlock(&inode->v);
+		locked = false;
+	}
 
 	bio = bio_alloc_bioset(NULL,
 			       iov_iter_npages(iter, BIO_MAX_VECS),
@@ -1924,8 +1945,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 	dio->req		= req;
 	dio->mm			= current->mm;
 	dio->loop		= false;
-	dio->sync		= is_sync_kiocb(req) ||
-		req->ki_pos + iter->count > inode->v.i_size;
+	dio->sync		= is_sync_kiocb(req) || extending;
 	dio->free_iov		= false;
 	dio->quota_res.sectors	= 0;
 	dio->iter		= *iter;
@@ -1944,7 +1964,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 	ret = bch2_quota_reservation_add(c, inode, &dio->quota_res,
 					 iter->count >> 9, true);
 	if (unlikely(ret))
-		goto err;
+		goto err_put_bio;
 
 	dio->op.nr_replicas	= dio->op.opts.data_replicas;
 
@@ -1955,55 +1975,54 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 					       req->ki_pos >> 9),
 					iter->count >> 9,
 					dio->op.opts.data_replicas))
-		goto err;
+		goto err_put_bio;
 
-	return bch2_dio_write_loop(dio);
+	ret = bch2_dio_write_loop(dio);
 err:
+	if (locked)
+		inode_unlock(&inode->v);
+	if (ret > 0)
+		req->ki_pos += ret;
+	return ret;
+err_put_bio:
+	bch2_pagecache_block_put(&inode->ei_pagecache_lock);
 	bch2_disk_reservation_put(c, &dio->op.res);
 	bch2_quota_reservation_put(c, inode, &dio->quota_res);
 	bio_put(bio);
-	return ret;
+	inode_dio_end(&inode->v);
+	goto err;
 }
 
-static ssize_t __bch2_write_iter(struct kiocb *iocb, struct iov_iter *from)
+ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
 	struct file *file = iocb->ki_filp;
+	struct bch_inode_info *inode = file_bch_inode(file);
 	ssize_t	ret;
 
 	if (iocb->ki_flags & IOCB_DIRECT)
 		return bch2_direct_write(iocb, from);
 
+	inode_lock(&inode->v);
+
+	ret = generic_write_checks(iocb, from);
+	if (ret <= 0)
+		goto unlock;
+
 	ret = file_remove_privs(file);
 	if (ret)
-		return ret;
+		goto unlock;
 
 	ret = file_update_time(file);
 	if (ret)
-		return ret;
-
-	ret = iocb->ki_flags & IOCB_DIRECT
-		? bch2_direct_write(iocb, from)
-		: bch2_buffered_write(iocb, from);
+		goto unlock;
 
+	ret = bch2_buffered_write(iocb, from);
 	if (likely(ret > 0))
 		iocb->ki_pos += ret;
-
-	return ret;
-}
-
-ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from)
-{
-	struct bch_inode_info *inode = file_bch_inode(iocb->ki_filp);
-	bool direct = iocb->ki_flags & IOCB_DIRECT;
-	ssize_t ret;
-
-	inode_lock(&inode->v);
-	ret = generic_write_checks(iocb, from);
-	if (ret > 0)
-		ret = __bch2_write_iter(iocb, from);
+unlock:
 	inode_unlock(&inode->v);
 
-	if (ret > 0 && !direct)
+	if (ret > 0)
 		ret = generic_write_sync(iocb, ret);
 
 	return ret;