bcachefs: Avoid atomics in write fast path

This adds some horrible hacks, but the atomic ops for closures were
getting to be a pretty expensive part of the write path. We don't want
to rip out closures entirely from the write path, because they're used
for e.g. waiting on the allocator, or waiting on the journal flush, and
that stuff would get really ugly without closures.

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

1 files changed, 44 insertions, 30 deletions

diff --git a/fs/bcachefs/io.c b/fs/bcachefs/io.c
index 567a553112d1..7a2368407a80 100644
--- a/fs/bcachefs/io.c
+++ b/fs/bcachefs/io.c
@@ -513,7 +513,12 @@ static void bch2_write_done(struct closure *cl)
 
 	bch2_time_stats_update(&c->times[BCH_TIME_data_write], op->start_time);
 
-	closure_return(cl);
+	if (op->end_io)
+		op->end_io(op);
+	if (cl->parent)
+		closure_return(cl);
+	else
+		closure_debug_destroy(cl);
 }
 
 /**
@@ -622,8 +627,10 @@ static void bch2_write_endio(struct bio *bio)
 
 	if (parent)
 		bio_endio(&parent->bio);
-	else
+	else if (!(op->flags & BCH_WRITE_SKIP_CLOSURE_PUT))
 		closure_put(cl);
+	else
+		continue_at_nobarrier(cl, bch2_write_index, index_update_wq(op));
 }
 
 static void init_append_extent(struct bch_write_op *op,
@@ -828,15 +835,14 @@ static enum prep_encoded_ret {
 	return PREP_ENCODED_OK;
 }
 
-static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp)
+static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp,
+			     struct bio **_dst)
 {
 	struct bch_fs *c = op->c;
 	struct bio *src = &op->wbio.bio, *dst = src;
 	struct bvec_iter saved_iter;
-	struct bkey_i *key_to_write;
 	void *ec_buf;
-	unsigned key_to_write_offset = op->insert_keys.top_p -
-		op->insert_keys.keys_p;
+	struct bpos ec_pos = op->pos;
 	unsigned total_output = 0, total_input = 0;
 	bool bounce = false;
 	bool page_alloc_failed = false;
@@ -855,6 +861,7 @@ static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp)
 	case PREP_ENCODED_CHECKSUM_ERR:
 		goto csum_err;
 	case PREP_ENCODED_DO_WRITE:
+		/* XXX look for bug here */
 		if (ec_buf) {
 			dst = bch2_write_bio_alloc(c, wp, src,
 						   &page_alloc_failed,
@@ -1004,31 +1011,15 @@ static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp)
 	dst->bi_iter.bi_size = total_output;
 do_write:
 	/* might have done a realloc... */
+	bch2_ec_add_backpointer(c, wp, ec_pos, total_input >> 9);
 
-	key_to_write = (void *) (op->insert_keys.keys_p + key_to_write_offset);
-
-	bch2_ec_add_backpointer(c, wp,
-				bkey_start_pos(&key_to_write->k),
-				total_input >> 9);
-
-	bch2_alloc_sectors_done(c, wp);
-
-	dst->bi_end_io	= bch2_write_endio;
-	dst->bi_private	= &op->cl;
-	dst->bi_opf	= REQ_OP_WRITE;
-
-	closure_get(dst->bi_private);
-
-	bch2_submit_wbio_replicas(to_wbio(dst), c, BCH_DATA_USER,
-				  key_to_write);
+	*_dst = dst;
 	return more;
 csum_err:
 	bch_err(c, "error verifying existing checksum while "
 		"rewriting existing data (memory corruption?)");
 	ret = -EIO;
 err:
-	bch2_alloc_sectors_done(c, wp);
-
 	if (to_wbio(dst)->bounce)
 		bch2_bio_free_pages_pool(c, dst);
 	if (to_wbio(dst)->put_bio)
@@ -1042,11 +1033,17 @@ static void __bch2_write(struct closure *cl)
 	struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
 	struct bch_fs *c = op->c;
 	struct write_point *wp;
+	struct bio *bio;
+	bool skip_put = true;
 	int ret;
 again:
 	memset(&op->failed, 0, sizeof(op->failed));
 
 	do {
+		struct bkey_i *key_to_write;
+		unsigned key_to_write_offset = op->insert_keys.top_p -
+			op->insert_keys.keys_p;
+
 		/* +1 for possible cache device: */
 		if (op->open_buckets.nr + op->nr_replicas + 1 >
 		    ARRAY_SIZE(op->open_buckets.v))
@@ -1080,21 +1077,38 @@ again:
 		}
 
 		bch2_open_bucket_get(c, wp, &op->open_buckets);
-
-		ret = bch2_write_extent(op, wp);
+		ret = bch2_write_extent(op, wp, &bio);
+		bch2_alloc_sectors_done(c, wp);
 
 		if (ret < 0)
 			goto err;
+
+		if (ret)
+			skip_put = false;
+
+		bio->bi_end_io	= bch2_write_endio;
+		bio->bi_private	= &op->cl;
+		bio->bi_opf	= REQ_OP_WRITE;
+
+		if (!skip_put)
+			closure_get(bio->bi_private);
+		else
+			op->flags |= BCH_WRITE_SKIP_CLOSURE_PUT;
+
+		key_to_write = (void *) (op->insert_keys.keys_p +
+					 key_to_write_offset);
+
+		bch2_submit_wbio_replicas(to_wbio(bio), c, BCH_DATA_USER,
+					  key_to_write);
 	} while (ret);
 
-	continue_at(cl, bch2_write_index, index_update_wq(op));
+	if (!skip_put)
+		continue_at(cl, bch2_write_index, index_update_wq(op));
 	return;
 err:
 	op->error = ret;
 
-	continue_at(cl, !bch2_keylist_empty(&op->insert_keys)
-		    ? bch2_write_index
-		    : bch2_write_done, index_update_wq(op));
+	continue_at(cl, bch2_write_index, index_update_wq(op));
 	return;
 flush_io:
 	closure_sync(cl);