summaryrefslogtreecommitdiff
path: root/fs/bcachefs
diff options
context:
space:
mode:
Diffstat (limited to 'fs/bcachefs')
-rw-r--r--fs/bcachefs/alloc_background.h10
-rw-r--r--fs/bcachefs/alloc_foreground.c1
-rw-r--r--fs/bcachefs/bcachefs.h2
-rw-r--r--fs/bcachefs/data_update.c1
-rw-r--r--fs/bcachefs/io.c18
-rw-r--r--fs/bcachefs/nocow_locking.c114
-rw-r--r--fs/bcachefs/nocow_locking.h54
-rw-r--r--fs/bcachefs/nocow_locking_types.h20
-rw-r--r--fs/bcachefs/super.c2
-rw-r--r--fs/bcachefs/sysfs.c19
10 files changed, 179 insertions, 62 deletions
diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h
index d4957b4557bf..a0c3c47b49b5 100644
--- a/fs/bcachefs/alloc_background.h
+++ b/fs/bcachefs/alloc_background.h
@@ -23,6 +23,16 @@ static inline bool bch2_dev_bucket_exists(struct bch_fs *c, struct bpos pos)
pos.offset < ca->mi.nbuckets;
}
+static inline u64 bucket_to_u64(struct bpos bucket)
+{
+ return (bucket.inode << 48) | bucket.offset;
+}
+
+static inline struct bpos u64_to_bucket(u64 bucket)
+{
+ return POS(bucket >> 48, bucket & ~(~0ULL << 48));
+}
+
static inline u8 alloc_gc_gen(struct bch_alloc_v4 a)
{
return a.gen - a.oldest_gen;
diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c
index f78eaa52c11f..9e1c236d57b8 100644
--- a/fs/bcachefs/alloc_foreground.c
+++ b/fs/bcachefs/alloc_foreground.c
@@ -28,6 +28,7 @@
#include "io.h"
#include "journal.h"
#include "movinggc.h"
+#include "nocow_locking.h"
#include "trace.h"
#include <linux/math64.h>
diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index acd4adaf475a..6089d9ed6c27 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -206,7 +206,7 @@
#include "bcachefs_format.h"
#include "errcode.h"
#include "fifo.h"
-#include "nocow_locking.h"
+#include "nocow_locking_types.h"
#include "opts.h"
#include "util.h"
diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c
index 190ad03910af..eb248968de48 100644
--- a/fs/bcachefs/data_update.c
+++ b/fs/bcachefs/data_update.c
@@ -11,6 +11,7 @@
#include "io.h"
#include "keylist.h"
#include "move.h"
+#include "nocow_locking.h"
#include "subvolume.h"
#include "trace.h"
diff --git a/fs/bcachefs/io.c b/fs/bcachefs/io.c
index d511bd664953..fe0c4b58e525 100644
--- a/fs/bcachefs/io.c
+++ b/fs/bcachefs/io.c
@@ -27,6 +27,7 @@
#include "journal.h"
#include "keylist.h"
#include "move.h"
+#include "nocow_locking.h"
#include "rebalance.h"
#include "subvolume.h"
#include "super.h"
@@ -1469,7 +1470,7 @@ static void bch2_nocow_write(struct bch_write_op *op)
struct {
struct bpos b;
unsigned gen;
- two_state_lock_t *l;
+ struct nocow_lock_bucket *l;
} buckets[BCH_REPLICAS_MAX];
unsigned nr_buckets = 0;
u32 snapshot;
@@ -1516,7 +1517,8 @@ retry:
buckets[nr_buckets].b = PTR_BUCKET_POS(c, ptr);
buckets[nr_buckets].gen = ptr->gen;
buckets[nr_buckets].l =
- bucket_nocow_lock(&c->nocow_locks, buckets[nr_buckets].b);
+ bucket_nocow_lock(&c->nocow_locks,
+ bucket_to_u64(buckets[nr_buckets].b));
prefetch(buckets[nr_buckets].l);
nr_buckets++;
@@ -1538,11 +1540,12 @@ retry:
for (i = 0; i < nr_buckets; i++) {
struct bch_dev *ca = bch_dev_bkey_exists(c, buckets[i].b.inode);
- two_state_lock_t *l = buckets[i].l;
+ struct nocow_lock_bucket *l = buckets[i].l;
bool stale;
- if (!bch2_two_state_trylock(l, BUCKET_NOCOW_LOCK_UPDATE))
- __bch2_bucket_nocow_lock(&c->nocow_locks, l, BUCKET_NOCOW_LOCK_UPDATE);
+ __bch2_bucket_nocow_lock(&c->nocow_locks, l,
+ bucket_to_u64(buckets[i].b),
+ BUCKET_NOCOW_LOCK_UPDATE);
rcu_read_lock();
stale = gen_after(*bucket_gen(ca, buckets[i].b.offset), buckets[i].gen);
@@ -2984,11 +2987,6 @@ void bch2_fs_io_exit(struct bch_fs *c)
int bch2_fs_io_init(struct bch_fs *c)
{
- unsigned i;
-
- for (i = 0; i < ARRAY_SIZE(c->nocow_locks.l); i++)
- two_state_lock_init(&c->nocow_locks.l[i]);
-
if (bioset_init(&c->bio_read, 1, offsetof(struct bch_read_bio, bio),
BIOSET_NEED_BVECS) ||
bioset_init(&c->bio_read_split, 1, offsetof(struct bch_read_bio, bio),
diff --git a/fs/bcachefs/nocow_locking.c b/fs/bcachefs/nocow_locking.c
index b325fb105322..53e5bc9fd585 100644
--- a/fs/bcachefs/nocow_locking.c
+++ b/fs/bcachefs/nocow_locking.c
@@ -4,12 +4,116 @@
#include "nocow_locking.h"
#include "util.h"
+#include <linux/closure.h>
+
+bool bch2_bucket_nocow_is_locked(struct bucket_nocow_lock_table *t, struct bpos bucket)
+{
+ u64 dev_bucket = bucket_to_u64(bucket);
+ struct nocow_lock_bucket *l = bucket_nocow_lock(t, dev_bucket);
+ unsigned i;
+
+ for (i = 0; i < ARRAY_SIZE(l->b); i++)
+ if (l->b[i] == dev_bucket && atomic_read(&l->l[i]))
+ return true;
+ return false;
+}
+
+void bch2_bucket_nocow_unlock(struct bucket_nocow_lock_table *t, struct bpos bucket, int flags)
+{
+ u64 dev_bucket = bucket_to_u64(bucket);
+ struct nocow_lock_bucket *l = bucket_nocow_lock(t, dev_bucket);
+ int lock_val = flags ? 1 : -1;
+ unsigned i;
+
+ for (i = 0; i < ARRAY_SIZE(l->b); i++)
+ if (l->b[i] == dev_bucket) {
+ if (!atomic_sub_return(lock_val, &l->l[i]))
+ closure_wake_up(&l->wait);
+ return;
+ }
+
+ BUG();
+}
+
+bool __bch2_bucket_nocow_trylock(struct nocow_lock_bucket *l,
+ u64 dev_bucket, int flags)
+{
+ int v, lock_val = flags ? 1 : -1;
+ unsigned i;
+
+ spin_lock(&l->lock);
+
+ for (i = 0; i < ARRAY_SIZE(l->b); i++)
+ if (l->b[i] == dev_bucket)
+ goto got_entry;
+
+ for (i = 0; i < ARRAY_SIZE(l->b); i++)
+ if (!atomic_read(&l->l[i])) {
+ l->b[i] = dev_bucket;
+ goto take_lock;
+ }
+fail:
+ spin_unlock(&l->lock);
+ return false;
+got_entry:
+ v = atomic_read(&l->l[i]);
+ if (lock_val > 0 ? v < 0 : v > 0)
+ goto fail;
+take_lock:
+ atomic_add(lock_val, &l->l[i]);
+ spin_unlock(&l->lock);
+ return true;
+}
+
void __bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *t,
- two_state_lock_t *l, int flags)
+ struct nocow_lock_bucket *l,
+ u64 dev_bucket, int flags)
+{
+ if (!__bch2_bucket_nocow_trylock(l, dev_bucket, flags)) {
+ struct bch_fs *c = container_of(t, struct bch_fs, nocow_locks);
+ u64 start_time = local_clock();
+
+ __closure_wait_event(&l->wait, __bch2_bucket_nocow_trylock(l, dev_bucket, flags));
+ bch2_time_stats_update(&c->times[BCH_TIME_nocow_lock_contended], start_time);
+ }
+}
+
+void bch2_nocow_locks_to_text(struct printbuf *out, struct bucket_nocow_lock_table *t)
{
- struct bch_fs *c = container_of(t, struct bch_fs, nocow_locks);
- u64 start_time = local_clock();
+ unsigned i, nr_zero = 0;
+ struct nocow_lock_bucket *l;
+
+ for (l = t->l; l < t->l + ARRAY_SIZE(t->l); l++) {
+ unsigned v = 0;
+
+ for (i = 0; i < ARRAY_SIZE(l->l); i++)
+ v |= atomic_read(&l->l[i]);
+
+ if (!v) {
+ nr_zero++;
+ continue;
+ }
+
+ if (nr_zero)
+ prt_printf(out, "(%u empty entries)\n", nr_zero);
+ nr_zero = 0;
+
+ for (i = 0; i < ARRAY_SIZE(l->l); i++)
+ if (atomic_read(&l->l[i]))
+ prt_printf(out, "%llu: %i ", l->b[i], atomic_read(&l->l[i]));
+ prt_newline(out);
+ }
+
+ if (nr_zero)
+ prt_printf(out, "(%u empty entries)\n", nr_zero);
+}
+
+int bch2_fs_nocow_locking_init(struct bch_fs *c)
+{
+ unsigned i;
+
+ for (i = 0; i < ARRAY_SIZE(c->nocow_locks.l); i++)
+ spin_lock_init(&c->nocow_locks.l[i].lock);
- __bch2_two_state_lock(l, flags & BUCKET_NOCOW_LOCK_UPDATE);
- bch2_time_stats_update(&c->times[BCH_TIME_nocow_lock_contended], start_time);
+ return 0;
}
diff --git a/fs/bcachefs/nocow_locking.h b/fs/bcachefs/nocow_locking.h
index 2a7a9f44e88e..ff8e4af52edc 100644
--- a/fs/bcachefs/nocow_locking.h
+++ b/fs/bcachefs/nocow_locking.h
@@ -2,54 +2,48 @@
#ifndef _BCACHEFS_NOCOW_LOCKING_H
#define _BCACHEFS_NOCOW_LOCKING_H
-#include "bcachefs_format.h"
-#include "two_state_shared_lock.h"
+#include "bcachefs.h"
+#include "alloc_background.h"
+#include "nocow_locking_types.h"
#include <linux/hash.h>
-#define BUCKET_NOCOW_LOCKS_BITS 10
-#define BUCKET_NOCOW_LOCKS (1U << BUCKET_NOCOW_LOCKS_BITS)
-
-struct bucket_nocow_lock_table {
- two_state_lock_t l[BUCKET_NOCOW_LOCKS];
-};
-
-#define BUCKET_NOCOW_LOCK_UPDATE (1 << 0)
-
-static inline two_state_lock_t *bucket_nocow_lock(struct bucket_nocow_lock_table *t,
- struct bpos bucket)
+static inline struct nocow_lock_bucket *bucket_nocow_lock(struct bucket_nocow_lock_table *t,
+ u64 dev_bucket)
{
- u64 dev_bucket = bucket.inode << 56 | bucket.offset;
unsigned h = hash_64(dev_bucket, BUCKET_NOCOW_LOCKS_BITS);
return t->l + (h & (BUCKET_NOCOW_LOCKS - 1));
}
-static inline bool bch2_bucket_nocow_is_locked(struct bucket_nocow_lock_table *t,
- struct bpos bucket)
-{
- two_state_lock_t *l = bucket_nocow_lock(t, bucket);
+#define BUCKET_NOCOW_LOCK_UPDATE (1 << 0)
- return atomic_long_read(&l->v) != 0;
-}
+bool bch2_bucket_nocow_is_locked(struct bucket_nocow_lock_table *, struct bpos);
+void bch2_bucket_nocow_unlock(struct bucket_nocow_lock_table *, struct bpos, int);
+bool __bch2_bucket_nocow_trylock(struct nocow_lock_bucket *, u64, int);
+void __bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *,
+ struct nocow_lock_bucket *, u64, int);
-static inline void bch2_bucket_nocow_unlock(struct bucket_nocow_lock_table *t,
- struct bpos bucket, int flags)
+static inline void bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *t,
+ struct bpos bucket, int flags)
{
- two_state_lock_t *l = bucket_nocow_lock(t, bucket);
+ u64 dev_bucket = bucket_to_u64(bucket);
+ struct nocow_lock_bucket *l = bucket_nocow_lock(t, dev_bucket);
- bch2_two_state_unlock(l, flags & BUCKET_NOCOW_LOCK_UPDATE);
+ __bch2_bucket_nocow_lock(t, l, dev_bucket, flags);
}
-void __bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *, two_state_lock_t *, int);
-
-static inline void bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *t,
+static inline bool bch2_bucket_nocow_trylock(struct bucket_nocow_lock_table *t,
struct bpos bucket, int flags)
{
- two_state_lock_t *l = bucket_nocow_lock(t, bucket);
+ u64 dev_bucket = bucket_to_u64(bucket);
+ struct nocow_lock_bucket *l = bucket_nocow_lock(t, dev_bucket);
- if (!bch2_two_state_trylock(l, flags & BUCKET_NOCOW_LOCK_UPDATE))
- __bch2_bucket_nocow_lock(t, l, flags);
+ return __bch2_bucket_nocow_trylock(l, dev_bucket, flags);
}
+void bch2_nocow_locks_to_text(struct printbuf *, struct bucket_nocow_lock_table *);
+
+int bch2_fs_nocow_locking_init(struct bch_fs *);
+
#endif /* _BCACHEFS_NOCOW_LOCKING_H */
diff --git a/fs/bcachefs/nocow_locking_types.h b/fs/bcachefs/nocow_locking_types.h
new file mode 100644
index 000000000000..bd12bf677924
--- /dev/null
+++ b/fs/bcachefs/nocow_locking_types.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _BCACHEFS_NOCOW_LOCKING_TYPES_H
+#define _BCACHEFS_NOCOW_LOCKING_TYPES_H
+
+#define BUCKET_NOCOW_LOCKS_BITS 10
+#define BUCKET_NOCOW_LOCKS (1U << BUCKET_NOCOW_LOCKS_BITS)
+
+struct nocow_lock_bucket {
+ struct closure_waitlist wait;
+ spinlock_t lock;
+ u64 b[4];
+ atomic_t l[4];
+} __aligned(SMP_CACHE_BYTES);
+
+struct bucket_nocow_lock_table {
+ struct nocow_lock_bucket l[BUCKET_NOCOW_LOCKS];
+};
+
+#endif /* _BCACHEFS_NOCOW_LOCKING_TYPES_H */
+
diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c
index 2fb7e6300ea5..e142de2a5527 100644
--- a/fs/bcachefs/super.c
+++ b/fs/bcachefs/super.c
@@ -39,6 +39,7 @@
#include "move.h"
#include "migrate.h"
#include "movinggc.h"
+#include "nocow_locking.h"
#include "quota.h"
#include "rebalance.h"
#include "recovery.h"
@@ -821,6 +822,7 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
bch2_fs_btree_write_buffer_init(c) ?:
bch2_fs_subvolumes_init(c) ?:
bch2_fs_io_init(c) ?:
+ bch2_fs_nocow_locking_init(c) ?:
bch2_fs_encryption_init(c) ?:
bch2_fs_compress_init(c) ?:
bch2_fs_ec_init(c) ?:
diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c
index 5b1f792243cd..6cbdf70f36bd 100644
--- a/fs/bcachefs/sysfs.c
+++ b/fs/bcachefs/sysfs.c
@@ -27,6 +27,7 @@
#include "journal.h"
#include "keylist.h"
#include "move.h"
+#include "nocow_locking.h"
#include "opts.h"
#include "rebalance.h"
#include "replicas.h"
@@ -477,22 +478,8 @@ SHOW(bch2_fs)
bch2_write_refs_to_text(out, c);
#endif
- if (attr == &sysfs_nocow_lock_table) {
- int i, count = 1;
- long last, curr = 0;
-
- last = atomic_long_read(&c->nocow_locks.l[0].v);
- for (i = 1; i < BUCKET_NOCOW_LOCKS; i++) {
- curr = atomic_long_read(&c->nocow_locks.l[i].v);
- if (last != curr) {
- prt_printf(out, "%li: %d\n", last, count);
- count = 1;
- last = curr;
- } else
- count++;
- }
- prt_printf(out, "%li: %d\n", last, count);
- }
+ if (attr == &sysfs_nocow_lock_table)
+ bch2_nocow_locks_to_text(out, &c->nocow_locks);
return 0;
}