summaryrefslogtreecommitdiff
path: root/fs/netfs
diff options
context:
space:
mode:
Diffstat (limited to 'fs/netfs')
-rw-r--r--fs/netfs/buffered_read.c2
-rw-r--r--fs/netfs/buffered_write.c2
-rw-r--r--fs/netfs/direct_write.c18
-rw-r--r--fs/netfs/internal.h12
-rw-r--r--fs/netfs/locking.c95
-rw-r--r--fs/netfs/read_retry.c7
-rw-r--r--fs/netfs/write_collect.c10
-rw-r--r--fs/netfs/write_issue.c55
-rw-r--r--fs/netfs/write_retry.c7
9 files changed, 162 insertions, 46 deletions
diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c
index 76d0f6a29aba..24a8a5418e31 100644
--- a/fs/netfs/buffered_read.c
+++ b/fs/netfs/buffered_read.c
@@ -659,7 +659,7 @@ retry:
* within the cache granule containing the EOF, in which case we need
* to preload the granule.
*/
- if (!netfs_is_cache_enabled(ctx) &&
+ if (!netfs_is_cache_maybe_enabled(ctx) &&
netfs_skip_folio_read(folio, pos, len, false)) {
netfs_stat(&netfs_n_rh_write_zskip);
goto have_folio_no_wait;
diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c
index 6bde3320bcec..2cdb68e6b16f 100644
--- a/fs/netfs/buffered_write.c
+++ b/fs/netfs/buffered_write.c
@@ -277,7 +277,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
* caching service temporarily because the backing store got
* culled.
*/
- if (netfs_is_cache_enabled(ctx)) {
+ if (netfs_is_cache_maybe_enabled(ctx)) {
if (finfo) {
netfs_stat(&netfs_n_wh_wstream_conflict);
goto flush_content;
diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c
index 25f8ceb15fad..c16fbad286a1 100644
--- a/fs/netfs/direct_write.c
+++ b/fs/netfs/direct_write.c
@@ -166,13 +166,16 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq)
*/
subreq->error = -EAGAIN;
trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
- if (subreq->transferred > 0)
+ if (subreq->transferred > 0) {
iov_iter_advance(&wreq->buffer.iter, subreq->transferred);
+ wreq->transferred += subreq->transferred;
+ }
if (stream->source == NETFS_UPLOAD_TO_SERVER &&
wreq->netfs_ops->retry_request)
wreq->netfs_ops->retry_request(wreq, stream);
+ __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
__clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
__clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
__clear_bit(NETFS_SREQ_FAILED, &subreq->flags);
@@ -186,17 +189,10 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq)
netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
- if (stream->prepare_write) {
+ if (stream->prepare_write)
stream->prepare_write(subreq);
- __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
- netfs_stat(&netfs_n_wh_retry_write_subreq);
- } else {
- struct iov_iter source;
-
- netfs_reset_iter(subreq);
- source = subreq->io_iter;
- netfs_reissue_write(stream, subreq, &source);
- }
+ __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
+ netfs_stat(&netfs_n_wh_retry_write_subreq);
}
netfs_unbuffered_write_done(wreq);
diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h
index 645996ecfc80..d889caa401dc 100644
--- a/fs/netfs/internal.h
+++ b/fs/netfs/internal.h
@@ -239,6 +239,18 @@ static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx)
#endif
}
+static inline bool netfs_is_cache_maybe_enabled(struct netfs_inode *ctx)
+{
+#if IS_ENABLED(CONFIG_FSCACHE)
+ struct fscache_cookie *cookie = ctx->cache;
+
+ return fscache_cookie_valid(cookie) &&
+ test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
+#else
+ return false;
+#endif
+}
+
/*
* Get a ref on a netfs group attached to a dirty page (e.g. a ceph snap).
*/
diff --git a/fs/netfs/locking.c b/fs/netfs/locking.c
index 2249ecd09d0a..4e3be2b81504 100644
--- a/fs/netfs/locking.c
+++ b/fs/netfs/locking.c
@@ -9,6 +9,11 @@
#include <linux/netfs.h>
#include "internal.h"
+struct netfs_wb_waiter {
+ struct list_head link; /* Link in ictx->wb_queue */
+ struct task_struct *waiter; /* Waiter task; cleared when lock granted */
+};
+
/*
* inode_dio_wait_interruptible - wait for outstanding DIO requests to finish
* @inode: inode to wait for
@@ -203,3 +208,93 @@ void netfs_end_io_direct(struct inode *inode)
up_read(&inode->i_rwsem);
}
EXPORT_SYMBOL(netfs_end_io_direct);
+
+/*
+ * Wait to have exclusive access to writeback.
+ */
+static bool netfs_wb_begin_wait(struct netfs_inode *ictx)
+{
+ struct netfs_wb_waiter waiter = {};
+ struct task_struct *tsk = current;
+ bool got = false;
+
+ spin_lock(&ictx->lock);
+
+ if (test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) {
+ get_task_struct(tsk);
+ waiter.waiter = tsk;
+ list_add_tail(&waiter.link, &ictx->wb_queue);
+ } else {
+ got = true;
+ }
+ spin_unlock(&ictx->lock);
+
+ if (!got) {
+ for (;;) {
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ /* Read waiter before accessing inode state. */
+ if (smp_load_acquire(&waiter.waiter) == NULL)
+ break;
+ schedule();
+ }
+ }
+ __set_current_state(TASK_RUNNING);
+ return true;
+}
+
+/**
+ * netfs_wb_begin - Begin writeback, waiting if need be
+ * @ictx: The inode to get writeback access on
+ * @nowait: Return failure immediately rather than waiting if true
+ *
+ * Begin writeback to an inode, waiting for exclusive access if @nowait is
+ * false. This prevents collection from being done out of order with respect
+ * to the issuance of write subrequests.
+ *
+ * Note that writeback may be ended in a different process (e.g. the collection
+ * function on a workqueue) than started it.
+ *
+ * Return: True if can proceed, false if denied.
+ */
+bool netfs_wb_begin(struct netfs_inode *ictx, bool nowait)
+{
+ if (!test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags))
+ return true;
+ if (nowait) {
+ netfs_stat(&netfs_n_wb_lock_skip);
+ return false;
+ }
+ netfs_stat(&netfs_n_wb_lock_wait);
+ return netfs_wb_begin_wait(ictx);
+}
+EXPORT_SYMBOL(netfs_wb_begin);
+
+/* netfs_wb_end - End writeback
+ * @ictx: The inode we have writeback access to
+ *
+ * End writeback access on an inode, waking up the next writeback request.
+ */
+void netfs_wb_end(struct netfs_inode *ictx)
+{
+ struct netfs_wb_waiter *waiter;
+ struct task_struct *tsk;
+
+ WARN_ON_ONCE(!test_bit(NETFS_ICTX_WB_LOCK, &ictx->flags));
+
+ spin_lock(&ictx->lock);
+
+ waiter = list_first_entry_or_null(&ictx->wb_queue, struct netfs_wb_waiter, link);
+ if (waiter) {
+ list_del(&waiter->link);
+ tsk = waiter->waiter;
+ /* Write inode state before clearing waiter. */
+ smp_store_release(&waiter->waiter, NULL);
+ wake_up_process(tsk);
+ put_task_struct(tsk);
+ } else {
+ clear_bit_unlock(NETFS_ICTX_WB_LOCK, &ictx->flags);
+ }
+
+ spin_unlock(&ictx->lock);
+}
+EXPORT_SYMBOL(netfs_wb_end);
diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c
index f59a70f3a086..2b42758e01ec 100644
--- a/fs/netfs/read_retry.c
+++ b/fs/netfs/read_retry.c
@@ -98,7 +98,12 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
goto abandon;
}
- list_for_each_continue(next, &stream->subrequests) {
+ for (;;) {
+ /* Read pointer to subreq before reading subreq state. */
+ next = smp_load_acquire(&next->next);
+ if (next == &stream->subrequests)
+ break;
+
subreq = list_entry(next, struct netfs_io_subrequest, rreq_link);
if (subreq->start + subreq->transferred != start + len ||
test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) ||
diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c
index 24fc2bb2f8a4..210eb8f3958d 100644
--- a/fs/netfs/write_collect.c
+++ b/fs/netfs/write_collect.c
@@ -408,6 +408,16 @@ bool netfs_write_collection(struct netfs_io_request *wreq)
netfs_wake_rreq_flag(wreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip);
/* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */
+ switch (wreq->origin) {
+ case NETFS_WRITEBACK:
+ case NETFS_WRITEBACK_SINGLE:
+ case NETFS_WRITETHROUGH:
+ netfs_wb_end(ictx);
+ break;
+ default:
+ break;
+ }
+
if (wreq->iocb) {
size_t written = min(wreq->transferred, wreq->len);
wreq->iocb->ki_pos += written;
diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c
index c03c7cc45e47..f2761c99795a 100644
--- a/fs/netfs/write_issue.c
+++ b/fs/netfs/write_issue.c
@@ -106,7 +106,7 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
_enter("R=%x", wreq->debug_id);
ictx = netfs_inode(wreq->inode);
- if (is_cacheable && netfs_is_cache_enabled(ictx))
+ if (is_cacheable)
fscache_begin_write_operation(&wreq->cache_resources, netfs_i_cookie(ictx));
if (rolling_buffer_init(&wreq->buffer, wreq->debug_id, ITER_SOURCE) < 0)
goto nomem;
@@ -551,14 +551,8 @@ int netfs_writepages(struct address_space *mapping,
struct folio *folio;
int error = 0;
- if (!mutex_trylock(&ictx->wb_lock)) {
- if (wbc->sync_mode == WB_SYNC_NONE) {
- netfs_stat(&netfs_n_wb_lock_skip);
- return 0;
- }
- netfs_stat(&netfs_n_wb_lock_wait);
- mutex_lock(&ictx->wb_lock);
- }
+ if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE))
+ return 0;
/* Need the first folio to be able to set up the op. */
folio = writeback_iter(mapping, wbc, NULL, &error);
@@ -588,13 +582,13 @@ int netfs_writepages(struct address_space *mapping,
}
error = netfs_write_folio(wreq, wbc, folio);
- if (error < 0)
- break;
+ if (error == -ENOMEM) {
+ folio_redirty_for_writepage(wbc, folio);
+ folio_unlock(folio);
+ }
} while ((folio = writeback_iter(mapping, wbc, folio, &error)));
netfs_end_issue_write(wreq);
-
- mutex_unlock(&ictx->wb_lock);
netfs_wake_collector(wreq);
netfs_put_request(wreq, netfs_rreq_trace_put_return);
@@ -602,9 +596,16 @@ int netfs_writepages(struct address_space *mapping,
return error;
couldnt_start:
- netfs_kill_dirty_pages(mapping, wbc, folio);
+ if (error == -ENOMEM) {
+ folio_redirty_for_writepage(wbc, folio);
+ folio_unlock(folio);
+ folio = writeback_iter(mapping, wbc, folio, &error);
+ WARN_ON_ONCE(folio != NULL);
+ } else {
+ netfs_kill_dirty_pages(mapping, wbc, folio);
+ }
out:
- mutex_unlock(&ictx->wb_lock);
+ netfs_wb_end(ictx);
_leave(" = %d", error);
return error;
}
@@ -618,16 +619,17 @@ struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len
struct netfs_io_request *wreq = NULL;
struct netfs_inode *ictx = netfs_inode(file_inode(iocb->ki_filp));
- mutex_lock(&ictx->wb_lock);
+ netfs_wb_begin(ictx, false);
wreq = netfs_create_write_req(iocb->ki_filp->f_mapping, iocb->ki_filp,
iocb->ki_pos, NETFS_WRITETHROUGH);
if (IS_ERR(wreq)) {
- mutex_unlock(&ictx->wb_lock);
+ netfs_wb_end(ictx);
return wreq;
}
wreq->io_streams[0].avail = true;
+ __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags);
trace_netfs_write(wreq, netfs_write_trace_writethrough);
return wreq;
}
@@ -685,7 +687,6 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c
ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
struct folio *writethrough_cache)
{
- struct netfs_inode *ictx = netfs_inode(wreq->inode);
ssize_t ret;
_enter("R=%x", wreq->debug_id);
@@ -699,8 +700,6 @@ ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_c
netfs_end_issue_write(wreq);
- mutex_unlock(&ictx->wb_lock);
-
if (wreq->iocb)
ret = -EIOCBQUEUED;
else
@@ -847,15 +846,10 @@ int netfs_writeback_single(struct address_space *mapping,
if (WARN_ON_ONCE(!iov_iter_is_folioq(iter)))
return -EIO;
- if (!mutex_trylock(&ictx->wb_lock)) {
- if (wbc->sync_mode == WB_SYNC_NONE) {
- /* The VFS will have undirtied the inode. */
- netfs_single_mark_inode_dirty(&ictx->inode);
- netfs_stat(&netfs_n_wb_lock_skip);
- return 1;
- }
- netfs_stat(&netfs_n_wb_lock_wait);
- mutex_lock(&ictx->wb_lock);
+ if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE)) {
+ /* The VFS will have undirtied the inode. */
+ netfs_single_mark_inode_dirty(&ictx->inode);
+ return 1;
}
wreq = netfs_create_write_req(mapping, NULL, 0, NETFS_WRITEBACK_SINGLE);
@@ -893,7 +887,6 @@ stop:
smp_wmb(); /* Write lists before ALL_QUEUED. */
set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
- mutex_unlock(&ictx->wb_lock);
netfs_wake_collector(wreq);
netfs_put_request(wreq, netfs_rreq_trace_put_return);
@@ -901,7 +894,7 @@ stop:
return ret;
couldnt_start:
- mutex_unlock(&ictx->wb_lock);
+ netfs_wb_end(ictx);
_leave(" = %d", ret);
return ret;
}
diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c
index 32735abfa03f..058bc7a166a5 100644
--- a/fs/netfs/write_retry.c
+++ b/fs/netfs/write_retry.c
@@ -72,7 +72,12 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
!test_bit(NETFS_SREQ_NEED_RETRY, &from->flags))
return;
- list_for_each_continue(next, &stream->subrequests) {
+ for (;;) {
+ /* Read pointer to subreq before reading subreq state. */
+ next = smp_load_acquire(&next->next);
+ if (next == &stream->subrequests)
+ break;
+
subreq = list_entry(next, struct netfs_io_subrequest, rreq_link);
if (subreq->start + subreq->transferred != start + len ||
test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) ||