summaryrefslogtreecommitdiff
path: root/block
diff options
context:
space:
mode:
Diffstat (limited to 'block')
-rw-r--r--block/bio.c64
-rw-r--r--block/blk-cgroup.c4
-rw-r--r--block/blk-map.c9
-rw-r--r--block/blk-mq.c3
-rw-r--r--block/blk-wbt.c21
-rw-r--r--block/blk-zoned.c42
-rw-r--r--block/blk.h17
-rw-r--r--block/elevator.c9
-rw-r--r--block/error-injection.c7
-rw-r--r--block/genhd.c6
-rw-r--r--block/partitions/aix.c9
11 files changed, 130 insertions, 61 deletions
diff --git a/block/bio.c b/block/bio.c
index f2a5f4d0a967..6a2f6fc3413e 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -555,6 +555,14 @@ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
bio = bio_alloc_percpu_cache(bs);
} else {
opf &= ~REQ_ALLOC_CACHE;
+ }
+
+ /*
+ * For a bioset without a percpu cache, or when the percpu cache was
+ * empty, try a slab allocation with optimistic GFP_ flags before
+ * falling back to the mempool.
+ */
+ if (!bio) {
p = kmem_cache_alloc(bs->bio_slab, gfp);
if (p)
bio = p + bs->front_pad;
@@ -1191,7 +1199,7 @@ void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter)
* for the next iteration.
*/
static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter,
- unsigned len_align_mask)
+ struct bio_vec *bv, unsigned len_align_mask)
{
size_t nbytes = bio->bi_iter.bi_size & len_align_mask;
@@ -1200,23 +1208,16 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter,
iov_iter_revert(iter, nbytes);
bio->bi_iter.bi_size -= nbytes;
- do {
- struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
-
- if (nbytes < bv->bv_len) {
- bv->bv_len -= nbytes;
- break;
- }
-
+ while (nbytes >= bv->bv_len) {
if (bio_flagged(bio, BIO_PAGE_PINNED))
unpin_user_page(bv->bv_page);
- bio->bi_vcnt--;
+ if (!--bio->bi_vcnt)
+ return -EFAULT;
nbytes -= bv->bv_len;
- } while (nbytes);
-
- if (!bio->bi_vcnt)
- return -EFAULT;
+ bv--;
+ }
+ bv->bv_len -= nbytes;
return 0;
}
@@ -1276,7 +1277,8 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page))
bio->bi_opf |= REQ_NOMERGE;
- return bio_iov_iter_align_down(bio, iter, len_align_mask);
+ return bio_iov_iter_align_down(bio, iter,
+ &bio->bi_io_vec[bio->bi_vcnt - 1], len_align_mask);
}
static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size,
@@ -1285,7 +1287,8 @@ static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size,
struct folio *folio;
while (*size > minsize) {
- folio = folio_alloc(gfp | __GFP_NORETRY, get_order(*size));
+ folio = folio_alloc(gfp | __GFP_NORETRY | __GFP_NOWARN,
+ get_order(*size));
if (folio)
return folio;
*size = rounddown_pow_of_two(*size - 1);
@@ -1302,7 +1305,7 @@ static void bio_free_folios(struct bio *bio)
bio_for_each_bvec_all(bv, bio, i) {
struct folio *folio = bvec_folio(bv);
- if (!is_zero_folio(folio))
+ if (!is_zero_folio(folio) && !is_huge_zero_folio(folio))
folio_put(folio);
}
}
@@ -1360,7 +1363,8 @@ static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
if (!bio->bi_iter.bi_size)
return -ENOMEM;
- return bio_iov_iter_align_down(bio, iter, minsize - 1);
+ return bio_iov_iter_align_down(bio, iter,
+ &bio->bi_io_vec[bio->bi_vcnt - 1], minsize - 1);
}
static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
@@ -1368,21 +1372,18 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
{
size_t len = min3(iov_iter_count(iter), maxlen, SZ_1M);
struct folio *folio;
+ ssize_t ret;
folio = folio_alloc_greedy(GFP_KERNEL, &len, minsize);
if (!folio)
return -ENOMEM;
do {
- ssize_t ret;
-
ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec + 1, len,
&bio->bi_vcnt, bio->bi_max_vecs - 1, 0);
if (ret <= 0) {
- if (!bio->bi_vcnt) {
- folio_put(folio);
- return ret;
- }
+ if (!bio->bi_vcnt)
+ goto out_folio_put;
break;
}
len -= ret;
@@ -1398,7 +1399,20 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
bvec_set_folio(&bio->bi_io_vec[0], folio, bio->bi_iter.bi_size, 0);
if (iov_iter_extract_will_pin(iter))
bio_set_flag(bio, BIO_PAGE_PINNED);
- return bio_iov_iter_align_down(bio, iter, minsize - 1);
+
+ /* The first vec stores the bounce buffer, so do not subtract 1 here. */
+ ret = bio_iov_iter_align_down(bio, iter,
+ &bio->bi_io_vec[bio->bi_vcnt], minsize - 1);
+ if (ret)
+ goto out_folio_put;
+
+ /* Update the bounc buffer bv_len to the aligned down size. */
+ bio->bi_io_vec[0].bv_len = bio->bi_iter.bi_size;
+ return 0;
+
+out_folio_put:
+ folio_put(folio);
+ return ret;
}
/**
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index d2a1f5903f24..d9676126c5b5 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -434,15 +434,15 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk,
blkg->pd[i]->online = true;
}
}
+ blkg->online = true;
}
- blkg->online = true;
spin_unlock(&blkcg->lock);
if (!ret)
return blkg;
/* @blkg failed fully initialized, use the usual release path */
- blkg_put(blkg);
+ percpu_ref_kill(&blkg->refcnt);
return ERR_PTR(ret);
err_free_blkg:
diff --git a/block/blk-map.c b/block/blk-map.c
index 768549f19f97..d1d6bbe0ecf1 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -653,6 +653,7 @@ int blk_rq_map_kern(struct request *rq, void *kbuf, unsigned int len,
gfp_t gfp_mask)
{
unsigned long addr = (unsigned long) kbuf;
+ bool do_copy;
struct bio *bio;
int ret;
@@ -661,7 +662,8 @@ int blk_rq_map_kern(struct request *rq, void *kbuf, unsigned int len,
if (!len || !kbuf)
return -EINVAL;
- if (!blk_rq_aligned(rq->q, addr, len) || object_is_on_stack(kbuf))
+ do_copy = !blk_rq_aligned(rq->q, addr, len) || object_is_on_stack(kbuf);
+ if (do_copy)
bio = bio_copy_kern(rq, kbuf, len, gfp_mask);
else
bio = bio_map_kern(rq, kbuf, len, gfp_mask);
@@ -670,8 +672,11 @@ int blk_rq_map_kern(struct request *rq, void *kbuf, unsigned int len,
return PTR_ERR(bio);
ret = blk_rq_append_bio(rq, bio);
- if (unlikely(ret))
+ if (unlikely(ret)) {
+ if (do_copy)
+ bio_free_pages(bio);
blk_mq_map_bio_put(bio);
+ }
return ret;
}
EXPORT_SYMBOL(blk_rq_map_kern);
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 88cb5acc4f39..2c850330a32b 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -5218,6 +5218,7 @@ static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
struct io_comp_batch *iob, unsigned int flags)
{
int ret;
+ unsigned long timeout = jiffies + 2;
do {
ret = q->mq_ops->poll(hctx, iob);
@@ -5228,7 +5229,7 @@ static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
if (ret < 0 || (flags & BLK_POLL_ONESHOT))
break;
cpu_relax();
- } while (!need_resched());
+ } while (!need_resched() && time_before(jiffies, timeout));
return 0;
}
diff --git a/block/blk-wbt.c b/block/blk-wbt.c
index dcc2438ca16d..953d400fd013 100644
--- a/block/blk-wbt.c
+++ b/block/blk-wbt.c
@@ -813,6 +813,21 @@ static void wbt_queue_depth_changed(struct rq_qos *rqos)
wbt_update_limits(RQWB(rqos));
}
+static bool wbt_set_lat_changed(struct request_queue *q, u64 val)
+{
+ struct rq_qos *rqos = wbt_rq_qos(q);
+ struct rq_wb *rwb;
+
+ if (!rqos)
+ return true;
+
+ rwb = RQWB(rqos);
+ if (rwb->min_lat_nsec != val)
+ return true;
+
+ return rwb_enabled(rwb) != !!val;
+}
+
static void wbt_exit(struct rq_qos *rqos)
{
struct rq_wb *rwb = RQWB(rqos);
@@ -1005,8 +1020,12 @@ int wbt_set_lat(struct gendisk *disk, s64 val)
else if (val >= 0)
val *= 1000ULL;
- if (wbt_get_min_lat(q) == val)
+ mutex_lock(&disk->rqos_state_mutex);
+ if (!wbt_set_lat_changed(q, val)) {
+ mutex_unlock(&disk->rqos_state_mutex);
goto out;
+ }
+ mutex_unlock(&disk->rqos_state_mutex);
blk_mq_quiesce_queue(q);
diff --git a/block/blk-zoned.c b/block/blk-zoned.c
index bea817f3de56..ca30caec838e 100644
--- a/block/blk-zoned.c
+++ b/block/blk-zoned.c
@@ -1923,11 +1923,20 @@ static int disk_alloc_zone_resources(struct gendisk *disk,
if (!disk->zone_wplugs_pool)
goto free_hash;
- disk->zone_wplugs_wq =
- alloc_workqueue("%s_zwplugs", WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU,
- pool_size, disk->disk_name);
- if (!disk->zone_wplugs_wq)
- goto destroy_pool;
+ /*
+ * We may already have a zone write plug workqueue as this function may
+ * be called after disk_free_zone_resources(), which does not destroy
+ * the workqueue (the zone write plugs workqueue is destroyed at
+ * disk_release() time).
+ */
+ if (!disk->zone_wplugs_wq) {
+ disk->zone_wplugs_wq =
+ alloc_workqueue("%s_zwplugs",
+ WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU,
+ pool_size, disk->disk_name);
+ if (!disk->zone_wplugs_wq)
+ goto destroy_pool;
+ }
disk->zone_wplugs_worker =
kthread_create(disk_zone_wplugs_worker, disk,
@@ -1935,15 +1944,12 @@ static int disk_alloc_zone_resources(struct gendisk *disk,
if (IS_ERR(disk->zone_wplugs_worker)) {
ret = PTR_ERR(disk->zone_wplugs_worker);
disk->zone_wplugs_worker = NULL;
- goto destroy_wq;
+ goto destroy_pool;
}
wake_up_process(disk->zone_wplugs_worker);
return 0;
-destroy_wq:
- destroy_workqueue(disk->zone_wplugs_wq);
- disk->zone_wplugs_wq = NULL;
destroy_pool:
mempool_destroy(disk->zone_wplugs_pool);
disk->zone_wplugs_pool = NULL;
@@ -1999,7 +2005,7 @@ static void disk_set_zones_cond_array(struct gendisk *disk, u8 *zones_cond)
kfree_rcu_mightsleep(zones_cond);
}
-void disk_free_zone_resources(struct gendisk *disk)
+static void disk_free_zone_resources(struct gendisk *disk)
{
if (disk->zone_wplugs_worker) {
kthread_stop(disk->zone_wplugs_worker);
@@ -2007,10 +2013,8 @@ void disk_free_zone_resources(struct gendisk *disk)
}
WARN_ON_ONCE(!list_empty(&disk->zone_wplugs_list));
- if (disk->zone_wplugs_wq) {
- destroy_workqueue(disk->zone_wplugs_wq);
- disk->zone_wplugs_wq = NULL;
- }
+ if (disk->zone_wplugs_wq)
+ drain_workqueue(disk->zone_wplugs_wq);
disk_destroy_zone_wplugs_hash_table(disk);
@@ -2020,6 +2024,16 @@ void disk_free_zone_resources(struct gendisk *disk)
disk->nr_zones = 0;
}
+void disk_release_zone_resources(struct gendisk *disk)
+{
+ if (disk->zone_wplugs_wq) {
+ destroy_workqueue(disk->zone_wplugs_wq);
+ disk->zone_wplugs_wq = NULL;
+ }
+
+ disk_free_zone_resources(disk);
+}
+
struct blk_revalidate_zone_args {
struct gendisk *disk;
u8 *zones_cond;
diff --git a/block/blk.h b/block/blk.h
index 25af8ac5ef0f..eaac05815cb0 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -528,7 +528,7 @@ static inline void ioc_clear_queue(struct request_queue *q)
#ifdef CONFIG_BLK_DEV_ZONED
void disk_init_zone_resources(struct gendisk *disk);
-void disk_free_zone_resources(struct gendisk *disk);
+void disk_release_zone_resources(struct gendisk *disk);
static inline bool bio_zone_write_plugging(struct bio *bio)
{
return bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING);
@@ -581,7 +581,7 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode,
static inline void disk_init_zone_resources(struct gendisk *disk)
{
}
-static inline void disk_free_zone_resources(struct gendisk *disk)
+static inline void disk_release_zone_resources(struct gendisk *disk)
{
}
static inline bool bio_zone_write_plugging(struct bio *bio)
@@ -717,6 +717,7 @@ static inline int req_ref_read(struct request *req)
static inline u64 blk_time_get_ns(void)
{
struct blk_plug *plug = current->plug;
+ u64 now;
if (!plug || !in_task())
return ktime_get_ns();
@@ -725,12 +726,18 @@ static inline u64 blk_time_get_ns(void)
* 0 could very well be a valid time, but rather than flag "this is
* a valid timestamp" separately, just accept that we'll do an extra
* ktime_get_ns() if we just happen to get 0 as the current time.
+ *
+ * cur_ktime can be zeroed by pre-emption the moment PF_BLOCK_TS is set.
*/
- if (!plug->cur_ktime) {
- plug->cur_ktime = ktime_get_ns();
+ now = READ_ONCE(plug->cur_ktime);
+ if (!now) {
+ now = ktime_get_ns();
+ WRITE_ONCE(plug->cur_ktime, now);
+ /* Ensure PF_BLOCK_TS is set after cur_ktime. */
+ barrier();
current->flags |= PF_BLOCK_TS;
}
- return plug->cur_ktime;
+ return now;
}
static inline ktime_t blk_time_get(void)
diff --git a/block/elevator.c b/block/elevator.c
index 3bcd37c2aa34..2161b6eea680 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -812,8 +812,13 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
* reference during concurrent disk deletion:
* update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del)
* kn->active -> update_nr_hwq_lock (via this sysfs write path)
+ *
+ * Use the writer lock instead of the reader lock of update_nr_hwq_lock
+ * to serialize the two-stage elevator switch steps in
+ * elevator_change(): the core switch step under the elevator lock and
+ * the elevator_change_done() step outside the elevator lock.
*/
- if (!down_read_trylock(&set->update_nr_hwq_lock)) {
+ if (!down_write_trylock(&set->update_nr_hwq_lock)) {
ret = -EBUSY;
goto out;
}
@@ -824,7 +829,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
} else {
ret = -ENOENT;
}
- up_read(&set->update_nr_hwq_lock);
+ up_write(&set->update_nr_hwq_lock);
out:
if (ctx.type)
diff --git a/block/error-injection.c b/block/error-injection.c
index cfb83138960c..e14bc4b723ef 100644
--- a/block/error-injection.c
+++ b/block/error-injection.c
@@ -276,9 +276,10 @@ static int blk_error_injection_show(struct seq_file *s, void *private)
rcu_read_lock();
list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) {
- seq_printf(s, "%llu:%llu status=%s,chance=%u",
- inj->start, inj->end,
- blk_status_to_tag(inj->status), inj->chance);
+ seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u",
+ inj->start, inj->end,
+ blk_op_str(inj->op),
+ blk_status_to_tag(inj->status), inj->chance);
seq_putc(s, '\n');
}
rcu_read_unlock();
diff --git a/block/genhd.c b/block/genhd.c
index f84b6a355b57..df2c3c69b467 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -407,10 +407,6 @@ static void add_disk_final(struct gendisk *disk)
struct device *ddev = disk_to_dev(disk);
if (!(disk->flags & GENHD_FL_HIDDEN)) {
- /* Make sure the first partition scan will be proceed */
- if (get_capacity(disk) && disk_has_partscan(disk))
- set_bit(GD_NEED_PART_SCAN, &disk->state);
-
bdev_add(disk->part0, ddev->devt);
if (get_capacity(disk))
disk_scan_partitions(disk, BLK_OPEN_READ);
@@ -1300,7 +1296,7 @@ static void disk_release(struct device *dev)
disk_release_events(disk);
kfree(disk->random);
- disk_free_zone_resources(disk);
+ disk_release_zone_resources(disk);
xa_destroy(&disk->part_tbl);
kobject_put(&disk->queue_kobj);
diff --git a/block/partitions/aix.c b/block/partitions/aix.c
index f3c4174e003e..689837deba27 100644
--- a/block/partitions/aix.c
+++ b/block/partitions/aix.c
@@ -208,7 +208,14 @@ int aix_partition(struct parsed_partitions *state)
if (n) {
int foundlvs = 0;
- for (i = 0; foundlvs < numlvs && i < state->limit; i += 1) {
+ /*
+ * The lvd array was read as a single sector; only the
+ * struct lvd entries that fit in it are valid. Bound the
+ * scan so an on-disk numlvs larger than that cannot walk
+ * the read buffer out of bounds.
+ */
+ for (i = 0; foundlvs < numlvs && i < state->limit &&
+ i < SECTOR_SIZE / (int)sizeof(struct lvd); i++) {
lvip[i].pps_per_lv = be16_to_cpu(p[i].num_lps);
if (lvip[i].pps_per_lv)
foundlvs += 1;