diff options
Diffstat (limited to 'block')
88 files changed, 8293 insertions, 4980 deletions
diff --git a/block/Kconfig b/block/Kconfig index df8973bc0539..70e4a66d941f 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -211,14 +211,6 @@ config BLK_INLINE_ENCRYPTION_FALLBACK source "block/partitions/Kconfig" -config BLK_MQ_PCI - def_bool PCI - -config BLK_MQ_VIRTIO - bool - depends on VIRTIO - default y - config BLK_PM def_bool PM @@ -229,6 +221,14 @@ config BLOCK_HOLDER_DEPRECATED config BLK_MQ_STACKING bool +config BLK_ERROR_INJECTION + bool "Enable block layer error injection" + select JUMP_LABEL if HAVE_ARCH_JUMP_LABEL + help + Enable inserting arbitrary block errors through a debugfs interface. + + See Documentation/block/error-injection.rst for details. + source "block/Kconfig.iosched" endif # BLOCK diff --git a/block/Makefile b/block/Makefile index 3a941dc0d27f..e7bd320e3d69 100644 --- a/block/Makefile +++ b/block/Makefile @@ -3,15 +3,17 @@ # Makefile for the kernel block layer # +CONTEXT_ANALYSIS := y + obj-y := bdev.o fops.o bio.o elevator.o blk-core.o blk-sysfs.o \ blk-flush.o blk-settings.o blk-ioc.o blk-map.o \ - blk-merge.o blk-timeout.o \ - blk-lib.o blk-mq.o blk-mq-tag.o blk-stat.o \ + blk-merge.o blk-timeout.o blk-lib.o blk-mq.o \ + blk-mq-tag.o blk-mq-dma.o blk-stat.o \ blk-mq-sysfs.o blk-mq-cpumap.o blk-mq-sched.o ioctl.o \ genhd.o ioprio.o badblocks.o partitions/ blk-rq-qos.o \ disk-events.o blk-ia-ranges.o early-lookup.o -obj-$(CONFIG_BOUNCE) += bounce.o +obj-$(CONFIG_BLK_ERROR_INJECTION) += error-injection.o obj-$(CONFIG_BLK_DEV_BSG_COMMON) += bsg.o obj-$(CONFIG_BLK_DEV_BSGLIB) += bsg-lib.o obj-$(CONFIG_BLK_CGROUP) += blk-cgroup.o @@ -27,7 +29,7 @@ bfq-y := bfq-iosched.o bfq-wf2q.o bfq-cgroup.o obj-$(CONFIG_IOSCHED_BFQ) += bfq.o obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o \ - bio-integrity-auto.o + bio-integrity-auto.o bio-integrity-fs.o obj-$(CONFIG_BLK_DEV_ZONED) += blk-zoned.o obj-$(CONFIG_BLK_WBT) += blk-wbt.o obj-$(CONFIG_BLK_DEBUG_FS) += blk-mq-debugfs.o diff --git a/block/bdev.c b/block/bdev.c index 4844d1e27b6f..85ce57bd2ae4 100644 --- a/block/bdev.c +++ b/block/bdev.c @@ -67,7 +67,7 @@ static void bdev_write_inode(struct block_device *bdev) int ret; spin_lock(&inode->i_lock); - while (inode->i_state & I_DIRTY) { + while (inode_state_read(inode) & I_DIRTY) { spin_unlock(&inode->i_lock); ret = write_inode_now(inode, true); if (ret) @@ -152,36 +152,90 @@ static void set_init_blocksize(struct block_device *bdev) get_order(bsize)); } -int set_blocksize(struct file *file, int size) +/** + * bdev_validate_blocksize - check that this block size is acceptable + * @bdev: blockdevice to check + * @block_size: block size to check + * + * For block device users that do not use buffer heads or the block device + * page cache, make sure that this block size can be used with the device. + * + * Return: On success zero is returned, negative error code on failure. + */ +int bdev_validate_blocksize(struct block_device *bdev, int block_size) { - struct inode *inode = file->f_mapping->host; - struct block_device *bdev = I_BDEV(inode); - - if (blk_validate_block_size(size)) + if (blk_validate_block_size(block_size)) return -EINVAL; /* Size cannot be smaller than the size supported by the device */ - if (size < bdev_logical_block_size(bdev)) + if (block_size < bdev_logical_block_size(bdev)) return -EINVAL; + return 0; +} +EXPORT_SYMBOL_GPL(bdev_validate_blocksize); + +int set_blocksize(struct file *file, int size) +{ + struct inode *inode = file->f_mapping->host; + struct block_device *bdev = I_BDEV(inode); + int ret; + + ret = bdev_validate_blocksize(bdev, size); + if (ret) + return ret; + if (!file->private_data) return -EINVAL; /* Don't change the size if it is same as current */ if (inode->i_blkbits != blksize_bits(size)) { + /* + * Flush and truncate the pagecache before we reconfigure the + * mapping geometry because folio sizes are variable now. If a + * reader has already allocated a folio whose size is smaller + * than the new min_order but invokes readahead after the new + * min_order becomes visible, readahead will think there are + * "zero" blocks per folio and crash. Take the inode and + * invalidation locks to avoid racing with + * read/write/fallocate. + */ + inode_lock(inode); + filemap_invalidate_lock(inode->i_mapping); + sync_blockdev(bdev); + kill_bdev(bdev); + inode->i_blkbits = blksize_bits(size); mapping_set_folio_min_order(inode->i_mapping, get_order(size)); - kill_bdev(bdev); + filemap_invalidate_unlock(inode->i_mapping); + inode_unlock(inode); } return 0; } EXPORT_SYMBOL(set_blocksize); +static int sb_validate_large_blocksize(struct super_block *sb, int size) +{ + const char *err_str = NULL; + + if (!(sb->s_type->fs_flags & FS_LBS)) + err_str = "not supported by filesystem"; + else if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) + err_str = "is only supported with CONFIG_TRANSPARENT_HUGEPAGE"; + + if (!err_str) + return 0; + + pr_warn_ratelimited("%s: block size(%d) > page size(%lu) %s\n", + sb->s_type->name, size, PAGE_SIZE, err_str); + return -EINVAL; +} + int sb_set_blocksize(struct super_block *sb, int size) { - if (!(sb->s_type->fs_flags & FS_LBS) && size > PAGE_SIZE) + if (size > PAGE_SIZE && sb_validate_large_blocksize(sb, size)) return 0; if (set_blocksize(sb->s_bdev_file, size)) return 0; @@ -193,7 +247,7 @@ int sb_set_blocksize(struct super_block *sb, int size) EXPORT_SYMBOL(sb_set_blocksize); -int sb_min_blocksize(struct super_block *sb, int size) +int __must_check sb_min_blocksize(struct super_block *sb, int size) { int minsize = bdev_logical_block_size(sb->s_bdev); if (size < minsize) @@ -363,19 +417,11 @@ static void init_once(void *data) inode_init_once(&ei->vfs_inode); } -static void bdev_evict_inode(struct inode *inode) -{ - truncate_inode_pages_final(&inode->i_data); - invalidate_inode_buffers(inode); /* is it needed here? */ - clear_inode(inode); -} - static const struct super_operations bdev_sops = { .statfs = simple_statfs, .alloc_inode = bdev_alloc_inode, .free_inode = bdev_free_inode, - .drop_inode = generic_delete_inode, - .evict_inode = bdev_evict_inode, + .drop_inode = inode_just_drop, }; static int bd_init_fs_context(struct fs_context *fc) @@ -400,15 +446,10 @@ EXPORT_SYMBOL_GPL(blockdev_superblock); void __init bdev_cache_init(void) { - int err; - bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| SLAB_ACCOUNT|SLAB_PANIC), init_once); - err = register_filesystem(&bd_type); - if (err) - panic("Cannot register bdev pseudo-fs"); blockdev_mnt = kern_mount(&bd_type); if (IS_ERR(blockdev_mnt)) panic("Cannot create bdev pseudo-fs"); @@ -777,13 +818,13 @@ static void blkdev_put_part(struct block_device *part) blkdev_put_whole(whole); } -struct block_device *blkdev_get_no_open(dev_t dev) +struct block_device *blkdev_get_no_open(dev_t dev, bool autoload) { struct block_device *bdev; struct inode *inode; inode = ilookup(blockdev_superblock, dev); - if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) { + if (!inode && autoload && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) { blk_request_module(dev); inode = ilookup(blockdev_superblock, dev); if (inode) @@ -1005,7 +1046,7 @@ struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, if (ret) return ERR_PTR(ret); - bdev = blkdev_get_no_open(dev); + bdev = blkdev_get_no_open(dev, true); if (!bdev) return ERR_PTR(-ENXIO); @@ -1204,7 +1245,13 @@ void bdev_mark_dead(struct block_device *bdev, bool surprise) bdev->bd_holder_ops->mark_dead(bdev, surprise); else { mutex_unlock(&bdev->bd_holder_lock); - sync_blockdev(bdev); + /* + * On surprise removal the device is already gone; syncing is + * futile and can hang forever waiting on I/O that will never + * complete. Match fs_bdev_mark_dead(), which also skips it. + */ + if (!surprise) + sync_blockdev(bdev); } invalidate_bdev(bdev); @@ -1227,7 +1274,7 @@ void sync_bdevs(bool wait) struct block_device *bdev; spin_lock(&inode->i_lock); - if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_NEW) || mapping->nrpages == 0) { spin_unlock(&inode->i_lock); continue; @@ -1272,21 +1319,17 @@ void sync_bdevs(bool wait) /* * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices. */ -void bdev_statx(struct path *path, struct kstat *stat, - u32 request_mask) +void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask) { - struct inode *backing_inode; struct block_device *bdev; - backing_inode = d_backing_inode(path->dentry); - /* - * Note that backing_inode is the inode of a block device node file, - * not the block device's internal inode. Therefore it is *not* valid - * to use I_BDEV() here; the block device has to be looked up by i_rdev + * Note that d_backing_inode() returns the block device node inode, not + * the block device's internal inode. Therefore it is *not* valid to + * use I_BDEV() here; the block device has to be looked up by i_rdev * instead. */ - bdev = blkdev_get_no_open(backing_inode->i_rdev); + bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev, false); if (!bdev) return; @@ -1301,7 +1344,8 @@ void bdev_statx(struct path *path, struct kstat *stat, generic_fill_statx_atomic_writes(stat, queue_atomic_write_unit_min_bytes(bd_queue), - queue_atomic_write_unit_max_bytes(bd_queue)); + queue_atomic_write_unit_max_bytes(bd_queue), + 0); } stat->blksize = bdev_io_min(bdev); diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 9fb9f3533150..e82ff03bda02 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -300,6 +300,25 @@ static struct bfq_group *bfqg_parent(struct bfq_group *bfqg) return pblkg ? blkg_to_bfqg(pblkg) : NULL; } +static void bfqg_stats_exit(struct bfqg_stats *stats) +{ + blkg_rwstat_exit(&stats->bytes); + blkg_rwstat_exit(&stats->ios); +#ifdef CONFIG_BFQ_CGROUP_DEBUG + blkg_rwstat_exit(&stats->merged); + blkg_rwstat_exit(&stats->service_time); + blkg_rwstat_exit(&stats->wait_time); + blkg_rwstat_exit(&stats->queued); + bfq_stat_exit(&stats->time); + bfq_stat_exit(&stats->avg_queue_size_sum); + bfq_stat_exit(&stats->avg_queue_size_samples); + bfq_stat_exit(&stats->dequeue); + bfq_stat_exit(&stats->group_wait_time); + bfq_stat_exit(&stats->idle_time); + bfq_stat_exit(&stats->empty_time); +#endif +} + struct bfq_group *bfqq_group(struct bfq_queue *bfqq) { struct bfq_entity *group_entity = bfqq->entity.parent; @@ -321,8 +340,10 @@ static void bfqg_get(struct bfq_group *bfqg) static void bfqg_put(struct bfq_group *bfqg) { - if (refcount_dec_and_test(&bfqg->ref)) + if (refcount_dec_and_test(&bfqg->ref)) { + bfqg_stats_exit(&bfqg->stats); kfree(bfqg); + } } static void bfqg_and_blkg_get(struct bfq_group *bfqg) @@ -380,7 +401,7 @@ static void bfqg_stats_add_aux(struct bfqg_stats *to, struct bfqg_stats *from) blkg_rwstat_add_aux(&to->merged, &from->merged); blkg_rwstat_add_aux(&to->service_time, &from->service_time); blkg_rwstat_add_aux(&to->wait_time, &from->wait_time); - bfq_stat_add_aux(&from->time, &from->time); + bfq_stat_add_aux(&to->time, &from->time); bfq_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum); bfq_stat_add_aux(&to->avg_queue_size_samples, &from->avg_queue_size_samples); @@ -433,25 +454,6 @@ void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg) entity->sched_data = &bfqg->sched_data; } -static void bfqg_stats_exit(struct bfqg_stats *stats) -{ - blkg_rwstat_exit(&stats->bytes); - blkg_rwstat_exit(&stats->ios); -#ifdef CONFIG_BFQ_CGROUP_DEBUG - blkg_rwstat_exit(&stats->merged); - blkg_rwstat_exit(&stats->service_time); - blkg_rwstat_exit(&stats->wait_time); - blkg_rwstat_exit(&stats->queued); - bfq_stat_exit(&stats->time); - bfq_stat_exit(&stats->avg_queue_size_sum); - bfq_stat_exit(&stats->avg_queue_size_samples); - bfq_stat_exit(&stats->dequeue); - bfq_stat_exit(&stats->group_wait_time); - bfq_stat_exit(&stats->idle_time); - bfq_stat_exit(&stats->empty_time); -#endif -} - static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp) { if (blkg_rwstat_init(&stats->bytes, gfp) || @@ -494,7 +496,7 @@ static struct blkcg_policy_data *bfq_cpd_alloc(gfp_t gfp) { struct bfq_group_data *bgd; - bgd = kzalloc(sizeof(*bgd), gfp); + bgd = kzalloc_obj(*bgd, gfp); if (!bgd) return NULL; @@ -548,14 +550,20 @@ static void bfq_pd_init(struct blkg_policy_data *pd) bfqg->rq_pos_tree = RB_ROOT; } -static void bfq_pd_free(struct blkg_policy_data *pd) +static void bfqg_release(struct rcu_head *rcu) { + struct blkg_policy_data *pd = + container_of(rcu, struct blkg_policy_data, rcu_head); struct bfq_group *bfqg = pd_to_bfqg(pd); - bfqg_stats_exit(&bfqg->stats); bfqg_put(bfqg); } +static void bfq_pd_free(struct blkg_policy_data *pd) +{ + call_rcu(&pd->rcu_head, bfqg_release); +} + static void bfq_pd_reset_stats(struct blkg_policy_data *pd) { struct bfq_group *bfqg = pd_to_bfqg(pd); @@ -935,14 +943,23 @@ put_async_queues: void bfq_end_wr_async(struct bfq_data *bfqd) { + struct request_queue *q = bfqd->queue; struct blkcg_gq *blkg; - list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) { + mutex_lock(&q->blkcg_mutex); + spin_lock_irq(&q->queue_lock); + spin_lock(&bfqd->lock); + + list_for_each_entry(blkg, &q->blkg_list, q_node) { struct bfq_group *bfqg = blkg_to_bfqg(blkg); bfq_end_wr_async_queues(bfqd, bfqg); } bfq_end_wr_async_queues(bfqd, bfqd->root_group); + + spin_unlock(&bfqd->lock); + spin_unlock_irq(&q->queue_lock); + mutex_unlock(&q->blkcg_mutex); } static int bfq_io_show_weight_legacy(struct seq_file *sf, void *v) @@ -1051,9 +1068,13 @@ static ssize_t bfq_io_set_device_weight(struct kernfs_open_file *of, blkg_conf_init(&ctx, buf); + ret = blkg_conf_open_bdev(&ctx); + if (ret) + return ret; + ret = blkg_conf_prep(blkcg, &blkcg_policy_bfq, &ctx); if (ret) - goto out; + goto close_bdev; if (sscanf(ctx.body, "%llu", &v) == 1) { /* require "default" on dfl */ @@ -1074,8 +1095,11 @@ static ssize_t bfq_io_set_device_weight(struct kernfs_open_file *of, bfq_group_set_weight(bfqg, bfqg->entity.weight, v); ret = 0; } + out: - blkg_conf_exit(&ctx); + blkg_conf_unprep(&ctx); +close_bdev: + blkg_conf_close_bdev(&ctx); return ret ?: nbytes; } @@ -1139,16 +1163,18 @@ static u64 bfqg_prfill_stat_recursive(struct seq_file *sf, struct cgroup_subsys_state *pos_css; u64 sum = 0; - lockdep_assert_held(&blkg->q->queue_lock); - rcu_read_lock(); blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { + struct blkg_policy_data *pd; struct bfq_stat *stat; if (!pos_blkg->online) continue; - stat = (void *)blkg_to_pd(pos_blkg, &blkcg_policy_bfq) + off; + pd = blkg_to_pd(pos_blkg, &blkcg_policy_bfq); + if (!pd) + continue; + stat = (void *)pd + off; sum += bfq_stat_read(stat) + atomic64_read(&stat->aux_cnt); } rcu_read_unlock(); @@ -1408,7 +1434,9 @@ void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) {} void bfq_end_wr_async(struct bfq_data *bfqd) { + spin_lock_irq(&bfqd->lock); bfq_end_wr_async_queues(bfqd, bfqd->root_group); + spin_unlock_irq(&bfqd->lock); } struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio) diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index abd80dc13562..0f75301b3115 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -231,7 +231,7 @@ static struct kmem_cache *bfq_pool; #define BFQ_RQ_SEEKY(bfqd, last_pos, rq) \ (get_sdist(last_pos, rq) > \ BFQQ_SEEK_THR && \ - (!blk_queue_nonrot(bfqd->queue) || \ + (blk_queue_rot(bfqd->queue) || \ blk_rq_sectors(rq) < BFQQ_SECT_THR_NONROT)) #define BFQQ_CLOSE_THR (sector_t)(8 * 1024) #define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 19) @@ -454,17 +454,10 @@ static struct bfq_io_cq *icq_to_bic(struct io_cq *icq) */ static struct bfq_io_cq *bfq_bic_lookup(struct request_queue *q) { - struct bfq_io_cq *icq; - unsigned long flags; - if (!current->io_context) return NULL; - spin_lock_irqsave(&q->queue_lock, flags); - icq = icq_to_bic(ioc_lookup_icq(q)); - spin_unlock_irqrestore(&q->queue_lock, flags); - - return icq; + return icq_to_bic(ioc_lookup_icq(q)); } /* @@ -613,7 +606,7 @@ retry: spin_unlock_irq(&bfqd->lock); if (entities != inline_entities) kfree(entities); - entities = kmalloc_array(depth, sizeof(*entities), GFP_NOIO); + entities = kmalloc_objs(*entities, depth, GFP_NOIO); if (!entities) return false; alloc_depth = depth; @@ -701,17 +694,13 @@ static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) { struct bfq_data *bfqd = data->q->elevator->elevator_data; struct bfq_io_cq *bic = bfq_bic_lookup(data->q); - int depth; - unsigned limit = data->q->nr_requests; - unsigned int act_idx; + unsigned int limit, act_idx; /* Sync reads have full depth available */ - if (op_is_sync(opf) && !op_is_write(opf)) { - depth = 0; - } else { - depth = bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(opf)]; - limit = (limit * depth) >> bfqd->full_depth_shift; - } + if (blk_mq_is_sync_read(opf)) + limit = data->q->nr_requests; + else + limit = bfqd->async_depths[!!bfqd->wr_busy_queues][op_is_sync(opf)]; for (act_idx = 0; bic && act_idx < bfqd->num_actuators; act_idx++) { /* Fast path to check if bfqq is already allocated. */ @@ -725,14 +714,16 @@ static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) * available requests and thus starve other entities. */ if (bfqq_request_over_limit(bfqd, bic, opf, act_idx, limit)) { - depth = 1; + limit = 1; break; } } + bfq_log(bfqd, "[%s] wr_busy %d sync %d depth %u", - __func__, bfqd->wr_busy_queues, op_is_sync(opf), depth); - if (depth) - data->shallow_depth = depth; + __func__, bfqd->wr_busy_queues, op_is_sync(opf), limit); + + if (limit < data->q->nr_requests) + data->shallow_depth = limit; } static struct bfq_queue * @@ -947,8 +938,8 @@ void bfq_weights_tree_add(struct bfq_queue *bfqq) } } - bfqq->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), - GFP_ATOMIC); + bfqq->weight_counter = kzalloc_obj(struct bfq_weight_counter, + GFP_ATOMIC); /* * In the unlucky event of an allocation failure, we just @@ -2457,15 +2448,8 @@ static bool bfq_bio_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs) { struct bfq_data *bfqd = q->elevator->elevator_data; - struct request *free = NULL; - /* - * bfq_bic_lookup grabs the queue_lock: invoke it now and - * store its return value for later use, to avoid nesting - * queue_lock inside the bfqd->lock. We assume that the bic - * returned by bfq_bic_lookup does not go away before - * bfqd->lock is taken. - */ struct bfq_io_cq *bic = bfq_bic_lookup(q); + struct request *free = NULL; bool ret; spin_lock_irq(&bfqd->lock); @@ -2669,9 +2653,10 @@ static void bfq_end_wr(struct bfq_data *bfqd) } list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) bfq_bfqq_end_wr(bfqq); - bfq_end_wr_async(bfqd); spin_unlock_irq(&bfqd->lock); + + bfq_end_wr_async(bfqd); } static sector_t bfq_io_struct_pos(void *io_struct, bool request) @@ -4181,7 +4166,7 @@ static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, /* don't use too short time intervals */ if (delta_usecs < 1000) { - if (blk_queue_nonrot(bfqd->queue)) + if (!blk_queue_rot(bfqd->queue)) /* * give same worst-case guarantees as idling * for seeky @@ -4503,7 +4488,7 @@ static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd, struct bfq_queue *bfqq) { bool rot_without_queueing = - !blk_queue_nonrot(bfqd->queue) && !bfqd->hw_tag, + blk_queue_rot(bfqd->queue) && !bfqd->hw_tag, bfqq_sequential_and_IO_bound, idling_boosts_thr; @@ -4537,7 +4522,7 @@ static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd, * flash-based device. */ idling_boosts_thr = rot_without_queueing || - ((!blk_queue_nonrot(bfqd->queue) || !bfqd->hw_tag) && + ((blk_queue_rot(bfqd->queue) || !bfqd->hw_tag) && bfqq_sequential_and_IO_bound); /* @@ -4738,7 +4723,7 @@ bfq_choose_bfqq_for_injection(struct bfq_data *bfqd) * there is only one in-flight large request * at a time. */ - if (blk_queue_nonrot(bfqd->queue) && + if (!blk_queue_rot(bfqd->queue) && blk_rq_sectors(bfqq->next_rq) >= BFQQ_SECT_THR_NONROT && bfqd->tot_rq_in_driver >= 1) @@ -5863,8 +5848,7 @@ static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, goto out; } - bfqq = kmem_cache_alloc_node(bfq_pool, - GFP_NOWAIT | __GFP_ZERO | __GFP_NOWARN, + bfqq = kmem_cache_alloc_node(bfq_pool, GFP_NOWAIT | __GFP_ZERO, bfqd->queue->node); if (bfqq) { @@ -6357,7 +6341,7 @@ static void bfq_update_hw_tag(struct bfq_data *bfqd) bfqd->hw_tag_samples = 0; bfqd->nonrot_with_queueing = - blk_queue_nonrot(bfqd->queue) && bfqd->hw_tag; + !blk_queue_rot(bfqd->queue) && bfqd->hw_tag; } static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) @@ -7126,56 +7110,33 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg) * See the comments on bfq_limit_depth for the purpose of * the depths set in the function. Return minimum shallow depth we'll use. */ -static void bfq_update_depths(struct bfq_data *bfqd, struct sbitmap_queue *bt) +static void bfq_depth_updated(struct request_queue *q) { - unsigned int depth = 1U << bt->sb.shift; + struct bfq_data *bfqd = q->elevator->elevator_data; + unsigned int async_depth = q->async_depth; - bfqd->full_depth_shift = bt->sb.shift; /* - * In-word depths if no bfq_queue is being weight-raised: - * leaving 25% of tags only for sync reads. + * By default: + * - sync reads are not limited + * If bfqq is not being weight-raised: + * - sync writes are limited to 75%(async depth default value) + * - async IO are limited to 50% + * If bfqq is being weight-raised: + * - sync writes are limited to ~37% + * - async IO are limited to ~18 * - * In next formulas, right-shift the value - * (1U<<bt->sb.shift), instead of computing directly - * (1U<<(bt->sb.shift - something)), to be robust against - * any possible value of bt->sb.shift, without having to - * limit 'something'. - */ - /* no more than 50% of tags for async I/O */ - bfqd->word_depths[0][0] = max(depth >> 1, 1U); - /* - * no more than 75% of tags for sync writes (25% extra tags - * w.r.t. async I/O, to prevent async I/O from starving sync - * writes) + * If request_queue->async_depth is updated by user, all limit are + * updated relatively. */ - bfqd->word_depths[0][1] = max((depth * 3) >> 2, 1U); + bfqd->async_depths[0][1] = async_depth; + bfqd->async_depths[0][0] = max(async_depth * 2 / 3, 1U); + bfqd->async_depths[1][1] = max(async_depth >> 1, 1U); + bfqd->async_depths[1][0] = max(async_depth >> 2, 1U); /* - * In-word depths in case some bfq_queue is being weight- - * raised: leaving ~63% of tags for sync reads. This is the - * highest percentage for which, in our tests, application - * start-up times didn't suffer from any regression due to tag - * shortage. + * Due to cgroup qos, the allowed request for bfqq might be 1 */ - /* no more than ~18% of tags for async I/O */ - bfqd->word_depths[1][0] = max((depth * 3) >> 4, 1U); - /* no more than ~37% of tags for sync writes (~20% extra tags) */ - bfqd->word_depths[1][1] = max((depth * 6) >> 4, 1U); -} - -static void bfq_depth_updated(struct blk_mq_hw_ctx *hctx) -{ - struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; - struct blk_mq_tags *tags = hctx->sched_tags; - - bfq_update_depths(bfqd, &tags->bitmap_tags); - sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, 1); -} - -static int bfq_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int index) -{ - bfq_depth_updated(hctx); - return 0; + blk_mq_set_min_shallow_depth(q, 1); } static void bfq_exit_queue(struct elevator_queue *e) @@ -7210,7 +7171,7 @@ static void bfq_exit_queue(struct elevator_queue *e) #endif blk_stat_disable_accounting(bfqd->queue); - clear_bit(ELEVATOR_FLAG_DISABLE_WBT, &e->flags); + blk_queue_flag_clear(QUEUE_FLAG_DISABLE_WBT_DEF, bfqd->queue); wbt_enable_default(bfqd->queue->disk); kfree(bfqd); @@ -7232,27 +7193,18 @@ static void bfq_init_root_group(struct bfq_group *root_group, root_group->sched_data.bfq_class_idle_last_service = jiffies; } -static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) +static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq) { struct bfq_data *bfqd; - struct elevator_queue *eq; unsigned int i; struct blk_independent_access_ranges *ia_ranges = q->disk->ia_ranges; - eq = elevator_alloc(q, e); - if (!eq) - return -ENOMEM; - bfqd = kzalloc_node(sizeof(*bfqd), GFP_KERNEL, q->node); - if (!bfqd) { - kobject_put(&eq->kobj); + if (!bfqd) return -ENOMEM; - } - eq->elevator_data = bfqd; - spin_lock_irq(&q->queue_lock); + eq->elevator_data = bfqd; q->elevator = eq; - spin_unlock_irq(&q->queue_lock); /* * Our fallback bfqq if bfq_find_alloc_queue() runs into OOM issues. @@ -7285,7 +7237,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) * If the disk supports multiple actuators, copy independent * access ranges from the request queue structure. */ - spin_lock_irq(&q->queue_lock); if (ia_ranges) { /* * Check if the disk ia_ranges size exceeds the current bfq @@ -7311,7 +7262,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) bfqd->sector[0] = 0; bfqd->nr_sectors[0] = get_capacity(q->disk); } - spin_unlock_irq(&q->queue_lock); INIT_LIST_HEAD(&bfqd->dispatch); @@ -7329,7 +7279,7 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) INIT_HLIST_HEAD(&bfqd->burst_list); bfqd->hw_tag = -1; - bfqd->nonrot_with_queueing = blk_queue_nonrot(bfqd->queue); + bfqd->nonrot_with_queueing = !blk_queue_rot(bfqd->queue); bfqd->bfq_max_budget = bfq_default_max_budget; @@ -7364,9 +7314,9 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) * Begin by assuming, optimistically, that the device peak * rate is equal to 2/3 of the highest reference rate. */ - bfqd->rate_dur_prod = ref_rate[blk_queue_nonrot(bfqd->queue)] * - ref_wr_duration[blk_queue_nonrot(bfqd->queue)]; - bfqd->peak_rate = ref_rate[blk_queue_nonrot(bfqd->queue)] * 2 / 3; + bfqd->rate_dur_prod = ref_rate[!blk_queue_rot(bfqd->queue)] * + ref_wr_duration[!blk_queue_rot(bfqd->queue)]; + bfqd->peak_rate = ref_rate[!blk_queue_rot(bfqd->queue)] * 2 / 3; /* see comments on the definition of next field inside bfq_data */ bfqd->actuator_load_threshold = 4; @@ -7393,19 +7343,20 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) goto out_free; bfq_init_root_group(bfqd->root_group, bfqd); bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group); + bfq_depth_updated(q); /* We dispatch from request queue wide instead of hw queue */ blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q); - set_bit(ELEVATOR_FLAG_DISABLE_WBT, &eq->flags); + blk_queue_flag_set(QUEUE_FLAG_DISABLE_WBT_DEF, q); wbt_disable_default(q->disk); blk_stat_enable_accounting(q); + q->async_depth = (q->nr_requests * 3) >> 2; return 0; out_free: kfree(bfqd); - kobject_put(&eq->kobj); return -ENOMEM; } @@ -7653,7 +7604,6 @@ static struct elevator_type iosched_bfq_mq = { .request_merged = bfq_request_merged, .has_work = bfq_has_work, .depth_updated = bfq_depth_updated, - .init_hctx = bfq_init_hctx, .init_sched = bfq_init_queue, .exit_sched = bfq_exit_queue, }, diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h index 687a3a7ba784..355a731e2c04 100644 --- a/block/bfq-iosched.h +++ b/block/bfq-iosched.h @@ -427,9 +427,6 @@ struct bfq_iocq_bfqq_data { */ bool saved_IO_bound; - u64 saved_io_start_time; - u64 saved_tot_idle_time; - /* * Same purpose as the previous fields for the values of the * field keeping the queue's belonging to a large burst @@ -450,6 +447,9 @@ struct bfq_iocq_bfqq_data { */ unsigned int saved_weight; + u64 saved_io_start_time; + u64 saved_tot_idle_time; + /* * Similar to previous fields: save wr information. */ @@ -457,13 +457,13 @@ struct bfq_iocq_bfqq_data { unsigned long saved_last_wr_start_finish; unsigned long saved_service_from_wr; unsigned long saved_wr_start_at_switch_to_srt; - unsigned int saved_wr_cur_max_time; struct bfq_ttime saved_ttime; + unsigned int saved_wr_cur_max_time; /* Save also injection state */ - u64 saved_last_serv_time_ns; unsigned int saved_inject_limit; unsigned long saved_decrease_time_jif; + u64 saved_last_serv_time_ns; /* candidate queue for a stable merge (due to close creation time) */ struct bfq_queue *stable_merge_bfqq; @@ -813,8 +813,7 @@ struct bfq_data { * Depth limits used in bfq_limit_depth (see comments on the * function) */ - unsigned int word_depths[2][2]; - unsigned int full_depth_shift; + unsigned int async_depths[2][2]; /* * Number of independent actuators. This is equal to 1 in @@ -985,7 +984,7 @@ struct bfq_group_data { * unused for the root group. Used to know whether there * are groups with more than one active @bfq_entity * (see the comments to the function - * bfq_bfqq_may_idle()). + * bfq_better_to_idle()). * @rq_pos_tree: rbtree sorted by next_request position, used when * determining if two or more queues have interleaving * requests (see bfq_find_close_cooperator()). diff --git a/block/bio-integrity-auto.c b/block/bio-integrity-auto.c index e524c609be50..b1c733ecfd2e 100644 --- a/block/bio-integrity-auto.c +++ b/block/bio-integrity-auto.c @@ -9,6 +9,7 @@ * not aware of PI. */ #include <linux/blk-integrity.h> +#include <linux/t10-pi.h> #include <linux/workqueue.h> #include "blk.h" @@ -28,7 +29,7 @@ static void bio_integrity_finish(struct bio_integrity_data *bid) { bid->bio->bi_integrity = NULL; bid->bio->bi_opf &= ~REQ_INTEGRITY; - kfree(bvec_virt(bid->bip.bip_vec)); + bio_integrity_free_buf(&bid->bip); mempool_free(bid, &bid_pool); } @@ -38,11 +39,17 @@ static void bio_integrity_verify_fn(struct work_struct *work) container_of(work, struct bio_integrity_data, work); struct bio *bio = bid->bio; - blk_integrity_verify_iter(bio, &bid->saved_bio_iter); + bio->bi_status = bio_integrity_verify(bio, &bid->saved_bio_iter); bio_integrity_finish(bid); bio_endio(bio); } +#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG) +static bool bip_should_check(struct bio_integrity_payload *bip) +{ + return bip->bip_flags & BIP_CHECK_FLAGS; +} + /** * __bio_integrity_endio - Integrity I/O completion function * @bio: Protected bio @@ -54,12 +61,12 @@ static void bio_integrity_verify_fn(struct work_struct *work) */ bool __bio_integrity_endio(struct bio *bio) { - struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_integrity_data *bid = container_of(bip, struct bio_integrity_data, bip); - if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && bi->csum_type) { + if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && + bip_should_check(bip)) { INIT_WORK(&bid->work, bio_integrity_verify_fn); queue_work(kintegrityd_wq, &bid->work); return false; @@ -72,95 +79,30 @@ bool __bio_integrity_endio(struct bio *bio) /** * bio_integrity_prep - Prepare bio for integrity I/O * @bio: bio to prepare + * @action: preparation action needed (BI_ACT_*) + * + * Allocate the integrity payload. For writes, generate the integrity metadata + * and for reads, setup the completion handler to verify the metadata. * - * Checks if the bio already has an integrity payload attached. If it does, the - * payload has been generated by another kernel subsystem, and we just pass it - * through. - * Otherwise allocates integrity payload and for writes the integrity metadata - * will be generated. For reads, the completion handler will verify the - * metadata. + * This is used for bios that do not have user integrity payloads attached. */ -bool bio_integrity_prep(struct bio *bio) +void bio_integrity_prep(struct bio *bio, unsigned int action) { - struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); struct bio_integrity_data *bid; - gfp_t gfp = GFP_NOIO; - unsigned int len; - void *buf; - - if (!bi) - return true; - - if (!bio_sectors(bio)) - return true; - - /* Already protected? */ - if (bio_integrity(bio)) - return true; - - switch (bio_op(bio)) { - case REQ_OP_READ: - if (bi->flags & BLK_INTEGRITY_NOVERIFY) - return true; - break; - case REQ_OP_WRITE: - if (bi->flags & BLK_INTEGRITY_NOGENERATE) - return true; - - /* - * Zero the memory allocated to not leak uninitialized kernel - * memory to disk for non-integrity metadata where nothing else - * initializes the memory. - */ - if (bi->csum_type == BLK_INTEGRITY_CSUM_NONE) - gfp |= __GFP_ZERO; - break; - default: - return true; - } - - if (WARN_ON_ONCE(bio_has_crypt_ctx(bio))) - return true; - /* Allocate kernel buffer for protection data */ - len = bio_integrity_bytes(bi, bio_sectors(bio)); - buf = kmalloc(len, gfp); - if (!buf) - goto err_end_io; bid = mempool_alloc(&bid_pool, GFP_NOIO); - if (!bid) - goto err_free_buf; bio_integrity_init(bio, &bid->bip, &bid->bvec, 1); - bid->bio = bio; - bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY; - bip_set_seed(&bid->bip, bio->bi_iter.bi_sector); - - if (bi->csum_type == BLK_INTEGRITY_CSUM_IP) - bid->bip.bip_flags |= BIP_IP_CHECKSUM; - if (bi->csum_type) - bid->bip.bip_flags |= BIP_CHECK_GUARD; - if (bi->flags & BLK_INTEGRITY_REF_TAG) - bid->bip.bip_flags |= BIP_CHECK_REFTAG; - - if (bio_integrity_add_page(bio, virt_to_page(buf), len, - offset_in_page(buf)) < len) - goto err_end_io; + bio_integrity_alloc_buf(bio, GFP_NOIO, action & BI_ACT_ZERO); + if (action & BI_ACT_CHECK) + bio_integrity_setup_default(bio); /* Auto-generate integrity metadata if this is a write */ - if (bio_data_dir(bio) == WRITE) - blk_integrity_generate(bio); + if (bio_data_dir(bio) == WRITE && bip_should_check(&bid->bip)) + bio_integrity_generate(bio); else bid->saved_bio_iter = bio->bi_iter; - return true; - -err_free_buf: - kfree(buf); -err_end_io: - bio->bi_status = BLK_STS_RESOURCE; - bio_endio(bio); - return false; } EXPORT_SYMBOL(bio_integrity_prep); @@ -183,7 +125,7 @@ static int __init blk_integrity_auto_init(void) * Make it highpri CPU intensive wq with max concurrency of 1. */ kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM | - WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1); + WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_PERCPU, 1); if (!kintegrityd_wq) panic("Failed to create kintegrityd\n"); return 0; diff --git a/block/bio-integrity-fs.c b/block/bio-integrity-fs.c new file mode 100644 index 000000000000..9c5fe5fa8f0d --- /dev/null +++ b/block/bio-integrity-fs.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2025 Christoph Hellwig. + */ +#include <linux/blk-integrity.h> +#include <linux/bio-integrity.h> +#include "blk.h" + +struct fs_bio_integrity_buf { + struct bio_integrity_payload bip; + struct bio_vec bvec; +}; + +static struct kmem_cache *fs_bio_integrity_cache; +static mempool_t fs_bio_integrity_pool; + +unsigned int fs_bio_integrity_alloc(struct bio *bio) +{ + struct fs_bio_integrity_buf *iib; + unsigned int action; + + action = bio_integrity_action(bio); + if (!action) + return 0; + + iib = mempool_alloc(&fs_bio_integrity_pool, GFP_NOFS); + bio_integrity_init(bio, &iib->bip, &iib->bvec, 1); + + bio_integrity_alloc_buf(bio, GFP_NOFS, action & BI_ACT_ZERO); + if (action & BI_ACT_CHECK) + bio_integrity_setup_default(bio); + return action; +} + +void fs_bio_integrity_free(struct bio *bio) +{ + struct bio_integrity_payload *bip = bio_integrity(bio); + + bio_integrity_free_buf(bip); + mempool_free(container_of(bip, struct fs_bio_integrity_buf, bip), + &fs_bio_integrity_pool); + + bio->bi_integrity = NULL; + bio->bi_opf &= ~REQ_INTEGRITY; +} + +void fs_bio_integrity_generate(struct bio *bio) +{ + if (fs_bio_integrity_alloc(bio)) + bio_integrity_generate(bio); +} +EXPORT_SYMBOL_GPL(fs_bio_integrity_generate); + +int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size) +{ + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); + struct bio_integrity_payload *bip = bio_integrity(bio); + struct bvec_iter data_iter = { + .bi_sector = sector, + .bi_size = size, + }; + + /* + * Reinitialize bip->bip_iter. + * + * This is for use in the submitter after the driver is done with the + * bio. Requires the submitter to remember the sector and the size. + */ + memset(&bip->bip_iter, 0, sizeof(bip->bip_iter)); + bip->bip_iter.bi_sector = sector; + bip->bip_iter.bi_size = bio_integrity_bytes(bi, size >> SECTOR_SHIFT); + return blk_status_to_errno(bio_integrity_verify(bio, &data_iter)); +} + +static int __init fs_bio_integrity_init(void) +{ + fs_bio_integrity_cache = kmem_cache_create("fs_bio_integrity", + sizeof(struct fs_bio_integrity_buf), 0, + SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); + if (mempool_init_slab_pool(&fs_bio_integrity_pool, BIO_POOL_SIZE, + fs_bio_integrity_cache)) + panic("fs_bio_integrity: can't create pool\n"); + return 0; +} +fs_initcall(fs_bio_integrity_init); diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 608594a154a5..b23e2434d80c 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -7,6 +7,7 @@ */ #include <linux/blk-integrity.h> +#include <linux/t10-pi.h> #include "blk.h" struct bio_integrity_alloc { @@ -14,6 +15,107 @@ struct bio_integrity_alloc { struct bio_vec bvecs[]; }; +static mempool_t integrity_buf_pool; + +static bool bi_offload_capable(struct blk_integrity *bi) +{ + return bi->metadata_size == bi->pi_tuple_size; +} + +unsigned int __bio_integrity_action(struct bio *bio) +{ + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); + + if (WARN_ON_ONCE(bio_has_crypt_ctx(bio))) + return 0; + + switch (bio_op(bio)) { + case REQ_OP_READ: + if (bi->flags & BLK_INTEGRITY_NOVERIFY) { + if (bi_offload_capable(bi)) + return 0; + return BI_ACT_BUFFER; + } + return BI_ACT_BUFFER | BI_ACT_CHECK; + case REQ_OP_WRITE: + case REQ_OP_ZONE_APPEND: + /* + * Flush masquerading as write? + */ + if (!bio_sectors(bio)) + return 0; + + /* + * Zero the memory allocated to not leak uninitialized kernel + * memory to disk for non-integrity metadata where nothing else + * initializes the memory. + */ + if (bi->flags & BLK_INTEGRITY_NOGENERATE) { + if (bi_offload_capable(bi)) + return 0; + return BI_ACT_BUFFER | BI_ACT_ZERO; + } + + if (bi->metadata_size > bi->pi_tuple_size) + return BI_ACT_BUFFER | BI_ACT_CHECK | BI_ACT_ZERO; + return BI_ACT_BUFFER | BI_ACT_CHECK; + default: + return 0; + } +} +EXPORT_SYMBOL_GPL(__bio_integrity_action); + +void bio_integrity_alloc_buf(struct bio *bio, gfp_t gfp, bool zero_buffer) +{ + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); + struct bio_integrity_payload *bip = bio_integrity(bio); + unsigned int len = bio_integrity_bytes(bi, bio_sectors(bio)); + void *buf; + + buf = kmalloc(len, gfp | __GFP_NOWARN | (zero_buffer ? __GFP_ZERO : 0)); + if (unlikely(!buf)) { + struct page *page; + + page = mempool_alloc(&integrity_buf_pool, gfp); + if (zero_buffer) + memset(page_address(page), 0, len); + bvec_set_page(&bip->bip_vec[0], page, len, 0); + bip->bip_flags |= BIP_MEMPOOL; + } else { + bvec_set_page(&bip->bip_vec[0], virt_to_page(buf), len, + offset_in_page(buf)); + } + + bip->bip_vcnt = 1; + bip->bip_iter.bi_size = len; +} + +void bio_integrity_free_buf(struct bio_integrity_payload *bip) +{ + struct bio_vec *bv = &bip->bip_vec[0]; + + if (bip->bip_flags & BIP_MEMPOOL) + mempool_free(bv->bv_page, &integrity_buf_pool); + else + kfree(bvec_virt(bv)); +} + +void bio_integrity_setup_default(struct bio *bio) +{ + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); + struct bio_integrity_payload *bip = bio_integrity(bio); + + bip_set_seed(bip, bio->bi_iter.bi_sector); + + if (bi->csum_type) { + bip->bip_flags |= BIP_CHECK_GUARD; + if (bi->csum_type == BLK_INTEGRITY_CSUM_IP) + bip->bip_flags |= BIP_IP_CHECKSUM; + } + if (bi->flags & BLK_INTEGRITY_REF_TAG) + bip->bip_flags |= BIP_CHECK_REFTAG; +} + /** * bio_integrity_free - Free bio integrity payload * @bio: bio containing bip to be freed @@ -58,7 +160,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, if (WARN_ON_ONCE(bio_has_crypt_ctx(bio))) return ERR_PTR(-EOPNOTSUPP); - bia = kmalloc(struct_size(bia, bvecs, nr_vecs), gfp_mask); + bia = kmalloc_flex(*bia, bvecs, nr_vecs, gfp_mask); if (unlikely(!bia)) return ERR_PTR(-ENOMEM); bio_integrity_init(bio, &bia->bip, bia->bvecs, nr_vecs); @@ -66,16 +168,12 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, } EXPORT_SYMBOL(bio_integrity_alloc); -static void bio_integrity_unpin_bvec(struct bio_vec *bv, int nr_vecs, - bool dirty) +static void bio_integrity_unpin_bvec(struct bio_vec *bv, int nr_vecs) { int i; - for (i = 0; i < nr_vecs; i++) { - if (dirty && !PageCompound(bv[i].bv_page)) - set_page_dirty_lock(bv[i].bv_page); + for (i = 0; i < nr_vecs; i++) unpin_user_page(bv[i].bv_page); - } } static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip) @@ -91,7 +189,7 @@ static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip) ret = copy_to_iter(bvec_virt(bounce_bvec), bytes, &orig_iter); WARN_ON_ONCE(ret != bytes); - bio_integrity_unpin_bvec(orig_bvecs, orig_nr_vecs, true); + bio_integrity_unpin_bvec(orig_bvecs, orig_nr_vecs); } /** @@ -111,8 +209,7 @@ void bio_integrity_unmap_user(struct bio *bio) return; } - bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt, - bio_data_dir(bio) == READ); + bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt); } /** @@ -132,10 +229,11 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, if (bip->bip_vcnt > 0) { struct bio_vec *bv = &bip->bip_vec[bip->bip_vcnt - 1]; - bool same_page = false; - if (bvec_try_merge_hw_page(q, bv, page, len, offset, - &same_page)) { + if (!zone_device_pages_compatible(bv->bv_page, page)) + return 0; + if (zone_device_pages_have_same_pgmap(bv->bv_page, page) && + bvec_try_merge_hw_page(q, bv, page, len, offset)) { bip->bip_iter.bi_size += len; return len; } @@ -161,10 +259,9 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, EXPORT_SYMBOL(bio_integrity_add_page); static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec, - int nr_vecs, unsigned int len, - unsigned int direction) + int nr_vecs, unsigned int len) { - bool write = direction == ITER_SOURCE; + bool write = op_is_write(bio_op(bio)); struct bio_integrity_payload *bip; struct iov_iter iter; void *buf; @@ -175,7 +272,7 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec, return -ENOMEM; if (write) { - iov_iter_bvec(&iter, direction, bvec, nr_vecs, len); + iov_iter_bvec(&iter, ITER_SOURCE, bvec, nr_vecs, len); if (!copy_from_iter_full(buf, len, &iter)) { ret = -EFAULT; goto free_buf; @@ -198,7 +295,7 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec, } if (write) - bio_integrity_unpin_bvec(bvec, nr_vecs, false); + bio_integrity_unpin_bvec(bvec, nr_vecs); else memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec)); @@ -210,7 +307,6 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec, } bip->bip_flags |= BIP_COPY_USER; - bip->bip_vcnt = nr_vecs; return 0; free_bip: bio_integrity_free(bio); @@ -235,7 +331,8 @@ static int bio_integrity_init_user(struct bio *bio, struct bio_vec *bvec, } static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages, - int nr_vecs, ssize_t bytes, ssize_t offset) + int nr_vecs, ssize_t bytes, ssize_t offset, + bool *is_p2p) { unsigned int nr_bvecs = 0; int i, j; @@ -256,6 +353,9 @@ static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages, bytes -= next; } + if (is_pci_p2pdma_page(pages[i])) + *is_p2p = true; + bvec_set_page(&bvec[nr_bvecs], pages[i], size, offset); offset = 0; nr_bvecs++; @@ -267,48 +367,69 @@ static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages, int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter) { struct request_queue *q = bdev_get_queue(bio->bi_bdev); - unsigned int align = blk_lim_dma_alignment_and_pad(&q->limits); struct page *stack_pages[UIO_FASTIOV], **pages = stack_pages; struct bio_vec stack_vec[UIO_FASTIOV], *bvec = stack_vec; + iov_iter_extraction_t extraction_flags = 0; size_t offset, bytes = iter->count; - unsigned int direction, nr_bvecs; + bool copy, is_p2p = false; + unsigned int nr_bvecs; int ret, nr_vecs; - bool copy; if (bio_integrity(bio)) return -EINVAL; if (bytes >> SECTOR_SHIFT > queue_max_hw_sectors(q)) return -E2BIG; - if (bio_data_dir(bio) == READ) - direction = ITER_DEST; - else - direction = ITER_SOURCE; - nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS + 1); if (nr_vecs > BIO_MAX_VECS) return -E2BIG; if (nr_vecs > UIO_FASTIOV) { - bvec = kcalloc(nr_vecs, sizeof(*bvec), GFP_KERNEL); + bvec = kzalloc_objs(*bvec, nr_vecs); if (!bvec) return -ENOMEM; pages = NULL; } - copy = !iov_iter_is_aligned(iter, align, align); - ret = iov_iter_extract_pages(iter, &pages, bytes, nr_vecs, 0, &offset); + copy = iov_iter_alignment(iter) & + blk_lim_dma_alignment_and_pad(&q->limits); + + if (blk_queue_pci_p2pdma(q)) + extraction_flags |= ITER_ALLOW_P2PDMA; + + ret = iov_iter_extract_pages(iter, &pages, bytes, nr_vecs, + extraction_flags, &offset); if (unlikely(ret < 0)) goto free_bvec; - nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset); + /* + * Handle partial pinning. This can happen when pin_user_pages_fast() + * returns fewer pages than requested. + */ + if (user_backed_iter(iter) && unlikely(ret != bytes)) { + if (ret > 0) { + int npinned = DIV_ROUND_UP(offset + ret, PAGE_SIZE); + int i; + + for (i = 0; i < npinned; i++) + unpin_user_page(pages[i]); + } + if (pages != stack_pages) + kvfree(pages); + ret = -EFAULT; + goto free_bvec; + } + + nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset, + &is_p2p); if (pages != stack_pages) kvfree(pages); if (nr_bvecs > queue_max_integrity_segments(q)) copy = true; + if (is_p2p) + bio->bi_opf |= REQ_NOMERGE; if (copy) - ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes, - direction); + ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes); else ret = bio_integrity_init_user(bio, bvec, nr_bvecs, bytes); if (ret) @@ -319,7 +440,7 @@ int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter) return 0; release_pages: - bio_integrity_unpin_bvec(bvec, nr_bvecs, false); + bio_integrity_unpin_bvec(bvec, nr_bvecs); free_bvec: if (bvec != stack_vec) kfree(bvec); @@ -436,3 +557,12 @@ int bio_integrity_clone(struct bio *bio, struct bio *bio_src, return 0; } + +static int __init bio_integrity_initfn(void) +{ + if (mempool_init_page_pool(&integrity_buf_pool, BIO_POOL_SIZE, + get_order(BLK_INTEGRITY_MAX_SIZE))) + panic("bio: can't create integrity buf pool\n"); + return 0; +} +subsys_initcall(bio_integrity_initfn); diff --git a/block/bio.c b/block/bio.c index 4e6c85a33d74..6a2f6fc3413e 100644 --- a/block/bio.c +++ b/block/bio.c @@ -18,6 +18,7 @@ #include <linux/highmem.h> #include <linux/blk-crypto.h> #include <linux/xarray.h> +#include <linux/kmemleak.h> #include <trace/events/block.h> #include "blk.h" @@ -34,6 +35,8 @@ struct bio_alloc_cache { unsigned int nr_irq; }; +#define BIO_INLINE_VECS 4 + static struct biovec_slab { int nr_vecs; char *name; @@ -84,7 +87,7 @@ static DEFINE_XARRAY(bio_slabs); static struct bio_slab *create_bio_slab(unsigned int size) { - struct bio_slab *bslab = kzalloc(sizeof(*bslab), GFP_KERNEL); + struct bio_slab *bslab = kzalloc_obj(*bslab); if (!bslab) return NULL; @@ -114,6 +117,11 @@ static inline unsigned int bs_bio_slab_size(struct bio_set *bs) return bs->front_pad + sizeof(struct bio) + bs->back_pad; } +static inline void *bio_slab_addr(struct bio *bio) +{ + return (void *)bio - bio->bi_pool->front_pad; +} + static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs) { unsigned int size = bs_bio_slab_size(bs); @@ -159,57 +167,16 @@ out: mutex_unlock(&bio_slab_lock); } -void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs) -{ - BUG_ON(nr_vecs > BIO_MAX_VECS); - - if (nr_vecs == BIO_MAX_VECS) - mempool_free(bv, pool); - else if (nr_vecs > BIO_INLINE_VECS) - kmem_cache_free(biovec_slab(nr_vecs)->slab, bv); -} - /* * Make the first allocation restricted and don't dump info on allocation * failures, since we'll fall back to the mempool in case of failure. */ -static inline gfp_t bvec_alloc_gfp(gfp_t gfp) +static inline gfp_t try_alloc_gfp(gfp_t gfp) { return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) | __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; } -struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, - gfp_t gfp_mask) -{ - struct biovec_slab *bvs = biovec_slab(*nr_vecs); - - if (WARN_ON_ONCE(!bvs)) - return NULL; - - /* - * Upgrade the nr_vecs request to take full advantage of the allocation. - * We also rely on this in the bvec_free path. - */ - *nr_vecs = bvs->nr_vecs; - - /* - * Try a slab allocation first for all smaller allocations. If that - * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool. - * The mempool is sized to handle up to BIO_MAX_VECS entries. - */ - if (*nr_vecs < BIO_MAX_VECS) { - struct bio_vec *bvl; - - bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask)); - if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) - return bvl; - *nr_vecs = BIO_MAX_VECS; - } - - return mempool_alloc(pool, gfp_mask); -} - void bio_uninit(struct bio *bio) { #ifdef CONFIG_BLK_CGROUP @@ -231,9 +198,14 @@ static void bio_free(struct bio *bio) void *p = bio; WARN_ON_ONCE(!bs); + WARN_ON_ONCE(bio->bi_max_vecs > BIO_MAX_VECS); bio_uninit(bio); - bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs); + if (bio->bi_max_vecs == BIO_MAX_VECS) + mempool_free(bio->bi_io_vec, &bs->bvec_pool); + else if (bio->bi_max_vecs > BIO_INLINE_VECS) + kmem_cache_free(biovec_slab(bio->bi_max_vecs)->slab, + bio->bi_io_vec); mempool_free(p - bs->front_pad, &bs->bio_pool); } @@ -251,7 +223,9 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, bio->bi_flags = 0; bio->bi_ioprio = 0; bio->bi_write_hint = 0; + bio->bi_write_stream = 0; bio->bi_status = 0; + bio->bi_bvec_gap_bit = 0; bio->bi_iter.bi_sector = 0; bio->bi_iter.bi_size = 0; bio->bi_iter.bi_idx = 0; @@ -260,7 +234,7 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, bio->bi_private = NULL; #ifdef CONFIG_BLK_CGROUP bio->bi_blkg = NULL; - bio->bi_issue.value = 0; + bio->issue_time_ns = 0; if (bdev) bio_associate_blkg(bio); #ifdef CONFIG_BLK_CGROUP_IOCOST @@ -299,9 +273,12 @@ EXPORT_SYMBOL(bio_init); */ void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf) { + struct bio_vec *bv = bio->bi_io_vec; + bio_uninit(bio); memset(bio, 0, BIO_RESET_BYTES); atomic_set(&bio->__bi_remaining, 1); + bio->bi_io_vec = bv; bio->bi_bdev = bdev; if (bio->bi_bdev) bio_associate_blkg(bio); @@ -309,6 +286,40 @@ void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf) } EXPORT_SYMBOL(bio_reset); +/** + * bio_reuse - reuse a bio with the payload left intact + * @bio: bio to reuse + * @opf: operation and flags for the next I/O + * + * Allow reusing an existing bio for another operation with all set up + * fields including the payload, device and end_io handler left intact. + * + * Typically used when @bio is first used to read data which is then written + * to another location without modification. @bio must not be in-flight and + * owned by the caller. Can't be used for cloned bios. + * + * Note: Can't be used when @bio has integrity or blk-crypto contexts for now. + * Feel free to add that support when you need it, though. + */ +void bio_reuse(struct bio *bio, blk_opf_t opf) +{ + unsigned short vcnt = bio->bi_vcnt, i; + bio_end_io_t *end_io = bio->bi_end_io; + void *private = bio->bi_private; + + WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); + WARN_ON_ONCE(bio_integrity(bio)); + WARN_ON_ONCE(bio_has_crypt_ctx(bio)); + + bio_reset(bio, bio->bi_bdev, opf); + for (i = 0; i < vcnt; i++) + bio->bi_iter.bi_size += bio->bi_io_vec[i].bv_len; + bio->bi_vcnt = vcnt; + bio->bi_private = private; + bio->bi_end_io = end_io; +} +EXPORT_SYMBOL_GPL(bio_reuse); + static struct bio *__bio_chain_endio(struct bio *bio) { struct bio *parent = bio->bi_private; @@ -319,9 +330,13 @@ static struct bio *__bio_chain_endio(struct bio *bio) return parent; } +/* + * This function should only be used as a flag and must never be called. + * If execution reaches here, it indicates a serious programming error. + */ static void bio_chain_endio(struct bio *bio) { - bio_endio(__bio_chain_endio(bio)); + BUG(); } /** @@ -387,13 +402,31 @@ static void bio_alloc_rescue(struct work_struct *work) } } +/* + * submit_bio_noacct() converts recursion to iteration; this means if we're + * running beneath it, any bios we allocate and submit will not be submitted + * (and thus freed) until after we return. + * + * This exposes us to a potential deadlock if we allocate multiple bios from the + * same bio_set while running underneath submit_bio_noacct(). If we were to + * allocate multiple bios (say a stacking block driver that was splitting bios), + * we would deadlock if we exhausted the mempool's reserve. + * + * We solve this, and guarantee forward progress by punting the bios on + * current->bio_list to a per bio_set rescuer workqueue before blocking to wait + * for elements being returned to the mempool. + */ static void punt_bios_to_rescuer(struct bio_set *bs) { struct bio_list punt, nopunt; struct bio *bio; - if (WARN_ON_ONCE(!bs->rescue_workqueue)) + if (!current->bio_list || !bs->rescue_workqueue) + return; + if (bio_list_empty(¤t->bio_list[0]) && + bio_list_empty(¤t->bio_list[1])) return; + /* * In order to guarantee forward progress we must punt only bios that * were allocated from this bio_set; otherwise, if there was a bio on @@ -440,9 +473,7 @@ static void bio_alloc_irq_cache_splice(struct bio_alloc_cache *cache) local_irq_restore(flags); } -static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, - unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp, - struct bio_set *bs) +static struct bio *bio_alloc_percpu_cache(struct bio_set *bs) { struct bio_alloc_cache *cache; struct bio *bio; @@ -460,9 +491,10 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, cache->free_list = bio->bi_next; cache->nr--; put_cpu(); - - bio_init(bio, bdev, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs, opf); bio->bi_pool = bs; + + kmemleak_alloc(bio_slab_addr(bio), + kmem_cache_size(bs->bio_slab), 1, GFP_NOIO); return bio; } @@ -471,7 +503,7 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, * @bdev: block device to allocate the bio for (can be %NULL) * @nr_vecs: number of bvecs to pre-allocate * @opf: operation and flags for bio - * @gfp_mask: the GFP_* mask given to the slab allocator + * @gfp: the GFP_* mask given to the slab allocator * @bs: the bio_set to allocate from. * * Allocate a bio from the mempools in @bs. @@ -501,93 +533,86 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, * Returns: Pointer to new bio on success, NULL on failure. */ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, - blk_opf_t opf, gfp_t gfp_mask, - struct bio_set *bs) + blk_opf_t opf, gfp_t gfp, struct bio_set *bs) { - gfp_t saved_gfp = gfp_mask; - struct bio *bio; + struct bio_vec *bvecs = NULL; + struct bio *bio = NULL; + gfp_t saved_gfp = gfp; void *p; /* should not use nobvec bioset for nr_vecs > 0 */ if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0)) return NULL; - if (opf & REQ_ALLOC_CACHE) { - if (bs->cache && nr_vecs <= BIO_INLINE_VECS) { - bio = bio_alloc_percpu_cache(bdev, nr_vecs, opf, - gfp_mask, bs); - if (bio) - return bio; - /* - * No cached bio available, bio returned below marked with - * REQ_ALLOC_CACHE to particpate in per-cpu alloc cache. - */ - } else { - opf &= ~REQ_ALLOC_CACHE; - } + if (saved_gfp & __GFP_DIRECT_RECLAIM) + gfp = try_alloc_gfp(gfp); + if (bs->cache && nr_vecs <= BIO_INLINE_VECS) { + /* + * Set REQ_ALLOC_CACHE even if no cached bio is available to + * return the allocated bio to the percpu cache when done. + */ + opf |= REQ_ALLOC_CACHE; + bio = bio_alloc_percpu_cache(bs); + } else { + opf &= ~REQ_ALLOC_CACHE; } /* - * submit_bio_noacct() converts recursion to iteration; this means if - * we're running beneath it, any bios we allocate and submit will not be - * submitted (and thus freed) until after we return. - * - * This exposes us to a potential deadlock if we allocate multiple bios - * from the same bio_set() while running underneath submit_bio_noacct(). - * If we were to allocate multiple bios (say a stacking block driver - * that was splitting bios), we would deadlock if we exhausted the - * mempool's reserve. - * - * We solve this, and guarantee forward progress, with a rescuer - * workqueue per bio_set. If we go to allocate and there are bios on - * current->bio_list, we first try the allocation without - * __GFP_DIRECT_RECLAIM; if that fails, we punt those bios we would be - * blocking to the rescuer workqueue before we retry with the original - * gfp_flags. + * For a bioset without a percpu cache, or when the percpu cache was + * empty, try a slab allocation with optimistic GFP_ flags before + * falling back to the mempool. */ - if (current->bio_list && - (!bio_list_empty(¤t->bio_list[0]) || - !bio_list_empty(¤t->bio_list[1])) && - bs->rescue_workqueue) - gfp_mask &= ~__GFP_DIRECT_RECLAIM; - - p = mempool_alloc(&bs->bio_pool, gfp_mask); - if (!p && gfp_mask != saved_gfp) { - punt_bios_to_rescuer(bs); - gfp_mask = saved_gfp; - p = mempool_alloc(&bs->bio_pool, gfp_mask); + if (!bio) { + p = kmem_cache_alloc(bs->bio_slab, gfp); + if (p) + bio = p + bs->front_pad; } - if (unlikely(!p)) - return NULL; - if (!mempool_is_saturated(&bs->bio_pool)) - opf &= ~REQ_ALLOC_CACHE; - bio = p + bs->front_pad; - if (nr_vecs > BIO_INLINE_VECS) { - struct bio_vec *bvl = NULL; - - bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask); - if (!bvl && gfp_mask != saved_gfp) { - punt_bios_to_rescuer(bs); - gfp_mask = saved_gfp; - bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask); + if (bio && nr_vecs > BIO_INLINE_VECS) { + struct biovec_slab *bvs = biovec_slab(nr_vecs); + + /* + * Upgrade nr_vecs to take full advantage of the allocation. + * We also rely on this in bio_free(). + */ + nr_vecs = bvs->nr_vecs; + bvecs = kmem_cache_alloc(bvs->slab, gfp); + if (unlikely(!bvecs)) { + kmem_cache_free(bs->bio_slab, p); + bio = NULL; } - if (unlikely(!bvl)) - goto err_free; + } - bio_init(bio, bdev, bvl, nr_vecs, opf); - } else if (nr_vecs) { - bio_init(bio, bdev, bio->bi_inline_vecs, BIO_INLINE_VECS, opf); - } else { - bio_init(bio, bdev, NULL, 0, opf); + if (unlikely(!bio)) { + /* + * Give up if we are not allow to sleep as non-blocking mempool + * allocations just go back to the slab allocation. + */ + if (!(saved_gfp & __GFP_DIRECT_RECLAIM)) + return NULL; + + punt_bios_to_rescuer(bs); + + /* + * Don't rob the mempools by returning to the per-CPU cache if + * we're tight on memory. + */ + opf &= ~REQ_ALLOC_CACHE; + + p = mempool_alloc(&bs->bio_pool, saved_gfp); + bio = p + bs->front_pad; + if (nr_vecs > BIO_INLINE_VECS) { + nr_vecs = BIO_MAX_VECS; + bvecs = mempool_alloc(&bs->bvec_pool, saved_gfp); + } } + if (nr_vecs && nr_vecs <= BIO_INLINE_VECS) + bio_init_inline(bio, bdev, nr_vecs, opf); + else + bio_init(bio, bdev, bvecs, nr_vecs, opf); bio->bi_pool = bs; return bio; - -err_free: - mempool_free(p, &bs->bio_pool); - return NULL; } EXPORT_SYMBOL(bio_alloc_bioset); @@ -611,21 +636,22 @@ struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask) { struct bio *bio; - if (nr_vecs > UIO_MAXIOV) + if (nr_vecs > BIO_MAX_INLINE_VECS) return NULL; - return kmalloc(struct_size(bio, bi_inline_vecs, nr_vecs), gfp_mask); + return kmalloc(sizeof(*bio) + nr_vecs * sizeof(struct bio_vec), + gfp_mask); } EXPORT_SYMBOL(bio_kmalloc); -void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start) +void zero_fill_bio(struct bio *bio) { struct bio_vec bv; struct bvec_iter iter; - __bio_for_each_segment(bv, bio, iter, start) + bio_for_each_segment(bv, bio, iter) memzero_bvec(&bv); } -EXPORT_SYMBOL(zero_fill_bio_iter); +EXPORT_SYMBOL(zero_fill_bio); /** * bio_truncate - truncate the bio to small size of @new_size @@ -652,13 +678,13 @@ static void bio_truncate(struct bio *bio, unsigned new_size) bio_for_each_segment(bv, bio, iter) { if (done + bv.bv_len > new_size) { - unsigned offset; + size_t offset; if (!truncated) offset = new_size - done; else offset = 0; - zero_user(bv.bv_page, bv.bv_offset + offset, + memzero_page(bv.bv_page, bv.bv_offset + offset, bv.bv_len - offset); truncated = true; } @@ -720,6 +746,9 @@ static int __bio_alloc_cache_prune(struct bio_alloc_cache *cache, while ((bio = cache->free_list) != NULL) { cache->free_list = bio->bi_next; cache->nr--; + kmemleak_alloc(bio_slab_addr(bio), + kmem_cache_size(bio->bi_pool->bio_slab), + 1, GFP_KERNEL); bio_free(bio); if (++i == nr) break; @@ -783,6 +812,7 @@ static inline void bio_put_percpu_cache(struct bio *bio) bio->bi_bdev = NULL; cache->free_list = bio; cache->nr++; + kmemleak_free(bio_slab_addr(bio)); } else if (in_hardirq()) { lockdep_assert_irqs_disabled(); @@ -790,6 +820,7 @@ static inline void bio_put_percpu_cache(struct bio *bio) bio->bi_next = cache->free_list_irq; cache->free_list_irq = bio; cache->nr_irq++; + kmemleak_free(bio_slab_addr(bio)); } else { goto out_free; } @@ -827,6 +858,7 @@ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp) bio_set_flag(bio, BIO_CLONED); bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; + bio->bi_write_stream = bio_src->bi_write_stream; bio->bi_iter = bio_src->bi_iter; if (bio->bi_bdev) { @@ -851,10 +883,11 @@ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp) * @gfp: allocation priority * @bs: bio_set to allocate from * - * Allocate a new bio that is a clone of @bio_src. The caller owns the returned - * bio, but not the actual data it points to. - * - * The caller must ensure that the return bio is not freed before @bio_src. + * Allocate a new bio that is a clone of @bio_src. This reuses the bio_vecs + * pointed to by @bio_src->bi_io_vec, and clones the iterator pointing to + * the current position in it. The caller owns the returned bio, but not + * the bio_vecs, and must ensure the bio is freed before the memory + * pointed to by @bio_Src->bi_io_vecs. */ struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src, gfp_t gfp, struct bio_set *bs) @@ -883,9 +916,7 @@ EXPORT_SYMBOL(bio_alloc_clone); * @gfp: allocation priority * * Initialize a new bio in caller provided memory that is a clone of @bio_src. - * The caller owns the returned bio, but not the actual data it points to. - * - * The caller must ensure that @bio_src is not freed before @bio. + * The same bio_vecs reuse and bio lifetime rules as bio_alloc_clone() apply. */ int bio_init_clone(struct block_device *bdev, struct bio *bio, struct bio *bio_src, gfp_t gfp) @@ -912,13 +943,13 @@ static inline bool bio_full(struct bio *bio, unsigned len) { if (bio->bi_vcnt >= bio->bi_max_vecs) return true; - if (bio->bi_iter.bi_size > UINT_MAX - len) + if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len) return true; return false; } static bool bvec_try_merge_page(struct bio_vec *bv, struct page *page, - unsigned int len, unsigned int off, bool *same_page) + unsigned int len, unsigned int off) { size_t bv_end = bv->bv_offset + bv->bv_len; phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + bv_end - 1; @@ -928,12 +959,8 @@ static bool bvec_try_merge_page(struct bio_vec *bv, struct page *page, return false; if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) return false; - if (!zone_device_pages_have_same_pgmap(bv->bv_page, page)) - return false; - *same_page = ((vec_end_addr & PAGE_MASK) == ((page_addr + off) & - PAGE_MASK)); - if (!*same_page) { + if ((vec_end_addr & PAGE_MASK) != ((page_addr + off) & PAGE_MASK)) { if (IS_ENABLED(CONFIG_KMSAN)) return false; if (bv->bv_page + bv_end / PAGE_SIZE != page + off / PAGE_SIZE) @@ -953,8 +980,7 @@ static bool bvec_try_merge_page(struct bio_vec *bv, struct page *page, * helpers to split. Hopefully this will go away soon. */ bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv, - struct page *page, unsigned len, unsigned offset, - bool *same_page) + struct page *page, unsigned len, unsigned offset) { unsigned long mask = queue_segment_boundary(q); phys_addr_t addr1 = bvec_phys(bv); @@ -964,7 +990,7 @@ bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv, return false; if (len > queue_max_segment_size(q) - bv->bv_len) return false; - return bvec_try_merge_page(bv, page, len, offset, same_page); + return bvec_try_merge_page(bv, page, len, offset); } /** @@ -983,6 +1009,9 @@ void __bio_add_page(struct bio *bio, struct page *page, WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); WARN_ON_ONCE(bio_full(bio, len)); + if (is_pci_p2pdma_page(page)) + bio->bi_opf |= REQ_NOMERGE; + bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt], page, len, off); bio->bi_iter.bi_size += len; bio->bi_vcnt++; @@ -990,6 +1019,22 @@ void __bio_add_page(struct bio *bio, struct page *page, EXPORT_SYMBOL_GPL(__bio_add_page); /** + * bio_add_virt_nofail - add data in the direct kernel mapping to a bio + * @bio: destination bio + * @vaddr: data to add + * @len: length of the data to add, may cross pages + * + * Add the data at @vaddr to @bio. The caller must have ensure a segment + * is available for the added data. No merging into an existing segment + * will be performed. + */ +void bio_add_virt_nofail(struct bio *bio, void *vaddr, unsigned len) +{ + __bio_add_page(bio, virt_to_page(vaddr), len, offset_in_page(vaddr)); +} +EXPORT_SYMBOL_GPL(bio_add_virt_nofail); + +/** * bio_add_page - attempt to add page(s) to bio * @bio: destination bio * @page: start page to add @@ -1002,18 +1047,23 @@ EXPORT_SYMBOL_GPL(__bio_add_page); int bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { - bool same_page = false; - if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return 0; - if (bio->bi_iter.bi_size > UINT_MAX - len) + if (WARN_ON_ONCE(len == 0)) + return 0; + if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len) return 0; - if (bio->bi_vcnt > 0 && - bvec_try_merge_page(&bio->bi_io_vec[bio->bi_vcnt - 1], - page, len, offset, &same_page)) { - bio->bi_iter.bi_size += len; - return len; + if (bio->bi_vcnt > 0) { + struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; + + if (!zone_device_pages_compatible(bv->bv_page, page)) + return 0; + if (zone_device_pages_have_same_pgmap(bv->bv_page, page) && + bvec_try_merge_page(bv, page, len, offset)) { + bio->bi_iter.bi_size += len; + return len; + } } if (bio->bi_vcnt >= bio->bi_max_vecs) @@ -1028,7 +1078,7 @@ void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len, { unsigned long nr = off / PAGE_SIZE; - WARN_ON_ONCE(len > UINT_MAX); + WARN_ON_ONCE(len > BIO_MAX_SIZE); __bio_add_page(bio, folio_page(folio, nr), len, off % PAGE_SIZE); } EXPORT_SYMBOL_GPL(bio_add_folio_nofail); @@ -1052,12 +1102,67 @@ bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len, { unsigned long nr = off / PAGE_SIZE; - if (len > UINT_MAX) + if (len > BIO_MAX_SIZE) return false; return bio_add_page(bio, folio_page(folio, nr), len, off % PAGE_SIZE) > 0; } EXPORT_SYMBOL(bio_add_folio); +/** + * bio_add_vmalloc_chunk - add a vmalloc chunk to a bio + * @bio: destination bio + * @vaddr: vmalloc address to add + * @len: total length in bytes of the data to add + * + * Add data starting at @vaddr to @bio and return how many bytes were added. + * This may be less than the amount originally asked. Returns 0 if no data + * could be added to @bio. + * + * This helper calls flush_kernel_vmap_range() for the range added. For reads + * the caller still needs to manually call invalidate_kernel_vmap_range() in + * the completion handler. + */ +unsigned int bio_add_vmalloc_chunk(struct bio *bio, void *vaddr, unsigned len) +{ + unsigned int offset = offset_in_page(vaddr); + + len = min(len, PAGE_SIZE - offset); + if (bio_add_page(bio, vmalloc_to_page(vaddr), len, offset) < len) + return 0; + if (op_is_write(bio_op(bio))) + flush_kernel_vmap_range(vaddr, len); + return len; +} +EXPORT_SYMBOL_GPL(bio_add_vmalloc_chunk); + +/** + * bio_add_vmalloc - add a vmalloc region to a bio + * @bio: destination bio + * @vaddr: vmalloc address to add + * @len: total length in bytes of the data to add + * + * Add data starting at @vaddr to @bio. Return %true on success or %false if + * @bio does not have enough space for the payload. + * + * This helper calls flush_kernel_vmap_range() for the range added. For reads + * the caller still needs to manually call invalidate_kernel_vmap_range() in + * the completion handler. + */ +bool bio_add_vmalloc(struct bio *bio, void *vaddr, unsigned int len) +{ + do { + unsigned int added = bio_add_vmalloc_chunk(bio, vaddr, len); + + if (!added) + return false; + vaddr += added; + len -= added; + } while (len); + + return true; +} +EXPORT_SYMBOL_GPL(bio_add_vmalloc); + void __bio_release_pages(struct bio *bio, bool mark_dirty) { struct folio_iter fi; @@ -1081,156 +1186,46 @@ void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter) { WARN_ON_ONCE(bio->bi_max_vecs); - bio->bi_vcnt = iter->nr_segs; bio->bi_io_vec = (struct bio_vec *)iter->bvec; + bio->bi_iter.bi_idx = 0; bio->bi_iter.bi_bvec_done = iter->iov_offset; bio->bi_iter.bi_size = iov_iter_count(iter); bio_set_flag(bio, BIO_CLONED); } -static int bio_iov_add_folio(struct bio *bio, struct folio *folio, size_t len, - size_t offset) +/* + * Aligns the bio size to the len_align_mask, releasing excessive bio vecs that + * __bio_iov_iter_get_pages may have inserted, and reverts the trimmed length + * for the next iteration. + */ +static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, + struct bio_vec *bv, unsigned len_align_mask) { - bool same_page = false; - - if (WARN_ON_ONCE(bio->bi_iter.bi_size > UINT_MAX - len)) - return -EIO; + size_t nbytes = bio->bi_iter.bi_size & len_align_mask; - if (bio->bi_vcnt > 0 && - bvec_try_merge_page(&bio->bi_io_vec[bio->bi_vcnt - 1], - folio_page(folio, 0), len, offset, - &same_page)) { - bio->bi_iter.bi_size += len; - if (same_page && bio_flagged(bio, BIO_PAGE_PINNED)) - unpin_user_folio(folio, 1); + if (!nbytes) return 0; - } - bio_add_folio_nofail(bio, folio, len, offset); - return 0; -} - -static unsigned int get_contig_folio_len(unsigned int *num_pages, - struct page **pages, unsigned int i, - struct folio *folio, size_t left, - size_t offset) -{ - size_t bytes = left; - size_t contig_sz = min_t(size_t, PAGE_SIZE - offset, bytes); - unsigned int j; - - /* - * We might COW a single page in the middle of - * a large folio, so we have to check that all - * pages belong to the same folio. - */ - bytes -= contig_sz; - for (j = i + 1; j < i + *num_pages; j++) { - size_t next = min_t(size_t, PAGE_SIZE, bytes); - - if (page_folio(pages[j]) != folio || - pages[j] != pages[j - 1] + 1) { - break; - } - contig_sz += next; - bytes -= next; - } - *num_pages = j - i; - - return contig_sz; -} - -#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) - -/** - * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio - * @bio: bio to add pages to - * @iter: iov iterator describing the region to be mapped - * - * Extracts pages from *iter and appends them to @bio's bvec array. The pages - * will have to be cleaned up in the way indicated by the BIO_PAGE_PINNED flag. - * For a multi-segment *iter, this function only adds pages from the next - * non-empty segment of the iov iterator. - */ -static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) -{ - iov_iter_extraction_t extraction_flags = 0; - unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; - unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; - struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; - struct page **pages = (struct page **)bv; - ssize_t size; - unsigned int num_pages, i = 0; - size_t offset, folio_offset, left, len; - int ret = 0; - - /* - * Move page array up in the allocated memory for the bio vecs as far as - * possible so that we can start filling biovecs from the beginning - * without overwriting the temporary page array. - */ - BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2); - pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); - - if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) - extraction_flags |= ITER_ALLOW_P2PDMA; - - /* - * Each segment in the iov is required to be a block size multiple. - * However, we may not be able to get the entire segment if it spans - * more pages than bi_max_vecs allows, so we have to ALIGN_DOWN the - * result to ensure the bio's total size is correct. The remainder of - * the iov data will be picked up in the next bio iteration. - */ - size = iov_iter_extract_pages(iter, &pages, - UINT_MAX - bio->bi_iter.bi_size, - nr_pages, extraction_flags, &offset); - if (unlikely(size <= 0)) - return size ? size : -EFAULT; - - nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE); - - if (bio->bi_bdev) { - size_t trim = size & (bdev_logical_block_size(bio->bi_bdev) - 1); - iov_iter_revert(iter, trim); - size -= trim; - } - - if (unlikely(!size)) { - ret = -EFAULT; - goto out; - } - - for (left = size, i = 0; left > 0; left -= len, i += num_pages) { - struct page *page = pages[i]; - struct folio *folio = page_folio(page); - - folio_offset = ((size_t)folio_page_idx(folio, page) << - PAGE_SHIFT) + offset; - - len = min(folio_size(folio) - folio_offset, left); - - num_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); - if (num_pages > 1) - len = get_contig_folio_len(&num_pages, pages, i, - folio, left, offset); + iov_iter_revert(iter, nbytes); + bio->bi_iter.bi_size -= nbytes; + while (nbytes >= bv->bv_len) { + if (bio_flagged(bio, BIO_PAGE_PINNED)) + unpin_user_page(bv->bv_page); - bio_iov_add_folio(bio, folio, len, folio_offset); - offset = 0; + if (!--bio->bi_vcnt) + return -EFAULT; + nbytes -= bv->bv_len; + bv--; } - - iov_iter_revert(iter, left); -out: - while (i < nr_pages) - bio_release_page(bio, pages[i++]); - - return ret; + bv->bv_len -= nbytes; + return 0; } /** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to * @iter: iov iterator describing the region to be added + * @len_align_mask: the mask to align the total size to, 0 for any length * * This takes either an iterator pointing to user memory, or one pointing to * kernel pages (BVEC iterator). If we're adding user pages, we pin them and @@ -1247,9 +1242,10 @@ out: * MM encounters an error pinning the requested pages, it stops. Error * is returned only if 0 pages could be pinned. */ -int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) +int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, + unsigned len_align_mask) { - int ret = 0; + iov_iter_extraction_t flags = 0; if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return -EIO; @@ -1262,20 +1258,279 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED); + if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) + flags |= ITER_ALLOW_P2PDMA; + do { - ret = __bio_iov_iter_get_pages(bio, iter); - } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); + ssize_t ret; + + ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec, + BIO_MAX_SIZE - bio->bi_iter.bi_size, + &bio->bi_vcnt, bio->bi_max_vecs, flags); + if (ret <= 0) { + if (!bio->bi_vcnt) + return ret; + break; + } + bio->bi_iter.bi_size += ret; + } while (iov_iter_count(iter) && !bio_full(bio, 0)); + + if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page)) + bio->bi_opf |= REQ_NOMERGE; + return bio_iov_iter_align_down(bio, iter, + &bio->bi_io_vec[bio->bi_vcnt - 1], len_align_mask); +} + +static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size, + size_t minsize) +{ + struct folio *folio; + + while (*size > minsize) { + folio = folio_alloc(gfp | __GFP_NORETRY | __GFP_NOWARN, + get_order(*size)); + if (folio) + return folio; + *size = rounddown_pow_of_two(*size - 1); + } + + return folio_alloc(gfp, get_order(*size)); +} + +static void bio_free_folios(struct bio *bio) +{ + struct bio_vec *bv; + int i; - return bio->bi_vcnt ? 0 : ret; + bio_for_each_bvec_all(bv, bio, i) { + struct folio *folio = bvec_folio(bv); + + if (!is_zero_folio(folio) && !is_huge_zero_folio(folio)) + folio_put(folio); + } +} + +static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter, + size_t maxlen, size_t minsize) +{ + size_t total_len = min(maxlen, iov_iter_count(iter)); + + if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) + return -EINVAL; + if (WARN_ON_ONCE(bio->bi_iter.bi_size)) + return -EINVAL; + if (WARN_ON_ONCE(bio->bi_vcnt >= bio->bi_max_vecs)) + return -EINVAL; + + do { + size_t this_len = min(total_len, SZ_1M); + size_t copied; + struct folio *folio; + + if (this_len > minsize * 2) + this_len = rounddown_pow_of_two(this_len); + + if (bio->bi_iter.bi_size > BIO_MAX_SIZE - this_len) + break; + + folio = folio_alloc_greedy(GFP_KERNEL, &this_len, minsize); + if (!folio) + break; + bio_add_folio_nofail(bio, folio, this_len, 0); + + if (iter->nofault) + copied = copy_folio_from_iter_atomic(folio, 0, this_len, + iter); + else + copied = copy_folio_from_iter(folio, 0, this_len, iter); + if (copied < this_len) { + /* + * Need to revert the iov iter for all bytes we have + * copied. + * + * However the bio size differs from the real copied + * bytes as @this_len is queued but only advanced + * less than that. + * Need to compensate that for the revert. + */ + iov_iter_revert(iter, bio->bi_iter.bi_size - this_len + + copied); + bio_free_folios(bio); + return -EFAULT; + } + total_len -= this_len; + } while (total_len && bio->bi_vcnt < bio->bi_max_vecs); + + if (!bio->bi_iter.bi_size) + return -ENOMEM; + return bio_iov_iter_align_down(bio, iter, + &bio->bi_io_vec[bio->bi_vcnt - 1], minsize - 1); } -EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages); -static void submit_bio_wait_endio(struct bio *bio) +static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter, + size_t maxlen, size_t minsize) +{ + size_t len = min3(iov_iter_count(iter), maxlen, SZ_1M); + struct folio *folio; + ssize_t ret; + + folio = folio_alloc_greedy(GFP_KERNEL, &len, minsize); + if (!folio) + return -ENOMEM; + + do { + ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec + 1, len, + &bio->bi_vcnt, bio->bi_max_vecs - 1, 0); + if (ret <= 0) { + if (!bio->bi_vcnt) + goto out_folio_put; + break; + } + len -= ret; + bio->bi_iter.bi_size += ret; + } while (len && bio->bi_vcnt < bio->bi_max_vecs - 1); + + /* + * Set the folio directly here. The above loop has already calculated + * the correct bi_size, and we use bi_vcnt for the user buffers. That + * is safe as bi_vcnt is only used by the submitter and not the actual + * I/O path. + */ + bvec_set_folio(&bio->bi_io_vec[0], folio, bio->bi_iter.bi_size, 0); + if (iov_iter_extract_will_pin(iter)) + bio_set_flag(bio, BIO_PAGE_PINNED); + + /* The first vec stores the bounce buffer, so do not subtract 1 here. */ + ret = bio_iov_iter_align_down(bio, iter, + &bio->bi_io_vec[bio->bi_vcnt], minsize - 1); + if (ret) + goto out_folio_put; + + /* Update the bounc buffer bv_len to the aligned down size. */ + bio->bi_io_vec[0].bv_len = bio->bi_iter.bi_size; + return 0; + +out_folio_put: + folio_put(folio); + return ret; +} + +/** + * bio_iov_iter_bounce - bounce buffer data from an iter into a bio + * @bio: bio to send + * @iter: iter to read from / write into + * @maxlen: maximum size to bounce + * @minsize: minimum folio allocation size + * + * Helper for direct I/O implementations that need to bounce buffer because + * we need to checksum the data or perform other operations that require + * consistency. Allocates folios to back the bounce buffer, and for writes + * copies the data into it. Needs to be paired with bio_iov_iter_unbounce() + * called on completion. + */ +int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen, + size_t minsize) +{ + if (op_is_write(bio_op(bio))) + return bio_iov_iter_bounce_write(bio, iter, maxlen, minsize); + return bio_iov_iter_bounce_read(bio, iter, maxlen, minsize); +} + +static void bvec_unpin(struct bio_vec *bv, bool mark_dirty) +{ + struct folio *folio = bvec_folio(bv); + size_t nr_pages = (bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE - + bv->bv_offset / PAGE_SIZE + 1; + + if (mark_dirty) + folio_mark_dirty_lock(folio); + unpin_user_folio(folio, nr_pages); +} + +static void bio_iov_iter_unbounce_read(struct bio *bio, bool is_error, + bool mark_dirty) +{ + unsigned int len = bio->bi_io_vec[0].bv_len; + + if (likely(!is_error)) { + void *buf = bvec_virt(&bio->bi_io_vec[0]); + struct iov_iter to; + + iov_iter_bvec(&to, ITER_DEST, bio->bi_io_vec + 1, bio->bi_vcnt, + len); + /* copying to pinned pages should always work */ + WARN_ON_ONCE(copy_to_iter(buf, len, &to) != len); + } else { + /* No need to mark folios dirty if never copied to them */ + mark_dirty = false; + } + + if (bio_flagged(bio, BIO_PAGE_PINNED)) { + int i; + + for (i = 0; i < bio->bi_vcnt; i++) + bvec_unpin(&bio->bi_io_vec[1 + i], mark_dirty); + } + + folio_put(bvec_folio(&bio->bi_io_vec[0])); +} + +/** + * bio_iov_iter_unbounce - finish a bounce buffer operation + * @bio: completed bio + * @is_error: %true if an I/O error occurred and data should not be copied + * @mark_dirty: If %true, folios will be marked dirty. + * + * Helper for direct I/O implementations that need to bounce buffer because + * we need to checksum the data or perform other operations that require + * consistency. Called to complete a bio set up by bio_iov_iter_bounce(). + * Copies data back for reads, and marks the original folios dirty if + * requested and then frees the bounce buffer. + */ +void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty) +{ + if (op_is_write(bio_op(bio))) + bio_free_folios(bio); + else + bio_iov_iter_unbounce_read(bio, is_error, mark_dirty); +} + +static void bio_wait_end_io(struct bio *bio) { complete(bio->bi_private); } /** + * bio_await - call a function on a bio, and wait until it completes + * @bio: the bio which describes the I/O + * @submit: function called to submit the bio + * @priv: private data passed to @submit + * + * Wait for the bio as well as any bio chained off it after executing the + * passed in callback @submit. The wait for the bio is set up before calling + * @submit to ensure that the completion is captured. If @submit is %NULL, + * submit_bio() is used instead to submit the bio. + * + * Note: this overrides the bi_private and bi_end_io fields in the bio. + */ +void bio_await(struct bio *bio, void *priv, + void (*submit)(struct bio *bio, void *priv)) +{ + DECLARE_COMPLETION_ONSTACK_MAP(done, + bio->bi_bdev->bd_disk->lockdep_map); + + bio->bi_private = &done; + bio->bi_end_io = bio_wait_end_io; + bio->bi_opf |= REQ_SYNC; + if (submit) + submit(bio, priv); + else + submit_bio(bio); + blk_wait_io(&done); +} +EXPORT_SYMBOL_GPL(bio_await); + +/** * submit_bio_wait - submit a bio, and wait until it completes * @bio: The &struct bio which describes the I/O * @@ -1288,39 +1543,60 @@ static void submit_bio_wait_endio(struct bio *bio) */ int submit_bio_wait(struct bio *bio) { - DECLARE_COMPLETION_ONSTACK_MAP(done, - bio->bi_bdev->bd_disk->lockdep_map); - - bio->bi_private = &done; - bio->bi_end_io = submit_bio_wait_endio; - bio->bi_opf |= REQ_SYNC; - submit_bio(bio); - blk_wait_io(&done); - + bio_await(bio, NULL, NULL); return blk_status_to_errno(bio->bi_status); } EXPORT_SYMBOL(submit_bio_wait); -static void bio_wait_end_io(struct bio *bio) +static void bio_endio_cb(struct bio *bio, void *priv) { - complete(bio->bi_private); - bio_put(bio); + bio_endio(bio); } /* - * bio_await_chain - ends @bio and waits for every chained bio to complete + * Submit @bio synchronously, or call bio_endio on it if the current process + * is being killed. */ -void bio_await_chain(struct bio *bio) +int bio_submit_or_kill(struct bio *bio, unsigned int flags) { - DECLARE_COMPLETION_ONSTACK_MAP(done, - bio->bi_bdev->bd_disk->lockdep_map); + if ((flags & BLKDEV_ZERO_KILLABLE) && fatal_signal_pending(current)) { + bio_await(bio, NULL, bio_endio_cb); + return -EINTR; + } - bio->bi_private = &done; - bio->bi_end_io = bio_wait_end_io; - bio_endio(bio); - blk_wait_io(&done); + return submit_bio_wait(bio); } +/** + * bdev_rw_virt - synchronously read into / write from kernel mapping + * @bdev: block device to access + * @sector: sector to access + * @data: data to read/write + * @len: length in byte to read/write + * @op: operation (e.g. REQ_OP_READ/REQ_OP_WRITE) + * + * Performs synchronous I/O to @bdev for @data/@len. @data must be in + * the kernel direct mapping and not a vmalloc address. + */ +int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data, + size_t len, enum req_op op) +{ + struct bio_vec bv; + struct bio bio; + int error; + + if (WARN_ON_ONCE(is_vmalloc_addr(data))) + return -EIO; + + bio_init(&bio, bdev, &bv, 1, op); + bio.bi_iter.bi_sector = sector; + bio_add_virt_nofail(&bio, data, len); + error = submit_bio_wait(&bio); + bio_uninit(&bio); + return error; +} +EXPORT_SYMBOL_GPL(bdev_rw_virt); + void __bio_advance(struct bio *bio, unsigned bytes) { if (bio_integrity(bio)) @@ -1331,26 +1607,6 @@ void __bio_advance(struct bio *bio, unsigned bytes) } EXPORT_SYMBOL(__bio_advance); -void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter, - struct bio *src, struct bvec_iter *src_iter) -{ - while (src_iter->bi_size && dst_iter->bi_size) { - struct bio_vec src_bv = bio_iter_iovec(src, *src_iter); - struct bio_vec dst_bv = bio_iter_iovec(dst, *dst_iter); - unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len); - void *src_buf = bvec_kmap_local(&src_bv); - void *dst_buf = bvec_kmap_local(&dst_bv); - - memcpy(dst_buf, src_buf, bytes); - - kunmap_local(dst_buf); - kunmap_local(src_buf); - - bio_advance_iter_single(src, src_iter, bytes); - bio_advance_iter_single(dst, dst_iter, bytes); - } -} -EXPORT_SYMBOL(bio_copy_data_iter); /** * bio_copy_data - copy contents of data buffers from one bio to another @@ -1365,7 +1621,21 @@ void bio_copy_data(struct bio *dst, struct bio *src) struct bvec_iter src_iter = src->bi_iter; struct bvec_iter dst_iter = dst->bi_iter; - bio_copy_data_iter(dst, &dst_iter, src, &src_iter); + while (src_iter.bi_size && dst_iter.bi_size) { + struct bio_vec src_bv = bio_iter_iovec(src, src_iter); + struct bio_vec dst_bv = bio_iter_iovec(dst, dst_iter); + unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len); + void *src_buf = bvec_kmap_local(&src_bv); + void *dst_buf = bvec_kmap_local(&dst_bv); + + memcpy(dst_buf, src_buf, bytes); + + kunmap_local(dst_buf); + kunmap_local(src_buf); + + bio_advance_iter_single(src, &src_iter, bytes); + bio_advance_iter_single(dst, &dst_iter, bytes); + } } EXPORT_SYMBOL(bio_copy_data); @@ -1412,7 +1682,6 @@ void bio_set_pages_dirty(struct bio *bio) folio_unlock(fi.folio); } } -EXPORT_SYMBOL_GPL(bio_set_pages_dirty); /* * bio_check_pages_dirty() will check that all the BIO's pages are still dirty. @@ -1471,7 +1740,6 @@ defer: spin_unlock_irqrestore(&bio_dirty_lock, flags); schedule_work(&bio_dirty_work); } -EXPORT_SYMBOL_GPL(bio_check_pages_dirty); static inline bool bio_remaining_done(struct bio *bio) { @@ -1637,7 +1905,7 @@ EXPORT_SYMBOL_GPL(bio_trim); * create memory pools for biovec's in a bio_set. * use the global biovec slabs created for general use. */ -int biovec_init_pool(mempool_t *pool, int pool_entries) +static int biovec_init_pool(mempool_t *pool, int pool_entries) { struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1; @@ -1715,7 +1983,7 @@ int bioset_init(struct bio_set *bs, if (flags & BIOSET_NEED_RESCUER) { bs->rescue_workqueue = alloc_workqueue("bioset", - WQ_MEM_RECLAIM, 0); + WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!bs->rescue_workqueue) goto bad; } diff --git a/block/blk-cgroup-rwstat.c b/block/blk-cgroup-rwstat.c index a55fb0c53558..aae910713814 100644 --- a/block/blk-cgroup-rwstat.c +++ b/block/blk-cgroup-rwstat.c @@ -101,24 +101,27 @@ void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol, struct cgroup_subsys_state *pos_css; unsigned int i; - lockdep_assert_held(&blkg->q->queue_lock); + WARN_ON_ONCE(!rcu_read_lock_held()); memset(sum, 0, sizeof(*sum)); - rcu_read_lock(); blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { struct blkg_rwstat *rwstat; if (!pos_blkg->online) continue; - if (pol) - rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off; - else + if (pol) { + struct blkg_policy_data *pd = blkg_to_pd(pos_blkg, pol); + + if (!pd) + continue; + rwstat = (void *)pd + off; + } else { rwstat = (void *)pos_blkg + off; + } for (i = 0; i < BLKG_RWSTAT_NR; i++) sum->cnt[i] += blkg_rwstat_read_counter(rwstat, i); } - rcu_read_unlock(); } EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum); diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 5905f277057b..d9676126c5b5 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -24,6 +24,7 @@ #include <linux/backing-dev.h> #include <linux/slab.h> #include <linux/delay.h> +#include <linux/wait_bit.h> #include <linux/atomic.h> #include <linux/ctype.h> #include <linux/resume_user_mode.h> @@ -110,12 +111,6 @@ static struct cgroup_subsys_state *blkcg_css(void) return task_css(current, io_cgrp_id); } -static bool blkcg_policy_enabled(struct request_queue *q, - const struct blkcg_policy *pol) -{ - return pol && test_bit(pol->plid, q->blkcg_pols); -} - static void blkg_free_workfn(struct work_struct *work) { struct blkcg_gq *blkg = container_of(work, struct blkcg_gq, @@ -141,6 +136,11 @@ static void blkg_free_workfn(struct work_struct *work) spin_unlock_irq(&q->queue_lock); mutex_unlock(&q->blkcg_mutex); + /* + * Release blkcg css ref only after blkg is removed from q->blkg_list, + * so concurrent iterators won't see a blkg with a freed blkcg. + */ + css_put(&blkg->blkcg->css); blk_put_queue(q); free_percpu(blkg->iostat_cpu); percpu_ref_exit(&blkg->refcnt); @@ -169,23 +169,11 @@ static void blkg_free(struct blkcg_gq *blkg) static void __blkg_release(struct rcu_head *rcu) { struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head); - struct blkcg *blkcg = blkg->blkcg; - int cpu; #ifdef CONFIG_BLK_CGROUP_PUNT_BIO WARN_ON(!bio_list_empty(&blkg->async_bios)); #endif - /* - * Flush all the non-empty percpu lockless lists before releasing - * us, given these stat belongs to us. - * - * blkg_stat_lock is for serializing blkg stat update - */ - for_each_possible_cpu(cpu) - __blkcg_rstat_flush(blkcg, cpu); - /* release the blkcg and parent blkg refs this blkg has been holding */ - css_put(&blkg->blkcg->css); blkg_free(blkg); } @@ -200,6 +188,17 @@ static void __blkg_release(struct rcu_head *rcu) static void blkg_release(struct percpu_ref *ref) { struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt); + struct blkcg *blkcg = blkg->blkcg; + int cpu; + + /* + * Flush all the non-empty percpu lockless lists before releasing + * us, given these stat belongs to us. + * + * blkg_stat_lock is for serializing blkg stat update + */ + for_each_possible_cpu(cpu) + __blkcg_rstat_flush(blkcg, cpu); call_rcu(&blkg->rcu_head, __blkg_release); } @@ -318,6 +317,9 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, goto out_exit_refcnt; if (!blk_get_queue(disk->queue)) goto out_free_iostat; + /* blkg holds a reference to blkcg */ + if (!css_tryget_online(&blkcg->css)) + goto out_put_queue; blkg->q = disk->queue; INIT_LIST_HEAD(&blkg->q_node); @@ -358,6 +360,8 @@ out_free_pds: while (--i >= 0) if (blkg->pd[i]) blkcg_policy[i]->pd_free_fn(blkg->pd[i]); + css_put(&blkcg->css); +out_put_queue: blk_put_queue(disk->queue); out_free_iostat: free_percpu(blkg->iostat_cpu); @@ -386,18 +390,12 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, goto err_free_blkg; } - /* blkg holds a reference to blkcg */ - if (!css_tryget_online(&blkcg->css)) { - ret = -ENODEV; - goto err_free_blkg; - } - /* allocate */ if (!new_blkg) { - new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT | __GFP_NOWARN); + new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT); if (unlikely(!new_blkg)) { ret = -ENOMEM; - goto err_put_css; + goto err_free_blkg; } } blkg = new_blkg; @@ -407,7 +405,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue); if (WARN_ON_ONCE(!blkg->parent)) { ret = -ENODEV; - goto err_put_css; + goto err_free_blkg; } blkg_get(blkg->parent); } @@ -436,19 +434,17 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, blkg->pd[i]->online = true; } } + blkg->online = true; } - blkg->online = true; spin_unlock(&blkcg->lock); if (!ret) return blkg; /* @blkg failed fully initialized, use the usual release path */ - blkg_put(blkg); + percpu_ref_kill(&blkg->refcnt); return ERR_PTR(ret); -err_put_css: - css_put(&blkcg->css); err_free_blkg: if (new_blkg) blkg_free(new_blkg); @@ -473,22 +469,17 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, { struct request_queue *q = disk->queue; struct blkcg_gq *blkg; - unsigned long flags; - WARN_ON_ONCE(!rcu_read_lock_held()); - - blkg = blkg_lookup(blkcg, q); - if (blkg) - return blkg; - - spin_lock_irqsave(&q->queue_lock, flags); + rcu_read_lock(); blkg = blkg_lookup(blkcg, q); if (blkg) { if (blkcg != &blkcg_root && blkg != rcu_dereference(blkcg->blkg_hint)) rcu_assign_pointer(blkcg->blkg_hint, blkg); - goto found; + rcu_read_unlock(); + return blkg; } + rcu_read_unlock(); /* * Create blkgs walking down from blkcg_root to @blkcg, so that all @@ -520,8 +511,6 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, break; } -found: - spin_unlock_irqrestore(&q->queue_lock, flags); return blkg; } @@ -617,6 +606,8 @@ restart: q->root_blkg = NULL; spin_unlock_irq(&q->queue_lock); + + wake_up_var(&q->root_blkg); } static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src) @@ -701,9 +692,9 @@ const char *blkg_dev_name(struct blkcg_gq *blkg) * * This function invokes @prfill on each blkg of @blkcg if pd for the * policy specified by @pol exists. @prfill is invoked with @sf, the - * policy data and @data and the matching queue lock held. If @show_total - * is %true, the sum of the return values from @prfill is printed with - * "Total" label at the end. + * policy data and @data under RCU read lock. If @show_total is %true, the + * sum of the return values from @prfill is printed with "Total" label at the + * end. * * This is to be used to construct print functions for * cftype->read_seq_string method. @@ -719,10 +710,14 @@ void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, rcu_read_lock(); hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { - spin_lock_irq(&blkg->q->queue_lock); - if (blkcg_policy_enabled(blkg->q, pol)) - total += prfill(sf, blkg->pd[pol->plid], data); - spin_unlock_irq(&blkg->q->queue_lock); + struct blkg_policy_data *pd; + + if (!blkcg_policy_enabled(blkg->q, pol)) + continue; + + pd = blkg_to_pd(blkg, pol); + if (pd) + total += prfill(sf, pd, data); } rcu_read_unlock(); @@ -758,7 +753,7 @@ EXPORT_SYMBOL_GPL(__blkg_prfill_u64); * * Initialize @ctx which can be used to parse blkg config input string @input. * Once initialized, @ctx can be used with blkg_conf_open_bdev() and - * blkg_conf_prep(), and must be cleaned up with blkg_conf_exit(). + * blkg_conf_prep(). */ void blkg_conf_init(struct blkg_conf_ctx *ctx, char *input) { @@ -774,10 +769,7 @@ EXPORT_SYMBOL_GPL(blkg_conf_init); * @ctx->input and get and store the matching bdev in @ctx->bdev. @ctx->body is * set to point past the device node prefix. * - * This function may be called multiple times on @ctx and the extra calls become - * NOOPs. blkg_conf_prep() implicitly calls this function. Use this function - * explicitly if bdev access is needed without resolving the blkcg / policy part - * of @ctx->input. Returns -errno on error. + * Returns: -errno on error. */ int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx) { @@ -786,8 +778,8 @@ int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx) struct block_device *bdev; int key_len; - if (ctx->bdev) - return 0; + if (WARN_ON_ONCE(ctx->bdev)) + return -EINVAL; if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2) return -EINVAL; @@ -797,7 +789,7 @@ int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx) return -EINVAL; input = skip_spaces(input); - bdev = blkdev_get_no_open(MKDEV(major, minor)); + bdev = blkdev_get_no_open(MKDEV(major, minor), false); if (!bdev) return -ENODEV; if (bdev_is_partition(bdev)) { @@ -816,41 +808,7 @@ int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx) ctx->bdev = bdev; return 0; } -/* - * Similar to blkg_conf_open_bdev, but additionally freezes the queue, - * acquires q->elevator_lock, and ensures the correct locking order - * between q->elevator_lock and q->rq_qos_mutex. - * - * This function returns negative error on failure. On success it returns - * memflags which must be saved and later passed to blkg_conf_exit_frozen - * for restoring the memalloc scope. - */ -unsigned long __must_check blkg_conf_open_bdev_frozen(struct blkg_conf_ctx *ctx) -{ - int ret; - unsigned long memflags; - - if (ctx->bdev) - return -EINVAL; - - ret = blkg_conf_open_bdev(ctx); - if (ret < 0) - return ret; - /* - * At this point, we haven’t started protecting anything related to QoS, - * so we release q->rq_qos_mutex here, which was first acquired in blkg_ - * conf_open_bdev. Later, we re-acquire q->rq_qos_mutex after freezing - * the queue and acquiring q->elevator_lock to maintain the correct - * locking order. - */ - mutex_unlock(&ctx->bdev->bd_queue->rq_qos_mutex); - - memflags = blk_mq_freeze_queue(ctx->bdev->bd_queue); - mutex_lock(&ctx->bdev->bd_queue->elevator_lock); - mutex_lock(&ctx->bdev->bd_queue->rq_qos_mutex); - - return memflags; -} +EXPORT_SYMBOL_GPL(blkg_conf_open_bdev); /** * blkg_conf_prep - parse and prepare for per-blkg config update @@ -863,34 +821,26 @@ unsigned long __must_check blkg_conf_open_bdev_frozen(struct blkg_conf_ctx *ctx) * following MAJ:MIN, @ctx->bdev points to the target block device and * @ctx->blkg to the blkg being configured. * - * blkg_conf_open_bdev() may be called on @ctx beforehand. On success, this + * blkg_conf_open_bdev() must be called on @ctx beforehand. On success, this * function returns with queue lock held and must be followed by - * blkg_conf_exit(). + * blkg_conf_close_bdev(). */ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, struct blkg_conf_ctx *ctx) - __acquires(&bdev->bd_queue->queue_lock) { struct gendisk *disk; struct request_queue *q; struct blkcg_gq *blkg; int ret; - ret = blkg_conf_open_bdev(ctx); - if (ret) - return ret; + if (WARN_ON_ONCE(!ctx->bdev)) + return -EINVAL; disk = ctx->bdev->bd_disk; q = disk->queue; - /* - * blkcg_deactivate_policy() requires queue to be frozen, we can grab - * q_usage_counter to prevent concurrent with blkcg_deactivate_policy(). - */ - ret = blk_queue_enter(q, 0); - if (ret) - goto fail; - + /* Prevent concurrent with blkcg_deactivate_policy() */ + mutex_lock(&q->blkcg_mutex); spin_lock_irq(&q->queue_lock); if (!blkcg_policy_enabled(q, pol)) { @@ -920,16 +870,16 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, /* Drop locks to do new blkg allocation with GFP_KERNEL. */ spin_unlock_irq(&q->queue_lock); - new_blkg = blkg_alloc(pos, disk, GFP_KERNEL); + new_blkg = blkg_alloc(pos, disk, GFP_NOIO); if (unlikely(!new_blkg)) { ret = -ENOMEM; - goto fail_exit_queue; + goto fail_exit; } if (radix_tree_preload(GFP_KERNEL)) { blkg_free(new_blkg); ret = -ENOMEM; - goto fail_exit_queue; + goto fail_exit; } spin_lock_irq(&q->queue_lock); @@ -957,7 +907,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, goto success; } success: - blk_queue_exit(q); + mutex_unlock(&q->blkcg_mutex); ctx->blkg = blkg; return 0; @@ -965,9 +915,8 @@ fail_preloaded: radix_tree_preload_end(); fail_unlock: spin_unlock_irq(&q->queue_lock); -fail_exit_queue: - blk_queue_exit(q); -fail: +fail_exit: + mutex_unlock(&q->blkcg_mutex); /* * If queue was bypassing, we should retry. Do so after a * short msleep(). It isn't strictly necessary but queue @@ -983,45 +932,29 @@ fail: EXPORT_SYMBOL_GPL(blkg_conf_prep); /** - * blkg_conf_exit - clean up per-blkg config update + * blkg_conf_unprep - counterpart of blkg_conf_prep() * @ctx: blkg_conf_ctx initialized with blkg_conf_init() - * - * Clean up after per-blkg config update. This function must be called on all - * blkg_conf_ctx's initialized with blkg_conf_init(). */ -void blkg_conf_exit(struct blkg_conf_ctx *ctx) - __releases(&ctx->bdev->bd_queue->queue_lock) - __releases(&ctx->bdev->bd_queue->rq_qos_mutex) +void blkg_conf_unprep(struct blkg_conf_ctx *ctx) { - if (ctx->blkg) { - spin_unlock_irq(&bdev_get_queue(ctx->bdev)->queue_lock); - ctx->blkg = NULL; - } - - if (ctx->bdev) { - mutex_unlock(&ctx->bdev->bd_queue->rq_qos_mutex); - blkdev_put_no_open(ctx->bdev); - ctx->body = NULL; - ctx->bdev = NULL; - } + WARN_ON_ONCE(!ctx->blkg); + spin_unlock_irq(&ctx->bdev->bd_disk->queue->queue_lock); + ctx->blkg = NULL; } -EXPORT_SYMBOL_GPL(blkg_conf_exit); +EXPORT_SYMBOL_GPL(blkg_conf_unprep); -/* - * Similar to blkg_conf_exit, but also unfreezes the queue and releases - * q->elevator_lock. Should be used when blkg_conf_open_bdev_frozen - * is used to open the bdev. +/** + * blkg_conf_close_bdev - counterpart of blkg_conf_open_bdev() + * @ctx: blkg_conf_ctx initialized with blkg_conf_init() */ -void blkg_conf_exit_frozen(struct blkg_conf_ctx *ctx, unsigned long memflags) +void blkg_conf_close_bdev(struct blkg_conf_ctx *ctx) { - if (ctx->bdev) { - struct request_queue *q = ctx->bdev->bd_queue; - - blkg_conf_exit(ctx); - mutex_unlock(&q->elevator_lock); - blk_mq_unfreeze_queue(q, memflags); - } + mutex_unlock(&ctx->bdev->bd_queue->rq_qos_mutex); + blkdev_put_no_open(ctx->bdev); + ctx->body = NULL; + ctx->bdev = NULL; } +EXPORT_SYMBOL_GPL(blkg_conf_close_bdev); static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src) { @@ -1074,8 +1007,8 @@ static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu) /* * For covering concurrent parent blkg update from blkg_release(). * - * When flushing from cgroup, cgroup_rstat_lock is always held, so - * this lock won't cause contention most of time. + * When flushing from cgroup, the subsystem rstat lock is always held, + * so this lock won't cause contention most of time. */ raw_spin_lock_irqsave(&blkg_stat_lock, flags); @@ -1144,7 +1077,7 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu) /* * We source root cgroup stats from the system-wide stats to avoid * tracking the same information twice and incurring overhead when no - * cgroups are defined. For that reason, cgroup_rstat_flush in + * cgroups are defined. For that reason, css_rstat_flush in * blkcg_print_stat does not actually fill out the iostat in the root * cgroup's blkcg_gq. * @@ -1253,15 +1186,12 @@ static int blkcg_print_stat(struct seq_file *sf, void *v) if (!seq_css(sf)->parent) blkcg_fill_root_iostats(); else - cgroup_rstat_flush(blkcg->css.cgroup); + css_rstat_flush(&blkcg->css); - rcu_read_lock(); - hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { - spin_lock_irq(&blkg->q->queue_lock); + guard(spinlock_irq)(&blkcg->lock); + hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) blkcg_print_one_stat(blkg, sf); - spin_unlock_irq(&blkg->q->queue_lock); - } - rcu_read_unlock(); + return 0; } @@ -1309,6 +1239,21 @@ struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css) * This finally frees the blkcg. */ +static struct blkcg_gq *blkcg_get_first_blkg(struct blkcg *blkcg) +{ + struct blkcg_gq *blkg = NULL; + + spin_lock_irq(&blkcg->lock); + if (!hlist_empty(&blkcg->blkg_list)) { + blkg = hlist_entry(blkcg->blkg_list.first, struct blkcg_gq, + blkcg_node); + blkg_get(blkg); + } + spin_unlock_irq(&blkcg->lock); + + return blkg; +} + /** * blkcg_destroy_blkgs - responsible for shooting down blkgs * @blkcg: blkcg of interest @@ -1322,32 +1267,24 @@ struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css) */ static void blkcg_destroy_blkgs(struct blkcg *blkcg) { - might_sleep(); + struct blkcg_gq *blkg; - spin_lock_irq(&blkcg->lock); + might_sleep(); - while (!hlist_empty(&blkcg->blkg_list)) { - struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, - struct blkcg_gq, blkcg_node); + while ((blkg = blkcg_get_first_blkg(blkcg))) { struct request_queue *q = blkg->q; - if (need_resched() || !spin_trylock(&q->queue_lock)) { - /* - * Given that the system can accumulate a huge number - * of blkgs in pathological cases, check to see if we - * need to rescheduling to avoid softlockup. - */ - spin_unlock_irq(&blkcg->lock); - cond_resched(); - spin_lock_irq(&blkcg->lock); - continue; - } + spin_lock_irq(&q->queue_lock); + spin_lock(&blkcg->lock); blkg_destroy(blkg); - spin_unlock(&q->queue_lock); - } - spin_unlock_irq(&blkcg->lock); + spin_unlock(&blkcg->lock); + spin_unlock_irq(&q->queue_lock); + + blkg_put(blkg); + cond_resched(); + } } /** @@ -1435,7 +1372,7 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css) if (!parent_css) { blkcg = &blkcg_root; } else { - blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL); + blkcg = kzalloc_obj(*blkcg); if (!blkcg) goto unlock; } @@ -1467,7 +1404,7 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css) spin_lock_init(&blkcg->lock); refcount_set(&blkcg->online_pin, 1); - INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN); + INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT); INIT_HLIST_HEAD(&blkcg->blkg_list); #ifdef CONFIG_CGROUP_WRITEBACK INIT_LIST_HEAD(&blkcg->cgwb_list); @@ -1516,6 +1453,18 @@ int blkcg_init_disk(struct gendisk *disk) struct blkcg_gq *new_blkg, *blkg; bool preloaded; + /* + * If the queue is shared across disk rebind (e.g., SCSI), the + * previous disk's blkcg state is cleaned up asynchronously via + * disk_release() -> blkcg_exit_disk(). Wait for that cleanup to + * finish (indicated by root_blkg becoming NULL) before setting up + * new blkcg state. Otherwise, we may overwrite q->root_blkg while + * the old one is still alive, and radix_tree_insert() in + * blkg_create() will fail with -EEXIST because the old entries + * still occupy the same queue id slot in blkcg->blkg_tree. + */ + wait_var_event(&q->root_blkg, !READ_ONCE(q->root_blkg)); + new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL); if (!new_blkg) return -ENOMEM; @@ -1630,7 +1579,7 @@ retry: pd_prealloc = NULL; } else { pd = pol->pd_alloc_fn(disk, blkg->blkcg, - GFP_NOWAIT | __GFP_NOWARN); + GFP_NOWAIT); } if (!pd) { @@ -1659,7 +1608,7 @@ retry: pd->blkg = blkg; pd->plid = pol->plid; - blkg->pd[pol->plid] = pd; + WRITE_ONCE(blkg->pd[pol->plid], pd); if (pol->pd_init_fn) pol->pd_init_fn(pd); @@ -1698,7 +1647,7 @@ enomem: pol->pd_offline_fn(pd); pd->online = false; pol->pd_free_fn(pd); - blkg->pd[pol->plid] = NULL; + WRITE_ONCE(blkg->pd[pol->plid], NULL); } spin_unlock(&blkcg->lock); } @@ -2040,6 +1989,7 @@ void blkcg_maybe_throttle_current(void) return; out: rcu_read_unlock(); + put_disk(disk); } /** @@ -2096,6 +2046,18 @@ void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta) atomic64_add(delta, &blkg->delay_nsec); } +static inline struct blkcg_gq *blkg_lookup_tryget(struct blkcg_gq *blkg) +{ +retry: + if (blkg_tryget(blkg)) + return blkg; + + blkg = blkg->parent; + if (blkg) + goto retry; + + return NULL; +} /** * blkg_tryget_closest - try and get a blkg ref on the closet blkg * @bio: target bio @@ -2108,20 +2070,30 @@ void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta) static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, struct cgroup_subsys_state *css) { - struct blkcg_gq *blkg, *ret_blkg = NULL; + struct request_queue *q = bio->bi_bdev->bd_queue; + struct blkcg *blkcg = css_to_blkcg(css); + struct blkcg_gq *blkg; rcu_read_lock(); - blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_bdev->bd_disk); - while (blkg) { - if (blkg_tryget(blkg)) { - ret_blkg = blkg; - break; - } - blkg = blkg->parent; - } + blkg = blkg_lookup(blkcg, q); + if (likely(blkg)) + blkg = blkg_lookup_tryget(blkg); rcu_read_unlock(); - return ret_blkg; + if (blkg) + return blkg; + + /* + * Fast path failed, we're probably issuing IO in this cgroup the first + * time, hold lock to create new blkg. + */ + spin_lock_irq(&q->queue_lock); + blkg = blkg_lookup_create(blkcg, bio->bi_bdev->bd_disk); + if (blkg) + blkg = blkg_lookup_tryget(blkg); + spin_unlock_irq(&q->queue_lock); + + return blkg; } /** @@ -2169,16 +2141,20 @@ void bio_associate_blkg(struct bio *bio) if (blk_op_is_passthrough(bio->bi_opf)) return; - rcu_read_lock(); - - if (bio->bi_blkg) + if (bio->bi_blkg) { css = bio_blkcg_css(bio); - else + bio_associate_blkg_from_css(bio, css); + } else { + rcu_read_lock(); css = blkcg_css(); + if (!css_tryget_online(css)) + css = NULL; + rcu_read_unlock(); - bio_associate_blkg_from_css(bio, css); - - rcu_read_unlock(); + bio_associate_blkg_from_css(bio, css); + if (css) + css_put(css); + } } EXPORT_SYMBOL_GPL(bio_associate_blkg); @@ -2243,7 +2219,7 @@ void blk_cgroup_bio_start(struct bio *bio) } u64_stats_update_end_irqrestore(&bis->sync, flags); - cgroup_rstat_updated(blkcg->css.cgroup, cpu); + __css_rstat_updated(&blkcg->css, cpu); put_cpu(); } diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index 81868ad86330..615390f751aa 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -140,6 +140,8 @@ struct blkg_policy_data { struct blkcg_gq *blkg; int plid; bool online; + + struct rcu_head rcu_head; }; /* @@ -218,12 +220,15 @@ struct blkg_conf_ctx { }; void blkg_conf_init(struct blkg_conf_ctx *ctx, char *input); -int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx); -unsigned long blkg_conf_open_bdev_frozen(struct blkg_conf_ctx *ctx); +int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx) + __cond_acquires(0, &ctx->bdev->bd_queue->rq_qos_mutex); int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, - struct blkg_conf_ctx *ctx); -void blkg_conf_exit(struct blkg_conf_ctx *ctx); -void blkg_conf_exit_frozen(struct blkg_conf_ctx *ctx, unsigned long memflags); + struct blkg_conf_ctx *ctx) + __cond_acquires(0, &ctx->bdev->bd_disk->queue->queue_lock); +void blkg_conf_unprep(struct blkg_conf_ctx *ctx) + __releases(ctx->bdev->bd_disk->queue->queue_lock); +void blkg_conf_close_bdev(struct blkg_conf_ctx *ctx) + __releases(&ctx->bdev->bd_queue->rq_qos_mutex); /** * bio_issue_as_root_blkg - see if this bio needs to be issued as root blkg @@ -279,9 +284,9 @@ static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, * Return pointer to private data associated with the @blkg-@pol pair. */ static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, - struct blkcg_policy *pol) + const struct blkcg_policy *pol) { - return blkg ? blkg->pd[pol->plid] : NULL; + return blkg ? READ_ONCE(blkg->pd[pol->plid]) : NULL; } static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg, @@ -370,11 +375,6 @@ static inline void blkg_put(struct blkcg_gq *blkg) if (((d_blkg) = blkg_lookup(css_to_blkcg(pos_css), \ (p_blkg)->q))) -static inline void blkcg_bio_issue_init(struct bio *bio) -{ - bio_issue_init(&bio->bi_issue, bio_sectors(bio)); -} - static inline void blkcg_use_delay(struct blkcg_gq *blkg) { if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0)) @@ -459,6 +459,12 @@ static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio) bio_issue_as_root_blkg(rq->bio) == bio_issue_as_root_blkg(bio); } +static inline bool blkcg_policy_enabled(struct request_queue *q, + const struct blkcg_policy *pol) +{ + return pol && test_bit(pol->plid, q->blkcg_pols); +} + void blk_cgroup_bio_start(struct bio *bio); void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta); #else /* CONFIG_BLK_CGROUP */ @@ -487,11 +493,10 @@ static inline void blkcg_deactivate_policy(struct gendisk *disk, const struct blkcg_policy *pol) { } static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, - struct blkcg_policy *pol) { return NULL; } + const struct blkcg_policy *pol) { return NULL; } static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; } static inline void blkg_get(struct blkcg_gq *blkg) { } static inline void blkg_put(struct blkcg_gq *blkg) { } -static inline void blkcg_bio_issue_init(struct bio *bio) { } static inline void blk_cgroup_bio_start(struct bio *bio) { } static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio) { return true; } diff --git a/block/blk-core.c b/block/blk-core.c index 4623de79effa..365641266c9e 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -50,6 +50,7 @@ #include "blk-cgroup.h" #include "blk-throttle.h" #include "blk-ioprio.h" +#include "error-injection.h" struct dentry *blk_debugfs_root; @@ -114,12 +115,12 @@ static const char *const blk_op_name[] = { #undef REQ_OP_NAME /** - * blk_op_str - Return string XXX in the REQ_OP_XXX. - * @op: REQ_OP_XXX. + * blk_op_str - Return the string "name" for an operation REQ_OP_name. + * @op: a request operation. * - * Description: Centralize block layer function to convert REQ_OP_XXX into - * string format. Useful in the debugging and tracing bio or request. For - * invalid REQ_OP_XXX it returns string "UNKNOWN". + * Convert a request operation REQ_OP_name into the string "name". Useful for + * debugging and tracing BIOs and requests. For an invalid request operation + * code, the string "UNKNOWN" is returned. */ inline const char *blk_op_str(enum req_op op) { @@ -132,39 +133,56 @@ inline const char *blk_op_str(enum req_op op) } EXPORT_SYMBOL_GPL(blk_op_str); +enum req_op str_to_blk_op(const char *op) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(blk_op_name); i++) + if (blk_op_name[i] && !strcmp(blk_op_name[i], op)) + return (enum req_op)i; + return REQ_OP_LAST; +} + +#define ENT(_tag, _errno, _desc) \ +[BLK_STS_##_tag] = { \ + .errno = _errno, \ + .tag = __stringify(_tag), \ + .name = _desc, \ +} static const struct { int errno; + const char *tag; const char *name; } blk_errors[] = { - [BLK_STS_OK] = { 0, "" }, - [BLK_STS_NOTSUPP] = { -EOPNOTSUPP, "operation not supported" }, - [BLK_STS_TIMEOUT] = { -ETIMEDOUT, "timeout" }, - [BLK_STS_NOSPC] = { -ENOSPC, "critical space allocation" }, - [BLK_STS_TRANSPORT] = { -ENOLINK, "recoverable transport" }, - [BLK_STS_TARGET] = { -EREMOTEIO, "critical target" }, - [BLK_STS_RESV_CONFLICT] = { -EBADE, "reservation conflict" }, - [BLK_STS_MEDIUM] = { -ENODATA, "critical medium" }, - [BLK_STS_PROTECTION] = { -EILSEQ, "protection" }, - [BLK_STS_RESOURCE] = { -ENOMEM, "kernel resource" }, - [BLK_STS_DEV_RESOURCE] = { -EBUSY, "device resource" }, - [BLK_STS_AGAIN] = { -EAGAIN, "nonblocking retry" }, - [BLK_STS_OFFLINE] = { -ENODEV, "device offline" }, + ENT(OK, 0, ""), + ENT(NOTSUPP, -EOPNOTSUPP, "operation not supported"), + ENT(TIMEOUT, -ETIMEDOUT, "timeout"), + ENT(NOSPC, -ENOSPC, "critical space allocation"), + ENT(TRANSPORT, -ENOLINK, "recoverable transport"), + ENT(TARGET, -EREMOTEIO, "critical target"), + ENT(RESV_CONFLICT, -EBADE, "reservation conflict"), + ENT(MEDIUM, -ENODATA, "critical medium"), + ENT(PROTECTION, -EILSEQ, "protection"), + ENT(RESOURCE, -ENOMEM, "kernel resource"), + ENT(DEV_RESOURCE, -EBUSY, "device resource"), + ENT(AGAIN, -EAGAIN, "nonblocking retry"), + ENT(OFFLINE, -ENODEV, "device offline"), /* device mapper special case, should not leak out: */ - [BLK_STS_DM_REQUEUE] = { -EREMCHG, "dm internal retry" }, + ENT(DM_REQUEUE, -EREMCHG, "dm internal retry"), /* zone device specific errors */ - [BLK_STS_ZONE_OPEN_RESOURCE] = { -ETOOMANYREFS, "open zones exceeded" }, - [BLK_STS_ZONE_ACTIVE_RESOURCE] = { -EOVERFLOW, "active zones exceeded" }, + ENT(ZONE_OPEN_RESOURCE, -ETOOMANYREFS, "open zones exceeded"), + ENT(ZONE_ACTIVE_RESOURCE, -EOVERFLOW, "active zones exceeded"), /* Command duration limit device-side timeout */ - [BLK_STS_DURATION_LIMIT] = { -ETIME, "duration limit exceeded" }, - - [BLK_STS_INVAL] = { -EINVAL, "invalid" }, + ENT(DURATION_LIMIT, -ETIME, "duration limit exceeded"), + ENT(INVAL, -EINVAL, "invalid"), /* everything else not covered above: */ - [BLK_STS_IOERR] = { -EIO, "I/O" }, + ENT(IOERR, -EIO, "I/O"), }; +#undef ENT blk_status_t errno_to_blk_status(int errno) { @@ -197,7 +215,32 @@ const char *blk_status_to_str(blk_status_t status) return "<null>"; return blk_errors[idx].name; } -EXPORT_SYMBOL_GPL(blk_status_to_str); + +const char *blk_status_to_tag(blk_status_t status) +{ + int idx = (__force int)status; + + if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors) || !blk_errors[idx].tag)) + return "<null>"; + return blk_errors[idx].tag; +} + +blk_status_t tag_to_blk_status(const char *tag) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(blk_errors); i++) { + if (blk_errors[i].tag && + !strcmp(blk_errors[i].tag, tag)) + return (__force blk_status_t)i; + } + + /* + * Return BLK_STS_OK for mismatches as this function is intended to + * parse error status values. + */ + return BLK_STS_OK; +} /** * blk_sync_queue - cancel any pending callbacks on a queue @@ -219,7 +262,7 @@ EXPORT_SYMBOL_GPL(blk_status_to_str); */ void blk_sync_queue(struct request_queue *q) { - del_timer_sync(&q->timeout); + timer_delete_sync(&q->timeout); cancel_work_sync(&q->timeout_work); } EXPORT_SYMBOL(blk_sync_queue); @@ -381,7 +424,7 @@ static void blk_queue_usage_counter_release(struct percpu_ref *ref) static void blk_rq_timed_out_timer(struct timer_list *t) { - struct request_queue *q = from_timer(q, t, timeout); + struct request_queue *q = timer_container_of(q, t, timeout); kblockd_schedule_work(&q->timeout_work); } @@ -463,6 +506,7 @@ struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id) fs_reclaim_release(GFP_KERNEL); q->nr_requests = BLKDEV_DEFAULT_RQ; + q->async_depth = BLKDEV_DEFAULT_RQ; return q; @@ -539,7 +583,7 @@ static inline void bio_check_ro(struct bio *bio) } } -static noinline int should_fail_bio(struct bio *bio) +int should_fail_bio(struct bio *bio) { if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size)) return -EIO; @@ -560,6 +604,8 @@ static inline int bio_check_eod(struct bio *bio) if (nr_sectors && (nr_sectors > maxsector || bio->bi_iter.bi_sector > maxsector - nr_sectors)) { + if (!maxsector) + return -EIO; pr_info_ratelimited("%s: attempt to access beyond end of device\n" "%pg: rw=%d, sector=%llu, nr_sectors = %u limit=%llu\n", current->comm, bio->bi_bdev, bio->bi_opf, @@ -623,30 +669,18 @@ static inline blk_status_t blk_check_zone_append(struct request_queue *q, static void __submit_bio(struct bio *bio) { - /* If plug is not used, add new plug here to cache nsecs time. */ - struct blk_plug plug; - - if (unlikely(!blk_crypto_bio_prep(&bio))) - return; - - blk_start_plug(&plug); - if (!bdev_test_flag(bio->bi_bdev, BD_HAS_SUBMIT_BIO)) { blk_mq_submit_bio(bio); } else if (likely(bio_queue_enter(bio) == 0)) { struct gendisk *disk = bio->bi_bdev->bd_disk; if ((bio->bi_opf & REQ_POLLED) && - !(disk->queue->limits.features & BLK_FEAT_POLL)) { - bio->bi_status = BLK_STS_NOTSUPP; - bio_endio(bio); - } else { + !(disk->queue->limits.features & BLK_FEAT_POLL)) + bio_endio_status(bio, BLK_STS_NOTSUPP); + else disk->fops->submit_bio(bio); - } blk_queue_exit(disk->queue); } - - blk_finish_plug(&plug); } /* @@ -660,13 +694,13 @@ static void __submit_bio(struct bio *bio) * bio_list of new bios to be added. ->submit_bio() may indeed add some more * bios through a recursive call to submit_bio_noacct. If it did, we find a * non-NULL value in bio_list and re-enter the loop from the top. - * - In this case we really did just take the bio of the top of the list (no + * - In this case we really did just take the bio off the top of the list (no * pretending) and so remove it from bio_list, and call into ->submit_bio() * again. * * bio_list_on_stack[0] contains bios submitted by the current ->submit_bio. * bio_list_on_stack[1] contains bios that were submitted before the current - * ->submit_bio, but that haven't been processed yet. + * ->submit_bio(), but that haven't been processed yet. */ static void __submit_bio_noacct(struct bio *bio) { @@ -725,10 +759,12 @@ static void __submit_bio_noacct_mq(struct bio *bio) current->bio_list = NULL; } -void submit_bio_noacct_nocheck(struct bio *bio) +void submit_bio_noacct_nocheck(struct bio *bio, bool split) { + if (unlikely(blk_error_inject(bio))) + return; + blk_cgroup_bio_start(bio); - blkcg_bio_issue_init(bio); if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) { trace_block_bio_queue(bio); @@ -742,15 +778,19 @@ void submit_bio_noacct_nocheck(struct bio *bio) /* * We only want one ->submit_bio to be active at a time, else stack * usage with stacked devices could be a problem. Use current->bio_list - * to collect a list of requests submited by a ->submit_bio method while - * it is active, and then process them after it returned. + * to collect a list of requests submitted by a ->submit_bio method + * while it is active, and then process them after it returned. */ - if (current->bio_list) - bio_list_add(¤t->bio_list[0], bio); - else if (!bdev_test_flag(bio->bi_bdev, BD_HAS_SUBMIT_BIO)) + if (current->bio_list) { + if (split) + bio_list_add_head(¤t->bio_list[0], bio); + else + bio_list_add(¤t->bio_list[0], bio); + } else if (!bdev_test_flag(bio->bi_bdev, BD_HAS_SUBMIT_BIO)) { __submit_bio_noacct_mq(bio); - else + } else { __submit_bio_noacct(bio); + } } static blk_status_t blk_validate_atomic_write_op_size(struct request_queue *q, @@ -789,6 +829,13 @@ void submit_bio_noacct(struct bio *bio) if ((bio->bi_opf & REQ_NOWAIT) && !bdev_nowait(bdev)) goto not_supported; + if (bio_has_crypt_ctx(bio)) { + if (WARN_ON_ONCE(!bio_has_data(bio))) + goto end_io; + if (!blk_crypto_supported(bio)) + goto not_supported; + } + if (should_fail_bio(bio)) goto end_io; bio_check_ro(bio); @@ -871,14 +918,13 @@ void submit_bio_noacct(struct bio *bio) if (blk_throtl_bio(bio)) return; - submit_bio_noacct_nocheck(bio); + submit_bio_noacct_nocheck(bio, false); return; not_supported: status = BLK_STS_NOTSUPP; end_io: - bio->bi_status = status; - bio_endio(bio); + bio_endio_status(bio, status); } EXPORT_SYMBOL(submit_bio_noacct); @@ -896,7 +942,7 @@ static void bio_set_ioprio(struct bio *bio) * * submit_bio() is used to submit I/O requests to block devices. It is passed a * fully set up &struct bio that describes the I/O that needs to be done. The - * bio will be send to the device described by the bi_bdev field. + * bio will be sent to the device described by the bi_bdev field. * * The success/failure status of the request, along with notification of * completion, is delivered asynchronously through the ->bi_end_io() callback @@ -986,7 +1032,7 @@ int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob, * point to a freshly allocated bio at this point. If that happens * we have a few cases to consider: * - * 1) the bio is beeing initialized and bi_bdev is NULL. We can just + * 1) the bio is being initialized and bi_bdev is NULL. We can just * simply nothing in this case * 2) the bio points to a not poll enabled device. bio_poll will catch * this and return 0 @@ -1018,7 +1064,7 @@ again: stamp = READ_ONCE(part->bd_stamp); if (unlikely(time_after(now, stamp)) && likely(try_cmpxchg(&part->bd_stamp, &stamp, now)) && - (end || part_in_flight(part))) + (end || bdev_count_inflight(part))) __part_stat_add(part, io_ticks, now - stamp); if (bdev_is_partition(part)) { @@ -1032,7 +1078,7 @@ unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op, { part_stat_lock(); update_io_ticks(bdev, start_time, false); - part_stat_local_inc(bdev, in_flight[op_is_write(op)]); + bdev_inc_in_flight(bdev, op); part_stat_unlock(); return start_time; @@ -1063,7 +1109,7 @@ void bdev_end_io_acct(struct block_device *bdev, enum req_op op, part_stat_inc(bdev, ios[sgrp]); part_stat_add(bdev, sectors[sgrp], sectors); part_stat_add(bdev, nsecs[sgrp], jiffies_to_nsecs(duration)); - part_stat_local_dec(bdev, in_flight[op_is_write(op)]); + bdev_dec_in_flight(bdev, op); part_stat_unlock(); } EXPORT_SYMBOL(bdev_end_io_acct); @@ -1260,7 +1306,6 @@ void blk_io_schedule(void) else io_schedule(); } -EXPORT_SYMBOL_GPL(blk_io_schedule); int __init blk_dev_init(void) { @@ -1272,7 +1317,7 @@ int __init blk_dev_init(void) /* used for unplugging and affects IO latency/throughput - HIGHPRI */ kblockd_workqueue = alloc_workqueue("kblockd", - WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0); if (!kblockd_workqueue) panic("Failed to create kblockd\n"); diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index f154be0b575a..2a5c52ab74b4 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -22,7 +22,7 @@ #include "blk-cgroup.h" #include "blk-crypto-internal.h" -static unsigned int num_prealloc_bounce_pg = 32; +static unsigned int num_prealloc_bounce_pg = BIO_MAX_VECS; module_param(num_prealloc_bounce_pg, uint, 0); MODULE_PARM_DESC(num_prealloc_bounce_pg, "Number of preallocated bounce pages for the blk-crypto crypto API fallback"); @@ -75,13 +75,13 @@ static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX]; static struct blk_crypto_fallback_keyslot { enum blk_crypto_mode_num crypto_mode; - struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; + struct crypto_sync_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; } *blk_crypto_keyslots; static struct blk_crypto_profile *blk_crypto_fallback_profile; static struct workqueue_struct *blk_crypto_wq; static mempool_t *blk_crypto_bounce_page_pool; -static struct bio_set crypto_bio_split; +static struct bio_set enc_bio_set; /* * This is the key we set when evicting a keyslot. This *should* be the all 0's @@ -98,7 +98,7 @@ static void blk_crypto_fallback_evict_keyslot(unsigned int slot) WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); /* Clear the key in the skcipher */ - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, + err = crypto_sync_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, blk_crypto_modes[crypto_mode].keysize); WARN_ON(err); slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; @@ -119,7 +119,7 @@ blk_crypto_fallback_keyslot_program(struct blk_crypto_profile *profile, blk_crypto_fallback_evict_keyslot(slot); slotp->crypto_mode = crypto_mode; - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->bytes, + err = crypto_sync_skcipher_setkey(slotp->tfms[crypto_mode], key->bytes, key->size); if (err) { blk_crypto_fallback_evict_keyslot(slot); @@ -144,98 +144,85 @@ static const struct blk_crypto_ll_ops blk_crypto_fallback_ll_ops = { static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio) { struct bio *src_bio = enc_bio->bi_private; - int i; + struct page **pages = (struct page **)enc_bio->bi_io_vec; + struct bio_vec *bv; + unsigned int i; - for (i = 0; i < enc_bio->bi_vcnt; i++) - mempool_free(enc_bio->bi_io_vec[i].bv_page, - blk_crypto_bounce_page_pool); + /* + * Use the same trick as the alloc side to avoid the need for an extra + * pages array. + */ + bio_for_each_bvec_all(bv, enc_bio, i) + pages[i] = bv->bv_page; - src_bio->bi_status = enc_bio->bi_status; + i = mempool_free_bulk(blk_crypto_bounce_page_pool, (void **)pages, + enc_bio->bi_vcnt); + if (i < enc_bio->bi_vcnt) + release_pages(pages + i, enc_bio->bi_vcnt - i); - bio_uninit(enc_bio); - kfree(enc_bio); + if (enc_bio->bi_status) + cmpxchg(&src_bio->bi_status, 0, enc_bio->bi_status); + + bio_put(enc_bio); bio_endio(src_bio); } -static struct bio *blk_crypto_fallback_clone_bio(struct bio *bio_src) +#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) + +static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src, + unsigned int nr_segs, struct page ***pages_ret) { - unsigned int nr_segs = bio_segments(bio_src); - struct bvec_iter iter; - struct bio_vec bv; + unsigned int memflags = memalloc_noio_save(); + unsigned int nr_allocated; + struct page **pages; struct bio *bio; - bio = bio_kmalloc(nr_segs, GFP_NOIO); - if (!bio) - return NULL; - bio_init(bio, bio_src->bi_bdev, bio->bi_inline_vecs, nr_segs, - bio_src->bi_opf); + bio = bio_alloc_bioset(bio_src->bi_bdev, nr_segs, bio_src->bi_opf, + GFP_NOIO, &enc_bio_set); if (bio_flagged(bio_src, BIO_REMAPPED)) bio_set_flag(bio, BIO_REMAPPED); + bio->bi_private = bio_src; + bio->bi_end_io = blk_crypto_fallback_encrypt_endio; bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; + bio->bi_write_stream = bio_src->bi_write_stream; bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; - bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; - - bio_for_each_segment(bv, bio_src, iter) - bio->bi_io_vec[bio->bi_vcnt++] = bv; - bio_clone_blkg_association(bio, bio_src); + /* + * Move page array up in the allocated memory for the bio vecs as far as + * possible so that we can start filling biovecs from the beginning + * without overwriting the temporary page array. + */ + static_assert(PAGE_PTRS_PER_BVEC > 1); + pages = (struct page **)bio->bi_io_vec; + pages += nr_segs * (PAGE_PTRS_PER_BVEC - 1); + + /* + * Try a bulk allocation first. This might not fill all allocated + * pages, but we'll fix that up later in mempool_alloc_bulk. + * + * Note: alloc_pages_bulk needs the array to be zeroed, as it assumes + * any non-zero slot already contains a valid allocation. + */ + memset(pages, 0, sizeof(struct page *) * nr_segs); + nr_allocated = alloc_pages_bulk(GFP_KERNEL, nr_segs, pages); + if (nr_allocated < nr_segs) + mempool_alloc_bulk(blk_crypto_bounce_page_pool, + (void **)pages + nr_allocated, + nr_segs - nr_allocated); + memalloc_noio_restore(memflags); + *pages_ret = pages; return bio; } -static bool -blk_crypto_fallback_alloc_cipher_req(struct blk_crypto_keyslot *slot, - struct skcipher_request **ciph_req_ret, - struct crypto_wait *wait) +static struct crypto_sync_skcipher * +blk_crypto_fallback_tfm(struct blk_crypto_keyslot *slot) { - struct skcipher_request *ciph_req; - const struct blk_crypto_fallback_keyslot *slotp; - int keyslot_idx = blk_crypto_keyslot_index(slot); - - slotp = &blk_crypto_keyslots[keyslot_idx]; - ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], - GFP_NOIO); - if (!ciph_req) - return false; - - skcipher_request_set_callback(ciph_req, - CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP, - crypto_req_done, wait); - *ciph_req_ret = ciph_req; + const struct blk_crypto_fallback_keyslot *slotp = + &blk_crypto_keyslots[blk_crypto_keyslot_index(slot)]; - return true; -} - -static bool blk_crypto_fallback_split_bio_if_needed(struct bio **bio_ptr) -{ - struct bio *bio = *bio_ptr; - unsigned int i = 0; - unsigned int num_sectors = 0; - struct bio_vec bv; - struct bvec_iter iter; - - bio_for_each_segment(bv, bio, iter) { - num_sectors += bv.bv_len >> SECTOR_SHIFT; - if (++i == BIO_MAX_VECS) - break; - } - if (num_sectors < bio_sectors(bio)) { - struct bio *split_bio; - - split_bio = bio_split(bio, num_sectors, GFP_NOIO, - &crypto_bio_split); - if (IS_ERR(split_bio)) { - bio->bi_status = BLK_STS_RESOURCE; - return false; - } - bio_chain(split_bio, bio); - submit_bio_noacct(bio); - *bio_ptr = split_bio; - } - - return true; + return slotp->tfms[slotp->crypto_mode]; } union blk_crypto_iv { @@ -252,59 +239,23 @@ static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], iv->dun[i] = cpu_to_le64(dun[i]); } -/* - * The crypto API fallback's encryption routine. - * Allocate a bounce bio for encryption, encrypt the input bio using crypto API, - * and replace *bio_ptr with the bounce bio. May split input bio if it's too - * large. Returns true on success. Returns false and sets bio->bi_status on - * error. - */ -static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) +static void __blk_crypto_fallback_encrypt_bio(struct bio *src_bio, + struct crypto_sync_skcipher *tfm) { - struct bio *src_bio, *enc_bio; - struct bio_crypt_ctx *bc; - struct blk_crypto_keyslot *slot; - int data_unit_size; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); + struct bio_crypt_ctx *bc = src_bio->bi_crypt_context; + int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; + SYNC_SKCIPHER_REQUEST_ON_STACK(ciph_req, tfm); u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; struct scatterlist src, dst; union blk_crypto_iv iv; - unsigned int i, j; - bool ret = false; - blk_status_t blk_st; - - /* Split the bio if it's too big for single page bvec */ - if (!blk_crypto_fallback_split_bio_if_needed(bio_ptr)) - return false; - - src_bio = *bio_ptr; - bc = src_bio->bi_crypt_context; - data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; - - /* Allocate bounce bio for encryption */ - enc_bio = blk_crypto_fallback_clone_bio(src_bio); - if (!enc_bio) { - src_bio->bi_status = BLK_STS_RESOURCE; - return false; - } - - /* - * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for - * this bio's algorithm and key. - */ - blk_st = blk_crypto_get_keyslot(blk_crypto_fallback_profile, - bc->bc_key, &slot); - if (blk_st != BLK_STS_OK) { - src_bio->bi_status = blk_st; - goto out_put_enc_bio; - } + unsigned int nr_enc_pages, enc_idx; + struct page **enc_pages; + struct bio *enc_bio; + unsigned int i; - /* and then allocate an skcipher_request for it */ - if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) { - src_bio->bi_status = BLK_STS_RESOURCE; - goto out_release_keyslot; - } + skcipher_request_set_callback(ciph_req, + CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); sg_init_table(&src, 1); @@ -313,101 +264,127 @@ static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size, iv.bytes); - /* Encrypt each page in the bounce bio */ - for (i = 0; i < enc_bio->bi_vcnt; i++) { - struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i]; - struct page *plaintext_page = enc_bvec->bv_page; - struct page *ciphertext_page = - mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO); + /* + * Encrypt each page in the source bio. Because the source bio could + * have bio_vecs that span more than a single page, but the encrypted + * bios are limited to a single page per bio_vec, this can generate + * more than a single encrypted bio per source bio. + */ +new_bio: + nr_enc_pages = min(bio_segments(src_bio), BIO_MAX_VECS); + enc_bio = blk_crypto_alloc_enc_bio(src_bio, nr_enc_pages, &enc_pages); + enc_idx = 0; + for (;;) { + struct bio_vec src_bv = + bio_iter_iovec(src_bio, src_bio->bi_iter); + struct page *enc_page = enc_pages[enc_idx]; + + if (!IS_ALIGNED(src_bv.bv_len | src_bv.bv_offset, + data_unit_size)) { + enc_bio->bi_status = BLK_STS_INVAL; + goto out_free_enc_bio; + } - enc_bvec->bv_page = ciphertext_page; + __bio_add_page(enc_bio, enc_page, src_bv.bv_len, + src_bv.bv_offset); - if (!ciphertext_page) { - src_bio->bi_status = BLK_STS_RESOURCE; - goto out_free_bounce_pages; - } + sg_set_page(&src, src_bv.bv_page, data_unit_size, + src_bv.bv_offset); + sg_set_page(&dst, enc_page, data_unit_size, src_bv.bv_offset); - sg_set_page(&src, plaintext_page, data_unit_size, - enc_bvec->bv_offset); - sg_set_page(&dst, ciphertext_page, data_unit_size, - enc_bvec->bv_offset); + /* + * Increment the index now that the encrypted page is added to + * the bio. This is important for the error unwind path. + */ + enc_idx++; - /* Encrypt each data unit in this page */ - for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { + /* + * Encrypt each data unit in this page. + */ + for (i = 0; i < src_bv.bv_len; i += data_unit_size) { blk_crypto_dun_to_iv(curr_dun, &iv); - if (crypto_wait_req(crypto_skcipher_encrypt(ciph_req), - &wait)) { - i++; - src_bio->bi_status = BLK_STS_IOERR; - goto out_free_bounce_pages; + if (crypto_skcipher_encrypt(ciph_req)) { + enc_bio->bi_status = BLK_STS_IOERR; + goto out_free_enc_bio; } bio_crypt_dun_increment(curr_dun, 1); src.offset += data_unit_size; dst.offset += data_unit_size; } + + bio_advance_iter_single(src_bio, &src_bio->bi_iter, + src_bv.bv_len); + if (!src_bio->bi_iter.bi_size) + break; + + if (enc_idx == nr_enc_pages) { + /* + * For each additional encrypted bio submitted, + * increment the source bio's remaining count. Each + * encrypted bio's completion handler calls bio_endio on + * the source bio, so this keeps the source bio from + * completing until the last encrypted bio does. + */ + bio_inc_remaining(src_bio); + submit_bio(enc_bio); + goto new_bio; + } } - enc_bio->bi_private = src_bio; - enc_bio->bi_end_io = blk_crypto_fallback_encrypt_endio; - *bio_ptr = enc_bio; - ret = true; - - enc_bio = NULL; - goto out_free_ciph_req; - -out_free_bounce_pages: - while (i > 0) - mempool_free(enc_bio->bi_io_vec[--i].bv_page, - blk_crypto_bounce_page_pool); -out_free_ciph_req: - skcipher_request_free(ciph_req); -out_release_keyslot: - blk_crypto_put_keyslot(slot); -out_put_enc_bio: - if (enc_bio) - bio_uninit(enc_bio); - kfree(enc_bio); - return ret; + submit_bio(enc_bio); + return; + +out_free_enc_bio: + /* + * Add the remaining pages to the bio so that the normal completion path + * in blk_crypto_fallback_encrypt_endio frees them. The exact data + * layout does not matter for that, so don't bother iterating the source + * bio. + */ + for (; enc_idx < nr_enc_pages; enc_idx++) + __bio_add_page(enc_bio, enc_pages[enc_idx], PAGE_SIZE, 0); + bio_endio(enc_bio); } /* - * The crypto API fallback's main decryption routine. - * Decrypts input bio in place, and calls bio_endio on the bio. + * The crypto API fallback's encryption routine. + * + * Allocate one or more bios for encryption, encrypt the input bio using the + * crypto API, and submit the encrypted bios. Sets bio->bi_status and + * completes the source bio on error */ -static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) +static void blk_crypto_fallback_encrypt_bio(struct bio *src_bio) { - struct bio_fallback_crypt_ctx *f_ctx = - container_of(work, struct bio_fallback_crypt_ctx, work); - struct bio *bio = f_ctx->bio; - struct bio_crypt_ctx *bc = &f_ctx->crypt_ctx; + struct bio_crypt_ctx *bc = src_bio->bi_crypt_context; struct blk_crypto_keyslot *slot; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); + blk_status_t status; + + status = blk_crypto_get_keyslot(blk_crypto_fallback_profile, + bc->bc_key, &slot); + if (status != BLK_STS_OK) { + bio_endio_status(src_bio, status); + return; + } + __blk_crypto_fallback_encrypt_bio(src_bio, + blk_crypto_fallback_tfm(slot)); + blk_crypto_put_keyslot(slot); +} + +static blk_status_t __blk_crypto_fallback_decrypt_bio(struct bio *bio, + struct bio_crypt_ctx *bc, struct bvec_iter iter, + struct crypto_sync_skcipher *tfm) +{ + SYNC_SKCIPHER_REQUEST_ON_STACK(ciph_req, tfm); u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; union blk_crypto_iv iv; struct scatterlist sg; struct bio_vec bv; - struct bvec_iter iter; const int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; unsigned int i; - blk_status_t blk_st; - /* - * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for - * this bio's algorithm and key. - */ - blk_st = blk_crypto_get_keyslot(blk_crypto_fallback_profile, - bc->bc_key, &slot); - if (blk_st != BLK_STS_OK) { - bio->bi_status = blk_st; - goto out_no_keyslot; - } - - /* and then allocate an skcipher_request for it */ - if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) { - bio->bi_status = BLK_STS_RESOURCE; - goto out; - } + skcipher_request_set_callback(ciph_req, + CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); sg_init_table(&sg, 1); @@ -415,30 +392,52 @@ static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) iv.bytes); /* Decrypt each segment in the bio */ - __bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) { + __bio_for_each_segment(bv, bio, iter, iter) { struct page *page = bv.bv_page; + if (!IS_ALIGNED(bv.bv_len | bv.bv_offset, data_unit_size)) + return BLK_STS_INVAL; + sg_set_page(&sg, page, data_unit_size, bv.bv_offset); /* Decrypt each data unit in the segment */ for (i = 0; i < bv.bv_len; i += data_unit_size) { blk_crypto_dun_to_iv(curr_dun, &iv); - if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), - &wait)) { - bio->bi_status = BLK_STS_IOERR; - goto out; - } + if (crypto_skcipher_decrypt(ciph_req)) + return BLK_STS_IOERR; bio_crypt_dun_increment(curr_dun, 1); sg.offset += data_unit_size; } } -out: - skcipher_request_free(ciph_req); - blk_crypto_put_keyslot(slot); -out_no_keyslot: + return BLK_STS_OK; +} + +/* + * The crypto API fallback's main decryption routine. + * + * Decrypts input bio in place, and calls bio_endio on the bio. + */ +static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) +{ + struct bio_fallback_crypt_ctx *f_ctx = + container_of(work, struct bio_fallback_crypt_ctx, work); + struct bio *bio = f_ctx->bio; + struct bio_crypt_ctx *bc = &f_ctx->crypt_ctx; + struct blk_crypto_keyslot *slot; + blk_status_t status; + + status = blk_crypto_get_keyslot(blk_crypto_fallback_profile, + bc->bc_key, &slot); + if (status == BLK_STS_OK) { + status = __blk_crypto_fallback_decrypt_bio(bio, bc, + f_ctx->crypt_iter, + blk_crypto_fallback_tfm(slot)); + blk_crypto_put_keyslot(slot); + } mempool_free(f_ctx, bio_fallback_crypt_ctx_pool); - bio_endio(bio); + + bio_endio_status(bio, status); } /** @@ -470,44 +469,43 @@ static void blk_crypto_fallback_decrypt_endio(struct bio *bio) /** * blk_crypto_fallback_bio_prep - Prepare a bio to use fallback en/decryption + * @bio: bio to prepare * - * @bio_ptr: pointer to the bio to prepare + * If bio is doing a WRITE operation, allocate one or more bios to contain the + * encrypted payload and submit them. * - * If bio is doing a WRITE operation, this splits the bio into two parts if it's - * too big (see blk_crypto_fallback_split_bio_if_needed()). It then allocates a - * bounce bio for the first part, encrypts it, and updates bio_ptr to point to - * the bounce bio. - * - * For a READ operation, we mark the bio for decryption by using bi_private and + * For a READ operation, mark the bio for decryption by using bi_private and * bi_end_io. * - * In either case, this function will make the bio look like a regular bio (i.e. - * as if no encryption context was ever specified) for the purposes of the rest - * of the stack except for blk-integrity (blk-integrity and blk-crypto are not - * currently supported together). + * In either case, this function will make the submitted bio(s) look like + * regular bios (i.e. as if no encryption context was ever specified) for the + * purposes of the rest of the stack except for blk-integrity (blk-integrity and + * blk-crypto are not currently supported together). * - * Return: true on success. Sets bio->bi_status and returns false on error. + * Return: true if @bio should be submitted to the driver by the caller, else + * false. Sets bio->bi_status, calls bio_endio and returns false on error. */ -bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr) +bool blk_crypto_fallback_bio_prep(struct bio *bio) { - struct bio *bio = *bio_ptr; struct bio_crypt_ctx *bc = bio->bi_crypt_context; struct bio_fallback_crypt_ctx *f_ctx; if (WARN_ON_ONCE(!tfms_inited[bc->bc_key->crypto_cfg.crypto_mode])) { /* User didn't call blk_crypto_start_using_key() first */ - bio->bi_status = BLK_STS_IOERR; + bio_io_error(bio); return false; } if (!__blk_crypto_cfg_supported(blk_crypto_fallback_profile, &bc->bc_key->crypto_cfg)) { - bio->bi_status = BLK_STS_NOTSUPP; + bio_endio_status(bio, BLK_STS_NOTSUPP); return false; } - if (bio_data_dir(bio) == WRITE) - return blk_crypto_fallback_encrypt_bio(bio_ptr); + if (bio_data_dir(bio) == WRITE) { + blk_crypto_fallback_encrypt_bio(bio); + return false; + } /* * bio READ case: Set up a f_ctx in the bio's bi_private and set the @@ -541,13 +539,12 @@ static int blk_crypto_fallback_init(void) get_random_bytes(blank_key, sizeof(blank_key)); - err = bioset_init(&crypto_bio_split, 64, 0, 0); + err = bioset_init(&enc_bio_set, 64, 0, BIOSET_NEED_BVECS); if (err) goto out; /* Dynamic allocation is needed because of lockdep_register_key(). */ - blk_crypto_fallback_profile = - kzalloc(sizeof(*blk_crypto_fallback_profile), GFP_KERNEL); + blk_crypto_fallback_profile = kzalloc_obj(*blk_crypto_fallback_profile); if (!blk_crypto_fallback_profile) { err = -ENOMEM; goto fail_free_bioset; @@ -574,9 +571,8 @@ static int blk_crypto_fallback_init(void) if (!blk_crypto_wq) goto fail_destroy_profile; - blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots, - sizeof(blk_crypto_keyslots[0]), - GFP_KERNEL); + blk_crypto_keyslots = kzalloc_objs(blk_crypto_keyslots[0], + blk_crypto_num_keyslots); if (!blk_crypto_keyslots) goto fail_free_wq; @@ -611,7 +607,7 @@ fail_destroy_profile: fail_free_profile: kfree(blk_crypto_fallback_profile); fail_free_bioset: - bioset_exit(&crypto_bio_split); + bioset_exit(&enc_bio_set); out: return err; } @@ -645,7 +641,8 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) for (i = 0; i < blk_crypto_num_keyslots; i++) { slotp = &blk_crypto_keyslots[i]; - slotp->tfms[mode_num] = crypto_alloc_skcipher(cipher_str, 0, 0); + slotp->tfms[mode_num] = crypto_alloc_sync_skcipher(cipher_str, + 0, 0); if (IS_ERR(slotp->tfms[mode_num])) { err = PTR_ERR(slotp->tfms[mode_num]); if (err == -ENOENT) { @@ -657,7 +654,7 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) goto out_free_tfms; } - crypto_skcipher_set_flags(slotp->tfms[mode_num], + crypto_sync_skcipher_set_flags(slotp->tfms[mode_num], CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); } @@ -671,7 +668,7 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) out_free_tfms: for (i = 0; i < blk_crypto_num_keyslots; i++) { slotp = &blk_crypto_keyslots[i]; - crypto_free_skcipher(slotp->tfms[mode_num]); + crypto_free_sync_skcipher(slotp->tfms[mode_num]); slotp->tfms[mode_num] = NULL; } out: diff --git a/block/blk-crypto-internal.h b/block/blk-crypto-internal.h index ccf6dff6ff6b..742694213529 100644 --- a/block/blk-crypto-internal.h +++ b/block/blk-crypto-internal.h @@ -86,6 +86,12 @@ bool __blk_crypto_cfg_supported(struct blk_crypto_profile *profile, int blk_crypto_ioctl(struct block_device *bdev, unsigned int cmd, void __user *argp); +static inline bool blk_crypto_supported(struct bio *bio) +{ + return blk_crypto_config_supported_natively(bio->bi_bdev, + &bio->bi_crypt_context->bc_key->crypto_cfg); +} + #else /* CONFIG_BLK_INLINE_ENCRYPTION */ static inline int blk_crypto_sysfs_register(struct gendisk *disk) @@ -139,6 +145,11 @@ static inline int blk_crypto_ioctl(struct block_device *bdev, unsigned int cmd, return -ENOTTY; } +static inline bool blk_crypto_supported(struct bio *bio) +{ + return false; +} + #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ void __bio_crypt_advance(struct bio *bio, unsigned int bytes); @@ -165,14 +176,6 @@ static inline void bio_crypt_do_front_merge(struct request *rq, #endif } -bool __blk_crypto_bio_prep(struct bio **bio_ptr); -static inline bool blk_crypto_bio_prep(struct bio **bio_ptr) -{ - if (bio_has_crypt_ctx(*bio_ptr)) - return __blk_crypto_bio_prep(bio_ptr); - return true; -} - blk_status_t __blk_crypto_rq_get_keyslot(struct request *rq); static inline blk_status_t blk_crypto_rq_get_keyslot(struct request *rq) { @@ -215,12 +218,12 @@ static inline int blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio, return 0; } +bool blk_crypto_fallback_bio_prep(struct bio *bio); + #ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num); -bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr); - int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key); #else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ @@ -232,13 +235,6 @@ blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) return -ENOPKG; } -static inline bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr) -{ - pr_warn_once("crypto API fallback disabled; failing request.\n"); - (*bio_ptr)->bi_status = BLK_STS_NOTSUPP; - return false; -} - static inline int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) { diff --git a/block/blk-crypto-profile.c b/block/blk-crypto-profile.c index 94a155912bf1..cf447ba4a66e 100644 --- a/block/blk-crypto-profile.c +++ b/block/blk-crypto-profile.c @@ -43,6 +43,7 @@ struct blk_crypto_keyslot { }; static inline void blk_crypto_hw_enter(struct blk_crypto_profile *profile) + __acquires(&profile->lock) { /* * Calling into the driver requires profile->lock held and the device @@ -55,6 +56,7 @@ static inline void blk_crypto_hw_enter(struct blk_crypto_profile *profile) } static inline void blk_crypto_hw_exit(struct blk_crypto_profile *profile) + __releases(&profile->lock) { up_write(&profile->lock); if (profile->dev) @@ -93,8 +95,7 @@ int blk_crypto_profile_init(struct blk_crypto_profile *profile, /* Initialize keyslot management data. */ - profile->slots = kvcalloc(num_slots, sizeof(profile->slots[0]), - GFP_KERNEL); + profile->slots = kvzalloc_objs(profile->slots[0], num_slots); if (!profile->slots) goto err_destroy; @@ -121,8 +122,7 @@ int blk_crypto_profile_init(struct blk_crypto_profile *profile, profile->log_slot_ht_size = ilog2(slot_hashtable_size); profile->slot_hashtable = - kvmalloc_array(slot_hashtable_size, - sizeof(profile->slot_hashtable[0]), GFP_KERNEL); + kvmalloc_objs(profile->slot_hashtable[0], slot_hashtable_size); if (!profile->slot_hashtable) goto err_destroy; for (i = 0; i < slot_hashtable_size; i++) @@ -501,6 +501,7 @@ int blk_crypto_derive_sw_secret(struct block_device *bdev, blk_crypto_hw_exit(profile); return err; } +EXPORT_SYMBOL_GPL(blk_crypto_derive_sw_secret); int blk_crypto_import_key(struct blk_crypto_profile *profile, const u8 *raw_key, size_t raw_key_size, @@ -520,6 +521,7 @@ int blk_crypto_import_key(struct blk_crypto_profile *profile, blk_crypto_hw_exit(profile); return ret; } +EXPORT_SYMBOL_GPL(blk_crypto_import_key); int blk_crypto_generate_key(struct blk_crypto_profile *profile, u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) @@ -537,6 +539,7 @@ int blk_crypto_generate_key(struct blk_crypto_profile *profile, blk_crypto_hw_exit(profile); return ret; } +EXPORT_SYMBOL_GPL(blk_crypto_generate_key); int blk_crypto_prepare_key(struct blk_crypto_profile *profile, const u8 *lt_key, size_t lt_key_size, @@ -556,6 +559,7 @@ int blk_crypto_prepare_key(struct blk_crypto_profile *profile, blk_crypto_hw_exit(profile); return ret; } +EXPORT_SYMBOL_GPL(blk_crypto_prepare_key); /** * blk_crypto_intersect_capabilities() - restrict supported crypto capabilities diff --git a/block/blk-crypto-sysfs.c b/block/blk-crypto-sysfs.c index e832f403f200..b069c418b6cc 100644 --- a/block/blk-crypto-sysfs.c +++ b/block/blk-crypto-sysfs.c @@ -18,7 +18,7 @@ struct blk_crypto_kobj { struct blk_crypto_attr { struct attribute attr; ssize_t (*show)(struct blk_crypto_profile *profile, - struct blk_crypto_attr *attr, char *page); + const struct blk_crypto_attr *attr, char *page); }; static struct blk_crypto_profile *kobj_to_crypto_profile(struct kobject *kobj) @@ -26,39 +26,39 @@ static struct blk_crypto_profile *kobj_to_crypto_profile(struct kobject *kobj) return container_of(kobj, struct blk_crypto_kobj, kobj)->profile; } -static struct blk_crypto_attr *attr_to_crypto_attr(struct attribute *attr) +static const struct blk_crypto_attr *attr_to_crypto_attr(const struct attribute *attr) { - return container_of(attr, struct blk_crypto_attr, attr); + return container_of_const(attr, struct blk_crypto_attr, attr); } static ssize_t hw_wrapped_keys_show(struct blk_crypto_profile *profile, - struct blk_crypto_attr *attr, char *page) + const struct blk_crypto_attr *attr, char *page) { /* Always show supported, since the file doesn't exist otherwise. */ return sysfs_emit(page, "supported\n"); } static ssize_t max_dun_bits_show(struct blk_crypto_profile *profile, - struct blk_crypto_attr *attr, char *page) + const struct blk_crypto_attr *attr, char *page) { return sysfs_emit(page, "%u\n", 8 * profile->max_dun_bytes_supported); } static ssize_t num_keyslots_show(struct blk_crypto_profile *profile, - struct blk_crypto_attr *attr, char *page) + const struct blk_crypto_attr *attr, char *page) { return sysfs_emit(page, "%u\n", profile->num_slots); } static ssize_t raw_keys_show(struct blk_crypto_profile *profile, - struct blk_crypto_attr *attr, char *page) + const struct blk_crypto_attr *attr, char *page) { /* Always show supported, since the file doesn't exist otherwise. */ return sysfs_emit(page, "supported\n"); } #define BLK_CRYPTO_RO_ATTR(_name) \ - static struct blk_crypto_attr _name##_attr = __ATTR_RO(_name) + static const struct blk_crypto_attr _name##_attr = __ATTR_RO(_name) BLK_CRYPTO_RO_ATTR(hw_wrapped_keys); BLK_CRYPTO_RO_ATTR(max_dun_bits); @@ -66,10 +66,10 @@ BLK_CRYPTO_RO_ATTR(num_keyslots); BLK_CRYPTO_RO_ATTR(raw_keys); static umode_t blk_crypto_is_visible(struct kobject *kobj, - struct attribute *attr, int n) + const struct attribute *attr, int n) { struct blk_crypto_profile *profile = kobj_to_crypto_profile(kobj); - struct blk_crypto_attr *a = attr_to_crypto_attr(attr); + const struct blk_crypto_attr *a = attr_to_crypto_attr(attr); if (a == &hw_wrapped_keys_attr && !(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED)) @@ -81,7 +81,7 @@ static umode_t blk_crypto_is_visible(struct kobject *kobj, return 0444; } -static struct attribute *blk_crypto_attrs[] = { +static const struct attribute *const blk_crypto_attrs[] = { &hw_wrapped_keys_attr.attr, &max_dun_bits_attr.attr, &num_keyslots_attr.attr, @@ -90,8 +90,8 @@ static struct attribute *blk_crypto_attrs[] = { }; static const struct attribute_group blk_crypto_attr_group = { - .attrs = blk_crypto_attrs, - .is_visible = blk_crypto_is_visible, + .attrs_const = blk_crypto_attrs, + .is_visible_const = blk_crypto_is_visible, }; /* @@ -99,13 +99,13 @@ static const struct attribute_group blk_crypto_attr_group = { * modes, these are initialized at boot time by blk_crypto_sysfs_init(). */ static struct blk_crypto_attr __blk_crypto_mode_attrs[BLK_ENCRYPTION_MODE_MAX]; -static struct attribute *blk_crypto_mode_attrs[BLK_ENCRYPTION_MODE_MAX + 1]; +static const struct attribute *blk_crypto_mode_attrs[BLK_ENCRYPTION_MODE_MAX + 1]; static umode_t blk_crypto_mode_is_visible(struct kobject *kobj, - struct attribute *attr, int n) + const struct attribute *attr, int n) { struct blk_crypto_profile *profile = kobj_to_crypto_profile(kobj); - struct blk_crypto_attr *a = attr_to_crypto_attr(attr); + const struct blk_crypto_attr *a = attr_to_crypto_attr(attr); int mode_num = a - __blk_crypto_mode_attrs; if (profile->modes_supported[mode_num]) @@ -114,7 +114,7 @@ static umode_t blk_crypto_mode_is_visible(struct kobject *kobj, } static ssize_t blk_crypto_mode_show(struct blk_crypto_profile *profile, - struct blk_crypto_attr *attr, char *page) + const struct blk_crypto_attr *attr, char *page) { int mode_num = attr - __blk_crypto_mode_attrs; @@ -123,8 +123,8 @@ static ssize_t blk_crypto_mode_show(struct blk_crypto_profile *profile, static const struct attribute_group blk_crypto_modes_attr_group = { .name = "modes", - .attrs = blk_crypto_mode_attrs, - .is_visible = blk_crypto_mode_is_visible, + .attrs_const = blk_crypto_mode_attrs, + .is_visible_const = blk_crypto_mode_is_visible, }; static const struct attribute_group *blk_crypto_attr_groups[] = { @@ -137,7 +137,7 @@ static ssize_t blk_crypto_attr_show(struct kobject *kobj, struct attribute *attr, char *page) { struct blk_crypto_profile *profile = kobj_to_crypto_profile(kobj); - struct blk_crypto_attr *a = attr_to_crypto_attr(attr); + const struct blk_crypto_attr *a = attr_to_crypto_attr(attr); return a->show(profile, a, page); } @@ -170,7 +170,7 @@ int blk_crypto_sysfs_register(struct gendisk *disk) if (!q->crypto_profile) return 0; - obj = kzalloc(sizeof(*obj), GFP_KERNEL); + obj = kzalloc_obj(*obj); if (!obj) return -ENOMEM; obj->profile = q->crypto_profile; diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 4b1ad84d1b5a..15e25e41b166 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -116,6 +116,7 @@ void bio_crypt_set_ctx(struct bio *bio, const struct blk_crypto_key *key, bio->bi_crypt_context = bc; } +EXPORT_SYMBOL_GPL(bio_crypt_set_ctx); void __bio_crypt_free_ctx(struct bio *bio) { @@ -219,22 +220,6 @@ bool bio_crypt_ctx_mergeable(struct bio_crypt_ctx *bc1, unsigned int bc1_bytes, return !bc1 || bio_crypt_dun_is_contiguous(bc1, bc1_bytes, bc2->bc_dun); } -/* Check that all I/O segments are data unit aligned. */ -static bool bio_crypt_check_alignment(struct bio *bio) -{ - const unsigned int data_unit_size = - bio->bi_crypt_context->bc_key->crypto_cfg.data_unit_size; - struct bvec_iter iter; - struct bio_vec bv; - - bio_for_each_segment(bv, bio, iter) { - if (!IS_ALIGNED(bv.bv_len | bv.bv_offset, data_unit_size)) - return false; - } - - return true; -} - blk_status_t __blk_crypto_rq_get_keyslot(struct request *rq) { return blk_crypto_get_keyslot(rq->q->crypto_profile, @@ -258,57 +243,40 @@ void __blk_crypto_free_request(struct request *rq) rq->crypt_ctx = NULL; } -/** - * __blk_crypto_bio_prep - Prepare bio for inline encryption - * - * @bio_ptr: pointer to original bio pointer - * - * If the bio crypt context provided for the bio is supported by the underlying - * device's inline encryption hardware, do nothing. - * - * Otherwise, try to perform en/decryption for this bio by falling back to the - * kernel crypto API. When the crypto API fallback is used for encryption, - * blk-crypto may choose to split the bio into 2 - the first one that will - * continue to be processed and the second one that will be resubmitted via - * submit_bio_noacct. A bounce bio will be allocated to encrypt the contents - * of the aforementioned "first one", and *bio_ptr will be updated to this - * bounce bio. - * - * Caller must ensure bio has bio_crypt_ctx. +/* + * Process a bio with a crypto context. Returns true if the caller should + * submit the passed in bio, false if the bio is consumed. * - * Return: true on success; false on error (and bio->bi_status will be set - * appropriately, and bio_endio() will have been called so bio - * submission should abort). + * See the kerneldoc comment for blk_crypto_submit_bio for further details. */ -bool __blk_crypto_bio_prep(struct bio **bio_ptr) +bool __blk_crypto_submit_bio(struct bio *bio) { - struct bio *bio = *bio_ptr; const struct blk_crypto_key *bc_key = bio->bi_crypt_context->bc_key; + struct block_device *bdev = bio->bi_bdev; /* Error if bio has no data. */ if (WARN_ON_ONCE(!bio_has_data(bio))) { - bio->bi_status = BLK_STS_IOERR; - goto fail; - } - - if (!bio_crypt_check_alignment(bio)) { - bio->bi_status = BLK_STS_IOERR; - goto fail; + bio_io_error(bio); + return false; } /* - * Success if device supports the encryption context, or if we succeeded - * in falling back to the crypto API. + * If the device does not natively support the encryption context, try to use + * the fallback if available. */ - if (blk_crypto_config_supported_natively(bio->bi_bdev, - &bc_key->crypto_cfg)) - return true; - if (blk_crypto_fallback_bio_prep(bio_ptr)) - return true; -fail: - bio_endio(*bio_ptr); - return false; + if (!blk_crypto_config_supported_natively(bdev, &bc_key->crypto_cfg)) { + if (!IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK)) { + pr_warn_once("%pg: crypto API fallback disabled; failing request.\n", + bdev); + bio_endio_status(bio, BLK_STS_NOTSUPP); + return false; + } + return blk_crypto_fallback_bio_prep(bio); + } + + return true; } +EXPORT_SYMBOL_GPL(__blk_crypto_submit_bio); int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio, gfp_t gfp_mask) @@ -381,6 +349,7 @@ int blk_crypto_init_key(struct blk_crypto_key *blk_key, return 0; } +EXPORT_SYMBOL_GPL(blk_crypto_init_key); bool blk_crypto_config_supported_natively(struct block_device *bdev, const struct blk_crypto_config *cfg) @@ -431,6 +400,7 @@ int blk_crypto_start_using_key(struct block_device *bdev, } return blk_crypto_fallback_start_using_mode(key->crypto_cfg.crypto_mode); } +EXPORT_SYMBOL_GPL(blk_crypto_start_using_key); /** * blk_crypto_evict_key() - Evict a blk_crypto_key from a block_device diff --git a/block/blk-flush.c b/block/blk-flush.c index 43d6152897a4..403a46c86411 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -199,7 +199,8 @@ static void blk_flush_complete_seq(struct request *rq, } static enum rq_end_io_ret flush_end_io(struct request *flush_rq, - blk_status_t error) + blk_status_t error, + const struct io_comp_batch *iob) { struct request_queue *q = flush_rq->q; struct list_head *running; @@ -335,7 +336,8 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, } static enum rq_end_io_ret mq_flush_data_end_io(struct request *rq, - blk_status_t error) + blk_status_t error, + const struct io_comp_batch *iob) { struct request_queue *q = rq->q; struct blk_mq_hw_ctx *hctx = rq->mq_hctx; diff --git a/block/blk-ia-ranges.c b/block/blk-ia-ranges.c index d479f5481b66..7be8b58893c9 100644 --- a/block/blk-ia-ranges.c +++ b/block/blk-ia-ranges.c @@ -30,17 +30,17 @@ struct blk_ia_range_sysfs_entry { ssize_t (*show)(struct blk_independent_access_range *iar, char *buf); }; -static struct blk_ia_range_sysfs_entry blk_ia_range_sector_entry = { +static const struct blk_ia_range_sysfs_entry blk_ia_range_sector_entry = { .attr = { .name = "sector", .mode = 0444 }, .show = blk_ia_range_sector_show, }; -static struct blk_ia_range_sysfs_entry blk_ia_range_nr_sectors_entry = { +static const struct blk_ia_range_sysfs_entry blk_ia_range_nr_sectors_entry = { .attr = { .name = "nr_sectors", .mode = 0444 }, .show = blk_ia_range_nr_sectors_show, }; -static struct attribute *blk_ia_range_attrs[] = { +static const struct attribute *const blk_ia_range_attrs[] = { &blk_ia_range_sector_entry.attr, &blk_ia_range_nr_sectors_entry.attr, NULL, diff --git a/block/blk-integrity.c b/block/blk-integrity.c index a1678f0a9f81..964eebbee14d 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -13,6 +13,7 @@ #include <linux/scatterlist.h> #include <linux/export.h> #include <linux/slab.h> +#include <linux/t10-pi.h> #include "blk.h" @@ -54,76 +55,78 @@ new_segment: return segments; } -/** - * blk_rq_map_integrity_sg - Map integrity metadata into a scatterlist - * @rq: request to map - * @sglist: target scatterlist - * - * Description: Map the integrity vectors in request into a - * scatterlist. The scatterlist must be big enough to hold all - * elements. I.e. sized using blk_rq_count_integrity_sg() or - * rq->nr_integrity_segments. - */ -int blk_rq_map_integrity_sg(struct request *rq, struct scatterlist *sglist) +int blk_get_meta_cap(struct block_device *bdev, unsigned int cmd, + struct logical_block_metadata_cap __user *argp) { - struct bio_vec iv, ivprv = { NULL }; - struct request_queue *q = rq->q; - struct scatterlist *sg = NULL; - struct bio *bio = rq->bio; - unsigned int segments = 0; - struct bvec_iter iter; - int prev = 0; + struct blk_integrity *bi; + struct logical_block_metadata_cap meta_cap = {}; + size_t usize = _IOC_SIZE(cmd); + + if (!extensible_ioctl_valid(cmd, FS_IOC_GETLBMD_CAP, LBMD_SIZE_VER0)) + return -ENOIOCTLCMD; + + bi = blk_get_integrity(bdev->bd_disk); + if (!bi) + goto out; + + if (bi->flags & BLK_INTEGRITY_DEVICE_CAPABLE) + meta_cap.lbmd_flags |= LBMD_PI_CAP_INTEGRITY; + if (bi->flags & BLK_INTEGRITY_REF_TAG) + meta_cap.lbmd_flags |= LBMD_PI_CAP_REFTAG; + meta_cap.lbmd_interval = 1 << bi->interval_exp; + meta_cap.lbmd_size = bi->metadata_size; + meta_cap.lbmd_pi_size = bi->pi_tuple_size; + meta_cap.lbmd_pi_offset = bi->pi_offset; + meta_cap.lbmd_opaque_size = bi->metadata_size - bi->pi_tuple_size; + if (meta_cap.lbmd_opaque_size && !bi->pi_offset) + meta_cap.lbmd_opaque_offset = bi->pi_tuple_size; - bio_for_each_integrity_vec(iv, bio, iter) { - if (prev) { - if (!biovec_phys_mergeable(q, &ivprv, &iv)) - goto new_segment; - if (sg->length + iv.bv_len > queue_max_segment_size(q)) - goto new_segment; + switch (bi->csum_type) { + case BLK_INTEGRITY_CSUM_NONE: + meta_cap.lbmd_guard_tag_type = LBMD_PI_CSUM_NONE; + break; + case BLK_INTEGRITY_CSUM_IP: + meta_cap.lbmd_guard_tag_type = LBMD_PI_CSUM_IP; + break; + case BLK_INTEGRITY_CSUM_CRC: + meta_cap.lbmd_guard_tag_type = LBMD_PI_CSUM_CRC16_T10DIF; + break; + case BLK_INTEGRITY_CSUM_CRC64: + meta_cap.lbmd_guard_tag_type = LBMD_PI_CSUM_CRC64_NVME; + break; + } - sg->length += iv.bv_len; - } else { -new_segment: - if (!sg) - sg = sglist; - else { - sg_unmark_end(sg); - sg = sg_next(sg); - } - - sg_set_page(sg, iv.bv_page, iv.bv_len, iv.bv_offset); - segments++; + if (bi->csum_type != BLK_INTEGRITY_CSUM_NONE) + meta_cap.lbmd_app_tag_size = 2; + + if (bi->flags & BLK_INTEGRITY_REF_TAG) { + switch (bi->csum_type) { + case BLK_INTEGRITY_CSUM_CRC64: + meta_cap.lbmd_ref_tag_size = + sizeof_field(struct crc64_pi_tuple, ref_tag); + break; + case BLK_INTEGRITY_CSUM_CRC: + case BLK_INTEGRITY_CSUM_IP: + meta_cap.lbmd_ref_tag_size = + sizeof_field(struct t10_pi_tuple, ref_tag); + break; + default: + break; } - - prev = 1; - ivprv = iv; } - if (sg) - sg_mark_end(sg); - - /* - * Something must have been wrong if the figured number of segment - * is bigger than number of req's physical integrity segments - */ - BUG_ON(segments > rq->nr_integrity_segments); - BUG_ON(segments > queue_max_integrity_segments(q)); - return segments; +out: + return copy_struct_to_user(argp, usize, &meta_cap, sizeof(meta_cap), + NULL); } -EXPORT_SYMBOL(blk_rq_map_integrity_sg); int blk_rq_integrity_map_user(struct request *rq, void __user *ubuf, ssize_t bytes) { int ret; struct iov_iter iter; - unsigned int direction; - if (op_is_write(req_op(rq))) - direction = ITER_DEST; - else - direction = ITER_SOURCE; - iov_iter_ubuf(&iter, direction, ubuf, bytes); + iov_iter_ubuf(&iter, rq_data_dir(rq), ubuf, bytes); ret = bio_integrity_map_user(rq->bio, &iter); if (ret) return ret; @@ -137,14 +140,21 @@ EXPORT_SYMBOL_GPL(blk_rq_integrity_map_user); bool blk_integrity_merge_rq(struct request_queue *q, struct request *req, struct request *next) { + struct bio_integrity_payload *bip, *bip_next; + if (blk_integrity_rq(req) == 0 && blk_integrity_rq(next) == 0) return true; if (blk_integrity_rq(req) == 0 || blk_integrity_rq(next) == 0) return false; - if (bio_integrity(req->bio)->bip_flags != - bio_integrity(next->bio)->bip_flags) + bip = bio_integrity(req->bio); + bip_next = bio_integrity(next->bio); + if (bip->bip_flags != bip_next->bip_flags) + return false; + + if (bip->bip_flags & BIP_CHECK_APPTAG && + bip->app_tag != bip_next->app_tag) return false; if (req->nr_integrity_segments + next->nr_integrity_segments > @@ -160,15 +170,21 @@ bool blk_integrity_merge_rq(struct request_queue *q, struct request *req, bool blk_integrity_merge_bio(struct request_queue *q, struct request *req, struct bio *bio) { + struct bio_integrity_payload *bip, *bip_bio = bio_integrity(bio); int nr_integrity_segs; - if (blk_integrity_rq(req) == 0 && bio_integrity(bio) == NULL) + if (blk_integrity_rq(req) == 0 && bip_bio == NULL) return true; - if (blk_integrity_rq(req) == 0 || bio_integrity(bio) == NULL) + if (blk_integrity_rq(req) == 0 || bip_bio == NULL) + return false; + + bip = bio_integrity(req->bio); + if (bip->bip_flags != bip_bio->bip_flags) return false; - if (bio_integrity(req->bio)->bip_flags != bio_integrity(bio)->bip_flags) + if (bip->bip_flags & BIP_CHECK_APPTAG && + bip->app_tag != bip_bio->app_tag) return false; nr_integrity_segs = blk_rq_count_integrity_sg(q, bio); @@ -244,7 +260,7 @@ static ssize_t format_show(struct device *dev, struct device_attribute *attr, { struct blk_integrity *bi = dev_to_bi(dev); - if (!bi->tuple_size) + if (!bi->metadata_size) return sysfs_emit(page, "none\n"); return sysfs_emit(page, "%s\n", blk_integrity_profile_name(bi)); } diff --git a/block/blk-ioc.c b/block/blk-ioc.c index ce82770c72ab..d15918d7fabb 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -286,7 +286,7 @@ out: } EXPORT_SYMBOL_GPL(set_task_ioprio); -int __copy_io(unsigned long clone_flags, struct task_struct *tsk) +int __copy_io(u64 clone_flags, struct task_struct *tsk) { struct io_context *ioc = current->io_context; @@ -308,24 +308,23 @@ int __copy_io(unsigned long clone_flags, struct task_struct *tsk) #ifdef CONFIG_BLK_ICQ /** - * ioc_lookup_icq - lookup io_cq from ioc + * ioc_lookup_icq - lookup io_cq from ioc in io issue path * @q: the associated request_queue * * Look up io_cq associated with @ioc - @q pair from @ioc. Must be called - * with @q->queue_lock held. + * from io issue path, either return NULL if current issue io to @q for the + * first time, or return a valid icq. */ struct io_cq *ioc_lookup_icq(struct request_queue *q) { struct io_context *ioc = current->io_context; struct io_cq *icq; - lockdep_assert_held(&q->queue_lock); - /* * icq's are indexed from @ioc using radix tree and hint pointer, - * both of which are protected with RCU. All removals are done - * holding both q and ioc locks, and we're holding q lock - if we - * find a icq which points to us, it's guaranteed to be valid. + * both of which are protected with RCU, io issue path ensures that + * both request_queue and current task are valid, the found icq + * is guaranteed to be valid until the io is done. */ rcu_read_lock(); icq = rcu_dereference(ioc->icq_hint); @@ -419,10 +418,7 @@ struct io_cq *ioc_find_get_icq(struct request_queue *q) task_unlock(current); } else { get_io_context(ioc); - - spin_lock_irq(&q->queue_lock); icq = ioc_lookup_icq(q); - spin_unlock_irq(&q->queue_lock); } if (!icq) { diff --git a/block/blk-iocost.c b/block/blk-iocost.c index 5bfd70311359..8b2aeba2e1e3 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -727,26 +727,6 @@ static void iocg_commit_bio(struct ioc_gq *iocg, struct bio *bio, put_cpu_ptr(gcs); } -static void iocg_lock(struct ioc_gq *iocg, bool lock_ioc, unsigned long *flags) -{ - if (lock_ioc) { - spin_lock_irqsave(&iocg->ioc->lock, *flags); - spin_lock(&iocg->waitq.lock); - } else { - spin_lock_irqsave(&iocg->waitq.lock, *flags); - } -} - -static void iocg_unlock(struct ioc_gq *iocg, bool unlock_ioc, unsigned long *flags) -{ - if (unlock_ioc) { - spin_unlock(&iocg->waitq.lock); - spin_unlock_irqrestore(&iocg->ioc->lock, *flags); - } else { - spin_unlock_irqrestore(&iocg->waitq.lock, *flags); - } -} - #define CREATE_TRACE_POINTS #include <trace/events/iocost.h> @@ -812,7 +792,7 @@ static int ioc_autop_idx(struct ioc *ioc, struct gendisk *disk) u64 now_ns; /* rotational? */ - if (!blk_queue_nonrot(disk->queue)) + if (blk_queue_rot(disk->queue)) return AUTOP_HDD; /* handle SATA SSDs w/ broken NCQ */ @@ -1589,14 +1569,23 @@ static enum hrtimer_restart iocg_waitq_timer_fn(struct hrtimer *timer) ioc_now(iocg->ioc, &now); - iocg_lock(iocg, pay_debt, &flags); - iocg_kick_waitq(iocg, pay_debt, &now); - iocg_unlock(iocg, pay_debt, &flags); + if (pay_debt) { + spin_lock_irqsave(&iocg->ioc->lock, flags); + spin_lock(&iocg->waitq.lock); + iocg_kick_waitq(iocg, pay_debt, &now); + spin_unlock(&iocg->waitq.lock); + spin_unlock_irqrestore(&iocg->ioc->lock, flags); + } else { + spin_lock_irqsave(&iocg->waitq.lock, flags); + iocg_kick_waitq(iocg, pay_debt, &now); + spin_unlock_irqrestore(&iocg->waitq.lock, flags); + } return HRTIMER_NORESTART; } -static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p) +static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p, + u32 *nr_done) { u32 nr_met[2] = { }; u32 nr_missed[2] = { }; @@ -1633,6 +1622,8 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p *rq_wait_pct_p = div64_u64(rq_wait_ns * 100, ioc->period_us * NSEC_PER_USEC); + + *nr_done = nr_met[READ] + nr_met[WRITE] + nr_missed[READ] + nr_missed[WRITE]; } /* was iocg idle this period? */ @@ -2250,12 +2241,12 @@ static void ioc_timer_fn(struct timer_list *timer) u64 usage_us_sum = 0; u32 ppm_rthr; u32 ppm_wthr; - u32 missed_ppm[2], rq_wait_pct; + u32 missed_ppm[2], rq_wait_pct, nr_done; u64 period_vtime; int prev_busy_level; /* how were the latencies during the period? */ - ioc_lat_stat(ioc, missed_ppm, &rq_wait_pct); + ioc_lat_stat(ioc, missed_ppm, &rq_wait_pct, &nr_done); /* take care of active iocgs */ spin_lock_irq(&ioc->lock); @@ -2334,10 +2325,8 @@ static void ioc_timer_fn(struct timer_list *timer) else usage_dur = max_t(u64, now.now - ioc->period_at, 1); - usage = clamp_t(u32, - DIV64_U64_ROUND_UP(usage_us * WEIGHT_ONE, - usage_dur), - 1, WEIGHT_ONE); + usage = clamp(DIV64_U64_ROUND_UP(usage_us * WEIGHT_ONE, usage_dur), + 1, WEIGHT_ONE); /* * Already donating or accumulated enough to start. @@ -2399,9 +2388,17 @@ static void ioc_timer_fn(struct timer_list *timer) * and should increase vtime rate. */ prev_busy_level = ioc->busy_level; - if (rq_wait_pct > RQ_WAIT_BUSY_PCT || - missed_ppm[READ] > ppm_rthr || - missed_ppm[WRITE] > ppm_wthr) { + if (!nr_done && nr_lagging) { + /* + * When there are lagging IOs but no completions, we don't + * know if the IO latency will meet the QoS targets. The + * disk might be saturated or not. We should not reset + * busy_level to 0 (which would prevent vrate from scaling + * up or down), but rather to keep it unchanged. + */ + } else if (rq_wait_pct > RQ_WAIT_BUSY_PCT || + missed_ppm[READ] > ppm_rthr || + missed_ppm[WRITE] > ppm_wthr) { /* clearly missing QoS targets, slow down vrate */ ioc->busy_level = max(ioc->busy_level, 0); ioc->busy_level++; @@ -2605,6 +2602,88 @@ static u64 calc_size_vtime_cost(struct request *rq, struct ioc *ioc) return cost; } +enum over_budget_action { + action_retry, + action_commit, + action_wait, + action_return, +}; + +static enum over_budget_action +iocg_handle_over_budget(struct rq_qos *rqos, struct ioc_gq *iocg, + struct bio *bio, struct ioc_now *now, + struct iocg_wait *wait, bool use_debt, bool ioc_locked, + u64 abs_cost, u64 cost) +{ + lockdep_assert_held(&iocg->waitq.lock); + + /* + * @iocg must stay activated for debt and waitq handling. Deactivation + * is synchronized against both ioc->lock and waitq.lock and we won't + * get deactivated as long as we're waiting or have debt, so we're good + * if we're activated here. In the unlikely cases that we aren't, just + * issue the IO. + */ + if (unlikely(list_empty(&iocg->active_list))) + return action_commit; + + /* + * We're over budget. If @bio has to be issued regardless, remember + * the abs_cost instead of advancing vtime. iocg_kick_waitq() will pay + * off the debt before waking more IOs. + * + * This way, the debt is continuously paid off each period with the + * actual budget available to the cgroup. If we just wound vtime, we + * would incorrectly use the current hw_inuse for the entire amount + * which, for example, can lead to the cgroup staying blocked for a + * long time even with substantially raised hw_inuse. + * + * An iocg with vdebt should stay online so that the timer can keep + * deducting its vdebt and [de]activate use_delay mechanism + * accordingly. We don't want to race against the timer trying to + * clear them and leave @iocg inactive w/ dangling use_delay heavily + * penalizing the cgroup and its descendants. + */ + if (use_debt) { + iocg_incur_debt(iocg, abs_cost, now); + if (iocg_kick_delay(iocg, now)) + blkcg_schedule_throttle(rqos->disk, + (bio->bi_opf & REQ_SWAP) == + REQ_SWAP); + return action_return; + } + + /* guarantee that iocgs w/ waiters have maximum inuse */ + if (!iocg->abs_vdebt && iocg->inuse != iocg->active) { + if (!ioc_locked) + return action_retry; + lockdep_assert_held(&iocg->ioc->lock); + propagate_weights(iocg, iocg->active, iocg->active, true, now); + } + + /* + * Append self to the waitq and schedule the wakeup timer if we're + * the first waiter. The timer duration is calculated based on the + * current vrate. vtime and hweight changes can make it too short + * or too long. Each wait entry records the absolute cost it's + * waiting for to allow re-evaluation using a custom wait entry. + * + * If too short, the timer simply reschedules itself. If too long, + * the period timer will notice and trigger wakeups. + * + * All waiters are on iocg->waitq and the wait states are + * synchronized using waitq.lock. + */ + init_wait_func(&wait->wait, iocg_wake_fn); + wait->bio = bio; + wait->abs_cost = abs_cost; + wait->committed = false; /* will be set true by waker */ + + __add_wait_queue_entry_tail(&iocg->waitq, &wait->wait); + iocg_kick_waitq(iocg, ioc_locked, now); + return action_wait; +} + static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio) { struct blkcg_gq *blkg = bio->bi_blkg; @@ -2614,6 +2693,7 @@ static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio) struct iocg_wait wait; u64 abs_cost, cost, vtime; bool use_debt, ioc_locked; + enum over_budget_action action; unsigned long flags; /* bypass IOs if disabled, still initializing, or for root cgroup */ @@ -2653,81 +2733,34 @@ static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio) use_debt = bio_issue_as_root_blkg(bio) || fatal_signal_pending(current); ioc_locked = use_debt || READ_ONCE(iocg->abs_vdebt); retry_lock: - iocg_lock(iocg, ioc_locked, &flags); - - /* - * @iocg must stay activated for debt and waitq handling. Deactivation - * is synchronized against both ioc->lock and waitq.lock and we won't - * get deactivated as long as we're waiting or has debt, so we're good - * if we're activated here. In the unlikely cases that we aren't, just - * issue the IO. - */ - if (unlikely(list_empty(&iocg->active_list))) { - iocg_unlock(iocg, ioc_locked, &flags); + if (ioc_locked) { + spin_lock_irqsave(&iocg->ioc->lock, flags); + spin_lock(&iocg->waitq.lock); + action = iocg_handle_over_budget(rqos, iocg, bio, &now, &wait, + use_debt, ioc_locked, abs_cost, + cost); + spin_unlock(&iocg->waitq.lock); + spin_unlock_irqrestore(&iocg->ioc->lock, flags); + } else { + spin_lock_irqsave(&iocg->waitq.lock, flags); + action = iocg_handle_over_budget(rqos, iocg, bio, &now, &wait, + use_debt, ioc_locked, abs_cost, + cost); + spin_unlock_irqrestore(&iocg->waitq.lock, flags); + } + switch (action) { + case action_retry: + ioc_locked = true; + goto retry_lock; + case action_commit: iocg_commit_bio(iocg, bio, abs_cost, cost); return; - } - - /* - * We're over budget. If @bio has to be issued regardless, remember - * the abs_cost instead of advancing vtime. iocg_kick_waitq() will pay - * off the debt before waking more IOs. - * - * This way, the debt is continuously paid off each period with the - * actual budget available to the cgroup. If we just wound vtime, we - * would incorrectly use the current hw_inuse for the entire amount - * which, for example, can lead to the cgroup staying blocked for a - * long time even with substantially raised hw_inuse. - * - * An iocg with vdebt should stay online so that the timer can keep - * deducting its vdebt and [de]activate use_delay mechanism - * accordingly. We don't want to race against the timer trying to - * clear them and leave @iocg inactive w/ dangling use_delay heavily - * penalizing the cgroup and its descendants. - */ - if (use_debt) { - iocg_incur_debt(iocg, abs_cost, &now); - if (iocg_kick_delay(iocg, &now)) - blkcg_schedule_throttle(rqos->disk, - (bio->bi_opf & REQ_SWAP) == REQ_SWAP); - iocg_unlock(iocg, ioc_locked, &flags); + case action_return: return; + case action_wait: + break; } - /* guarantee that iocgs w/ waiters have maximum inuse */ - if (!iocg->abs_vdebt && iocg->inuse != iocg->active) { - if (!ioc_locked) { - iocg_unlock(iocg, false, &flags); - ioc_locked = true; - goto retry_lock; - } - propagate_weights(iocg, iocg->active, iocg->active, true, - &now); - } - - /* - * Append self to the waitq and schedule the wakeup timer if we're - * the first waiter. The timer duration is calculated based on the - * current vrate. vtime and hweight changes can make it too short - * or too long. Each wait entry records the absolute cost it's - * waiting for to allow re-evaluation using a custom wait entry. - * - * If too short, the timer simply reschedules itself. If too long, - * the period timer will notice and trigger wakeups. - * - * All waiters are on iocg->waitq and the wait states are - * synchronized using waitq.lock. - */ - init_wait_func(&wait.wait, iocg_wake_fn); - wait.bio = bio; - wait.abs_cost = abs_cost; - wait.committed = false; /* will be set true by waker */ - - __add_wait_queue_entry_tail(&iocg->waitq, &wait.wait); - iocg_kick_waitq(iocg, ioc_locked, &now); - - iocg_unlock(iocg, ioc_locked, &flags); - while (true) { set_current_state(TASK_UNINTERRUPTIBLE); if (wait.committed) @@ -2884,7 +2917,7 @@ static int blk_iocost_init(struct gendisk *disk) struct ioc *ioc; int i, cpu, ret; - ioc = kzalloc(sizeof(*ioc), GFP_KERNEL); + ioc = kzalloc_obj(*ioc); if (!ioc) return -ENOMEM; @@ -2948,7 +2981,7 @@ static struct blkcg_policy_data *ioc_cpd_alloc(gfp_t gfp) { struct ioc_cgrp *iocc; - iocc = kzalloc(sizeof(struct ioc_cgrp), gfp); + iocc = kzalloc_obj(struct ioc_cgrp, gfp); if (!iocc) return NULL; @@ -3017,6 +3050,16 @@ static void ioc_pd_init(struct blkg_policy_data *pd) spin_unlock_irqrestore(&ioc->lock, flags); } +static void iocg_release(struct rcu_head *rcu) +{ + struct blkg_policy_data *pd = + container_of(rcu, struct blkg_policy_data, rcu_head); + struct ioc_gq *iocg = pd_to_iocg(pd); + + free_percpu(iocg->pcpu_stat); + kfree(iocg); +} + static void ioc_pd_free(struct blkg_policy_data *pd) { struct ioc_gq *iocg = pd_to_iocg(pd); @@ -3041,8 +3084,8 @@ static void ioc_pd_free(struct blkg_policy_data *pd) hrtimer_cancel(&iocg->waitq_timer); } - free_percpu(iocg->pcpu_stat); - kfree(iocg); + + call_rcu(&pd->rcu_head, iocg_release); } static void ioc_pd_stat(struct blkg_policy_data *pd, struct seq_file *s) @@ -3131,19 +3174,25 @@ static ssize_t ioc_weight_write(struct kernfs_open_file *of, char *buf, blkg_conf_init(&ctx, buf); + ret = blkg_conf_open_bdev(&ctx); + if (ret) + return ret; + ret = blkg_conf_prep(blkcg, &blkcg_policy_iocost, &ctx); if (ret) - goto err; + goto close_bdev; iocg = blkg_to_iocg(ctx.blkg); + ret = -EINVAL; + if (!strncmp(ctx.body, "default", 7)) { v = 0; } else { if (!sscanf(ctx.body, "%u", &v)) - goto einval; + goto unprep; if (v < CGROUP_WEIGHT_MIN || v > CGROUP_WEIGHT_MAX) - goto einval; + goto unprep; } spin_lock(&iocg->ioc->lock); @@ -3152,14 +3201,15 @@ static ssize_t ioc_weight_write(struct kernfs_open_file *of, char *buf, weight_updated(iocg, &now); spin_unlock(&iocg->ioc->lock); - blkg_conf_exit(&ctx); - return nbytes; + ret = 0; -einval: - ret = -EINVAL; -err: - blkg_conf_exit(&ctx); - return ret; +unprep: + blkg_conf_unprep(&ctx); + +close_bdev: + blkg_conf_close_bdev(&ctx); + + return ret ?: nbytes; } static u64 ioc_qos_prfill(struct seq_file *sf, struct blkg_policy_data *pd, @@ -3171,7 +3221,7 @@ static u64 ioc_qos_prfill(struct seq_file *sf, struct blkg_policy_data *pd, if (!dname) return 0; - spin_lock(&ioc->lock); + spin_lock_irq(&ioc->lock); seq_printf(sf, "%s enable=%d ctrl=%s rpct=%u.%02u rlat=%u wpct=%u.%02u wlat=%u min=%u.%02u max=%u.%02u\n", dname, ioc->enabled, ioc->user_qos_params ? "user" : "auto", ioc->params.qos[QOS_RPPM] / 10000, @@ -3184,7 +3234,7 @@ static u64 ioc_qos_prfill(struct seq_file *sf, struct blkg_policy_data *pd, ioc->params.qos[QOS_MIN] % 10000 / 100, ioc->params.qos[QOS_MAX] / 10000, ioc->params.qos[QOS_MAX] % 10000 / 100); - spin_unlock(&ioc->lock); + spin_unlock_irq(&ioc->lock); return 0; } @@ -3217,34 +3267,43 @@ static ssize_t ioc_qos_write(struct kernfs_open_file *of, char *input, size_t nbytes, loff_t off) { struct blkg_conf_ctx ctx; + struct request_queue *q; struct gendisk *disk; struct ioc *ioc; u32 qos[NR_QOS_PARAMS]; bool enable, user; char *body, *p; - unsigned long memflags; + unsigned int memflags; int ret; blkg_conf_init(&ctx, input); - memflags = blkg_conf_open_bdev_frozen(&ctx); - if (IS_ERR_VALUE(memflags)) { - ret = memflags; - goto err; - } + ret = blkg_conf_open_bdev(&ctx); + if (ret) + return ret; + /* + * At this point, we haven’t started protecting anything related to QoS, + * so we release q->rq_qos_mutex here, which was first acquired in blkg_ + * conf_open_bdev. Later, we re-acquire q->rq_qos_mutex after freezing + * the queue to maintain the correct locking order. + */ + mutex_unlock(&ctx.bdev->bd_queue->rq_qos_mutex); + + memflags = blk_mq_freeze_queue(ctx.bdev->bd_queue); + mutex_lock(&ctx.bdev->bd_queue->rq_qos_mutex); body = ctx.body; disk = ctx.bdev->bd_disk; if (!queue_is_mq(disk->queue)) { ret = -EOPNOTSUPP; - goto err; + goto close_bdev; } ioc = q_to_ioc(disk->queue); if (!ioc) { ret = blk_iocost_init(disk); if (ret) - goto err; + goto close_bdev; ioc = q_to_ioc(disk->queue); } @@ -3348,15 +3407,17 @@ static ssize_t ioc_qos_write(struct kernfs_open_file *of, char *input, blk_mq_unquiesce_queue(disk->queue); - blkg_conf_exit_frozen(&ctx, memflags); - return nbytes; +close_bdev: + q = ctx.bdev->bd_queue; + blkg_conf_close_bdev(&ctx); + blk_mq_unfreeze_queue(q, memflags); + return ret ?: nbytes; + einval: spin_unlock_irq(&ioc->lock); blk_mq_unquiesce_queue(disk->queue); ret = -EINVAL; -err: - blkg_conf_exit_frozen(&ctx, memflags); - return ret; + goto close_bdev; } static u64 ioc_cost_model_prfill(struct seq_file *sf, @@ -3369,14 +3430,14 @@ static u64 ioc_cost_model_prfill(struct seq_file *sf, if (!dname) return 0; - spin_lock(&ioc->lock); + spin_lock_irq(&ioc->lock); seq_printf(sf, "%s ctrl=%s model=linear " "rbps=%llu rseqiops=%llu rrandiops=%llu " "wbps=%llu wseqiops=%llu wrandiops=%llu\n", dname, ioc->user_cost_model ? "user" : "auto", u[I_LCOEF_RBPS], u[I_LCOEF_RSEQIOPS], u[I_LCOEF_RRANDIOPS], u[I_LCOEF_WBPS], u[I_LCOEF_WSEQIOPS], u[I_LCOEF_WRANDIOPS]); - spin_unlock(&ioc->lock); + spin_unlock_irq(&ioc->lock); return 0; } @@ -3421,20 +3482,20 @@ static ssize_t ioc_cost_model_write(struct kernfs_open_file *of, char *input, ret = blkg_conf_open_bdev(&ctx); if (ret) - goto err; + return ret; body = ctx.body; q = bdev_get_queue(ctx.bdev); if (!queue_is_mq(q)) { ret = -EOPNOTSUPP; - goto err; + goto close_bdev; } ioc = q_to_ioc(q); if (!ioc) { ret = blk_iocost_init(ctx.bdev->bd_disk); if (ret) - goto err; + goto close_bdev; ioc = q_to_ioc(q); } @@ -3445,6 +3506,8 @@ static ssize_t ioc_cost_model_write(struct kernfs_open_file *of, char *input, memcpy(u, ioc->params.i_lcoefs, sizeof(u)); user = ioc->user_cost_model; + ret = -EINVAL; + while ((p = strsep(&body, " \t\n"))) { substring_t args[MAX_OPT_ARGS]; char buf[32]; @@ -3462,20 +3525,20 @@ static ssize_t ioc_cost_model_write(struct kernfs_open_file *of, char *input, else if (!strcmp(buf, "user")) user = true; else - goto einval; + goto unlock; continue; case COST_MODEL: match_strlcpy(buf, &args[0], sizeof(buf)); if (strcmp(buf, "linear")) - goto einval; + goto unlock; continue; } tok = match_token(p, i_lcoef_tokens, args); if (tok == NR_I_LCOEFS) - goto einval; + goto unlock; if (match_u64(&args[0], &v)) - goto einval; + goto unlock; u[tok] = v; user = true; } @@ -3487,24 +3550,18 @@ static ssize_t ioc_cost_model_write(struct kernfs_open_file *of, char *input, ioc->user_cost_model = false; } ioc_refresh_params(ioc, true); - spin_unlock_irq(&ioc->lock); - blk_mq_unquiesce_queue(q); - blk_mq_unfreeze_queue(q, memflags); - - blkg_conf_exit(&ctx); - return nbytes; + ret = 0; -einval: +unlock: spin_unlock_irq(&ioc->lock); blk_mq_unquiesce_queue(q); blk_mq_unfreeze_queue(q, memflags); - ret = -EINVAL; -err: - blkg_conf_exit(&ctx); - return ret; +close_bdev: + blkg_conf_close_bdev(&ctx); + return ret ?: nbytes; } static struct cftype ioc_files[] = { diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index 42c1e0b9a68f..cef02b6c5fa9 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -485,19 +485,11 @@ static void blkcg_iolatency_throttle(struct rq_qos *rqos, struct bio *bio) mod_timer(&blkiolat->timer, jiffies + HZ); } -static void iolatency_record_time(struct iolatency_grp *iolat, - struct bio_issue *issue, u64 now, - bool issue_as_root) +static void iolatency_record_time(struct iolatency_grp *iolat, u64 start, + u64 now, bool issue_as_root) { - u64 start = bio_issue_time(issue); u64 req_time; - /* - * Have to do this so we are truncated to the correct time that our - * issue is truncated to. - */ - now = __bio_issue_time(now); - if (now <= start) return; @@ -625,7 +617,7 @@ static void blkcg_iolatency_done_bio(struct rq_qos *rqos, struct bio *bio) * submitted, so do not account for it. */ if (iolat->min_lat_nsec && bio->bi_status != BLK_STS_AGAIN) { - iolatency_record_time(iolat, &bio->bi_issue, now, + iolatency_record_time(iolat, bio->issue_time_ns, now, issue_as_root); window_start = atomic64_read(&iolat->window_start); if (now > window_start && @@ -658,7 +650,8 @@ static const struct rq_qos_ops blkcg_iolatency_ops = { static void blkiolatency_timer_fn(struct timer_list *t) { - struct blk_iolatency *blkiolat = from_timer(blkiolat, t, timer); + struct blk_iolatency *blkiolat = timer_container_of(blkiolat, t, + timer); struct blkcg_gq *blkg; struct cgroup_subsys_state *pos_css; u64 now = blk_time_get_ns(); @@ -749,10 +742,15 @@ static void blkiolatency_enable_work_fn(struct work_struct *work) */ enabled = atomic_read(&blkiolat->enable_cnt); if (enabled != blkiolat->enabled) { + struct request_queue *q = blkiolat->rqos.disk->queue; unsigned int memflags; memflags = blk_mq_freeze_queue(blkiolat->rqos.disk->queue); blkiolat->enabled = enabled; + if (enabled) + blk_queue_flag_set(QUEUE_FLAG_BIO_ISSUE_TIME, q); + else + blk_queue_flag_clear(QUEUE_FLAG_BIO_ISSUE_TIME, q); blk_mq_unfreeze_queue(blkiolat->rqos.disk->queue, memflags); } } @@ -762,7 +760,7 @@ static int blk_iolatency_init(struct gendisk *disk) struct blk_iolatency *blkiolat; int ret; - blkiolat = kzalloc(sizeof(*blkiolat), GFP_KERNEL); + blkiolat = kzalloc_obj(*blkiolat); if (!blkiolat) return -ENOMEM; @@ -842,7 +840,7 @@ static ssize_t iolatency_set_limit(struct kernfs_open_file *of, char *buf, ret = blkg_conf_open_bdev(&ctx); if (ret) - goto out; + return ret; /* * blk_iolatency_init() may fail after rq_qos_add() succeeds which can @@ -852,11 +850,11 @@ static ssize_t iolatency_set_limit(struct kernfs_open_file *of, char *buf, if (!iolat_rq_qos(ctx.bdev->bd_queue)) ret = blk_iolatency_init(ctx.bdev->bd_disk); if (ret) - goto out; + goto close_bdev; ret = blkg_conf_prep(blkcg, &blkcg_policy_iolatency, &ctx); if (ret) - goto out; + goto close_bdev; iolat = blkg_to_lat(ctx.blkg); p = ctx.body; @@ -867,7 +865,7 @@ static ssize_t iolatency_set_limit(struct kernfs_open_file *of, char *buf, char val[21]; /* 18446744073709551616 */ if (sscanf(tok, "%15[^=]=%20s", key, val) != 2) - goto out; + goto unprep; if (!strcmp(key, "target")) { u64 v; @@ -877,9 +875,9 @@ static ssize_t iolatency_set_limit(struct kernfs_open_file *of, char *buf, else if (sscanf(val, "%llu", &v) == 1) lat_val = v * NSEC_PER_USEC; else - goto out; + goto unprep; } else { - goto out; + goto unprep; } } @@ -891,8 +889,11 @@ static ssize_t iolatency_set_limit(struct kernfs_open_file *of, char *buf, if (oldval != iolat->min_lat_nsec) iolatency_clear_scaling(blkg); ret = 0; -out: - blkg_conf_exit(&ctx); + +unprep: + blkg_conf_unprep(&ctx); +close_bdev: + blkg_conf_close_bdev(&ctx); return ret ?: nbytes; } @@ -990,10 +991,7 @@ static void iolatency_pd_init(struct blkg_policy_data *pd) u64 now = blk_time_get_ns(); int cpu; - if (blk_queue_nonrot(blkg->q)) - iolat->ssd = true; - else - iolat->ssd = false; + iolat->ssd = !blk_queue_rot(blkg->q); for_each_possible_cpu(cpu) { struct latency_stat *stat; @@ -1033,13 +1031,21 @@ static void iolatency_pd_offline(struct blkg_policy_data *pd) iolatency_clear_scaling(blkg); } -static void iolatency_pd_free(struct blkg_policy_data *pd) +static void iolat_release(struct rcu_head *rcu) { + struct blkg_policy_data *pd = + container_of(rcu, struct blkg_policy_data, rcu_head); struct iolatency_grp *iolat = pd_to_lat(pd); + free_percpu(iolat->stats); kfree(iolat); } +static void iolatency_pd_free(struct blkg_policy_data *pd) +{ + call_rcu(&pd->rcu_head, iolat_release); +} + static struct cftype iolatency_files[] = { { .name = "latency", diff --git a/block/blk-ioprio.c b/block/blk-ioprio.c index 13659dc15c3f..8fa8bca35062 100644 --- a/block/blk-ioprio.c +++ b/block/blk-ioprio.c @@ -99,7 +99,7 @@ static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp) { struct ioprio_blkcg *blkcg; - blkcg = kzalloc(sizeof(*blkcg), gfp); + blkcg = kzalloc_obj(*blkcg, gfp); if (!blkcg) return NULL; blkcg->prio_policy = POLICY_NO_CHANGE; diff --git a/block/blk-lib.c b/block/blk-lib.c index 4c9f20a689f7..688bc67cbf73 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -32,7 +32,7 @@ static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector) * Align the bio size to the discard granularity to make splitting the bio * at discard granularity boundaries easier in the driver if needed. */ - return round_down(UINT_MAX, discard_granularity) >> SECTOR_SHIFT; + return round_down(BIO_MAX_SIZE, discard_granularity) >> SECTOR_SHIFT; } struct bio *blk_alloc_discard_bio(struct block_device *bdev, @@ -60,7 +60,7 @@ struct bio *blk_alloc_discard_bio(struct block_device *bdev, return bio; } -int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, +void __blkdev_issue_discard(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct bio **biop) { struct bio *bio; @@ -68,7 +68,6 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, while ((bio = blk_alloc_discard_bio(bdev, §or, &nr_sects, gfp_mask))) *biop = bio_chain_and_submit(*biop, bio); - return 0; } EXPORT_SYMBOL(__blkdev_issue_discard); @@ -87,11 +86,11 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, { struct bio *bio = NULL; struct blk_plug plug; - int ret; + int ret = 0; blk_start_plug(&plug); - ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio); - if (!ret && bio) { + __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio); + if (bio) { ret = submit_bio_wait(bio); if (ret == -EOPNOTSUPP) ret = 0; @@ -107,8 +106,7 @@ static sector_t bio_write_zeroes_limit(struct block_device *bdev) { sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; - return min(bdev_write_zeroes_sectors(bdev), - (UINT_MAX >> SECTOR_SHIFT) & ~bs_mask); + return min(bdev_write_zeroes_sectors(bdev), BIO_MAX_SECTORS & ~bs_mask); } /* @@ -157,13 +155,7 @@ static int blkdev_issue_write_zeroes(struct block_device *bdev, sector_t sector, __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp, &bio, flags, limit); if (bio) { - if ((flags & BLKDEV_ZERO_KILLABLE) && - fatal_signal_pending(current)) { - bio_await_chain(bio); - blk_finish_plug(&plug); - return -EINTR; - } - ret = submit_bio_wait(bio); + ret = bio_submit_or_kill(bio, flags); bio_put(bio); } blk_finish_plug(&plug); @@ -196,27 +188,28 @@ static void __blkdev_issue_zero_pages(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, unsigned int flags) { + struct folio *zero_folio = largest_zero_folio(); + while (nr_sects) { unsigned int nr_vecs = __blkdev_sectors_to_bio_pages(nr_sects); struct bio *bio; - bio = bio_alloc(bdev, nr_vecs, REQ_OP_WRITE, gfp_mask); - bio->bi_iter.bi_sector = sector; - if ((flags & BLKDEV_ZERO_KILLABLE) && fatal_signal_pending(current)) break; + bio = bio_alloc(bdev, nr_vecs, REQ_OP_WRITE, gfp_mask); + bio->bi_iter.bi_sector = sector; + do { - unsigned int len, added; + unsigned int len; - len = min_t(sector_t, - PAGE_SIZE, nr_sects << SECTOR_SHIFT); - added = bio_add_page(bio, ZERO_PAGE(0), len, 0); - if (added < len) + len = min_t(sector_t, folio_size(zero_folio), + nr_sects << SECTOR_SHIFT); + if (!bio_add_folio(bio, zero_folio, len, 0)) break; - nr_sects -= added >> SECTOR_SHIFT; - sector += added >> SECTOR_SHIFT; + nr_sects -= len >> SECTOR_SHIFT; + sector += len >> SECTOR_SHIFT; } while (nr_sects); *biop = bio_chain_and_submit(*biop, bio); @@ -237,13 +230,7 @@ static int blkdev_issue_zero_pages(struct block_device *bdev, sector_t sector, blk_start_plug(&plug); __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp, &bio, flags); if (bio) { - if ((flags & BLKDEV_ZERO_KILLABLE) && - fatal_signal_pending(current)) { - bio_await_chain(bio); - blk_finish_plug(&plug); - return -EINTR; - } - ret = submit_bio_wait(bio); + ret = bio_submit_or_kill(bio, flags); bio_put(bio); } blk_finish_plug(&plug); @@ -336,8 +323,8 @@ int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, int ret = 0; /* make sure that "len << SECTOR_SHIFT" doesn't overflow */ - if (max_sectors > UINT_MAX >> SECTOR_SHIFT) - max_sectors = UINT_MAX >> SECTOR_SHIFT; + if (max_sectors > BIO_MAX_SECTORS) + max_sectors = BIO_MAX_SECTORS; max_sectors &= ~bs_mask; if (max_sectors == 0) diff --git a/block/blk-map.c b/block/blk-map.c index d2f22744b3d1..d1d6bbe0ecf1 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -26,7 +26,7 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, if (data->nr_segs > UIO_MAXIOV) return NULL; - bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); + bmd = kmalloc_flex(*bmd, iov, data->nr_segs, gfp_mask); if (!bmd) return NULL; bmd->iter = *data; @@ -37,6 +37,25 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, return bmd; } +static inline void blk_mq_map_bio_put(struct bio *bio) +{ + bio_put(bio); +} + +static struct bio *blk_rq_map_bio_alloc(struct request *rq, + unsigned int nr_vecs, gfp_t gfp_mask) +{ + struct block_device *bdev = rq->q->disk ? rq->q->disk->part0 : NULL; + struct bio *bio; + + bio = bio_alloc_bioset(bdev, nr_vecs, rq->cmd_flags, gfp_mask, + &fs_bio_set); + if (!bio) + return NULL; + + return bio; +} + /** * bio_copy_from_iter - copy all pages from iov_iter to bio * @bio: The &struct bio which describes the I/O as destination @@ -154,10 +173,9 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); ret = -ENOMEM; - bio = bio_kmalloc(nr_pages, gfp_mask); + bio = blk_rq_map_bio_alloc(rq, nr_pages, gfp_mask); if (!bio) goto out_bmd; - bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, req_op(rq)); if (map_data) { nr_pages = 1U << map_data->page_order; @@ -233,42 +251,12 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, cleanup: if (!map_data) bio_free_pages(bio); - bio_uninit(bio); - kfree(bio); + blk_mq_map_bio_put(bio); out_bmd: kfree(bmd); return ret; } -static void blk_mq_map_bio_put(struct bio *bio) -{ - if (bio->bi_opf & REQ_ALLOC_CACHE) { - bio_put(bio); - } else { - bio_uninit(bio); - kfree(bio); - } -} - -static struct bio *blk_rq_map_bio_alloc(struct request *rq, - unsigned int nr_vecs, gfp_t gfp_mask) -{ - struct bio *bio; - - if (rq->cmd_flags & REQ_ALLOC_CACHE && (nr_vecs <= BIO_INLINE_VECS)) { - bio = bio_alloc_bioset(NULL, nr_vecs, rq->cmd_flags, gfp_mask, - &fs_bio_set); - if (!bio) - return NULL; - } else { - bio = bio_kmalloc(nr_vecs, gfp_mask); - if (!bio) - return NULL; - bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); - } - return bio; -} - static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, gfp_t gfp_mask) { @@ -282,7 +270,11 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask); if (!bio) return -ENOMEM; - ret = bio_iov_iter_get_pages(bio, iter); + /* + * No alignment requirements on our part to support arbitrary + * passthrough commands. + */ + ret = bio_iov_iter_get_pages(bio, iter, 0); if (ret) goto out_put; ret = blk_rq_append_bio(rq, bio); @@ -313,68 +305,28 @@ static void bio_invalidate_vmalloc_pages(struct bio *bio) static void bio_map_kern_endio(struct bio *bio) { bio_invalidate_vmalloc_pages(bio); - bio_uninit(bio); - kfree(bio); + blk_mq_map_bio_put(bio); } -/** - * bio_map_kern - map kernel address into bio - * @q: the struct request_queue for the bio - * @data: pointer to buffer to map - * @len: length in bytes - * @gfp_mask: allocation flags for bio allocation - * - * Map the kernel address into a bio suitable for io to a block - * device. Returns an error pointer in case of error. - */ -static struct bio *bio_map_kern(struct request_queue *q, void *data, - unsigned int len, gfp_t gfp_mask) +static struct bio *bio_map_kern(struct request *rq, void *data, unsigned int len, + gfp_t gfp_mask) { - unsigned long kaddr = (unsigned long)data; - unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; - unsigned long start = kaddr >> PAGE_SHIFT; - const int nr_pages = end - start; - bool is_vmalloc = is_vmalloc_addr(data); - struct page *page; - int offset, i; + unsigned int nr_vecs = bio_add_max_vecs(data, len); struct bio *bio; - bio = bio_kmalloc(nr_pages, gfp_mask); + bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask); if (!bio) return ERR_PTR(-ENOMEM); - bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); - if (is_vmalloc) { - flush_kernel_vmap_range(data, len); + if (is_vmalloc_addr(data)) { bio->bi_private = data; - } - - offset = offset_in_page(kaddr); - for (i = 0; i < nr_pages; i++) { - unsigned int bytes = PAGE_SIZE - offset; - - if (len <= 0) - break; - - if (bytes > len) - bytes = len; - - if (!is_vmalloc) - page = virt_to_page(data); - else - page = vmalloc_to_page(data); - if (bio_add_page(bio, page, bytes, offset) < bytes) { - /* we don't support partial mappings */ - bio_uninit(bio); - kfree(bio); + if (!bio_add_vmalloc(bio, data, len)) { + blk_mq_map_bio_put(bio); return ERR_PTR(-EINVAL); } - - data += bytes; - len -= bytes; - offset = 0; + } else { + bio_add_virt_nofail(bio, data, len); } - bio->bi_end_io = bio_map_kern_endio; return bio; } @@ -382,8 +334,7 @@ static struct bio *bio_map_kern(struct request_queue *q, void *data, static void bio_copy_kern_endio(struct bio *bio) { bio_free_pages(bio); - bio_uninit(bio); - kfree(bio); + blk_mq_map_bio_put(bio); } static void bio_copy_kern_endio_read(struct bio *bio) @@ -402,18 +353,19 @@ static void bio_copy_kern_endio_read(struct bio *bio) /** * bio_copy_kern - copy kernel address into bio - * @q: the struct request_queue for the bio + * @rq: request to fill * @data: pointer to buffer to copy * @len: length in bytes + * @op: bio/request operation * @gfp_mask: allocation flags for bio and page allocation - * @reading: data direction is READ * * copy the kernel address into a bio suitable for io to a block * device. Returns an error pointer in case of error. */ -static struct bio *bio_copy_kern(struct request_queue *q, void *data, - unsigned int len, gfp_t gfp_mask, int reading) +static struct bio *bio_copy_kern(struct request *rq, void *data, unsigned int len, + gfp_t gfp_mask) { + enum req_op op = req_op(rq); unsigned long kaddr = (unsigned long)data; unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; unsigned long start = kaddr >> PAGE_SHIFT; @@ -428,10 +380,9 @@ static struct bio *bio_copy_kern(struct request_queue *q, void *data, return ERR_PTR(-EINVAL); nr_pages = end - start; - bio = bio_kmalloc(nr_pages, gfp_mask); + bio = blk_rq_map_bio_alloc(rq, nr_pages, gfp_mask); if (!bio) return ERR_PTR(-ENOMEM); - bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); while (len) { struct page *page; @@ -444,29 +395,27 @@ static struct bio *bio_copy_kern(struct request_queue *q, void *data, if (!page) goto cleanup; - if (!reading) + if (op_is_write(op)) memcpy(page_address(page), p, bytes); - if (bio_add_page(bio, page, bytes, 0) < bytes) - break; + __bio_add_page(bio, page, bytes, 0); len -= bytes; p += bytes; } - if (reading) { + if (op_is_write(op)) { + bio->bi_end_io = bio_copy_kern_endio; + } else { bio->bi_end_io = bio_copy_kern_endio_read; bio->bi_private = data; - } else { - bio->bi_end_io = bio_copy_kern_endio; } return bio; cleanup: bio_free_pages(bio); - bio_uninit(bio); - kfree(bio); + blk_mq_map_bio_put(bio); return ERR_PTR(-ENOMEM); } @@ -482,7 +431,7 @@ int blk_rq_append_bio(struct request *rq, struct bio *bio) int ret; /* check that the data layout matches the hardware restrictions */ - ret = bio_split_rw_at(bio, lim, &nr_segs, max_bytes); + ret = bio_split_io_at(bio, lim, &nr_segs, max_bytes, 0); if (ret) { /* if we would have to split the bio, copy instead */ if (ret > 0) @@ -493,6 +442,8 @@ int blk_rq_append_bio(struct request *rq, struct bio *bio) if (rq->bio) { if (!ll_back_merge_fn(rq, bio, nr_segs)) return -EINVAL; + rq->phys_gap_bit = bio_seg_gap(rq->q, rq->biotail, bio, + rq->phys_gap_bit); rq->biotail->bi_next = bio; rq->biotail = bio; rq->__data_len += bio->bi_iter.bi_size; @@ -503,6 +454,7 @@ int blk_rq_append_bio(struct request *rq, struct bio *bio) rq->nr_phys_segments = nr_segs; rq->bio = rq->biotail = bio; rq->__data_len = bio->bi_iter.bi_size; + rq->phys_gap_bit = bio->bi_bvec_gap_bit; return 0; } EXPORT_SYMBOL(blk_rq_append_bio); @@ -556,8 +508,6 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, if (map_data) copy = true; - else if (blk_queue_may_bounce(q)) - copy = true; else if (iov_iter_alignment(iter) & align) copy = true; else if (iov_iter_is_bvec(iter)) @@ -689,7 +639,6 @@ EXPORT_SYMBOL(blk_rq_unmap_user); /** * blk_rq_map_kern - map kernel data to a request, for passthrough requests - * @q: request queue where request should be inserted * @rq: request to fill * @kbuf: the kernel buffer * @len: length of user data @@ -700,35 +649,33 @@ EXPORT_SYMBOL(blk_rq_unmap_user); * buffer is used. Can be called multiple times to append multiple * buffers. */ -int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, - unsigned int len, gfp_t gfp_mask) +int blk_rq_map_kern(struct request *rq, void *kbuf, unsigned int len, + gfp_t gfp_mask) { - int reading = rq_data_dir(rq) == READ; unsigned long addr = (unsigned long) kbuf; + bool do_copy; struct bio *bio; int ret; - if (len > (queue_max_hw_sectors(q) << 9)) + if (len > (queue_max_hw_sectors(rq->q) << SECTOR_SHIFT)) return -EINVAL; if (!len || !kbuf) return -EINVAL; - if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) || - blk_queue_may_bounce(q)) - bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); + do_copy = !blk_rq_aligned(rq->q, addr, len) || object_is_on_stack(kbuf); + if (do_copy) + bio = bio_copy_kern(rq, kbuf, len, gfp_mask); else - bio = bio_map_kern(q, kbuf, len, gfp_mask); + bio = bio_map_kern(rq, kbuf, len, gfp_mask); if (IS_ERR(bio)) return PTR_ERR(bio); - bio->bi_opf &= ~REQ_OP_MASK; - bio->bi_opf |= req_op(rq); - ret = blk_rq_append_bio(rq, bio); if (unlikely(ret)) { - bio_uninit(bio); - kfree(bio); + if (do_copy) + bio_free_pages(bio); + blk_mq_map_bio_put(bio); } return ret; } diff --git a/block/blk-merge.c b/block/blk-merge.c index fdd4efb54c6c..ab1161ca69f1 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -7,7 +7,6 @@ #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/blk-integrity.h> -#include <linux/scatterlist.h> #include <linux/part_stat.h> #include <linux/blk-cgroup.h> @@ -96,47 +95,70 @@ static inline bool req_gap_front_merge(struct request *req, struct bio *bio) } /* - * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size - * is defined as 'unsigned int', meantime it has to be aligned to with the + * The maximum size that a bio can fit has to be aligned down to the * logical block size, which is the minimum accepted unit by hardware. */ static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim) { - return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT; + return round_down(BIO_MAX_SIZE, lim->logical_block_size) >> + SECTOR_SHIFT; } +/* + * bio_submit_split_bioset - Submit a bio, splitting it at a designated sector + * @bio: the original bio to be submitted and split + * @split_sectors: the sector count at which to split + * @bs: the bio set used for allocating the new split bio + * + * The original bio is modified to contain the remaining sectors and submitted. + * The caller is responsible for submitting the returned bio. + * + * If succeed, the newly allocated bio representing the initial part will be + * returned, on failure NULL will be returned and original bio will fail. + */ +struct bio *bio_submit_split_bioset(struct bio *bio, unsigned int split_sectors, + struct bio_set *bs) +{ + struct bio *split = bio_split(bio, split_sectors, GFP_NOIO, bs); + + if (IS_ERR(split)) { + bio_endio_status(bio, errno_to_blk_status(PTR_ERR(split))); + return NULL; + } + + bio_chain(split, bio); + trace_block_split(split, bio->bi_iter.bi_sector); + WARN_ON_ONCE(bio_zone_write_plugging(bio)); + + if (should_fail_bio(bio)) + bio_io_error(bio); + else if (!blk_throtl_bio(bio)) + submit_bio_noacct_nocheck(bio, true); + + return split; +} +EXPORT_SYMBOL_GPL(bio_submit_split_bioset); + static struct bio *bio_submit_split(struct bio *bio, int split_sectors) { - if (unlikely(split_sectors < 0)) - goto error; + if (unlikely(split_sectors < 0)) { + bio_endio_status(bio, errno_to_blk_status(split_sectors)); + return NULL; + } if (split_sectors) { - struct bio *split; - - split = bio_split(bio, split_sectors, GFP_NOIO, + bio = bio_submit_split_bioset(bio, split_sectors, &bio->bi_bdev->bd_disk->bio_split); - if (IS_ERR(split)) { - split_sectors = PTR_ERR(split); - goto error; - } - split->bi_opf |= REQ_NOMERGE; - blkcg_bio_issue_init(split); - bio_chain(split, bio); - trace_block_split(split, bio->bi_iter.bi_sector); - WARN_ON_ONCE(bio_zone_write_plugging(bio)); - submit_bio_noacct(bio); - return split; + if (bio) + bio->bi_opf |= REQ_NOMERGE; } return bio; -error: - bio->bi_status = errno_to_blk_status(split_sectors); - bio_endio(bio); - return NULL; } -struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, - unsigned *nsegs) +static struct bio *__bio_split_discard(struct bio *bio, + const struct queue_limits *lim, unsigned *nsegs, + unsigned int max_sectors) { unsigned int max_discard_sectors, granularity; sector_t tmp; @@ -146,8 +168,7 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, granularity = max(lim->discard_granularity >> 9, 1U); - max_discard_sectors = - min(lim->max_discard_sectors, bio_allowed_max_sectors(lim)); + max_discard_sectors = min(max_sectors, bio_allowed_max_sectors(lim)); max_discard_sectors -= max_discard_sectors % granularity; if (unlikely(!max_discard_sectors)) return bio; @@ -171,6 +192,19 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, return bio_submit_split(bio, split_sectors); } +struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, + unsigned *nsegs) +{ + unsigned int max_sectors; + + if (bio_op(bio) == REQ_OP_SECURE_ERASE) + max_sectors = lim->max_secure_erase_sectors; + else + max_sectors = lim->max_discard_sectors; + + return __bio_split_discard(bio, lim, nsegs, max_sectors); +} + static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim, bool is_atomic) { @@ -226,27 +260,6 @@ static inline unsigned get_max_io_size(struct bio *bio, } /** - * get_max_segment_size() - maximum number of bytes to add as a single segment - * @lim: Request queue limits. - * @paddr: address of the range to add - * @len: maximum length available to add at @paddr - * - * Returns the maximum number of bytes of the range starting at @paddr that can - * be added to a single segment. - */ -static inline unsigned get_max_segment_size(const struct queue_limits *lim, - phys_addr_t paddr, unsigned int len) -{ - /* - * Prevent an overflow if mask = ULONG_MAX and offset = 0 by adding 1 - * after having calculated the minimum. - */ - return min_t(unsigned long, len, - min(lim->seg_boundary_mask - (lim->seg_boundary_mask & paddr), - (unsigned long)lim->max_segment_size - 1) + 1); -} - -/** * bvec_split_segs - verify whether or not a bvec should be split in the middle * @lim: [in] queue limits to split based on * @bv: [in] bvec to examine @@ -300,36 +313,57 @@ static unsigned int bio_split_alignment(struct bio *bio, return lim->logical_block_size; } +static inline unsigned int bvec_seg_gap(struct bio_vec *bvprv, + struct bio_vec *bv) +{ + return bv->bv_offset | (bvprv->bv_offset + bvprv->bv_len); +} + /** - * bio_split_rw_at - check if and where to split a read/write bio + * bio_split_io_at - check if and where to split a bio * @bio: [in] bio to be split * @lim: [in] queue limits to split based on * @segs: [out] number of segments in the bio with the first half of the sectors * @max_bytes: [in] maximum number of bytes per bio + * @len_align_mask: [in] length alignment mask for each vector * * Find out if @bio needs to be split to fit the queue limits in @lim and a * maximum size of @max_bytes. Returns a negative error number if @bio can't be * split, 0 if the bio doesn't have to be split, or a positive sector offset if * @bio needs to be split. */ -int bio_split_rw_at(struct bio *bio, const struct queue_limits *lim, - unsigned *segs, unsigned max_bytes) +int bio_split_io_at(struct bio *bio, const struct queue_limits *lim, + unsigned *segs, unsigned max_bytes, unsigned len_align_mask) { + struct bio_crypt_ctx *bc = bio_crypt_ctx(bio); struct bio_vec bv, bvprv, *bvprvp = NULL; + unsigned nsegs = 0, bytes = 0, gaps = 0; struct bvec_iter iter; - unsigned nsegs = 0, bytes = 0; + unsigned start_align_mask = lim->dma_alignment; + + if (bc) { + start_align_mask |= (bc->bc_key->crypto_cfg.data_unit_size - 1); + len_align_mask |= (bc->bc_key->crypto_cfg.data_unit_size - 1); + } bio_for_each_bvec(bv, bio, iter) { + if (bv.bv_offset & start_align_mask || + bv.bv_len & len_align_mask) + return -EINVAL; + /* * If the queue doesn't support SG gaps and adding this * offset would create a gap, disallow it. */ - if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset)) - goto split; + if (bvprvp) { + if (bvec_gap_to_prev(lim, bvprvp, bv.bv_offset)) + goto split; + gaps |= bvec_seg_gap(bvprvp, &bv); + } if (nsegs < lim->max_segments && bytes + bv.bv_len <= max_bytes && - bv.bv_offset + bv.bv_len <= lim->min_segment_size) { + bv.bv_offset + bv.bv_len <= lim->max_fast_segment_size) { nsegs++; bytes += bv.bv_len; } else { @@ -343,6 +377,7 @@ int bio_split_rw_at(struct bio *bio, const struct queue_limits *lim, } *segs = nsegs; + bio->bi_bvec_gap_bit = ffs(gaps); return 0; split: if (bio->bi_opf & REQ_ATOMIC) @@ -361,8 +396,16 @@ split: * Individual bvecs might not be logical block aligned. Round down the * split size so that each bio is properly block size aligned, even if * we do not use the full hardware limits. + * + * It is possible to submit a bio that can't be split into a valid io: + * there may either be too many discontiguous vectors for the max + * segments limit, or contain virtual boundary gaps without having a + * valid block sized split. A zero byte result means one of those + * conditions occured. */ bytes = ALIGN_DOWN(bytes, bio_split_alignment(bio, lim)); + if (!bytes) + return -EINVAL; /* * Bio splitting may cause subtle trouble such as hang when doing sync @@ -370,9 +413,10 @@ split: * big IO can be trival, disable iopoll when split needed. */ bio_clear_polled(bio); + bio->bi_bvec_gap_bit = ffs(gaps); return bytes >> SECTOR_SHIFT; } -EXPORT_SYMBOL_GPL(bio_split_rw_at); +EXPORT_SYMBOL_GPL(bio_split_io_at); struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim, unsigned *nr_segs) @@ -469,121 +513,10 @@ unsigned int blk_recalc_rq_segments(struct request *rq) rq_for_each_bvec(bv, rq, iter) bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes, - UINT_MAX, UINT_MAX); + UINT_MAX, BIO_MAX_SIZE); return nr_phys_segs; } -struct phys_vec { - phys_addr_t paddr; - u32 len; -}; - -static bool blk_map_iter_next(struct request *req, - struct req_iterator *iter, struct phys_vec *vec) -{ - unsigned int max_size; - struct bio_vec bv; - - if (req->rq_flags & RQF_SPECIAL_PAYLOAD) { - if (!iter->bio) - return false; - vec->paddr = bvec_phys(&req->special_vec); - vec->len = req->special_vec.bv_len; - iter->bio = NULL; - return true; - } - - if (!iter->iter.bi_size) - return false; - - bv = mp_bvec_iter_bvec(iter->bio->bi_io_vec, iter->iter); - vec->paddr = bvec_phys(&bv); - max_size = get_max_segment_size(&req->q->limits, vec->paddr, UINT_MAX); - bv.bv_len = min(bv.bv_len, max_size); - bio_advance_iter_single(iter->bio, &iter->iter, bv.bv_len); - - /* - * If we are entirely done with this bi_io_vec entry, check if the next - * one could be merged into it. This typically happens when moving to - * the next bio, but some callers also don't pack bvecs tight. - */ - while (!iter->iter.bi_size || !iter->iter.bi_bvec_done) { - struct bio_vec next; - - if (!iter->iter.bi_size) { - if (!iter->bio->bi_next) - break; - iter->bio = iter->bio->bi_next; - iter->iter = iter->bio->bi_iter; - } - - next = mp_bvec_iter_bvec(iter->bio->bi_io_vec, iter->iter); - if (bv.bv_len + next.bv_len > max_size || - !biovec_phys_mergeable(req->q, &bv, &next)) - break; - - bv.bv_len += next.bv_len; - bio_advance_iter_single(iter->bio, &iter->iter, next.bv_len); - } - - vec->len = bv.bv_len; - return true; -} - -static inline struct scatterlist *blk_next_sg(struct scatterlist **sg, - struct scatterlist *sglist) -{ - if (!*sg) - return sglist; - - /* - * If the driver previously mapped a shorter list, we could see a - * termination bit prematurely unless it fully inits the sg table - * on each mapping. We KNOW that there must be more entries here - * or the driver would be buggy, so force clear the termination bit - * to avoid doing a full sg_init_table() in drivers for each command. - */ - sg_unmark_end(*sg); - return sg_next(*sg); -} - -/* - * Map a request to scatterlist, return number of sg entries setup. Caller - * must make sure sg can hold rq->nr_phys_segments entries. - */ -int __blk_rq_map_sg(struct request *rq, struct scatterlist *sglist, - struct scatterlist **last_sg) -{ - struct req_iterator iter = { - .bio = rq->bio, - }; - struct phys_vec vec; - int nsegs = 0; - - /* the internal flush request may not have bio attached */ - if (iter.bio) - iter.iter = iter.bio->bi_iter; - - while (blk_map_iter_next(rq, &iter, &vec)) { - *last_sg = blk_next_sg(last_sg, sglist); - sg_set_page(*last_sg, phys_to_page(vec.paddr), vec.len, - offset_in_page(vec.paddr)); - nsegs++; - } - - if (*last_sg) - sg_mark_end(*last_sg); - - /* - * Something must have been wrong if the figured number of - * segment is bigger than number of req's physical segments - */ - WARN_ON(nsegs > blk_rq_nr_phys_segments(rq)); - - return nsegs; -} -EXPORT_SYMBOL(__blk_rq_map_sg); - static inline unsigned int blk_rq_get_max_sectors(struct request *rq, sector_t offset) { @@ -612,7 +545,7 @@ static inline int ll_new_hw_segment(struct request *req, struct bio *bio, if (!blk_cgroup_mergeable(req, bio)) goto no_merge; - if (blk_integrity_merge_bio(req->q, req, bio) == false) + if (unlikely(!blk_integrity_merge_bio(req->q, req, bio))) goto no_merge; /* discard request merge won't add new segment */ @@ -714,7 +647,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, if (!blk_cgroup_mergeable(req, next->bio)) return 0; - if (blk_integrity_merge_rq(q, req, next) == false) + if (unlikely(!blk_integrity_merge_rq(q, req, next))) return 0; if (!bio_crypt_ctx_merge_rq(req, next)) @@ -788,8 +721,7 @@ static void blk_account_io_merge_request(struct request *req) if (req->rq_flags & RQF_IO_STAT) { part_stat_lock(); part_stat_inc(req->part, merges[op_stat_group(req_op(req))]); - part_stat_local_dec(req->part, - in_flight[op_is_write(req_op(req))]); + bdev_dec_in_flight(req->part, req_op(req)); part_stat_unlock(); } } @@ -817,6 +749,24 @@ static bool blk_atomic_write_mergeable_rqs(struct request *rq, return (rq->cmd_flags & REQ_ATOMIC) == (next->cmd_flags & REQ_ATOMIC); } +u8 bio_seg_gap(struct request_queue *q, struct bio *prev, struct bio *next, + u8 gaps_bit) +{ + struct bio_vec pb, nb; + + if (!bio_has_data(prev)) + return 0; + + gaps_bit = min_not_zero(gaps_bit, prev->bi_bvec_gap_bit); + gaps_bit = min_not_zero(gaps_bit, next->bi_bvec_gap_bit); + + bio_get_last_bvec(prev, &pb); + bio_get_first_bvec(next, &nb); + if (!biovec_phys_mergeable(q, &pb, &nb)) + gaps_bit = min_not_zero(gaps_bit, ffs(bvec_seg_gap(&pb, &nb))); + return gaps_bit; +} + /* * For non-mq, this has to be called with the request spinlock acquired. * For mq with scheduling, the appropriate queue wide lock should be held. @@ -832,6 +782,8 @@ static struct request *attempt_merge(struct request_queue *q, if (req->bio->bi_write_hint != next->bio->bi_write_hint) return NULL; + if (req->bio->bi_write_stream != next->bio->bi_write_stream) + return NULL; if (req->bio->bi_ioprio != next->bio->bi_ioprio) return NULL; if (!blk_atomic_write_mergeable_rqs(req, next)) @@ -879,6 +831,9 @@ static struct request *attempt_merge(struct request_queue *q, if (next->start_time_ns < req->start_time_ns) req->start_time_ns = next->start_time_ns; + req->phys_gap_bit = bio_seg_gap(req->q, req->biotail, next->bio, + min_not_zero(next->phys_gap_bit, + req->phys_gap_bit)); req->biotail->bi_next = next->bio; req->biotail = next->biotail; @@ -947,15 +902,17 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) if (!blk_cgroup_mergeable(rq, bio)) return false; - if (blk_integrity_merge_bio(rq->q, rq, bio) == false) + if (unlikely(!blk_integrity_merge_bio(rq->q, rq, bio))) return false; if (!bio_crypt_rq_ctx_compatible(rq, bio)) return false; if (rq->bio->bi_write_hint != bio->bi_write_hint) return false; + if (rq->bio->bi_write_stream != bio->bi_write_stream) + return false; if (rq->bio->bi_ioprio != bio->bi_ioprio) return false; - if (blk_atomic_write_mergeable_rq_bio(rq, bio) == false) + if (unlikely(!blk_atomic_write_mergeable_rq_bio(rq, bio))) return false; return true; @@ -1000,6 +957,8 @@ enum bio_merge_status bio_attempt_back_merge(struct request *req, if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING) blk_zone_write_plug_bio_merged(bio); + req->phys_gap_bit = bio_seg_gap(req->q, req->biotail, bio, + req->phys_gap_bit); req->biotail->bi_next = bio; req->biotail = bio; req->__data_len += bio->bi_iter.bi_size; @@ -1034,6 +993,8 @@ static enum bio_merge_status bio_attempt_front_merge(struct request *req, blk_update_mixed_merge(req, bio, true); + req->phys_gap_bit = bio_seg_gap(req->q, bio, req->bio, + req->phys_gap_bit); bio->bi_next = req->bio; req->bio = bio; @@ -1127,20 +1088,20 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, if (!plug || rq_list_empty(&plug->mq_list)) return false; - rq_list_for_each(&plug->mq_list, rq) { - if (rq->q == q) { - if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) == - BIO_MERGE_OK) - return true; - break; - } + rq = plug->mq_list.tail; + if (rq->q == q) + return blk_attempt_bio_merge(q, rq, bio, nr_segs, false) == + BIO_MERGE_OK; + else if (!plug->multiple_queues) + return false; - /* - * Only keep iterating plug list for merges if we have multiple - * queues - */ - if (!plug->multiple_queues) - break; + rq_list_for_each(&plug->mq_list, rq) { + if (rq->q != q) + continue; + if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) == + BIO_MERGE_OK) + return true; + break; } return false; } diff --git a/block/blk-mq-cpumap.c b/block/blk-mq-cpumap.c index 444798c5374f..705da074ad6c 100644 --- a/block/blk-mq-cpumap.c +++ b/block/blk-mq-cpumap.c @@ -12,16 +12,56 @@ #include <linux/cpu.h> #include <linux/group_cpus.h> #include <linux/device/bus.h> +#include <linux/sched/isolation.h> #include "blk.h" #include "blk-mq.h" +static unsigned int blk_mq_num_queues(const struct cpumask *mask, + unsigned int max_queues) +{ + unsigned int num; + + num = cpumask_weight(mask); + return min_not_zero(num, max_queues); +} + +/** + * blk_mq_num_possible_queues - Calc nr of queues for multiqueue devices + * @max_queues: The maximum number of queues the hardware/driver + * supports. If max_queues is 0, the argument is + * ignored. + * + * Calculates the number of queues to be used for a multiqueue + * device based on the number of possible CPUs. + */ +unsigned int blk_mq_num_possible_queues(unsigned int max_queues) +{ + return blk_mq_num_queues(cpu_possible_mask, max_queues); +} +EXPORT_SYMBOL_GPL(blk_mq_num_possible_queues); + +/** + * blk_mq_num_online_queues - Calc nr of queues for multiqueue devices + * @max_queues: The maximum number of queues the hardware/driver + * supports. If max_queues is 0, the argument is + * ignored. + * + * Calculates the number of queues to be used for a multiqueue + * device based on the number of online CPUs. + */ +unsigned int blk_mq_num_online_queues(unsigned int max_queues) +{ + return blk_mq_num_queues(cpu_online_mask, max_queues); +} +EXPORT_SYMBOL_GPL(blk_mq_num_online_queues); + void blk_mq_map_queues(struct blk_mq_queue_map *qmap) { const struct cpumask *masks; - unsigned int queue, cpu; + unsigned int queue, cpu, nr_masks; - masks = group_cpus_evenly(qmap->nr_queues); + masks = group_cpus_evenly(qmap->nr_queues, &nr_masks); if (!masks) { for_each_possible_cpu(cpu) qmap->mq_map[cpu] = qmap->queue_offset; @@ -29,7 +69,7 @@ void blk_mq_map_queues(struct blk_mq_queue_map *qmap) } for (queue = 0; queue < qmap->nr_queues; queue++) { - for_each_cpu(cpu, &masks[queue]) + for_each_cpu(cpu, &masks[queue % nr_masks]) qmap->mq_map[cpu] = qmap->queue_offset + queue; } kfree(masks); diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 3421b5521fe2..6754d8f9449c 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -19,8 +19,10 @@ static int queue_poll_stat_show(void *data, struct seq_file *m) return 0; } +#define TO_REQUEST_QUEUE(m) ((struct request_queue *)m->private) + static void *queue_requeue_list_start(struct seq_file *m, loff_t *pos) - __acquires(&q->requeue_lock) + __acquires(&TO_REQUEST_QUEUE(m)->requeue_lock) { struct request_queue *q = m->private; @@ -36,13 +38,15 @@ static void *queue_requeue_list_next(struct seq_file *m, void *v, loff_t *pos) } static void queue_requeue_list_stop(struct seq_file *m, void *v) - __releases(&q->requeue_lock) + __releases(&TO_REQUEST_QUEUE(m)->requeue_lock) { struct request_queue *q = m->private; spin_unlock_irq(&q->requeue_lock); } +#undef TO_REQUEST_QUEUE + static const struct seq_operations queue_requeue_list_seq_ops = { .start = queue_requeue_list_start, .next = queue_requeue_list_next, @@ -93,6 +97,11 @@ static const char *const blk_queue_flag_name[] = { QUEUE_FLAG_NAME(RQ_ALLOC_TIME), QUEUE_FLAG_NAME(HCTX_ACTIVE), QUEUE_FLAG_NAME(SQ_SCHED), + QUEUE_FLAG_NAME(DISABLE_WBT_DEF), + QUEUE_FLAG_NAME(NO_ELV_SWITCH), + QUEUE_FLAG_NAME(QOS_ENABLED), + QUEUE_FLAG_NAME(BIO_ISSUE_TIME), + QUEUE_FLAG_NAME(ZONED_QD1_WRITES), }; #undef QUEUE_FLAG_NAME @@ -292,8 +301,10 @@ int blk_mq_debugfs_rq_show(struct seq_file *m, void *v) } EXPORT_SYMBOL_GPL(blk_mq_debugfs_rq_show); +#define TO_HCTX(m) ((struct blk_mq_hw_ctx *)m->private) + static void *hctx_dispatch_start(struct seq_file *m, loff_t *pos) - __acquires(&hctx->lock) + __acquires(&TO_HCTX(m)->lock) { struct blk_mq_hw_ctx *hctx = m->private; @@ -309,13 +320,15 @@ static void *hctx_dispatch_next(struct seq_file *m, void *v, loff_t *pos) } static void hctx_dispatch_stop(struct seq_file *m, void *v) - __releases(&hctx->lock) + __releases(&TO_HCTX(m)->lock) { struct blk_mq_hw_ctx *hctx = m->private; spin_unlock(&hctx->lock); } +#undef TO_HCTX + static const struct seq_operations hctx_dispatch_seq_ops = { .start = hctx_dispatch_start, .next = hctx_dispatch_next, @@ -479,9 +492,11 @@ static int hctx_dispatch_busy_show(void *data, struct seq_file *m) return 0; } +#define TO_CTX(m) ((struct blk_mq_ctx *)m->private) + #define CTX_RQ_SEQ_OPS(name, type) \ static void *ctx_##name##_rq_list_start(struct seq_file *m, loff_t *pos) \ - __acquires(&ctx->lock) \ + __acquires(&TO_CTX(m)->lock) \ { \ struct blk_mq_ctx *ctx = m->private; \ \ @@ -498,7 +513,7 @@ static void *ctx_##name##_rq_list_next(struct seq_file *m, void *v, \ } \ \ static void ctx_##name##_rq_list_stop(struct seq_file *m, void *v) \ - __releases(&ctx->lock) \ + __releases(&TO_CTX(m)->lock) \ { \ struct blk_mq_ctx *ctx = m->private; \ \ @@ -516,10 +531,12 @@ CTX_RQ_SEQ_OPS(default, HCTX_TYPE_DEFAULT); CTX_RQ_SEQ_OPS(read, HCTX_TYPE_READ); CTX_RQ_SEQ_OPS(poll, HCTX_TYPE_POLL); +#undef TO_CTX + static int blk_mq_debugfs_show(struct seq_file *m, void *v) { const struct blk_mq_debugfs_attr *attr = m->private; - void *data = d_inode(m->file->f_path.dentry->d_parent)->i_private; + void *data = debugfs_get_aux(m->file); return attr->show(data, m); } @@ -529,7 +546,7 @@ static ssize_t blk_mq_debugfs_write(struct file *file, const char __user *buf, { struct seq_file *m = file->private_data; const struct blk_mq_debugfs_attr *attr = m->private; - void *data = d_inode(file->f_path.dentry->d_parent)->i_private; + void *data = debugfs_get_aux(file); /* * Attributes that only implement .seq_ops are read-only and 'attr' is @@ -544,7 +561,7 @@ static ssize_t blk_mq_debugfs_write(struct file *file, const char __user *buf, static int blk_mq_debugfs_open(struct inode *inode, struct file *file) { const struct blk_mq_debugfs_attr *attr = inode->i_private; - void *data = d_inode(file->f_path.dentry->d_parent)->i_private; + void *data = debugfs_get_aux(file); struct seq_file *m; int ret; @@ -604,17 +621,24 @@ static const struct blk_mq_debugfs_attr blk_mq_debugfs_ctx_attrs[] = { {}, }; -static void debugfs_create_files(struct dentry *parent, void *data, +static void debugfs_create_files(struct request_queue *q, struct dentry *parent, + void *data, const struct blk_mq_debugfs_attr *attr) { + lockdep_assert_held(&q->debugfs_mutex); + /* + * debugfs_mutex should not be nested under other locks that can be + * grabbed while queue is frozen. + */ + lockdep_assert_not_held(&q->elevator_lock); + lockdep_assert_not_held(&q->rq_qos_mutex); + if (IS_ERR_OR_NULL(parent)) return; - d_inode(parent)->i_private = data; - for (; attr->name; attr++) - debugfs_create_file(attr->name, attr->mode, parent, - (void *)attr, &blk_mq_debugfs_fops); + debugfs_create_file_aux(attr->name, attr->mode, parent, + (void *)attr, data, &blk_mq_debugfs_fops); } void blk_mq_debugfs_register(struct request_queue *q) @@ -622,32 +646,14 @@ void blk_mq_debugfs_register(struct request_queue *q) struct blk_mq_hw_ctx *hctx; unsigned long i; - debugfs_create_files(q->debugfs_dir, q, blk_mq_debugfs_queue_attrs); + debugfs_create_files(q, q->debugfs_dir, q, blk_mq_debugfs_queue_attrs); - /* - * blk_mq_init_sched() attempted to do this already, but q->debugfs_dir - * didn't exist yet (because we don't know what to name the directory - * until the queue is registered to a gendisk). - */ - if (q->elevator && !q->sched_debugfs_dir) - blk_mq_debugfs_register_sched(q); - - /* Similarly, blk_mq_init_hctx() couldn't do this previously. */ queue_for_each_hw_ctx(q, hctx, i) { if (!hctx->debugfs_dir) blk_mq_debugfs_register_hctx(q, hctx); - if (q->elevator && !hctx->sched_debugfs_dir) - blk_mq_debugfs_register_sched_hctx(q, hctx); } - if (q->rq_qos) { - struct rq_qos *rqos = q->rq_qos; - - while (rqos) { - blk_mq_debugfs_register_rqos(rqos); - rqos = rqos->next; - } - } + blk_mq_debugfs_register_rq_qos(q); } static void blk_mq_debugfs_register_ctx(struct blk_mq_hw_ctx *hctx, @@ -659,7 +665,8 @@ static void blk_mq_debugfs_register_ctx(struct blk_mq_hw_ctx *hctx, snprintf(name, sizeof(name), "cpu%u", ctx->cpu); ctx_dir = debugfs_create_dir(name, hctx->debugfs_dir); - debugfs_create_files(ctx_dir, ctx, blk_mq_debugfs_ctx_attrs); + debugfs_create_files(hctx->queue, ctx_dir, ctx, + blk_mq_debugfs_ctx_attrs); } void blk_mq_debugfs_register_hctx(struct request_queue *q, @@ -675,7 +682,8 @@ void blk_mq_debugfs_register_hctx(struct request_queue *q, snprintf(name, sizeof(name), "hctx%u", hctx->queue_num); hctx->debugfs_dir = debugfs_create_dir(name, q->debugfs_dir); - debugfs_create_files(hctx->debugfs_dir, hctx, blk_mq_debugfs_hctx_attrs); + debugfs_create_files(q, hctx->debugfs_dir, hctx, + blk_mq_debugfs_hctx_attrs); hctx_for_each_ctx(hctx, ctx, i) blk_mq_debugfs_register_ctx(hctx, ctx); @@ -693,10 +701,13 @@ void blk_mq_debugfs_unregister_hctx(struct blk_mq_hw_ctx *hctx) void blk_mq_debugfs_register_hctxs(struct request_queue *q) { struct blk_mq_hw_ctx *hctx; + unsigned int memflags; unsigned long i; + memflags = blk_debugfs_lock(q); queue_for_each_hw_ctx(q, hctx, i) blk_mq_debugfs_register_hctx(q, hctx); + blk_debugfs_unlock(q, memflags); } void blk_mq_debugfs_unregister_hctxs(struct request_queue *q) @@ -726,7 +737,7 @@ void blk_mq_debugfs_register_sched(struct request_queue *q) q->sched_debugfs_dir = debugfs_create_dir("sched", q->debugfs_dir); - debugfs_create_files(q->sched_debugfs_dir, q, e->queue_debugfs_attrs); + debugfs_create_files(q, q->sched_debugfs_dir, q, e->queue_debugfs_attrs); } void blk_mq_debugfs_unregister_sched(struct request_queue *q) @@ -750,17 +761,7 @@ static const char *rq_qos_id_to_name(enum rq_qos_id id) return "unknown"; } -void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos) -{ - lockdep_assert_held(&rqos->disk->queue->debugfs_mutex); - - if (!rqos->disk->queue->debugfs_dir) - return; - debugfs_remove_recursive(rqos->debugfs_dir); - rqos->debugfs_dir = NULL; -} - -void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) +static void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) { struct request_queue *q = rqos->disk->queue; const char *dir_name = rq_qos_id_to_name(rqos->id); @@ -775,7 +776,22 @@ void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) q->debugfs_dir); rqos->debugfs_dir = debugfs_create_dir(dir_name, q->rqos_debugfs_dir); - debugfs_create_files(rqos->debugfs_dir, rqos, rqos->ops->debugfs_attrs); + debugfs_create_files(q, rqos->debugfs_dir, rqos, + rqos->ops->debugfs_attrs); +} + +void blk_mq_debugfs_register_rq_qos(struct request_queue *q) +{ + lockdep_assert_held(&q->debugfs_mutex); + + if (q->rq_qos) { + struct rq_qos *rqos = q->rq_qos; + + while (rqos) { + blk_mq_debugfs_register_rqos(rqos); + rqos = rqos->next; + } + } } void blk_mq_debugfs_register_sched_hctx(struct request_queue *q, @@ -798,7 +814,7 @@ void blk_mq_debugfs_register_sched_hctx(struct request_queue *q, hctx->sched_debugfs_dir = debugfs_create_dir("sched", hctx->debugfs_dir); - debugfs_create_files(hctx->sched_debugfs_dir, hctx, + debugfs_create_files(q, hctx->sched_debugfs_dir, hctx, e->hctx_debugfs_attrs); } diff --git a/block/blk-mq-debugfs.h b/block/blk-mq-debugfs.h index c80e453e3014..49bb1aaa83dc 100644 --- a/block/blk-mq-debugfs.h +++ b/block/blk-mq-debugfs.h @@ -33,8 +33,7 @@ void blk_mq_debugfs_register_sched_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx); void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx); -void blk_mq_debugfs_register_rqos(struct rq_qos *rqos); -void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos); +void blk_mq_debugfs_register_rq_qos(struct request_queue *q); #else static inline void blk_mq_debugfs_register(struct request_queue *q) { @@ -74,13 +73,10 @@ static inline void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hc { } -static inline void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) +static inline void blk_mq_debugfs_register_rq_qos(struct request_queue *q) { } -static inline void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos) -{ -} #endif #if defined(CONFIG_BLK_DEV_ZONED) && defined(CONFIG_BLK_DEBUG_FS) diff --git a/block/blk-mq-dma.c b/block/blk-mq-dma.c new file mode 100644 index 000000000000..bfdb9ed70741 --- /dev/null +++ b/block/blk-mq-dma.c @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 Christoph Hellwig + */ +#include <linux/blk-integrity.h> +#include <linux/blk-mq-dma.h> +#include "blk.h" + +static bool __blk_map_iter_next(struct blk_map_iter *iter) +{ + if (iter->iter.bi_size) + return true; + if (!iter->bio || !iter->bio->bi_next) + return false; + + iter->bio = iter->bio->bi_next; + if (iter->is_integrity) { + iter->iter = bio_integrity(iter->bio)->bip_iter; + iter->bvecs = bio_integrity(iter->bio)->bip_vec; + } else { + iter->iter = iter->bio->bi_iter; + iter->bvecs = iter->bio->bi_io_vec; + } + return true; +} + +static bool blk_map_iter_next(struct request *req, struct blk_map_iter *iter, + struct phys_vec *vec) +{ + unsigned int max_size; + struct bio_vec bv; + + if (!iter->iter.bi_size) + return false; + + bv = mp_bvec_iter_bvec(iter->bvecs, iter->iter); + vec->paddr = bvec_phys(&bv); + max_size = get_max_segment_size(&req->q->limits, vec->paddr, UINT_MAX); + bv.bv_len = min(bv.bv_len, max_size); + bvec_iter_advance_single(iter->bvecs, &iter->iter, bv.bv_len); + + /* + * If we are entirely done with this bi_io_vec entry, check if the next + * one could be merged into it. This typically happens when moving to + * the next bio, but some callers also don't pack bvecs tight. + */ + while (!iter->iter.bi_size || !iter->iter.bi_bvec_done) { + struct bio_vec next; + + if (!__blk_map_iter_next(iter)) + break; + + next = mp_bvec_iter_bvec(iter->bvecs, iter->iter); + if (bv.bv_len + next.bv_len > max_size || + !biovec_phys_mergeable(req->q, &bv, &next)) + break; + + bv.bv_len += next.bv_len; + bvec_iter_advance_single(iter->bvecs, &iter->iter, next.bv_len); + } + + vec->len = bv.bv_len; + return true; +} + +/* + * The IOVA-based DMA API wants to be able to coalesce at the minimal IOMMU page + * size granularity (which is guaranteed to be <= PAGE_SIZE and usually 4k), so + * we need to ensure our segments are aligned to this as well. + * + * Note that there is no point in using the slightly more complicated IOVA based + * path for single segment mappings. + */ +static inline bool blk_can_dma_map_iova(struct request *req, + struct device *dma_dev) +{ + return !(req_phys_gap_mask(req) & dma_get_merge_boundary(dma_dev)); +} + +static bool blk_dma_map_bus(struct blk_dma_iter *iter, struct phys_vec *vec) +{ + iter->addr = pci_p2pdma_bus_addr_map(iter->p2pdma.mem, vec->paddr); + iter->len = vec->len; + return true; +} + +static bool blk_dma_map_direct(struct request *req, struct device *dma_dev, + struct blk_dma_iter *iter, struct phys_vec *vec) +{ + unsigned int attrs = 0; + + if (iter->p2pdma.map == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE) + attrs |= DMA_ATTR_MMIO; + + iter->addr = dma_map_phys(dma_dev, vec->paddr, vec->len, + rq_dma_dir(req), attrs); + if (dma_mapping_error(dma_dev, iter->addr)) { + iter->status = BLK_STS_RESOURCE; + return false; + } + iter->len = vec->len; + return true; +} + +static bool blk_rq_dma_map_iova(struct request *req, struct device *dma_dev, + struct dma_iova_state *state, struct blk_dma_iter *iter, + struct phys_vec *vec) +{ + enum dma_data_direction dir = rq_dma_dir(req); + unsigned int attrs = 0; + size_t mapped = 0; + int error; + + iter->addr = state->addr; + iter->len = dma_iova_size(state); + + if (iter->p2pdma.map == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE) + attrs |= DMA_ATTR_MMIO; + + do { + error = dma_iova_link(dma_dev, state, vec->paddr, mapped, + vec->len, dir, attrs); + if (error) + goto out_unlink; + mapped += vec->len; + } while (blk_map_iter_next(req, &iter->iter, vec)); + + error = dma_iova_sync(dma_dev, state, 0, mapped); + if (error) + goto out_unlink; + + return true; + +out_unlink: + dma_iova_destroy(dma_dev, state, mapped, dir, attrs); + iter->status = errno_to_blk_status(error); + return false; +} + +static inline void blk_rq_map_iter_init(struct request *rq, + struct blk_map_iter *iter) +{ + struct bio *bio = rq->bio; + + if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { + *iter = (struct blk_map_iter) { + .bvecs = &rq->special_vec, + .iter = { + .bi_size = rq->special_vec.bv_len, + } + }; + } else if (bio) { + *iter = (struct blk_map_iter) { + .bio = bio, + .bvecs = bio->bi_io_vec, + .iter = bio->bi_iter, + }; + } else { + /* the internal flush request may not have bio attached */ + *iter = (struct blk_map_iter) {}; + } +} + +static bool blk_dma_map_iter_start(struct request *req, struct device *dma_dev, + struct dma_iova_state *state, struct blk_dma_iter *iter, + unsigned int total_len) +{ + struct phys_vec vec; + + memset(&iter->p2pdma, 0, sizeof(iter->p2pdma)); + iter->status = BLK_STS_OK; + iter->p2pdma.map = PCI_P2PDMA_MAP_NONE; + + /* + * Grab the first segment ASAP because we'll need it to check for P2P + * transfers. + */ + if (!blk_map_iter_next(req, &iter->iter, &vec)) + return false; + + switch (pci_p2pdma_state(&iter->p2pdma, dma_dev, + phys_to_page(vec.paddr))) { + case PCI_P2PDMA_MAP_BUS_ADDR: + return blk_dma_map_bus(iter, &vec); + case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: + /* + * P2P transfers through the host bridge are treated the + * same as non-P2P transfers below and during unmap. + */ + case PCI_P2PDMA_MAP_NONE: + break; + default: + iter->status = BLK_STS_INVAL; + return false; + } + + if (blk_can_dma_map_iova(req, dma_dev) && + dma_iova_try_alloc(dma_dev, state, vec.paddr, total_len)) + return blk_rq_dma_map_iova(req, dma_dev, state, iter, &vec); + memset(state, 0, sizeof(*state)); + return blk_dma_map_direct(req, dma_dev, iter, &vec); +} + +/** + * blk_rq_dma_map_iter_start - map the first DMA segment for a request + * @req: request to map + * @dma_dev: device to map to + * @state: DMA IOVA state + * @iter: block layer DMA iterator + * + * Start DMA mapping @req to @dma_dev. @state and @iter are provided by the + * caller and don't need to be initialized. @state needs to be stored for use + * at unmap time, @iter is only needed at map time. + * + * Returns %false if there is no segment to map, including due to an error, or + * %true ft it did map a segment. + * + * If a segment was mapped, the DMA address for it is returned in @iter.addr and + * the length in @iter.len. If no segment was mapped the status code is + * returned in @iter.status. + * + * The caller can call blk_rq_dma_map_coalesce() to check if further segments + * need to be mapped after this, or go straight to blk_rq_dma_map_iter_next() + * to try to map the following segments. + */ +bool blk_rq_dma_map_iter_start(struct request *req, struct device *dma_dev, + struct dma_iova_state *state, struct blk_dma_iter *iter) +{ + blk_rq_map_iter_init(req, &iter->iter); + return blk_dma_map_iter_start(req, dma_dev, state, iter, + blk_rq_payload_bytes(req)); +} +EXPORT_SYMBOL_GPL(blk_rq_dma_map_iter_start); + +/** + * blk_rq_dma_map_iter_next - map the next DMA segment for a request + * @req: request to map + * @dma_dev: device to map to + * @iter: block layer DMA iterator + * + * Iterate to the next mapping after a previous call to + * blk_rq_dma_map_iter_start(). See there for a detailed description of the + * arguments. + * + * Returns %false if there is no segment to map, including due to an error, or + * %true ft it did map a segment. + * + * If a segment was mapped, the DMA address for it is returned in @iter.addr and + * the length in @iter.len. If no segment was mapped the status code is + * returned in @iter.status. + */ +bool blk_rq_dma_map_iter_next(struct request *req, struct device *dma_dev, + struct blk_dma_iter *iter) +{ + struct phys_vec vec; + + if (!blk_map_iter_next(req, &iter->iter, &vec)) + return false; + + if (iter->p2pdma.map == PCI_P2PDMA_MAP_BUS_ADDR) + return blk_dma_map_bus(iter, &vec); + return blk_dma_map_direct(req, dma_dev, iter, &vec); +} +EXPORT_SYMBOL_GPL(blk_rq_dma_map_iter_next); + +static inline struct scatterlist * +blk_next_sg(struct scatterlist **sg, struct scatterlist *sglist) +{ + if (!*sg) + return sglist; + + /* + * If the driver previously mapped a shorter list, we could see a + * termination bit prematurely unless it fully inits the sg table + * on each mapping. We KNOW that there must be more entries here + * or the driver would be buggy, so force clear the termination bit + * to avoid doing a full sg_init_table() in drivers for each command. + */ + sg_unmark_end(*sg); + return sg_next(*sg); +} + +/* + * Map a request to scatterlist, return number of sg entries setup. Caller + * must make sure sg can hold rq->nr_phys_segments entries. + */ +int __blk_rq_map_sg(struct request *rq, struct scatterlist *sglist, + struct scatterlist **last_sg) +{ + struct blk_map_iter iter; + struct phys_vec vec; + int nsegs = 0; + + blk_rq_map_iter_init(rq, &iter); + while (blk_map_iter_next(rq, &iter, &vec)) { + *last_sg = blk_next_sg(last_sg, sglist); + + WARN_ON_ONCE(overflows_type(vec.len, unsigned int)); + sg_set_page(*last_sg, phys_to_page(vec.paddr), vec.len, + offset_in_page(vec.paddr)); + nsegs++; + } + + if (*last_sg) + sg_mark_end(*last_sg); + + /* + * Something must have been wrong if the figured number of + * segment is bigger than number of req's physical segments + */ + WARN_ON(nsegs > blk_rq_nr_phys_segments(rq)); + + return nsegs; +} +EXPORT_SYMBOL(__blk_rq_map_sg); + +#ifdef CONFIG_BLK_DEV_INTEGRITY +/** + * blk_rq_integrity_dma_map_iter_start - map the first integrity DMA segment + * for a request + * @req: request to map + * @dma_dev: device to map to + * @state: DMA IOVA state + * @iter: block layer DMA iterator + * + * Start DMA mapping @req integrity data to @dma_dev. @state and @iter are + * provided by the caller and don't need to be initialized. @state needs to be + * stored for use at unmap time, @iter is only needed at map time. + * + * Returns %false if there is no segment to map, including due to an error, or + * %true if it did map a segment. + * + * If a segment was mapped, the DMA address for it is returned in @iter.addr + * and the length in @iter.len. If no segment was mapped the status code is + * returned in @iter.status. + * + * The caller can call blk_rq_dma_map_coalesce() to check if further segments + * need to be mapped after this, or go straight to blk_rq_dma_map_iter_next() + * to try to map the following segments. + */ +bool blk_rq_integrity_dma_map_iter_start(struct request *req, + struct device *dma_dev, struct dma_iova_state *state, + struct blk_dma_iter *iter) +{ + unsigned len = bio_integrity_bytes(&req->q->limits.integrity, + blk_rq_sectors(req)); + struct bio *bio = req->bio; + + iter->iter = (struct blk_map_iter) { + .bio = bio, + .iter = bio_integrity(bio)->bip_iter, + .bvecs = bio_integrity(bio)->bip_vec, + .is_integrity = true, + }; + return blk_dma_map_iter_start(req, dma_dev, state, iter, len); +} +EXPORT_SYMBOL_GPL(blk_rq_integrity_dma_map_iter_start); + +/** + * blk_rq_integrity_dma_map_iter_next - map the next integrity DMA segment for + * a request + * @req: request to map + * @dma_dev: device to map to + * @state: DMA IOVA state + * @iter: block layer DMA iterator + * + * Iterate to the next integrity mapping after a previous call to + * blk_rq_integrity_dma_map_iter_start(). See there for a detailed description + * of the arguments. + * + * Returns %false if there is no segment to map, including due to an error, or + * %true if it did map a segment. + * + * If a segment was mapped, the DMA address for it is returned in @iter.addr and + * the length in @iter.len. If no segment was mapped the status code is + * returned in @iter.status. + */ +bool blk_rq_integrity_dma_map_iter_next(struct request *req, + struct device *dma_dev, struct blk_dma_iter *iter) +{ + struct phys_vec vec; + + if (!blk_map_iter_next(req, &iter->iter, &vec)) + return false; + + if (iter->p2pdma.map == PCI_P2PDMA_MAP_BUS_ADDR) + return blk_dma_map_bus(iter, &vec); + return blk_dma_map_direct(req, dma_dev, iter, &vec); +} +EXPORT_SYMBOL_GPL(blk_rq_integrity_dma_map_iter_next); + +/** + * blk_rq_map_integrity_sg - Map integrity metadata into a scatterlist + * @rq: request to map + * @sglist: target scatterlist + * + * Description: Map the integrity vectors in request into a + * scatterlist. The scatterlist must be big enough to hold all + * elements. I.e. sized using blk_rq_count_integrity_sg() or + * rq->nr_integrity_segments. + */ +int blk_rq_map_integrity_sg(struct request *rq, struct scatterlist *sglist) +{ + struct request_queue *q = rq->q; + struct scatterlist *sg = NULL; + struct bio *bio = rq->bio; + unsigned int segments = 0; + struct phys_vec vec; + + struct blk_map_iter iter = { + .bio = bio, + .iter = bio_integrity(bio)->bip_iter, + .bvecs = bio_integrity(bio)->bip_vec, + .is_integrity = true, + }; + + while (blk_map_iter_next(rq, &iter, &vec)) { + sg = blk_next_sg(&sg, sglist); + + WARN_ON_ONCE(overflows_type(vec.len, unsigned int)); + sg_set_page(sg, phys_to_page(vec.paddr), vec.len, + offset_in_page(vec.paddr)); + segments++; + } + + if (sg) + sg_mark_end(sg); + + /* + * Something must have been wrong if the figured number of segment + * is bigger than number of req's physical integrity segments + */ + BUG_ON(segments > rq->nr_integrity_segments); + BUG_ON(segments > queue_max_integrity_segments(q)); + return segments; +} +EXPORT_SYMBOL(blk_rq_map_integrity_sg); +#endif diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c index 109611445d40..0a00f5a76f5a 100644 --- a/block/blk-mq-sched.c +++ b/block/blk-mq-sched.c @@ -59,19 +59,17 @@ static bool blk_mq_dispatch_hctx_list(struct list_head *rq_list) list_first_entry(rq_list, struct request, queuelist)->mq_hctx; struct request *rq; LIST_HEAD(hctx_list); - unsigned int count = 0; list_for_each_entry(rq, rq_list, queuelist) { if (rq->mq_hctx != hctx) { list_cut_before(&hctx_list, rq_list, &rq->queuelist); goto dispatch; } - count++; } list_splice_tail_init(rq_list, &hctx_list); dispatch: - return blk_mq_dispatch_rq_list(hctx, &hctx_list, count); + return blk_mq_dispatch_rq_list(hctx, &hctx_list, false); } #define BLK_MQ_BUDGET_DELAY 3 /* ms units */ @@ -167,7 +165,7 @@ static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx) dispatched |= blk_mq_dispatch_hctx_list(&rq_list); } while (!list_empty(&rq_list)); } else { - dispatched = blk_mq_dispatch_rq_list(hctx, &rq_list, count); + dispatched = blk_mq_dispatch_rq_list(hctx, &rq_list, false); } if (busy) @@ -261,7 +259,7 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx) /* round robin for fair dispatch */ ctx = blk_mq_next_ctx(hctx, rq->mq_ctx); - } while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, 1)); + } while (blk_mq_dispatch_rq_list(rq->mq_hctx, &rq_list, false)); WRITE_ONCE(hctx->dispatch_from, ctx); return ret; @@ -298,7 +296,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx) */ if (!list_empty(&rq_list)) { blk_mq_sched_mark_restart_hctx(hctx); - if (!blk_mq_dispatch_rq_list(hctx, &rq_list, 0)) + if (!blk_mq_dispatch_rq_list(hctx, &rq_list, true)) return 0; need_dispatch = true; } else { @@ -312,7 +310,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx) if (need_dispatch) return blk_mq_do_dispatch_ctx(hctx); blk_mq_flush_busy_ctxs(hctx, &rq_list); - blk_mq_dispatch_rq_list(hctx, &rq_list, 0); + blk_mq_dispatch_rq_list(hctx, &rq_list, true); return 0; } @@ -376,125 +374,291 @@ bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq, } EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge); -static int blk_mq_sched_alloc_map_and_rqs(struct request_queue *q, - struct blk_mq_hw_ctx *hctx, - unsigned int hctx_idx) +/* called in queue's release handler, tagset has gone away */ +static void blk_mq_sched_tags_teardown(struct request_queue *q, unsigned int flags) { - if (blk_mq_is_shared_tags(q->tag_set->flags)) { - hctx->sched_tags = q->sched_shared_tags; - return 0; - } + struct blk_mq_hw_ctx *hctx; + unsigned long i; - hctx->sched_tags = blk_mq_alloc_map_and_rqs(q->tag_set, hctx_idx, - q->nr_requests); + queue_for_each_hw_ctx(q, hctx, i) + hctx->sched_tags = NULL; - if (!hctx->sched_tags) - return -ENOMEM; - return 0; + if (blk_mq_is_shared_tags(flags)) + q->sched_shared_tags = NULL; } -static void blk_mq_exit_sched_shared_tags(struct request_queue *queue) +void blk_mq_sched_reg_debugfs(struct request_queue *q) { - blk_mq_free_rq_map(queue->sched_shared_tags); - queue->sched_shared_tags = NULL; + struct blk_mq_hw_ctx *hctx; + unsigned int memflags; + unsigned long i; + + memflags = blk_debugfs_lock(q); + blk_mq_debugfs_register_sched(q); + queue_for_each_hw_ctx(q, hctx, i) + blk_mq_debugfs_register_sched_hctx(q, hctx); + blk_debugfs_unlock(q, memflags); } -/* called in queue's release handler, tagset has gone away */ -static void blk_mq_sched_tags_teardown(struct request_queue *q, unsigned int flags) +void blk_mq_sched_unreg_debugfs(struct request_queue *q) { struct blk_mq_hw_ctx *hctx; unsigned long i; - queue_for_each_hw_ctx(q, hctx, i) { - if (hctx->sched_tags) { - if (!blk_mq_is_shared_tags(flags)) - blk_mq_free_rq_map(hctx->sched_tags); - hctx->sched_tags = NULL; + blk_debugfs_lock_nomemsave(q); + queue_for_each_hw_ctx(q, hctx, i) + blk_mq_debugfs_unregister_sched_hctx(hctx); + blk_mq_debugfs_unregister_sched(q); + blk_debugfs_unlock_nomemrestore(q); +} + +void blk_mq_free_sched_tags(struct elevator_tags *et, + struct blk_mq_tag_set *set) +{ + unsigned long i; + + /* Shared tags are stored at index 0 in @tags. */ + if (blk_mq_is_shared_tags(set->flags)) + blk_mq_free_map_and_rqs(set, et->tags[0], BLK_MQ_NO_HCTX_IDX); + else { + for (i = 0; i < et->nr_hw_queues; i++) + blk_mq_free_map_and_rqs(set, et->tags[i], i); + } + + kfree(et); +} + +void blk_mq_free_sched_res(struct elevator_resources *res, + struct elevator_type *type, + struct blk_mq_tag_set *set) +{ + if (res->et) { + blk_mq_free_sched_tags(res->et, set); + res->et = NULL; + } + if (res->data) { + blk_mq_free_sched_data(type, res->data); + res->data = NULL; + } +} + +void blk_mq_free_sched_res_batch(struct xarray *elv_tbl, + struct blk_mq_tag_set *set) +{ + struct request_queue *q; + struct elv_change_ctx *ctx; + + lockdep_assert_held_write(&set->update_nr_hwq_lock); + + list_for_each_entry(q, &set->tag_list, tag_set_list) { + /* + * Accessing q->elevator without holding q->elevator_lock is + * safe because we're holding here set->update_nr_hwq_lock in + * the writer context. So, scheduler update/switch code (which + * acquires the same lock but in the reader context) can't run + * concurrently. + */ + if (q->elevator) { + ctx = xa_load(elv_tbl, q->id); + if (!ctx) { + WARN_ON_ONCE(1); + continue; + } + blk_mq_free_sched_res(&ctx->res, ctx->type, set); } } +} - if (blk_mq_is_shared_tags(flags)) - blk_mq_exit_sched_shared_tags(q); +void blk_mq_free_sched_ctx_batch(struct xarray *elv_tbl) +{ + unsigned long i; + struct elv_change_ctx *ctx; + + xa_for_each(elv_tbl, i, ctx) { + xa_erase(elv_tbl, i); + kfree(ctx); + } } -static int blk_mq_init_sched_shared_tags(struct request_queue *queue) +int blk_mq_alloc_sched_ctx_batch(struct xarray *elv_tbl, + struct blk_mq_tag_set *set) { - struct blk_mq_tag_set *set = queue->tag_set; + struct request_queue *q; + struct elv_change_ctx *ctx; - /* - * Set initial depth at max so that we don't need to reallocate for - * updating nr_requests. - */ - queue->sched_shared_tags = blk_mq_alloc_map_and_rqs(set, - BLK_MQ_NO_HCTX_IDX, - MAX_SCHED_RQ); - if (!queue->sched_shared_tags) + lockdep_assert_held_write(&set->update_nr_hwq_lock); + + list_for_each_entry(q, &set->tag_list, tag_set_list) { + ctx = kzalloc_obj(struct elv_change_ctx); + if (!ctx) + return -ENOMEM; + + if (xa_insert(elv_tbl, q->id, ctx, GFP_KERNEL)) { + kfree(ctx); + return -ENOMEM; + } + } + return 0; +} + +struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set, + unsigned int nr_hw_queues, unsigned int nr_requests) +{ + unsigned int nr_tags; + int i; + struct elevator_tags *et; + gfp_t gfp = GFP_NOIO | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY; + + if (blk_mq_is_shared_tags(set->flags)) + nr_tags = 1; + else + nr_tags = nr_hw_queues; + + et = kmalloc_flex(*et, tags, nr_tags, gfp); + if (!et) + return NULL; + + et->nr_requests = nr_requests; + et->nr_hw_queues = nr_hw_queues; + + if (blk_mq_is_shared_tags(set->flags)) { + /* Shared tags are stored at index 0 in @tags. */ + et->tags[0] = blk_mq_alloc_map_and_rqs(set, BLK_MQ_NO_HCTX_IDX, + MAX_SCHED_RQ); + if (!et->tags[0]) + goto out; + } else { + for (i = 0; i < et->nr_hw_queues; i++) { + et->tags[i] = blk_mq_alloc_map_and_rqs(set, i, + et->nr_requests); + if (!et->tags[i]) + goto out_unwind; + } + } + + return et; +out_unwind: + while (--i >= 0) + blk_mq_free_map_and_rqs(set, et->tags[i], i); +out: + kfree(et); + return NULL; +} + +int blk_mq_alloc_sched_res(struct request_queue *q, + struct elevator_type *type, + struct elevator_resources *res, + unsigned int nr_hw_queues) +{ + struct blk_mq_tag_set *set = q->tag_set; + + res->et = blk_mq_alloc_sched_tags(set, nr_hw_queues, + blk_mq_default_nr_requests(set)); + if (!res->et) return -ENOMEM; - blk_mq_tag_update_sched_shared_tags(queue); + res->data = blk_mq_alloc_sched_data(q, type); + if (IS_ERR(res->data)) { + blk_mq_free_sched_tags(res->et, set); + return -ENOMEM; + } + + return 0; +} + +int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl, + struct blk_mq_tag_set *set, unsigned int nr_hw_queues) +{ + struct elv_change_ctx *ctx; + struct request_queue *q; + int ret = -ENOMEM; + + lockdep_assert_held_write(&set->update_nr_hwq_lock); + list_for_each_entry(q, &set->tag_list, tag_set_list) { + /* + * Accessing q->elevator without holding q->elevator_lock is + * safe because we're holding here set->update_nr_hwq_lock in + * the writer context. So, scheduler update/switch code (which + * acquires the same lock but in the reader context) can't run + * concurrently. + */ + if (q->elevator) { + ctx = xa_load(elv_tbl, q->id); + if (WARN_ON_ONCE(!ctx)) { + ret = -ENOENT; + goto out_unwind; + } + + ret = blk_mq_alloc_sched_res(q, q->elevator->type, + &ctx->res, nr_hw_queues); + if (ret) + goto out_unwind; + } + } return 0; + +out_unwind: + list_for_each_entry_continue_reverse(q, &set->tag_list, tag_set_list) { + if (q->elevator) { + ctx = xa_load(elv_tbl, q->id); + if (ctx) + blk_mq_free_sched_res(&ctx->res, + ctx->type, set); + } + } + return ret; } /* caller must have a reference to @e, will grab another one if successful */ -int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e) +int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e, + struct elevator_resources *res) { unsigned int flags = q->tag_set->flags; + struct elevator_tags *et = res->et; struct blk_mq_hw_ctx *hctx; struct elevator_queue *eq; unsigned long i; int ret; - /* - * Default to double of smaller one between hw queue_depth and 128, - * since we don't split into sync/async like the old code did. - * Additionally, this is a per-hw queue depth. - */ - q->nr_requests = 2 * min_t(unsigned int, q->tag_set->queue_depth, - BLKDEV_DEFAULT_RQ); + eq = elevator_alloc(q, e, res); + if (!eq) + return -ENOMEM; + + q->nr_requests = et->nr_requests; if (blk_mq_is_shared_tags(flags)) { - ret = blk_mq_init_sched_shared_tags(q); - if (ret) - return ret; + /* Shared tags are stored at index 0 in @et->tags. */ + q->sched_shared_tags = et->tags[0]; + blk_mq_tag_update_sched_shared_tags(q, et->nr_requests); } queue_for_each_hw_ctx(q, hctx, i) { - ret = blk_mq_sched_alloc_map_and_rqs(q, hctx, i); - if (ret) - goto err_free_map_and_rqs; + if (blk_mq_is_shared_tags(flags)) + hctx->sched_tags = q->sched_shared_tags; + else + hctx->sched_tags = et->tags[i]; } - ret = e->ops.init_sched(q, e); + ret = e->ops.init_sched(q, eq); if (ret) - goto err_free_map_and_rqs; - - mutex_lock(&q->debugfs_mutex); - blk_mq_debugfs_register_sched(q); - mutex_unlock(&q->debugfs_mutex); + goto out; queue_for_each_hw_ctx(q, hctx, i) { if (e->ops.init_hctx) { ret = e->ops.init_hctx(hctx, i); if (ret) { - eq = q->elevator; - blk_mq_sched_free_rqs(q); blk_mq_exit_sched(q, eq); kobject_put(&eq->kobj); return ret; } } - mutex_lock(&q->debugfs_mutex); - blk_mq_debugfs_register_sched_hctx(q, hctx); - mutex_unlock(&q->debugfs_mutex); } - return 0; -err_free_map_and_rqs: - blk_mq_sched_free_rqs(q); +out: blk_mq_sched_tags_teardown(q, flags); - + kobject_put(&eq->kobj); q->elevator = NULL; return ret; } @@ -527,10 +691,6 @@ void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e) unsigned int flags = 0; queue_for_each_hw_ctx(q, hctx, i) { - mutex_lock(&q->debugfs_mutex); - blk_mq_debugfs_unregister_sched_hctx(hctx); - mutex_unlock(&q->debugfs_mutex); - if (e->type->ops.exit_hctx && hctx->sched_data) { e->type->ops.exit_hctx(hctx, i); hctx->sched_data = NULL; @@ -538,12 +698,9 @@ void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e) flags = hctx->flags; } - mutex_lock(&q->debugfs_mutex); - blk_mq_debugfs_unregister_sched(q); - mutex_unlock(&q->debugfs_mutex); - if (e->type->ops.exit_sched) e->type->ops.exit_sched(e); blk_mq_sched_tags_teardown(q, flags); + set_bit(ELEVATOR_FLAG_DYING, &q->elevator->flags); q->elevator = NULL; } diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h index 1326526bb733..5678e15bd33c 100644 --- a/block/blk-mq-sched.h +++ b/block/blk-mq-sched.h @@ -18,10 +18,54 @@ void __blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx); void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx); -int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e); +int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e, + struct elevator_resources *res); void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e); void blk_mq_sched_free_rqs(struct request_queue *q); +struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set, + unsigned int nr_hw_queues, unsigned int nr_requests); +int blk_mq_alloc_sched_res(struct request_queue *q, + struct elevator_type *type, + struct elevator_resources *res, + unsigned int nr_hw_queues); +int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl, + struct blk_mq_tag_set *set, unsigned int nr_hw_queues); +int blk_mq_alloc_sched_ctx_batch(struct xarray *elv_tbl, + struct blk_mq_tag_set *set); +void blk_mq_free_sched_ctx_batch(struct xarray *elv_tbl); +void blk_mq_free_sched_tags(struct elevator_tags *et, + struct blk_mq_tag_set *set); +void blk_mq_free_sched_res(struct elevator_resources *res, + struct elevator_type *type, + struct blk_mq_tag_set *set); +void blk_mq_free_sched_res_batch(struct xarray *et_table, + struct blk_mq_tag_set *set); +/* + * blk_mq_alloc_sched_data() - Allocates scheduler specific data + * Returns: + * - Pointer to allocated data on success + * - NULL if no allocation needed + * - ERR_PTR(-ENOMEM) in case of failure + */ +static inline void *blk_mq_alloc_sched_data(struct request_queue *q, + struct elevator_type *e) +{ + void *sched_data; + + if (!e || !e->ops.alloc_sched_data) + return NULL; + + sched_data = e->ops.alloc_sched_data(q); + return (sched_data) ?: ERR_PTR(-ENOMEM); +} + +static inline void blk_mq_free_sched_data(struct elevator_type *e, void *data) +{ + if (e && e->ops.free_sched_data) + e->ops.free_sched_data(data); +} + static inline void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx) { if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) @@ -82,4 +126,20 @@ static inline bool blk_mq_sched_needs_restart(struct blk_mq_hw_ctx *hctx) return test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); } +static inline void blk_mq_set_min_shallow_depth(struct request_queue *q, + unsigned int depth) +{ + struct blk_mq_hw_ctx *hctx; + unsigned long i; + + queue_for_each_hw_ctx(q, hctx, i) + sbitmap_queue_min_shallow_depth(&hctx->sched_tags->bitmap_tags, + depth); +} + +static inline bool blk_mq_is_sync_read(blk_opf_t opf) +{ + return op_is_sync(opf) && !op_is_write(opf); +} + #endif diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c index 24656980f443..895397831ecc 100644 --- a/block/blk-mq-sysfs.c +++ b/block/blk-mq-sysfs.c @@ -34,7 +34,6 @@ static void blk_mq_hw_sysfs_release(struct kobject *kobj) struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); - blk_free_flush_queue(hctx->fq); sbitmap_free(&hctx->ctx_map); free_cpumask_var(hctx->cpumask); kfree(hctx->ctxs); @@ -54,7 +53,7 @@ static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj, struct request_queue *q; ssize_t res; - entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr); + entry = container_of_const(attr, struct blk_mq_hw_ctx_sysfs_entry, attr); hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); q = hctx->queue; @@ -102,20 +101,20 @@ static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page) return pos + ret; } -static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = { +static const struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = { .attr = {.name = "nr_tags", .mode = 0444 }, .show = blk_mq_hw_sysfs_nr_tags_show, }; -static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = { +static const struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = { .attr = {.name = "nr_reserved_tags", .mode = 0444 }, .show = blk_mq_hw_sysfs_nr_reserved_tags_show, }; -static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = { +static const struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = { .attr = {.name = "cpu_list", .mode = 0444 }, .show = blk_mq_hw_sysfs_cpus_show, }; -static struct attribute *default_hw_ctx_attrs[] = { +static const struct attribute *const default_hw_ctx_attrs[] = { &blk_mq_hw_sysfs_nr_tags.attr, &blk_mq_hw_sysfs_nr_reserved_tags.attr, &blk_mq_hw_sysfs_cpus.attr, @@ -150,9 +149,11 @@ static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx) return; hctx_for_each_ctx(hctx, ctx, i) - kobject_del(&ctx->kobj); + if (ctx->kobj.state_in_sysfs) + kobject_del(&ctx->kobj); - kobject_del(&hctx->kobj); + if (hctx->kobj.state_in_sysfs) + kobject_del(&hctx->kobj); } static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx) diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index d880c50629d6..35deee5bbc73 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -8,8 +8,12 @@ */ #include <linux/kernel.h> #include <linux/module.h> +#include <linux/slab.h> +#include <linux/mm.h> +#include <linux/kmemleak.h> #include <linux/delay.h> +#include <trace/events/block.h> #include "blk.h" #include "blk-mq.h" #include "blk-mq-sched.h" @@ -178,6 +182,11 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) if (tag != BLK_MQ_NO_TAG) break; + /* Log the starvation event before altering task state */ + trace_block_rq_tag_wait(data->q, data->hctx, + data->rq_flags & RQF_SCHED_TAGS, + data->flags); + sbitmap_prepare_to_wait(bt, ws, &wait, TASK_UNINTERRUPTIBLE); tag = __blk_mq_get_tag(data, bt); @@ -253,13 +262,10 @@ static struct request *blk_mq_find_and_get_req(struct blk_mq_tags *tags, unsigned int bitnr) { struct request *rq; - unsigned long flags; - spin_lock_irqsave(&tags->lock, flags); rq = tags->rqs[bitnr]; if (!rq || rq->tag != bitnr || !req_ref_inc_not_zero(rq)) rq = NULL; - spin_unlock_irqrestore(&tags->lock, flags); return rq; } @@ -297,15 +303,15 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) /** * bt_for_each - iterate over the requests associated with a hardware queue * @hctx: Hardware queue to examine. - * @q: Request queue to examine. + * @q: Request queue @hctx is associated with (@hctx->queue). * @bt: sbitmap to examine. This is either the breserved_tags member * or the bitmap_tags member of struct blk_mq_tags. * @fn: Pointer to the function that will be called for each request * associated with @hctx that has been assigned a driver tag. - * @fn will be called as follows: @fn(@hctx, rq, @data, @reserved) - * where rq is a pointer to a request. Return true to continue - * iterating tags, false to stop. - * @data: Will be passed as third argument to @fn. + * @fn will be called as follows: @fn(rq, @data) where rq is a + * pointer to a request. Return %true to continue iterating tags; + * %false to stop. + * @data: Will be passed as second argument to @fn. * @reserved: Indicates whether @bt is the breserved_tags member or the * bitmap_tags member of struct blk_mq_tags. */ @@ -371,9 +377,9 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) * @bt: sbitmap to examine. This is either the breserved_tags member * or the bitmap_tags member of struct blk_mq_tags. * @fn: Pointer to the function that will be called for each started - * request. @fn will be called as follows: @fn(rq, @data, - * @reserved) where rq is a pointer to a request. Return true - * to continue iterating tags, false to stop. + * request. @fn will be called as follows: @fn(rq, @data) where rq + * is a pointer to a request. Return %true to continue iterating + * tags; %false to stop. * @data: Will be passed as second argument to @fn. * @flags: BT_TAG_ITER_* */ @@ -406,10 +412,9 @@ static void __blk_mq_all_tag_iter(struct blk_mq_tags *tags, * blk_mq_all_tag_iter - iterate over all requests in a tag map * @tags: Tag map to iterate over. * @fn: Pointer to the function that will be called for each - * request. @fn will be called as follows: @fn(rq, @priv, - * reserved) where rq is a pointer to a request. 'reserved' - * indicates whether or not @rq is a reserved request. Return - * true to continue iterating tags, false to stop. + * request. @fn will be called as follows: @fn(rq, @priv) where rq + * is a pointer to a request. Return %true to continue iterating + * tags; %false to stop. * @priv: Will be passed as second argument to @fn. * * Caller has to pass the tag map from which requests are allocated. @@ -424,10 +429,9 @@ void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, * blk_mq_tagset_busy_iter - iterate over all started requests in a tag set * @tagset: Tag set to iterate over. * @fn: Pointer to the function that will be called for each started - * request. @fn will be called as follows: @fn(rq, @priv, - * reserved) where rq is a pointer to a request. 'reserved' - * indicates whether or not @rq is a reserved request. Return - * true to continue iterating tags, false to stop. + * request. @fn will be called as follows: @fn(rq, @priv) where + * rq is a pointer to a request. Return true to continue iterating + * tags, false to stop. * @priv: Will be passed as second argument to @fn. * * We grab one request reference before calling @fn and release it after @@ -437,7 +441,9 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, busy_tag_iter_fn *fn, void *priv) { unsigned int flags = tagset->flags; - int i, nr_tags; + int i, nr_tags, srcu_idx; + + srcu_idx = srcu_read_lock(&tagset->tags_srcu); nr_tags = blk_mq_is_shared_tags(flags) ? 1 : tagset->nr_hw_queues; @@ -446,6 +452,7 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, __blk_mq_all_tag_iter(tagset->tags[i], fn, priv, BT_TAG_ITER_STARTED); } + srcu_read_unlock(&tagset->tags_srcu, srcu_idx); } EXPORT_SYMBOL(blk_mq_tagset_busy_iter); @@ -483,11 +490,10 @@ EXPORT_SYMBOL(blk_mq_tagset_wait_completed_request); * blk_mq_queue_tag_busy_iter - iterate over all requests with a driver tag * @q: Request queue to examine. * @fn: Pointer to the function that will be called for each request - * on @q. @fn will be called as follows: @fn(hctx, rq, @priv, - * reserved) where rq is a pointer to a request and hctx points - * to the hardware queue associated with the request. 'reserved' - * indicates whether or not @rq is a reserved request. - * @priv: Will be passed as third argument to @fn. + * on @q. @fn will be called as follows: @fn(rq, @priv) where rq + * is a pointer to a request and hctx points to the hardware queue + * associated with the request. + * @priv: Will be passed as second argument to @fn. * * Note: if @q->tag_set is shared with other request queues then @fn will be * called for all requests on all queues that share that tag set and not only @@ -496,14 +502,17 @@ EXPORT_SYMBOL(blk_mq_tagset_wait_completed_request); void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn, void *priv) { + int srcu_idx; + /* - * __blk_mq_update_nr_hw_queues() updates nr_hw_queues and hctx_table + * __blk_mq_update_nr_hw_queues() updates nr_hw_queues and queue_hw_ctx * while the queue is frozen. So we can use q_usage_counter to avoid * racing with it. */ if (!percpu_ref_tryget(&q->q_usage_counter)) return; + srcu_idx = srcu_read_lock(&q->tag_set->tags_srcu); if (blk_mq_is_shared_tags(q->tag_set->flags)) { struct blk_mq_tags *tags = q->tag_set->shared_tags; struct sbitmap_queue *bresv = &tags->breserved_tags; @@ -533,6 +542,7 @@ void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn, bt_for_each(hctx, q, btags, fn, priv, false); } } + srcu_read_unlock(&q->tag_set->tags_srcu, srcu_idx); blk_queue_exit(q); } @@ -562,6 +572,8 @@ struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, tags->nr_tags = total_tags; tags->nr_reserved_tags = reserved_tags; spin_lock_init(&tags->lock); + INIT_LIST_HEAD(&tags->page_list); + if (bt_alloc(&tags->bitmap_tags, depth, round_robin, node)) goto out_free_tags; if (bt_alloc(&tags->breserved_tags, reserved_tags, round_robin, node)) @@ -576,63 +588,37 @@ out_free_tags: return NULL; } -void blk_mq_free_tags(struct blk_mq_tags *tags) -{ - sbitmap_queue_free(&tags->bitmap_tags); - sbitmap_queue_free(&tags->breserved_tags); - kfree(tags); -} - -int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, - struct blk_mq_tags **tagsptr, unsigned int tdepth, - bool can_grow) +static void blk_mq_free_tags_callback(struct rcu_head *head) { - struct blk_mq_tags *tags = *tagsptr; - - if (tdepth <= tags->nr_reserved_tags) - return -EINVAL; - - /* - * If we are allowed to grow beyond the original size, allocate - * a new set of tags before freeing the old one. - */ - if (tdepth > tags->nr_tags) { - struct blk_mq_tag_set *set = hctx->queue->tag_set; - struct blk_mq_tags *new; - - if (!can_grow) - return -EINVAL; + struct blk_mq_tags *tags = container_of(head, struct blk_mq_tags, + rcu_head); + struct page *page; + while (!list_empty(&tags->page_list)) { + page = list_first_entry(&tags->page_list, struct page, lru); + list_del_init(&page->lru); /* - * We need some sort of upper limit, set it high enough that - * no valid use cases should require more. + * Remove kmemleak object previously allocated in + * blk_mq_alloc_rqs(). */ - if (tdepth > MAX_SCHED_RQ) - return -EINVAL; - - /* - * Only the sbitmap needs resizing since we allocated the max - * initially. - */ - if (blk_mq_is_shared_tags(set->flags)) - return 0; + kmemleak_free(page_address(page)); + __free_pages(page, page->private); + } + kfree(tags); +} - new = blk_mq_alloc_map_and_rqs(set, hctx->queue_num, tdepth); - if (!new) - return -ENOMEM; +void blk_mq_free_tags(struct blk_mq_tag_set *set, struct blk_mq_tags *tags) +{ + sbitmap_queue_free(&tags->bitmap_tags); + sbitmap_queue_free(&tags->breserved_tags); - blk_mq_free_map_and_rqs(set, *tagsptr, hctx->queue_num); - *tagsptr = new; - } else { - /* - * Don't need (or can't) update reserved tags here, they - * remain static and should never need resizing. - */ - sbitmap_queue_resize(&tags->bitmap_tags, - tdepth - tags->nr_reserved_tags); + /* if tags pages is not allocated yet, free tags directly */ + if (list_empty(&tags->page_list)) { + kfree(tags); + return; } - return 0; + call_srcu(&set->tags_srcu, &tags->rcu_head, blk_mq_free_tags_callback); } void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, unsigned int size) @@ -642,10 +628,11 @@ void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, unsigned int size sbitmap_queue_resize(&tags->bitmap_tags, size - set->reserved_tags); } -void blk_mq_tag_update_sched_shared_tags(struct request_queue *q) +void blk_mq_tag_update_sched_shared_tags(struct request_queue *q, + unsigned int nr) { sbitmap_queue_resize(&q->sched_shared_tags->bitmap_tags, - q->nr_requests - q->tag_set->reserved_tags); + nr - q->tag_set->reserved_tags); } /** diff --git a/block/blk-mq.c b/block/blk-mq.c index c2697db59109..2c850330a32b 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -23,6 +23,7 @@ #include <linux/cache.h> #include <linux/sched/topology.h> #include <linux/sched/signal.h> +#include <linux/suspend.h> #include <linux/delay.h> #include <linux/crash_dump.h> #include <linux/prefetch.h> @@ -89,7 +90,7 @@ struct mq_inflight { unsigned int inflight[2]; }; -static bool blk_mq_check_inflight(struct request *rq, void *priv) +static bool blk_mq_check_in_driver(struct request *rq, void *priv) { struct mq_inflight *mi = priv; @@ -101,24 +102,14 @@ static bool blk_mq_check_inflight(struct request *rq, void *priv) return true; } -unsigned int blk_mq_in_flight(struct request_queue *q, - struct block_device *part) +void blk_mq_in_driver_rw(struct block_device *part, unsigned int inflight[2]) { struct mq_inflight mi = { .part = part }; - blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); - - return mi.inflight[0] + mi.inflight[1]; -} - -void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part, - unsigned int inflight[2]) -{ - struct mq_inflight mi = { .part = part }; - - blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); - inflight[0] = mi.inflight[0]; - inflight[1] = mi.inflight[1]; + blk_mq_queue_tag_busy_iter(bdev_get_queue(part), blk_mq_check_in_driver, + &mi); + inflight[READ] = mi.inflight[READ]; + inflight[WRITE] = mi.inflight[WRITE]; } #ifdef CONFIG_LOCKDEP @@ -345,12 +336,12 @@ void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set) { struct request_queue *q; - mutex_lock(&set->tag_list_lock); - list_for_each_entry(q, &set->tag_list, tag_set_list) { + rcu_read_lock(); + list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) { if (!blk_queue_skip_tagset_quiesce(q)) blk_mq_quiesce_queue_nowait(q); } - mutex_unlock(&set->tag_list_lock); + rcu_read_unlock(); blk_mq_wait_quiesce_done(set); } @@ -360,12 +351,12 @@ void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set) { struct request_queue *q; - mutex_lock(&set->tag_list_lock); - list_for_each_entry(q, &set->tag_list, tag_set_list) { + rcu_read_lock(); + list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) { if (!blk_queue_skip_tagset_quiesce(q)) blk_mq_unquiesce_queue(q); } - mutex_unlock(&set->tag_list_lock); + rcu_read_unlock(); } EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset); @@ -386,6 +377,7 @@ void blk_rq_init(struct request_queue *q, struct request *rq) INIT_LIST_HEAD(&rq->queuelist); rq->q = q; rq->__sector = (sector_t) -1; + rq->phys_gap_bit = 0; INIT_HLIST_NODE(&rq->hash); RB_CLEAR_NODE(&rq->rb_node); rq->tag = BLK_MQ_NO_TAG; @@ -406,6 +398,15 @@ static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns) #endif } +static inline void blk_mq_bio_issue_init(struct request_queue *q, + struct bio *bio) +{ +#ifdef CONFIG_BLK_CGROUP + if (test_bit(QUEUE_FLAG_BIO_ISSUE_TIME, &q->queue_flags)) + bio->issue_time_ns = blk_time_get_ns(); +#endif +} + static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, struct blk_mq_tags *tags, unsigned int tag) { @@ -468,21 +469,26 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) unsigned long tag_mask; int i, nr = 0; - tag_mask = blk_mq_get_tags(data, data->nr_tags, &tag_offset); - if (unlikely(!tag_mask)) - return NULL; + do { + tag_mask = blk_mq_get_tags(data, data->nr_tags - nr, &tag_offset); + if (unlikely(!tag_mask)) { + if (nr == 0) + return NULL; + break; + } + tags = blk_mq_tags_from_data(data); + for (i = 0; tag_mask; i++) { + if (!(tag_mask & (1UL << i))) + continue; + tag = tag_offset + i; + prefetch(tags->static_rqs[tag]); + tag_mask &= ~(1UL << i); + rq = blk_mq_rq_ctx_init(data, tags, tag); + rq_list_add_head(data->cached_rqs, rq); + nr++; + } + } while (data->nr_tags > nr); - tags = blk_mq_tags_from_data(data); - for (i = 0; tag_mask; i++) { - if (!(tag_mask & (1UL << i))) - continue; - tag = tag_offset + i; - prefetch(tags->static_rqs[tag]); - tag_mask &= ~(1UL << i); - rq = blk_mq_rq_ctx_init(data, tags, tag); - rq_list_add_head(data->cached_rqs, rq); - nr++; - } if (!(data->rq_flags & RQF_SCHED_TAGS)) blk_mq_add_active_requests(data->hctx, nr); /* caller already holds a reference, add for remainder */ @@ -492,6 +498,42 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) return rq_list_pop(data->cached_rqs); } +static void blk_mq_limit_depth(struct blk_mq_alloc_data *data) +{ + struct elevator_mq_ops *ops; + + /* If no I/O scheduler has been configured, don't limit requests */ + if (!data->q->elevator) { + blk_mq_tag_busy(data->hctx); + return; + } + + /* + * All requests use scheduler tags when an I/O scheduler is + * enabled for the queue. + */ + data->rq_flags |= RQF_SCHED_TAGS; + + /* + * Flush/passthrough requests are special and go directly to the + * dispatch list, they are not subject to the async_depth limit. + */ + if ((data->cmd_flags & REQ_OP_MASK) == REQ_OP_FLUSH || + blk_op_is_passthrough(data->cmd_flags)) + return; + + WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED); + data->rq_flags |= RQF_USE_SCHED; + + /* + * By default, sync requests have no limit, and async requests are + * limited to async_depth. + */ + ops = &data->q->elevator->type->ops; + if (ops->limit_depth) + ops->limit_depth(data->cmd_flags, data); +} + static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) { struct request_queue *q = data->q; @@ -510,31 +552,7 @@ retry: data->ctx = blk_mq_get_ctx(q); data->hctx = blk_mq_map_queue(data->cmd_flags, data->ctx); - if (q->elevator) { - /* - * All requests use scheduler tags when an I/O scheduler is - * enabled for the queue. - */ - data->rq_flags |= RQF_SCHED_TAGS; - - /* - * Flush/passthrough requests are special and go directly to the - * dispatch list. - */ - if ((data->cmd_flags & REQ_OP_MASK) != REQ_OP_FLUSH && - !blk_op_is_passthrough(data->cmd_flags)) { - struct elevator_mq_ops *ops = &q->elevator->type->ops; - - WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED); - - data->rq_flags |= RQF_USE_SCHED; - if (ops->limit_depth) - ops->limit_depth(data->cmd_flags, data); - } - } else { - blk_mq_tag_busy(data->hctx); - } - + blk_mq_limit_depth(data); if (data->flags & BLK_MQ_REQ_RESERVED) data->rq_flags |= RQF_RESV; @@ -584,9 +602,13 @@ static struct request *blk_mq_rq_cache_fill(struct request_queue *q, struct blk_mq_alloc_data data = { .q = q, .flags = flags, + .shallow_depth = 0, .cmd_flags = opf, + .rq_flags = 0, .nr_tags = plug->nr_ios, .cached_rqs = &plug->cached_rqs, + .ctx = NULL, + .hctx = NULL }; struct request *rq; @@ -646,8 +668,13 @@ struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, struct blk_mq_alloc_data data = { .q = q, .flags = flags, + .shallow_depth = 0, .cmd_flags = opf, + .rq_flags = 0, .nr_tags = 1, + .cached_rqs = NULL, + .ctx = NULL, + .hctx = NULL }; int ret; @@ -660,6 +687,7 @@ struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, goto out_queue_exit; } rq->__data_len = 0; + rq->phys_gap_bit = 0; rq->__sector = (sector_t) -1; rq->bio = rq->biotail = NULL; return rq; @@ -675,8 +703,13 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, struct blk_mq_alloc_data data = { .q = q, .flags = flags, + .shallow_depth = 0, .cmd_flags = opf, + .rq_flags = 0, .nr_tags = 1, + .cached_rqs = NULL, + .ctx = NULL, + .hctx = NULL }; u64 alloc_time_ns = 0; struct request *rq; @@ -710,7 +743,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, * If not tell the caller that it should skip this queue. */ ret = -EXDEV; - data.hctx = xa_load(&q->hctx_table, hctx_idx); + data.hctx = q->queue_hw_ctx[hctx_idx]; if (!blk_mq_hw_queue_mapped(data.hctx)) goto out_queue_exit; cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask); @@ -735,6 +768,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag); blk_mq_rq_time_init(rq, alloc_time_ns); rq->__data_len = 0; + rq->phys_gap_bit = 0; rq->__sector = (sector_t) -1; rq->bio = rq->biotail = NULL; return rq; @@ -789,9 +823,6 @@ void blk_mq_free_request(struct request *rq) blk_mq_finish_request(rq); - if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq))) - laptop_io_completion(q->disk->bdi); - rq_qos_done(q, rq); WRITE_ONCE(rq->state, MQ_RQ_IDLE); @@ -879,7 +910,8 @@ static void blk_complete_request(struct request *req) /* Completion has already been traced */ bio_clear_flag(bio, BIO_TRACE_COMPLETION); - blk_zone_update_request_bio(req, bio); + if (blk_req_bio_is_zone_append(req, bio)) + blk_zone_append_update_request_bio(req, bio); if (!is_flush) bio_endio(bio); @@ -978,7 +1010,8 @@ bool blk_update_request(struct request *req, blk_status_t error, /* Don't actually finish bio if it's part of flush sequence */ if (!bio->bi_iter.bi_size) { - blk_zone_update_request_bio(req, bio); + if (blk_req_bio_is_zone_append(req, bio)) + blk_zone_append_update_request_bio(req, bio); if (!is_flush) bio_endio(bio); } @@ -1049,49 +1082,18 @@ static inline void blk_account_io_done(struct request *req, u64 now) update_io_ticks(req->part, jiffies, true); part_stat_inc(req->part, ios[sgrp]); part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns); - part_stat_local_dec(req->part, - in_flight[op_is_write(req_op(req))]); + bdev_dec_in_flight(req->part, req_op(req)); part_stat_unlock(); } } -static inline bool blk_rq_passthrough_stats(struct request *req) -{ - struct bio *bio = req->bio; - - if (!blk_queue_passthrough_stat(req->q)) - return false; - - /* Requests without a bio do not transfer data. */ - if (!bio) - return false; - - /* - * Stats are accumulated in the bdev, so must have one attached to a - * bio to track stats. Most drivers do not set the bdev for passthrough - * requests, but nvme is one that will set it. - */ - if (!bio->bi_bdev) - return false; - - /* - * We don't know what a passthrough command does, but we know the - * payload size and data direction. Ensuring the size is aligned to the - * block size filters out most commands with payloads that don't - * represent sector access. - */ - if (blk_rq_bytes(req) & (bdev_logical_block_size(bio->bi_bdev) - 1)) - return false; - return true; -} - static inline void blk_account_io_start(struct request *req) { trace_block_io_start(req); if (!blk_queue_io_stat(req->q)) return; - if (blk_rq_is_passthrough(req) && !blk_rq_passthrough_stats(req)) + if (blk_rq_is_passthrough(req) && !blk_rq_passthrough_stats(req, req->q)) return; req->rq_flags |= RQF_IO_STAT; @@ -1110,7 +1112,7 @@ static inline void blk_account_io_start(struct request *req) part_stat_lock(); update_io_ticks(req->part, jiffies, false); - part_stat_local_inc(req->part, in_flight[op_is_write(req_op(req))]); + bdev_inc_in_flight(req->part, req_op(req)); part_stat_unlock(); } @@ -1132,7 +1134,7 @@ inline void __blk_mq_end_request(struct request *rq, blk_status_t error) if (rq->end_io) { rq_qos_done(rq->q, rq); - if (rq->end_io(rq, error) == RQ_END_IO_FREE) + if (rq->end_io(rq, error, NULL) == RQ_END_IO_FREE) blk_mq_free_request(rq); } else { blk_mq_free_request(rq); @@ -1187,7 +1189,7 @@ void blk_mq_end_request_batch(struct io_comp_batch *iob) * If end_io handler returns NONE, then it still has * ownership of the request. */ - if (rq->end_io && rq->end_io(rq, 0) == RQ_END_IO_NONE) + if (rq->end_io && rq->end_io(rq, 0, iob) == RQ_END_IO_NONE) continue; WRITE_ONCE(rq->state, MQ_RQ_IDLE); @@ -1434,7 +1436,8 @@ struct blk_rq_wait { blk_status_t ret; }; -static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret) +static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret, + const struct io_comp_batch *iob) { struct blk_rq_wait *wait = rq->end_io_data; @@ -1456,7 +1459,7 @@ EXPORT_SYMBOL_GPL(blk_rq_is_poll); static void blk_rq_poll_completion(struct request *rq, struct completion *wait) { do { - blk_hctx_poll(rq->q, rq->mq_hctx, NULL, 0); + blk_hctx_poll(rq->q, rq->mq_hctx, NULL, BLK_POLL_ONESHOT); cond_resched(); } while (!completion_done(wait)); } @@ -1664,7 +1667,7 @@ static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expi void blk_mq_put_rq_ref(struct request *rq) { if (is_flush_rq(rq)) { - if (rq->end_io(rq, 0) == RQ_END_IO_FREE) + if (rq->end_io(rq, 0, NULL) == RQ_END_IO_FREE) blk_mq_free_request(rq); } else if (req_ref_put_and_test(rq)) { __blk_mq_free_request(rq); @@ -2080,7 +2083,7 @@ static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued, * Returns true if we did some work AND can potentially do more. */ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, - unsigned int nr_budgets) + bool get_budget) { enum prep_dispatch prep; struct request_queue *q = hctx->queue; @@ -2102,7 +2105,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, rq = list_first_entry(list, struct request, queuelist); WARN_ON_ONCE(hctx != rq->mq_hctx); - prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets); + prep = blk_mq_prep_dispatch_rq(rq, get_budget); if (prep != PREP_DISPATCH_OK) break; @@ -2111,12 +2114,6 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, bd.rq = rq; bd.last = list_empty(list); - /* - * once the request is queued to lld, no need to cover the - * budget any more - */ - if (nr_budgets) - nr_budgets--; ret = q->mq_ops->queue_rq(hctx, &bd); switch (ret) { case BLK_STS_OK: @@ -2150,7 +2147,11 @@ out: ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) || blk_mq_is_shared_tags(hctx->flags)); - if (nr_budgets) + /* + * If the caller allocated budgets, free the budgets of the + * requests that have not yet been passed to the block driver. + */ + if (!get_budget) blk_mq_release_budgets(q, list); spin_lock(&hctx->lock); @@ -2661,6 +2662,8 @@ static void blk_mq_bio_to_request(struct request *rq, struct bio *bio, rq->bio = rq->biotail = bio; rq->__sector = bio->bi_iter.bi_sector; rq->__data_len = bio->bi_iter.bi_size; + rq->phys_gap_bit = bio->bi_bvec_gap_bit; + rq->nr_phys_segments = nr_segs; if (bio_integrity(bio)) rq->nr_integrity_segments = blk_rq_count_integrity_sg(rq->q, @@ -2778,15 +2781,15 @@ static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last) return __blk_mq_issue_directly(hctx, rq, last); } -static void blk_mq_plug_issue_direct(struct blk_plug *plug) +static void blk_mq_issue_direct(struct rq_list *rqs) { struct blk_mq_hw_ctx *hctx = NULL; struct request *rq; int queued = 0; blk_status_t ret = BLK_STS_OK; - while ((rq = rq_list_pop(&plug->mq_list))) { - bool last = rq_list_empty(&plug->mq_list); + while ((rq = rq_list_pop(rqs))) { + bool last = rq_list_empty(rqs); if (hctx != rq->mq_hctx) { if (hctx) { @@ -2817,15 +2820,64 @@ out: blk_mq_commit_rqs(hctx, queued, false); } -static void __blk_mq_flush_plug_list(struct request_queue *q, - struct blk_plug *plug) +static void __blk_mq_flush_list(struct request_queue *q, struct rq_list *rqs) { if (blk_queue_quiesced(q)) return; - q->mq_ops->queue_rqs(&plug->mq_list); + q->mq_ops->queue_rqs(rqs); +} + +static unsigned blk_mq_extract_queue_requests(struct rq_list *rqs, + struct rq_list *queue_rqs) +{ + struct request *rq = rq_list_pop(rqs); + struct request_queue *this_q = rq->q; + struct request **prev = &rqs->head; + struct rq_list matched_rqs = {}; + struct request *last = NULL; + unsigned depth = 1; + + rq_list_add_tail(&matched_rqs, rq); + while ((rq = *prev)) { + if (rq->q == this_q) { + /* move rq from rqs to matched_rqs */ + *prev = rq->rq_next; + rq_list_add_tail(&matched_rqs, rq); + depth++; + } else { + /* leave rq in rqs */ + prev = &rq->rq_next; + last = rq; + } + } + + rqs->tail = last; + *queue_rqs = matched_rqs; + return depth; +} + +static void blk_mq_dispatch_queue_requests(struct rq_list *rqs, unsigned depth) +{ + struct request_queue *q = rq_list_peek(rqs)->q; + + trace_block_unplug(q, depth, true); + + /* + * Peek first request and see if we have a ->queue_rqs() hook. + * If we do, we can dispatch the whole list in one go. + * We already know at this point that all requests belong to the + * same queue, caller must ensure that's the case. + */ + if (q->mq_ops->queue_rqs) { + blk_mq_run_dispatch_ops(q, __blk_mq_flush_list(q, rqs)); + if (rq_list_empty(rqs)) + return; + } + + blk_mq_run_dispatch_ops(q, blk_mq_issue_direct(rqs)); } -static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) +static void blk_mq_dispatch_list(struct rq_list *rqs, bool from_sched) { struct blk_mq_hw_ctx *this_hctx = NULL; struct blk_mq_ctx *this_ctx = NULL; @@ -2835,7 +2887,7 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) LIST_HEAD(list); do { - struct request *rq = rq_list_pop(&plug->mq_list); + struct request *rq = rq_list_pop(rqs); if (!this_hctx) { this_hctx = rq->mq_hctx; @@ -2848,9 +2900,9 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) } list_add_tail(&rq->queuelist, &list); depth++; - } while (!rq_list_empty(&plug->mq_list)); + } while (!rq_list_empty(rqs)); - plug->mq_list = requeue_list; + *rqs = requeue_list; trace_block_unplug(this_hctx->queue, depth, !from_sched); percpu_ref_get(&this_hctx->queue->q_usage_counter); @@ -2870,9 +2922,21 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) percpu_ref_put(&this_hctx->queue->q_usage_counter); } +static void blk_mq_dispatch_multiple_queue_requests(struct rq_list *rqs) +{ + do { + struct rq_list queue_rqs; + unsigned depth; + + depth = blk_mq_extract_queue_requests(rqs, &queue_rqs); + blk_mq_dispatch_queue_requests(&queue_rqs, depth); + while (!rq_list_empty(&queue_rqs)) + blk_mq_dispatch_list(&queue_rqs, false); + } while (!rq_list_empty(rqs)); +} + void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) { - struct request *rq; unsigned int depth; /* @@ -2887,34 +2951,19 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) depth = plug->rq_count; plug->rq_count = 0; - if (!plug->multiple_queues && !plug->has_elevator && !from_schedule) { - struct request_queue *q; - - rq = rq_list_peek(&plug->mq_list); - q = rq->q; - trace_block_unplug(q, depth, true); - - /* - * Peek first request and see if we have a ->queue_rqs() hook. - * If we do, we can dispatch the whole plug list in one go. We - * already know at this point that all requests belong to the - * same queue, caller must ensure that's the case. - */ - if (q->mq_ops->queue_rqs) { - blk_mq_run_dispatch_ops(q, - __blk_mq_flush_plug_list(q, plug)); - if (rq_list_empty(&plug->mq_list)) - return; + if (!plug->has_elevator && !from_schedule) { + if (plug->multiple_queues) { + blk_mq_dispatch_multiple_queue_requests(&plug->mq_list); + return; } - blk_mq_run_dispatch_ops(q, - blk_mq_plug_issue_direct(plug)); + blk_mq_dispatch_queue_requests(&plug->mq_list, depth); if (rq_list_empty(&plug->mq_list)) return; } do { - blk_mq_dispatch_plug_list(plug, from_schedule); + blk_mq_dispatch_list(&plug->mq_list, from_schedule); } while (!rq_list_empty(&plug->mq_list)); } @@ -2969,8 +3018,14 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, { struct blk_mq_alloc_data data = { .q = q, - .nr_tags = 1, + .flags = 0, + .shallow_depth = 0, .cmd_flags = bio->bi_opf, + .rq_flags = 0, + .nr_tags = 1, + .cached_rqs = NULL, + .ctx = NULL, + .hctx = NULL }; struct request *rq; @@ -2991,7 +3046,7 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, /* * Check if there is a suitable cached request and return it. */ -static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, +static struct request *blk_mq_get_cached_request(struct blk_plug *plug, struct request_queue *q, blk_opf_t opf) { enum hctx_type type = blk_mq_get_hctx_type(opf); @@ -3007,27 +3062,10 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, return NULL; if (op_is_flush(rq->cmd_flags) != op_is_flush(opf)) return NULL; + rq_list_pop(&plug->cached_rqs); return rq; } -static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug, - struct bio *bio) -{ - if (rq_list_pop(&plug->cached_rqs) != rq) - WARN_ON_ONCE(1); - - /* - * If any qos ->throttle() end up blocking, we will have flushed the - * plug and hence killed the cached_rq list as well. Pop this entry - * before we throttle. - */ - rq_qos_throttle(rq->q, bio); - - blk_mq_rq_time_init(rq, blk_time_get_ns()); - rq->cmd_flags = bio->bi_opf; - INIT_LIST_HEAD(&rq->queuelist); -} - static bool bio_unaligned(const struct bio *bio, struct request_queue *q) { unsigned int bs_mask = queue_logical_block_size(q) - 1; @@ -3057,6 +3095,7 @@ void blk_mq_submit_bio(struct bio *bio) struct request_queue *q = bdev_get_queue(bio->bi_bdev); struct blk_plug *plug = current->plug; const int is_sync = op_is_sync(bio->bi_opf); + unsigned int integrity_action; struct blk_mq_hw_ctx *hctx; unsigned int nr_segs; struct request *rq; @@ -3065,7 +3104,7 @@ void blk_mq_submit_bio(struct bio *bio) /* * If the plug has a cached request for this queue, try to use it. */ - rq = blk_mq_peek_cached_request(plug, q, bio->bi_opf); + rq = blk_mq_get_cached_request(plug, q, bio->bi_opf); /* * A BIO that was released from a zone write plug has already been @@ -3080,8 +3119,6 @@ void blk_mq_submit_bio(struct bio *bio) goto new_request; } - bio = blk_queue_bounce(bio, q); - /* * The cached request already holds a q_usage_counter reference and we * don't have to acquire a new one if we use it. @@ -3102,8 +3139,7 @@ void blk_mq_submit_bio(struct bio *bio) } if ((bio->bi_opf & REQ_POLLED) && !blk_mq_can_poll(q)) { - bio->bi_status = BLK_STS_NOTSUPP; - bio_endio(bio); + bio_endio_status(bio, BLK_STS_NOTSUPP); goto queue_exit; } @@ -3111,18 +3147,25 @@ void blk_mq_submit_bio(struct bio *bio) if (!bio) goto queue_exit; - if (!bio_integrity_prep(bio)) - goto queue_exit; + integrity_action = bio_integrity_action(bio); + if (integrity_action) + bio_integrity_prep(bio, integrity_action); + blk_mq_bio_issue_init(q, bio); if (blk_mq_attempt_bio_merge(q, bio, nr_segs)) goto queue_exit; - if (blk_queue_is_zoned(q) && blk_zone_plug_bio(bio, nr_segs)) - goto queue_exit; + if (bio_needs_zone_write_plugging(bio)) { + if (blk_zone_plug_bio(bio, nr_segs)) + goto queue_exit; + } new_request: if (rq) { - blk_mq_use_cached_rq(rq, plug, bio); + rq_qos_throttle(rq->q, bio); + blk_mq_rq_time_init(rq, blk_time_get_ns()); + rq->cmd_flags = bio->bi_opf; + INIT_LIST_HEAD(&rq->queuelist); } else { rq = blk_mq_get_new_requests(q, plug, bio); if (unlikely(!rq)) { @@ -3140,8 +3183,7 @@ new_request: ret = blk_crypto_rq_get_keyslot(rq); if (ret != BLK_STS_OK) { - bio->bi_status = ret; - bio_endio(bio); + bio_endio_status(bio, ret); blk_mq_free_request(rq); return; } @@ -3168,12 +3210,10 @@ new_request: return; queue_exit: - /* - * Don't drop the queue reference if we were trying to use a cached - * request and thus didn't acquire one. - */ if (!rq) blk_queue_exit(q); + else + rq_list_add_head(&plug->cached_rqs, rq); } #ifdef CONFIG_BLK_MQ_STACKING @@ -3218,6 +3258,25 @@ blk_status_t blk_insert_cloned_request(struct request *rq) return BLK_STS_IOERR; } + /* + * Integrity segment counting depends on the same queue limits + * (virt_boundary_mask, seg_boundary_mask, max_segment_size) that + * vary across stacked queues, so recompute against the bottom + * queue just like nr_phys_segments above. + */ + if (blk_integrity_rq(rq) && rq->bio) { + unsigned short max_int_segs = queue_max_integrity_segments(q); + + rq->nr_integrity_segments = + blk_rq_count_integrity_sg(rq->q, rq->bio); + if (rq->nr_integrity_segments > max_int_segs) { + printk(KERN_ERR "%s: over max integrity segments limit. (%u > %u)\n", + __func__, rq->nr_integrity_segments, + max_int_segs); + return BLK_STS_IOERR; + } + } + if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq))) return BLK_STS_IOERR; @@ -3314,6 +3373,7 @@ int blk_rq_prep_clone(struct request *rq, struct request *rq_src, } rq->nr_phys_segments = rq_src->nr_phys_segments; rq->nr_integrity_segments = rq_src->nr_integrity_segments; + rq->phys_gap_bit = rq_src->phys_gap_bit; if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0) goto free_and_out; @@ -3334,6 +3394,25 @@ EXPORT_SYMBOL_GPL(blk_rq_prep_clone); */ void blk_steal_bios(struct bio_list *list, struct request *rq) { + struct bio *bio; + + for (bio = rq->bio; bio; bio = bio->bi_next) { + if (bio->bi_opf & REQ_POLLED) { + bio->bi_opf &= ~REQ_POLLED; + bio->bi_cookie = BLK_QC_T_NONE; + } + /* + * The alternate request queue that we may end up submitting + * the bio to may be frozen temporarily, in this case REQ_NOWAIT + * will fail the I/O immediately with EAGAIN to the issuer. + * We are not in the issuer context which cannot block. Clear + * the flag to avoid spurious EAGAIN I/O failures. + */ + bio->bi_opf &= ~REQ_NOWAIT; + bio_clear_flag(bio, BIO_QOS_THROTTLED); + bio_clear_flag(bio, BIO_QOS_MERGED); + } + if (rq->bio) { if (list->tail) list->tail->bi_next = rq->bio; @@ -3359,7 +3438,6 @@ static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags, struct blk_mq_tags *tags) { struct page *page; - unsigned long flags; /* * There is no need to clear mapping if driver tags is not initialized @@ -3383,22 +3461,12 @@ static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags, } } } - - /* - * Wait until all pending iteration is done. - * - * Request reference is cleared and it is guaranteed to be observed - * after the ->lock is released. - */ - spin_lock_irqsave(&drv_tags->lock, flags); - spin_unlock_irqrestore(&drv_tags->lock, flags); } void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, unsigned int hctx_idx) { struct blk_mq_tags *drv_tags; - struct page *page; if (list_empty(&tags->page_list)) return; @@ -3422,27 +3490,20 @@ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, } blk_mq_clear_rq_mapping(drv_tags, tags); - - while (!list_empty(&tags->page_list)) { - page = list_first_entry(&tags->page_list, struct page, lru); - list_del_init(&page->lru); - /* - * Remove kmemleak object previously allocated in - * blk_mq_alloc_rqs(). - */ - kmemleak_free(page_address(page)); - __free_pages(page, page->private); - } + /* + * Free request pages in SRCU callback, which is called from + * blk_mq_free_tags(). + */ } -void blk_mq_free_rq_map(struct blk_mq_tags *tags) +void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags) { kfree(tags->rqs); tags->rqs = NULL; kfree(tags->static_rqs); tags->static_rqs = NULL; - blk_mq_free_tags(tags); + blk_mq_free_tags(set, tags); } static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set, @@ -3504,7 +3565,7 @@ static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, err_free_rqs: kfree(tags->rqs); err_free_tags: - blk_mq_free_tags(tags); + blk_mq_free_tags(set, tags); return NULL; } @@ -3534,8 +3595,6 @@ static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, if (node == NUMA_NO_NODE) node = set->numa_node; - INIT_LIST_HEAD(&tags->page_list); - /* * rq_size is the size of the request plus driver payload, rounded * to the cacheline size @@ -3622,8 +3681,12 @@ static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx) struct rq_iter_data data = { .hctx = hctx, }; + int srcu_idx; + srcu_idx = srcu_read_lock(&hctx->queue->tag_set->tags_srcu); blk_mq_all_tag_iter(tags, blk_mq_has_request, &data); + srcu_read_unlock(&hctx->queue->tag_set->tags_srcu, srcu_idx); + return data.has_rq; } @@ -3657,8 +3720,9 @@ static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node) { struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_online); + int ret = 0; - if (blk_mq_hctx_has_online_cpu(hctx, cpu)) + if (!hctx->nr_ctx || blk_mq_hctx_has_online_cpu(hctx, cpu)) return 0; /* @@ -3677,12 +3741,24 @@ static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node) * frozen and there are no requests. */ if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) { - while (blk_mq_hctx_has_requests(hctx)) + while (blk_mq_hctx_has_requests(hctx)) { + /* + * The wakeup capable IRQ handler of block device is + * not called during suspend. Skip the loop by checking + * pm_wakeup_pending to prevent the deadlock and improve + * suspend latency. + */ + if (pm_wakeup_pending()) { + clear_bit(BLK_MQ_S_INACTIVE, &hctx->state); + ret = -EBUSY; + break; + } msleep(5); + } percpu_ref_put(&hctx->queue->q_usage_counter); } - return 0; + return ret; } /* @@ -3843,7 +3919,6 @@ static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags, unsigned int queue_depth, struct request *flush_rq) { int i; - unsigned long flags; /* The hw queue may not be mapped yet */ if (!tags) @@ -3853,15 +3928,14 @@ static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags, for (i = 0; i < queue_depth; i++) cmpxchg(&tags->rqs[i], flush_rq, NULL); +} - /* - * Wait until all pending iteration is done. - * - * Request reference is cleared and it is guaranteed to be observed - * after the ->lock is released. - */ - spin_lock_irqsave(&tags->lock, flags); - spin_unlock_irqrestore(&tags->lock, flags); +static void blk_free_flush_queue_callback(struct rcu_head *head) +{ + struct blk_flush_queue *fq = + container_of(head, struct blk_flush_queue, rcu_head); + + blk_free_flush_queue(fq); } /* hctx->ctxs will be freed in queue's release handler */ @@ -3883,7 +3957,9 @@ static void blk_mq_exit_hctx(struct request_queue *q, if (set->ops->exit_hctx) set->ops->exit_hctx(hctx, hctx_idx); - xa_erase(&q->hctx_table, hctx_idx); + call_srcu(&set->tags_srcu, &hctx->fq->rcu_head, + blk_free_flush_queue_callback); + hctx->fq = NULL; spin_lock(&q->unused_hctx_lock); list_add(&hctx->hctx_list, &q->unused_hctx_list); @@ -3908,29 +3984,32 @@ static int blk_mq_init_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) { + gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY; + + hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp); + if (!hctx->fq) + goto fail; + hctx->queue_num = hctx_idx; hctx->tags = set->tags[hctx_idx]; if (set->ops->init_hctx && set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) - goto fail; + goto fail_free_fq; if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx, hctx->numa_node)) goto exit_hctx; - if (xa_insert(&q->hctx_table, hctx_idx, hctx, GFP_KERNEL)) - goto exit_flush_rq; - return 0; - exit_flush_rq: - if (set->ops->exit_request) - set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx); exit_hctx: if (set->ops->exit_hctx) set->ops->exit_hctx(hctx, hctx_idx); + fail_free_fq: + blk_free_flush_queue(hctx->fq); + hctx->fq = NULL; fail: return -1; } @@ -3982,16 +4061,10 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); INIT_LIST_HEAD(&hctx->dispatch_wait.entry); - hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp); - if (!hctx->fq) - goto free_bitmap; - blk_mq_hctx_kobj_init(hctx); return hctx; - free_bitmap: - sbitmap_free(&hctx->ctx_map); free_ctxs: kfree(hctx->ctxs); free_cpumask: @@ -4045,7 +4118,7 @@ struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set, ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth); if (ret) { - blk_mq_free_rq_map(tags); + blk_mq_free_rq_map(set, tags); return NULL; } @@ -4073,7 +4146,7 @@ void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set, { if (tags) { blk_mq_free_rqs(set, tags, hctx_idx); - blk_mq_free_rq_map(tags); + blk_mq_free_rq_map(set, tags); } } @@ -4094,8 +4167,6 @@ static void blk_mq_map_swqueue(struct request_queue *q) struct blk_mq_ctx *ctx; struct blk_mq_tag_set *set = q->tag_set; - mutex_lock(&q->elevator_lock); - queue_for_each_hw_ctx(q, hctx, i) { cpumask_clear(hctx->cpumask); hctx->nr_ctx = 0; @@ -4187,12 +4258,16 @@ static void blk_mq_map_swqueue(struct request_queue *q) /* * Rule out isolated CPUs from hctx->cpumask to avoid - * running block kworker on isolated CPUs + * running block kworker on isolated CPUs. + * FIXME: cpuset should propagate further changes to isolated CPUs + * here. */ + rcu_read_lock(); for_each_cpu(cpu, hctx->cpumask) { if (cpu_is_isolated(cpu)) cpumask_clear_cpu(cpu, hctx->cpumask); } + rcu_read_unlock(); /* * Initialize batch roundrobin counts @@ -4200,8 +4275,6 @@ static void blk_mq_map_swqueue(struct request_queue *q) hctx->next_cpu = blk_mq_first_mapped_cpu(hctx); hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; } - - mutex_unlock(&q->elevator_lock); } /* @@ -4243,7 +4316,7 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q) struct blk_mq_tag_set *set = q->tag_set; mutex_lock(&set->tag_list_lock); - list_del(&q->tag_set_list); + list_del_rcu(&q->tag_set_list); if (list_is_singular(&set->tag_list)) { /* just transitioned to unshared */ set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED; @@ -4251,7 +4324,6 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q) blk_mq_update_tag_set_shared(set, false); } mutex_unlock(&set->tag_list_lock); - INIT_LIST_HEAD(&q->tag_set_list); } static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, @@ -4270,7 +4342,7 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, } if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED) queue_set_hctx_shared(q, true); - list_add_tail(&q->tag_set_list, &set->tag_list); + list_add_tail_rcu(&q->tag_set_list, &set->tag_list); mutex_unlock(&set->tag_list_lock); } @@ -4281,7 +4353,7 @@ static int blk_mq_alloc_ctxs(struct request_queue *q) struct blk_mq_ctxs *ctxs; int cpu; - ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL); + ctxs = kzalloc_obj(*ctxs); if (!ctxs) return -ENOMEM; @@ -4323,7 +4395,7 @@ void blk_mq_release(struct request_queue *q) kobject_put(&hctx->kobj); } - xa_destroy(&q->hctx_table); + kfree(q->queue_hw_ctx); /* * release .mq_kobj and sw queue's kobject now because @@ -4467,26 +4539,48 @@ static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx( static void __blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, struct request_queue *q) { - struct blk_mq_hw_ctx *hctx; - unsigned long i, j; + int i, j, end; + struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; + + if (q->nr_hw_queues < set->nr_hw_queues) { + struct blk_mq_hw_ctx **new_hctxs; + + new_hctxs = kcalloc_node(set->nr_hw_queues, + sizeof(*new_hctxs), GFP_KERNEL, + set->numa_node); + if (!new_hctxs) + return; + if (hctxs) + memcpy(new_hctxs, hctxs, q->nr_hw_queues * + sizeof(*hctxs)); + rcu_assign_pointer(q->queue_hw_ctx, new_hctxs); + /* + * Make sure reading the old queue_hw_ctx from other + * context concurrently won't trigger uaf. + */ + kfree_rcu_mightsleep(hctxs); + hctxs = new_hctxs; + } for (i = 0; i < set->nr_hw_queues; i++) { int old_node; int node = blk_mq_get_hctx_node(set, i); - struct blk_mq_hw_ctx *old_hctx = xa_load(&q->hctx_table, i); + struct blk_mq_hw_ctx *old_hctx = hctxs[i]; if (old_hctx) { old_node = old_hctx->numa_node; blk_mq_exit_hctx(q, set, old_hctx, i); } - if (!blk_mq_alloc_and_init_hctx(set, q, i, node)) { + hctxs[i] = blk_mq_alloc_and_init_hctx(set, q, i, node); + if (!hctxs[i]) { if (!old_hctx) break; pr_warn("Allocate new hctx on node %d fails, fallback to previous one on node %d\n", node, old_node); - hctx = blk_mq_alloc_and_init_hctx(set, q, i, old_node); - WARN_ON_ONCE(!hctx); + hctxs[i] = blk_mq_alloc_and_init_hctx(set, q, i, + old_node); + WARN_ON_ONCE(!hctxs[i]); } } /* @@ -4495,26 +4589,27 @@ static void __blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, */ if (i != set->nr_hw_queues) { j = q->nr_hw_queues; + end = i; } else { j = i; + end = q->nr_hw_queues; q->nr_hw_queues = set->nr_hw_queues; } - xa_for_each_start(&q->hctx_table, j, hctx, j) - blk_mq_exit_hctx(q, set, hctx, j); + for (; j < end; j++) { + struct blk_mq_hw_ctx *hctx = hctxs[j]; + + if (hctx) { + blk_mq_exit_hctx(q, set, hctx, j); + hctxs[j] = NULL; + } + } } static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, - struct request_queue *q, bool lock) + struct request_queue *q) { - if (lock) { - /* protect against switching io scheduler */ - mutex_lock(&q->elevator_lock); - __blk_mq_realloc_hw_ctxs(set, q); - mutex_unlock(&q->elevator_lock); - } else { - __blk_mq_realloc_hw_ctxs(set, q); - } + __blk_mq_realloc_hw_ctxs(set, q); /* unregister cpuhp callbacks for exited hctxs */ blk_mq_remove_hw_queues_cpuhp(q); @@ -4544,9 +4639,7 @@ int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, INIT_LIST_HEAD(&q->unused_hctx_list); spin_lock_init(&q->unused_hctx_lock); - xa_init(&q->hctx_table); - - blk_mq_realloc_hw_ctxs(set, q, false); + blk_mq_realloc_hw_ctxs(set, q); if (!q->nr_hw_queues) goto err_hctxs; @@ -4561,10 +4654,11 @@ int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, spin_lock_init(&q->requeue_lock); q->nr_requests = set->queue_depth; + q->async_depth = set->queue_depth; blk_mq_init_cpu_queues(q, set->nr_hw_queues); - blk_mq_add_queue_tag_set(set, q); blk_mq_map_swqueue(q); + blk_mq_add_queue_tag_set(set, q); return 0; err_hctxs: @@ -4690,38 +4784,45 @@ static void blk_mq_update_queue_map(struct blk_mq_tag_set *set) } } -static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set, - int new_nr_hw_queues) +static struct blk_mq_tags **blk_mq_prealloc_tag_set_tags( + struct blk_mq_tag_set *set, + int new_nr_hw_queues) { struct blk_mq_tags **new_tags; int i; if (set->nr_hw_queues >= new_nr_hw_queues) - goto done; + return NULL; new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *), GFP_KERNEL, set->numa_node); if (!new_tags) - return -ENOMEM; + return ERR_PTR(-ENOMEM); if (set->tags) memcpy(new_tags, set->tags, set->nr_hw_queues * sizeof(*set->tags)); - kfree(set->tags); - set->tags = new_tags; for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) { - if (!__blk_mq_alloc_map_and_rqs(set, i)) { - while (--i >= set->nr_hw_queues) - __blk_mq_free_map_and_rqs(set, i); - return -ENOMEM; + if (blk_mq_is_shared_tags(set->flags)) { + new_tags[i] = set->shared_tags; + } else { + new_tags[i] = blk_mq_alloc_map_and_rqs(set, i, + set->queue_depth); + if (!new_tags[i]) + goto out_unwind; } cond_resched(); } -done: - set->nr_hw_queues = new_nr_hw_queues; - return 0; + return new_tags; +out_unwind: + while (--i >= set->nr_hw_queues) { + if (!blk_mq_is_shared_tags(set->flags)) + blk_mq_free_map_and_rqs(set, new_tags[i], i); + } + kfree(new_tags); + return ERR_PTR(-ENOMEM); } /* @@ -4776,20 +4877,25 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) set->nr_hw_queues = nr_cpu_ids; if (set->flags & BLK_MQ_F_BLOCKING) { - set->srcu = kmalloc(sizeof(*set->srcu), GFP_KERNEL); + set->srcu = kmalloc_obj(*set->srcu); if (!set->srcu) return -ENOMEM; ret = init_srcu_struct(set->srcu); if (ret) goto out_free_srcu; } + ret = init_srcu_struct(&set->tags_srcu); + if (ret) + goto out_cleanup_srcu; + + init_rwsem(&set->update_nr_hwq_lock); ret = -ENOMEM; set->tags = kcalloc_node(set->nr_hw_queues, sizeof(struct blk_mq_tags *), GFP_KERNEL, set->numa_node); if (!set->tags) - goto out_cleanup_srcu; + goto out_cleanup_tags_srcu; for (i = 0; i < set->nr_maps; i++) { set->map[i].mq_map = kcalloc_node(nr_cpu_ids, @@ -4818,6 +4924,8 @@ out_free_mq_map: } kfree(set->tags); set->tags = NULL; +out_cleanup_tags_srcu: + cleanup_srcu_struct(&set->tags_srcu); out_cleanup_srcu: if (set->flags & BLK_MQ_F_BLOCKING) cleanup_srcu_struct(set->srcu); @@ -4863,6 +4971,9 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set) kfree(set->tags); set->tags = NULL; + + srcu_barrier(&set->tags_srcu); + cleanup_srcu_struct(&set->tags_srcu); if (set->flags & BLK_MQ_F_BLOCKING) { cleanup_srcu_struct(set->srcu); kfree(set->srcu); @@ -4870,144 +4981,138 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set) } EXPORT_SYMBOL(blk_mq_free_tag_set); -int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) +struct elevator_tags *blk_mq_update_nr_requests(struct request_queue *q, + struct elevator_tags *et, + unsigned int nr) { struct blk_mq_tag_set *set = q->tag_set; + struct elevator_tags *old_et = NULL; struct blk_mq_hw_ctx *hctx; - int ret; unsigned long i; - if (WARN_ON_ONCE(!q->mq_freeze_depth)) - return -EINVAL; - - if (!set) - return -EINVAL; - - if (q->nr_requests == nr) - return 0; - blk_mq_quiesce_queue(q); - ret = 0; - queue_for_each_hw_ctx(q, hctx, i) { - if (!hctx->tags) - continue; + if (blk_mq_is_shared_tags(set->flags)) { /* - * If we're using an MQ scheduler, just update the scheduler - * queue depth. This is similar to what the old code would do. + * Shared tags, for sched tags, we allocate max initially hence + * tags can't grow, see blk_mq_alloc_sched_tags(). */ - if (hctx->sched_tags) { - ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, - nr, true); - } else { - ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr, - false); + if (q->elevator) + blk_mq_tag_update_sched_shared_tags(q, nr); + else + blk_mq_tag_resize_shared_tags(set, nr); + } else if (!q->elevator) { + /* + * Non-shared hardware tags, nr is already checked from + * queue_requests_store() and tags can't grow. + */ + queue_for_each_hw_ctx(q, hctx, i) { + if (!hctx->tags) + continue; + sbitmap_queue_resize(&hctx->tags->bitmap_tags, + nr - hctx->tags->nr_reserved_tags); } - if (ret) - break; - if (q->elevator && q->elevator->type->ops.depth_updated) - q->elevator->type->ops.depth_updated(hctx); - } - if (!ret) { - q->nr_requests = nr; - if (blk_mq_is_shared_tags(set->flags)) { - if (q->elevator) - blk_mq_tag_update_sched_shared_tags(q); - else - blk_mq_tag_resize_shared_tags(set, nr); + } else if (nr <= q->elevator->et->nr_requests) { + /* Non-shared sched tags, and tags don't grow. */ + queue_for_each_hw_ctx(q, hctx, i) { + if (!hctx->sched_tags) + continue; + sbitmap_queue_resize(&hctx->sched_tags->bitmap_tags, + nr - hctx->sched_tags->nr_reserved_tags); } + } else { + /* Non-shared sched tags, and tags grow */ + queue_for_each_hw_ctx(q, hctx, i) + hctx->sched_tags = et->tags[i]; + old_et = q->elevator->et; + q->elevator->et = et; } - blk_mq_unquiesce_queue(q); + /* + * Preserve relative value, both nr and async_depth are at most 16 bit + * value, no need to worry about overflow. + */ + q->async_depth = max(q->async_depth * nr / q->nr_requests, 1); + q->nr_requests = nr; + if (q->elevator && q->elevator->type->ops.depth_updated) + q->elevator->type->ops.depth_updated(q); - return ret; + blk_mq_unquiesce_queue(q); + return old_et; } /* - * request_queue and elevator_type pair. - * It is just used by __blk_mq_update_nr_hw_queues to cache - * the elevator_type associated with a request_queue. + * Switch back to the elevator type stored in the xarray. */ -struct blk_mq_qe_pair { - struct list_head node; - struct request_queue *q; - struct elevator_type *type; -}; - -/* - * Cache the elevator_type in qe pair list and switch the - * io scheduler to 'none' - */ -static bool blk_mq_elv_switch_none(struct list_head *head, - struct request_queue *q) +static void blk_mq_elv_switch_back(struct request_queue *q, + struct xarray *elv_tbl) { - struct blk_mq_qe_pair *qe; + struct elv_change_ctx *ctx = xa_load(elv_tbl, q->id); - qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY); - if (!qe) - return false; - - /* Accessing q->elevator needs protection from ->elevator_lock. */ - mutex_lock(&q->elevator_lock); - - if (!q->elevator) { - kfree(qe); - goto unlock; - } + if (WARN_ON_ONCE(!ctx)) + return; - INIT_LIST_HEAD(&qe->node); - qe->q = q; - qe->type = q->elevator->type; - /* keep a reference to the elevator module as we'll switch back */ - __elevator_get(qe->type); - list_add(&qe->node, head); - elevator_disable(q); -unlock: - mutex_unlock(&q->elevator_lock); + /* The elv_update_nr_hw_queues unfreezes the queue. */ + elv_update_nr_hw_queues(q, ctx); - return true; + /* Drop the reference acquired in blk_mq_elv_switch_none. */ + if (ctx->type) + elevator_put(ctx->type); } -static struct blk_mq_qe_pair *blk_lookup_qe_pair(struct list_head *head, - struct request_queue *q) +/* + * Stores elevator name and type in ctx and set current elevator to none. + */ +static int blk_mq_elv_switch_none(struct request_queue *q, + struct xarray *elv_tbl) { - struct blk_mq_qe_pair *qe; + struct elv_change_ctx *ctx; - list_for_each_entry(qe, head, node) - if (qe->q == q) - return qe; + lockdep_assert_held_write(&q->tag_set->update_nr_hwq_lock); - return NULL; -} + /* + * Accessing q->elevator without holding q->elevator_lock is safe here + * because we're called from nr_hw_queue update which is protected by + * set->update_nr_hwq_lock in the writer context. So, scheduler update/ + * switch code (which acquires the same lock in the reader context) + * can't run concurrently. + */ + if (q->elevator) { + ctx = xa_load(elv_tbl, q->id); + if (WARN_ON_ONCE(!ctx)) + return -ENOENT; -static void blk_mq_elv_switch_back(struct list_head *head, - struct request_queue *q) -{ - struct blk_mq_qe_pair *qe; - struct elevator_type *t; + ctx->name = q->elevator->type->elevator_name; - qe = blk_lookup_qe_pair(head, q); - if (!qe) - return; - t = qe->type; - list_del(&qe->node); - kfree(qe); + /* + * Before we switch elevator to 'none', take a reference to + * the elevator module so that while nr_hw_queue update is + * running, no one can remove elevator module. We'd put the + * reference to elevator module later when we switch back + * elevator. + */ + __elevator_get(q->elevator->type); - mutex_lock(&q->elevator_lock); - elevator_switch(q, t); - /* drop the reference acquired in blk_mq_elv_switch_none */ - elevator_put(t); - mutex_unlock(&q->elevator_lock); + /* + * Store elevator type so that we can release the reference + * taken above later. + */ + ctx->type = q->elevator->type; + elevator_set_none(q); + } + return 0; } static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) { struct request_queue *q; - LIST_HEAD(head); int prev_nr_hw_queues = set->nr_hw_queues; unsigned int memflags; int i; + struct xarray elv_tbl; + struct blk_mq_tags **new_tags; + bool queues_frozen = false; lockdep_assert_held(&set->tag_list_lock); @@ -5019,8 +5124,18 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, return; memflags = memalloc_noio_save(); - list_for_each_entry(q, &set->tag_list, tag_set_list) - blk_mq_freeze_queue_nomemsave(q); + + xa_init(&elv_tbl); + if (blk_mq_alloc_sched_ctx_batch(&elv_tbl, set) < 0) + goto out_free_ctx; + + if (blk_mq_alloc_sched_res_batch(&elv_tbl, set, nr_hw_queues) < 0) + goto out_free_ctx; + + list_for_each_entry(q, &set->tag_list, tag_set_list) { + blk_mq_debugfs_unregister_hctxs(q); + blk_mq_sysfs_unregister_hctxs(q); + } /* * Switch IO scheduler to 'none', cleaning up the data associated @@ -5028,21 +5143,26 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, * updating the new sw to hw queue mappings. */ list_for_each_entry(q, &set->tag_list, tag_set_list) - if (!blk_mq_elv_switch_none(&head, q)) + if (blk_mq_elv_switch_none(q, &elv_tbl)) goto switch_back; - list_for_each_entry(q, &set->tag_list, tag_set_list) { - blk_mq_debugfs_unregister_hctxs(q); - blk_mq_sysfs_unregister_hctxs(q); - } + new_tags = blk_mq_prealloc_tag_set_tags(set, nr_hw_queues); + if (IS_ERR(new_tags)) + goto switch_back; - if (blk_mq_realloc_tag_set_tags(set, nr_hw_queues) < 0) - goto reregister; + list_for_each_entry(q, &set->tag_list, tag_set_list) + blk_mq_freeze_queue_nomemsave(q); + queues_frozen = true; + if (new_tags) { + kfree(set->tags); + set->tags = new_tags; + } + set->nr_hw_queues = nr_hw_queues; fallback: blk_mq_update_queue_map(set); list_for_each_entry(q, &set->tag_list, tag_set_list) { - blk_mq_realloc_hw_ctxs(set, q, true); + __blk_mq_realloc_hw_ctxs(set, q); if (q->nr_hw_queues != set->nr_hw_queues) { int i = prev_nr_hw_queues; @@ -5057,19 +5177,26 @@ fallback: } blk_mq_map_swqueue(q); } +switch_back: + /* The blk_mq_elv_switch_back unfreezes queue for us. */ + list_for_each_entry(q, &set->tag_list, tag_set_list) { + /* switch_back expects queue to be frozen */ + if (!queues_frozen) + blk_mq_freeze_queue_nomemsave(q); + blk_mq_elv_switch_back(q, &elv_tbl); + } -reregister: list_for_each_entry(q, &set->tag_list, tag_set_list) { blk_mq_sysfs_register_hctxs(q); blk_mq_debugfs_register_hctxs(q); - } -switch_back: - list_for_each_entry(q, &set->tag_list, tag_set_list) - blk_mq_elv_switch_back(&head, q); + blk_mq_remove_hw_queues_cpuhp(q); + blk_mq_add_hw_queues_cpuhp(q); + } - list_for_each_entry(q, &set->tag_list, tag_set_list) - blk_mq_unfreeze_queue_nomemrestore(q); +out_free_ctx: + blk_mq_free_sched_ctx_batch(&elv_tbl); + xa_destroy(&elv_tbl); memalloc_noio_restore(memflags); /* Free the excess tags when nr_hw_queues shrink. */ @@ -5079,36 +5206,31 @@ switch_back: void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) { + down_write(&set->update_nr_hwq_lock); mutex_lock(&set->tag_list_lock); __blk_mq_update_nr_hw_queues(set, nr_hw_queues); mutex_unlock(&set->tag_list_lock); + up_write(&set->update_nr_hwq_lock); } EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob, unsigned int flags) { - long state = get_current_state(); int ret; + unsigned long timeout = jiffies + 2; do { ret = q->mq_ops->poll(hctx, iob); - if (ret > 0) { - __set_current_state(TASK_RUNNING); + if (ret > 0) return ret; - } - - if (signal_pending_state(state, current)) - __set_current_state(TASK_RUNNING); - if (task_is_running(current)) + if (task_sigpending(current)) return 1; - if (ret < 0 || (flags & BLK_POLL_ONESHOT)) break; cpu_relax(); - } while (!need_resched()); + } while (!need_resched() && time_before(jiffies, timeout)); - __set_current_state(TASK_RUNNING); return 0; } @@ -5117,7 +5239,7 @@ int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, { if (!blk_mq_can_poll(q)) return 0; - return blk_hctx_poll(q, xa_load(&q->hctx_table, cookie), iob, flags); + return blk_hctx_poll(q, q->queue_hw_ctx[cookie], iob, flags); } int blk_rq_poll(struct request *rq, struct io_comp_batch *iob, diff --git a/block/blk-mq.h b/block/blk-mq.h index 3011a78cf16a..aa15d31aaae9 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -6,6 +6,7 @@ #include "blk-stat.h" struct blk_mq_tag_set; +struct elevator_tags; struct blk_mq_ctxs { struct kobject kobj; @@ -45,10 +46,12 @@ void blk_mq_submit_bio(struct bio *bio); int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, struct io_comp_batch *iob, unsigned int flags); void blk_mq_exit_queue(struct request_queue *q); -int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr); +struct elevator_tags *blk_mq_update_nr_requests(struct request_queue *q, + struct elevator_tags *tags, + unsigned int nr); void blk_mq_wake_waiters(struct request_queue *q); bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *, - unsigned int); + bool); void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list); struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *start); @@ -59,7 +62,7 @@ void blk_mq_put_rq_ref(struct request *rq); */ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, unsigned int hctx_idx); -void blk_mq_free_rq_map(struct blk_mq_tags *tags); +void blk_mq_free_rq_map(struct blk_mq_tag_set *set, struct blk_mq_tags *tags); struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set, unsigned int hctx_idx, unsigned int depth); void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set, @@ -81,7 +84,7 @@ static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue * enum hctx_type type, unsigned int cpu) { - return xa_load(&q->hctx_table, q->tag_set->map[type].mq_map[cpu]); + return queue_hctx((q), (q->tag_set->map[type].mq_map[cpu])); } static inline enum hctx_type blk_mq_get_hctx_type(blk_opf_t opf) @@ -110,6 +113,17 @@ static inline struct blk_mq_hw_ctx *blk_mq_map_queue(blk_opf_t opf, } /* + * Default to double of smaller one between hw queue_depth and + * 128, since we don't split into sync/async like the old code + * did. Additionally, this is a per-hw queue depth. + */ +static inline unsigned int blk_mq_default_nr_requests( + struct blk_mq_tag_set *set) +{ + return 2 * min_t(unsigned int, set->queue_depth, BLKDEV_DEFAULT_RQ); +} + +/* * sysfs helpers */ extern void blk_mq_sysfs_init(struct request_queue *q); @@ -162,7 +176,7 @@ struct blk_mq_alloc_data { struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int reserved_tags, unsigned int flags, int node); -void blk_mq_free_tags(struct blk_mq_tags *tags); +void blk_mq_free_tags(struct blk_mq_tag_set *set, struct blk_mq_tags *tags); unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data); unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags, @@ -170,11 +184,10 @@ unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags, void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, unsigned int tag); void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags); -int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, - struct blk_mq_tags **tags, unsigned int depth, bool can_grow); void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, unsigned int size); -void blk_mq_tag_update_sched_shared_tags(struct request_queue *q); +void blk_mq_tag_update_sched_shared_tags(struct request_queue *q, + unsigned int nr); void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool); void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn, @@ -246,10 +259,7 @@ static inline bool blk_mq_hw_queue_mapped(struct blk_mq_hw_ctx *hctx) return hctx->nr_ctx && hctx->tags; } -unsigned int blk_mq_in_flight(struct request_queue *q, - struct block_device *part); -void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part, - unsigned int inflight[2]); +void blk_mq_in_driver_rw(struct block_device *part, unsigned int inflight[2]); static inline void blk_mq_put_dispatch_budget(struct request_queue *q, int budget_token) diff --git a/block/blk-rq-qos.c b/block/blk-rq-qos.c index 95982bc46ba1..85cf74402a09 100644 --- a/block/blk-rq-qos.c +++ b/block/blk-rq-qos.c @@ -318,6 +318,7 @@ void rq_qos_exit(struct request_queue *q) q->rq_qos = rqos->next; rqos->ops->exit(rqos); } + blk_queue_flag_clear(QUEUE_FLAG_QOS_ENABLED, q); mutex_unlock(&q->rq_qos_mutex); } @@ -343,15 +344,9 @@ int rq_qos_add(struct rq_qos *rqos, struct gendisk *disk, enum rq_qos_id id, goto ebusy; rqos->next = q->rq_qos; q->rq_qos = rqos; + blk_queue_flag_set(QUEUE_FLAG_QOS_ENABLED, q); blk_mq_unfreeze_queue(q, memflags); - - if (rqos->ops->debugfs_attrs) { - mutex_lock(&q->debugfs_mutex); - blk_mq_debugfs_register_rqos(rqos); - mutex_unlock(&q->debugfs_mutex); - } - return 0; ebusy: blk_mq_unfreeze_queue(q, memflags); @@ -373,9 +368,7 @@ void rq_qos_del(struct rq_qos *rqos) break; } } + if (!q->rq_qos) + blk_queue_flag_clear(QUEUE_FLAG_QOS_ENABLED, q); blk_mq_unfreeze_queue(q, memflags); - - mutex_lock(&q->debugfs_mutex); - blk_mq_debugfs_unregister_rqos(rqos); - mutex_unlock(&q->debugfs_mutex); } diff --git a/block/blk-rq-qos.h b/block/blk-rq-qos.h index 37245c97ee61..a747a504fe42 100644 --- a/block/blk-rq-qos.h +++ b/block/blk-rq-qos.h @@ -112,41 +112,54 @@ void __rq_qos_queue_depth_changed(struct rq_qos *rqos); static inline void rq_qos_cleanup(struct request_queue *q, struct bio *bio) { - if (q->rq_qos) + if (test_bit(QUEUE_FLAG_QOS_ENABLED, &q->queue_flags) && q->rq_qos) __rq_qos_cleanup(q->rq_qos, bio); } static inline void rq_qos_done(struct request_queue *q, struct request *rq) { - if (q->rq_qos && !blk_rq_is_passthrough(rq)) + if (test_bit(QUEUE_FLAG_QOS_ENABLED, &q->queue_flags) && + q->rq_qos && !blk_rq_is_passthrough(rq)) __rq_qos_done(q->rq_qos, rq); } static inline void rq_qos_issue(struct request_queue *q, struct request *rq) { - if (q->rq_qos) + if (test_bit(QUEUE_FLAG_QOS_ENABLED, &q->queue_flags) && q->rq_qos) __rq_qos_issue(q->rq_qos, rq); } static inline void rq_qos_requeue(struct request_queue *q, struct request *rq) { - if (q->rq_qos) + if (test_bit(QUEUE_FLAG_QOS_ENABLED, &q->queue_flags) && q->rq_qos) __rq_qos_requeue(q->rq_qos, rq); } static inline void rq_qos_done_bio(struct bio *bio) { - if (bio->bi_bdev && (bio_flagged(bio, BIO_QOS_THROTTLED) || - bio_flagged(bio, BIO_QOS_MERGED))) { - struct request_queue *q = bdev_get_queue(bio->bi_bdev); - if (q->rq_qos) - __rq_qos_done_bio(q->rq_qos, bio); - } + struct request_queue *q; + + if (!bio->bi_bdev || (!bio_flagged(bio, BIO_QOS_THROTTLED) && + !bio_flagged(bio, BIO_QOS_MERGED))) + return; + + q = bdev_get_queue(bio->bi_bdev); + + /* + * A BIO may carry BIO_QOS_* flags even if the associated request_queue + * does not have rq_qos enabled. This can happen with stacked block + * devices — for example, NVMe multipath, where it's possible that the + * bottom device has QoS enabled but the top device does not. Therefore, + * always verify that q->rq_qos is present and QoS is enabled before + * calling __rq_qos_done_bio(). + */ + if (test_bit(QUEUE_FLAG_QOS_ENABLED, &q->queue_flags) && q->rq_qos) + __rq_qos_done_bio(q->rq_qos, bio); } static inline void rq_qos_throttle(struct request_queue *q, struct bio *bio) { - if (q->rq_qos) { + if (test_bit(QUEUE_FLAG_QOS_ENABLED, &q->queue_flags) && q->rq_qos) { bio_set_flag(bio, BIO_QOS_THROTTLED); __rq_qos_throttle(q->rq_qos, bio); } @@ -155,14 +168,14 @@ static inline void rq_qos_throttle(struct request_queue *q, struct bio *bio) static inline void rq_qos_track(struct request_queue *q, struct request *rq, struct bio *bio) { - if (q->rq_qos) + if (test_bit(QUEUE_FLAG_QOS_ENABLED, &q->queue_flags) && q->rq_qos) __rq_qos_track(q->rq_qos, rq, bio); } static inline void rq_qos_merge(struct request_queue *q, struct request *rq, struct bio *bio) { - if (q->rq_qos) { + if (test_bit(QUEUE_FLAG_QOS_ENABLED, &q->queue_flags) && q->rq_qos) { bio_set_flag(bio, BIO_QOS_MERGED); __rq_qos_merge(q->rq_qos, rq, bio); } @@ -170,7 +183,7 @@ static inline void rq_qos_merge(struct request_queue *q, struct request *rq, static inline void rq_qos_queue_depth_changed(struct request_queue *q) { - if (q->rq_qos) + if (test_bit(QUEUE_FLAG_QOS_ENABLED, &q->queue_flags) && q->rq_qos) __rq_qos_queue_depth_changed(q->rq_qos); } diff --git a/block/blk-settings.c b/block/blk-settings.c index 6b2dbe645d23..8274631290db 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -14,6 +14,8 @@ #include <linux/jiffies.h> #include <linux/gfp.h> #include <linux/dma-mapping.h> +#include <linux/t10-pi.h> +#include <linux/crc64.h> #include "blk.h" #include "blk-rq-qos.h" @@ -50,19 +52,36 @@ void blk_set_stacking_limits(struct queue_limits *lim) lim->max_sectors = UINT_MAX; lim->max_dev_sectors = UINT_MAX; lim->max_write_zeroes_sectors = UINT_MAX; + lim->max_hw_wzeroes_unmap_sectors = UINT_MAX; + lim->max_user_wzeroes_unmap_sectors = UINT_MAX; lim->max_hw_zone_append_sectors = UINT_MAX; lim->max_user_discard_sectors = UINT_MAX; + lim->atomic_write_hw_max = UINT_MAX; } EXPORT_SYMBOL(blk_set_stacking_limits); void blk_apply_bdi_limits(struct backing_dev_info *bdi, struct queue_limits *lim) { + u64 io_opt = lim->io_opt; + /* * For read-ahead of large files to be effective, we need to read ahead - * at least twice the optimal I/O size. + * at least twice the optimal I/O size. For rotational devices that do + * not report an optimal I/O size (e.g. ATA HDDs), use the maximum I/O + * size to avoid falling back to the (rather inefficient) small default + * read-ahead size. + * + * There is no hardware limitation for the read-ahead size and the user + * might have increased the read-ahead size through sysfs, so don't ever + * decrease it. */ - bdi->ra_pages = max(lim->io_opt * 2 / PAGE_SIZE, VM_READAHEAD_PAGES); + if (!io_opt && (lim->features & BLK_FEAT_ROTATIONAL)) + io_opt = (u64)lim->max_sectors << SECTOR_SHIFT; + + bdi->ra_pages = max3(bdi->ra_pages, + io_opt * 2 >> PAGE_SHIFT, + VM_READAHEAD_PAGES); bdi->io_pages = lim->max_sectors >> PAGE_SECTORS_SHIFT; } @@ -108,7 +127,7 @@ static int blk_validate_integrity_limits(struct queue_limits *lim) { struct blk_integrity *bi = &lim->integrity; - if (!bi->tuple_size) { + if (!bi->metadata_size) { if (bi->csum_type != BLK_INTEGRITY_CSUM_NONE || bi->tag_size || ((bi->flags & BLK_INTEGRITY_REF_TAG))) { pr_warn("invalid PI settings.\n"); @@ -118,11 +137,6 @@ static int blk_validate_integrity_limits(struct queue_limits *lim) return 0; } - if (lim->features & BLK_FEAT_BOUNCE_HIGH) { - pr_warn("no bounce buffer support for integrity metadata\n"); - return -EINVAL; - } - if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) { pr_warn("integrity support disabled.\n"); return -EINVAL; @@ -134,8 +148,63 @@ static int blk_validate_integrity_limits(struct queue_limits *lim) return -EINVAL; } - if (!bi->interval_exp) + if (bi->pi_offset + bi->pi_tuple_size > bi->metadata_size) { + pr_warn("pi_offset (%u) + pi_tuple_size (%u) exceeds metadata_size (%u)\n", + bi->pi_offset, bi->pi_tuple_size, bi->metadata_size); + return -EINVAL; + } + + switch (bi->csum_type) { + case BLK_INTEGRITY_CSUM_NONE: + if (bi->pi_tuple_size) { + pr_warn("pi_tuple_size must be 0 when checksum type is none\n"); + return -EINVAL; + } + break; + case BLK_INTEGRITY_CSUM_CRC: + case BLK_INTEGRITY_CSUM_IP: + if (bi->pi_tuple_size != sizeof(struct t10_pi_tuple)) { + pr_warn("pi_tuple_size mismatch for T10 PI: expected %zu, got %u\n", + sizeof(struct t10_pi_tuple), + bi->pi_tuple_size); + return -EINVAL; + } + break; + case BLK_INTEGRITY_CSUM_CRC64: + if (bi->pi_tuple_size != sizeof(struct crc64_pi_tuple)) { + pr_warn("pi_tuple_size mismatch for CRC64 PI: expected %zu, got %u\n", + sizeof(struct crc64_pi_tuple), + bi->pi_tuple_size); + return -EINVAL; + } + break; + } + + if (!bi->interval_exp) { bi->interval_exp = ilog2(lim->logical_block_size); + } else if (bi->interval_exp < SECTOR_SHIFT || + bi->interval_exp > ilog2(lim->logical_block_size)) { + pr_warn("invalid interval_exp %u\n", bi->interval_exp); + return -EINVAL; + } + + /* + * Some IO controllers can not handle data intervals straddling + * multiple bio_vecs. For those, enforce alignment so that those are + * never generated, and that each buffer is aligned as expected. + */ + if (!(bi->flags & BLK_SPLIT_INTERVAL_CAPABLE) && bi->csum_type) { + lim->dma_alignment = max(lim->dma_alignment, + (1U << bi->interval_exp) - 1); + } + + /* + * The block layer automatically adds integrity data for bios that don't + * already have it. Limit the I/O size so that a single maximum size + * metadata segment can cover the integrity data for the entire I/O. + */ + lim->max_sectors = min(lim->max_sectors, + max_integrity_io_size(lim) >> SECTOR_SHIFT); return 0; } @@ -177,13 +246,40 @@ static void blk_atomic_writes_update_limits(struct queue_limits *lim) lim->atomic_write_hw_boundary >> SECTOR_SHIFT; } +/* + * Test whether any boundary is aligned with any chunk size. Stacked + * devices store any stripe size in t->chunk_sectors. + */ +static bool blk_valid_atomic_writes_boundary(unsigned int chunk_sectors, + unsigned int boundary_sectors) +{ + if (!chunk_sectors || !boundary_sectors) + return true; + + if (boundary_sectors > chunk_sectors && + boundary_sectors % chunk_sectors) + return false; + + if (chunk_sectors > boundary_sectors && + chunk_sectors % boundary_sectors) + return false; + + return true; +} + static void blk_validate_atomic_write_limits(struct queue_limits *lim) { unsigned int boundary_sectors; + unsigned int atomic_write_hw_max_sectors = + lim->atomic_write_hw_max >> SECTOR_SHIFT; if (!(lim->features & BLK_FEAT_ATOMIC_WRITES)) goto unsupported; + /* UINT_MAX indicates stacked limits in initial state */ + if (lim->atomic_write_hw_max == UINT_MAX) + goto unsupported; + if (!lim->atomic_write_hw_max) goto unsupported; @@ -201,26 +297,19 @@ static void blk_validate_atomic_write_limits(struct queue_limits *lim) lim->atomic_write_hw_max)) goto unsupported; + if (WARN_ON_ONCE(lim->chunk_sectors && + atomic_write_hw_max_sectors > lim->chunk_sectors)) + goto unsupported; + boundary_sectors = lim->atomic_write_hw_boundary >> SECTOR_SHIFT; if (boundary_sectors) { if (WARN_ON_ONCE(lim->atomic_write_hw_max > lim->atomic_write_hw_boundary)) goto unsupported; - /* - * A feature of boundary support is that it disallows bios to - * be merged which would result in a merged request which - * crosses either a chunk sector or atomic write HW boundary, - * even though chunk sectors may be just set for performance. - * For simplicity, disallow atomic writes for a chunk sector - * which is non-zero and smaller than atomic write HW boundary. - * Furthermore, chunk sectors must be a multiple of atomic - * write HW boundary. Otherwise boundary support becomes - * complicated. - * Devices which do not conform to these rules can be dealt - * with if and when they show up. - */ - if (WARN_ON_ONCE(lim->chunk_sectors % boundary_sectors)) + + if (WARN_ON_ONCE(!blk_valid_atomic_writes_boundary( + lim->chunk_sectors, boundary_sectors))) goto unsupported; /* @@ -265,8 +354,12 @@ int blk_validate_limits(struct queue_limits *lim) pr_warn("Invalid logical block size (%d)\n", lim->logical_block_size); return -EINVAL; } - if (lim->physical_block_size < lim->logical_block_size) + if (lim->physical_block_size < lim->logical_block_size) { lim->physical_block_size = lim->logical_block_size; + } else if (!is_power_of_2(lim->physical_block_size)) { + pr_warn("Invalid physical block size (%d)\n", lim->physical_block_size); + return -EINVAL; + } /* * The minimum I/O size defaults to the physical block size unless @@ -332,15 +425,28 @@ int blk_validate_limits(struct queue_limits *lim) if (!lim->max_segments) lim->max_segments = BLK_MAX_SEGMENTS; + if (lim->max_hw_wzeroes_unmap_sectors && + lim->max_hw_wzeroes_unmap_sectors != lim->max_write_zeroes_sectors) + return -EINVAL; + lim->max_wzeroes_unmap_sectors = min(lim->max_hw_wzeroes_unmap_sectors, + lim->max_user_wzeroes_unmap_sectors); + lim->max_discard_sectors = min(lim->max_hw_discard_sectors, lim->max_user_discard_sectors); + /* + * When discard is not supported, discard_granularity should be reported + * as 0 to userspace. + */ + if (lim->max_discard_sectors) + lim->discard_granularity = + max(lim->discard_granularity, lim->physical_block_size); + else + lim->discard_granularity = 0; + if (!lim->max_discard_segments) lim->max_discard_segments = 1; - if (lim->discard_granularity < lim->physical_block_size) - lim->discard_granularity = lim->physical_block_size; - /* * By default there is no limit on the segment boundary alignment, * but if there is one it can't be smaller than the page size as @@ -373,12 +479,12 @@ int blk_validate_limits(struct queue_limits *lim) return -EINVAL; } - /* setup min segment size for building new segment in fast path */ + /* setup max segment size for building new segment in fast path */ if (lim->seg_boundary_mask > lim->max_segment_size - 1) seg_size = lim->max_segment_size; else seg_size = lim->seg_boundary_mask + 1; - lim->min_segment_size = min_t(unsigned int, seg_size, PAGE_SIZE); + lim->max_fast_segment_size = min_t(unsigned int, seg_size, PAGE_SIZE); /* * We require drivers to at least do logical block aligned I/O, but @@ -417,10 +523,11 @@ int blk_set_default_limits(struct queue_limits *lim) { /* * Most defaults are set by capping the bounds in blk_validate_limits, - * but max_user_discard_sectors is special and needs an explicit - * initialization to the max value here. + * but these limits are special and need an explicit initialization to + * the max value here. */ lim->max_user_discard_sectors = UINT_MAX; + lim->max_user_wzeroes_unmap_sectors = UINT_MAX; return blk_validate_limits(lim); } @@ -440,6 +547,8 @@ int queue_limits_commit_update(struct request_queue *q, { int error; + lockdep_assert_held(&q->limits_lock); + error = blk_validate_limits(lim); if (error) goto out_unlock; @@ -569,60 +678,51 @@ static bool blk_stack_atomic_writes_tail(struct queue_limits *t, return true; } -/* Check for valid boundary of first bottom device */ -static bool blk_stack_atomic_writes_boundary_head(struct queue_limits *t, - struct queue_limits *b) +static void blk_stack_atomic_writes_chunk_sectors(struct queue_limits *t) { - /* - * Ensure atomic write boundary is aligned with chunk sectors. Stacked - * devices store chunk sectors in t->io_min. - */ - if (b->atomic_write_hw_boundary > t->io_min && - b->atomic_write_hw_boundary % t->io_min) - return false; - if (t->io_min > b->atomic_write_hw_boundary && - t->io_min % b->atomic_write_hw_boundary) - return false; + unsigned int chunk_bytes; - t->atomic_write_hw_boundary = b->atomic_write_hw_boundary; - return true; -} - - -/* Check stacking of first bottom device */ -static bool blk_stack_atomic_writes_head(struct queue_limits *t, - struct queue_limits *b) -{ - if (b->atomic_write_hw_boundary && - !blk_stack_atomic_writes_boundary_head(t, b)) - return false; + if (!t->chunk_sectors) + return; - if (t->io_min <= SECTOR_SIZE) { - /* No chunk sectors, so use bottom device values directly */ - t->atomic_write_hw_unit_max = b->atomic_write_hw_unit_max; - t->atomic_write_hw_unit_min = b->atomic_write_hw_unit_min; - t->atomic_write_hw_max = b->atomic_write_hw_max; - return true; - } + /* + * If chunk sectors is so large that its value in bytes overflows + * UINT_MAX, then just shift it down so it definitely will fit. + * We don't support atomic writes of such a large size anyway. + */ + if (check_shl_overflow(t->chunk_sectors, SECTOR_SHIFT, &chunk_bytes)) + chunk_bytes = t->chunk_sectors; /* * Find values for limits which work for chunk size. * b->atomic_write_hw_unit_{min, max} may not be aligned with chunk - * size (t->io_min), as chunk size is not restricted to a power-of-2. + * size, as the chunk size is not restricted to a power-of-2. * So we need to find highest power-of-2 which works for the chunk * size. - * As an example scenario, we could have b->unit_max = 16K and - * t->io_min = 24K. For this case, reduce t->unit_max to a value - * aligned with both limits, i.e. 8K in this example. + * As an example scenario, we could have t->unit_max = 16K and + * t->chunk_sectors = 24KB. For this case, reduce t->unit_max to a + * value aligned with both limits, i.e. 8K in this example. */ - t->atomic_write_hw_unit_max = b->atomic_write_hw_unit_max; - while (t->io_min % t->atomic_write_hw_unit_max) - t->atomic_write_hw_unit_max /= 2; + t->atomic_write_hw_unit_max = min(t->atomic_write_hw_unit_max, + max_pow_of_two_factor(chunk_bytes)); - t->atomic_write_hw_unit_min = min(b->atomic_write_hw_unit_min, + t->atomic_write_hw_unit_min = min(t->atomic_write_hw_unit_min, t->atomic_write_hw_unit_max); - t->atomic_write_hw_max = min(b->atomic_write_hw_max, t->io_min); + t->atomic_write_hw_max = min(t->atomic_write_hw_max, chunk_bytes); +} + +/* Check stacking of first bottom device */ +static bool blk_stack_atomic_writes_head(struct queue_limits *t, + struct queue_limits *b) +{ + if (!blk_valid_atomic_writes_boundary(t->chunk_sectors, + b->atomic_write_hw_boundary >> SECTOR_SHIFT)) + return false; + t->atomic_write_hw_unit_max = b->atomic_write_hw_unit_max; + t->atomic_write_hw_unit_min = b->atomic_write_hw_unit_min; + t->atomic_write_hw_max = b->atomic_write_hw_max; + t->atomic_write_hw_boundary = b->atomic_write_hw_boundary; return true; } @@ -638,18 +738,15 @@ static void blk_stack_atomic_writes_limits(struct queue_limits *t, if (!blk_atomic_write_start_sect_aligned(start, b)) goto unsupported; - /* - * If atomic_write_hw_max is set, we have already stacked 1x bottom - * device, so check for compliance. - */ - if (t->atomic_write_hw_max) { + /* UINT_MAX indicates no stacking of bottom devices yet */ + if (t->atomic_write_hw_max == UINT_MAX) { + if (!blk_stack_atomic_writes_head(t, b)) + goto unsupported; + } else { if (!blk_stack_atomic_writes_tail(t, b)) goto unsupported; - return; } - - if (!blk_stack_atomic_writes_head(t, b)) - goto unsupported; + blk_stack_atomic_writes_chunk_sectors(t); return; unsupported: @@ -683,7 +780,8 @@ unsupported: int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, sector_t start) { - unsigned int top, bottom, alignment, ret = 0; + unsigned int top, bottom, alignment; + int ret = 0; t->features |= (b->features & BLK_FEAT_INHERIT_MASK); @@ -697,6 +795,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, t->features &= ~BLK_FEAT_NOWAIT; if (!(b->features & BLK_FEAT_POLL)) t->features &= ~BLK_FEAT_POLL; + if (!(b->features & BLK_FEAT_PCI_P2PDMA)) + t->features &= ~BLK_FEAT_PCI_P2PDMA; t->flags |= (b->flags & BLK_FLAG_MISALIGNED); @@ -707,6 +807,13 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, t->max_dev_sectors = min_not_zero(t->max_dev_sectors, b->max_dev_sectors); t->max_write_zeroes_sectors = min(t->max_write_zeroes_sectors, b->max_write_zeroes_sectors); + t->max_user_wzeroes_unmap_sectors = + min(t->max_user_wzeroes_unmap_sectors, + b->max_user_wzeroes_unmap_sectors); + t->max_hw_wzeroes_unmap_sectors = + min(t->max_hw_wzeroes_unmap_sectors, + b->max_hw_wzeroes_unmap_sectors); + t->max_hw_zone_append_sectors = min(t->max_hw_zone_append_sectors, b->max_hw_zone_append_sectors); @@ -778,7 +885,7 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, } /* chunk_sectors a multiple of the physical block size? */ - if ((t->chunk_sectors << 9) & (t->physical_block_size - 1)) { + if (t->chunk_sectors % (t->physical_block_size >> SECTOR_SHIFT)) { t->chunk_sectors = 0; t->flags |= BLK_FLAG_MISALIGNED; ret = -1; @@ -874,7 +981,7 @@ bool queue_limits_stack_integrity(struct queue_limits *t, return true; if (ti->flags & BLK_INTEGRITY_STACKED) { - if (ti->tuple_size != bi->tuple_size) + if (ti->metadata_size != bi->metadata_size) goto incompatible; if (ti->interval_exp != bi->interval_exp) goto incompatible; @@ -882,15 +989,22 @@ bool queue_limits_stack_integrity(struct queue_limits *t, goto incompatible; if (ti->csum_type != bi->csum_type) goto incompatible; + if (ti->pi_tuple_size != bi->pi_tuple_size) + goto incompatible; if ((ti->flags & BLK_INTEGRITY_REF_TAG) != (bi->flags & BLK_INTEGRITY_REF_TAG)) goto incompatible; + if ((ti->flags & BLK_SPLIT_INTERVAL_CAPABLE) && + !(bi->flags & BLK_SPLIT_INTERVAL_CAPABLE)) + ti->flags &= ~BLK_SPLIT_INTERVAL_CAPABLE; } else { ti->flags = BLK_INTEGRITY_STACKED; ti->flags |= (bi->flags & BLK_INTEGRITY_DEVICE_CAPABLE) | - (bi->flags & BLK_INTEGRITY_REF_TAG); + (bi->flags & BLK_INTEGRITY_REF_TAG) | + (bi->flags & BLK_SPLIT_INTERVAL_CAPABLE); ti->csum_type = bi->csum_type; - ti->tuple_size = bi->tuple_size; + ti->pi_tuple_size = bi->pi_tuple_size; + ti->metadata_size = bi->metadata_size; ti->pi_offset = bi->pi_offset; ti->interval_exp = bi->interval_exp; ti->tag_size = bi->tag_size; diff --git a/block/blk-stat.c b/block/blk-stat.c index eaf60097bbe1..de126e1ea5ac 100644 --- a/block/blk-stat.c +++ b/block/blk-stat.c @@ -76,7 +76,7 @@ void blk_stat_add(struct request *rq, u64 now) static void blk_stat_timer_fn(struct timer_list *t) { - struct blk_stat_callback *cb = from_timer(cb, t, timer); + struct blk_stat_callback *cb = timer_container_of(cb, t, timer); unsigned int bucket; int cpu; @@ -103,12 +103,11 @@ blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *), { struct blk_stat_callback *cb; - cb = kmalloc(sizeof(*cb), GFP_KERNEL); + cb = kmalloc_obj(*cb); if (!cb) return NULL; - cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat), - GFP_KERNEL); + cb->stat = kmalloc_objs(struct blk_rq_stat, buckets); if (!cb->stat) { kfree(cb); return NULL; @@ -162,7 +161,7 @@ void blk_stat_remove_callback(struct request_queue *q, blk_queue_flag_clear(QUEUE_FLAG_STATS, q); spin_unlock_irqrestore(&q->stats->lock, flags); - del_timer_sync(&cb->timer); + timer_delete_sync(&cb->timer); } static void blk_stat_free_callback_rcu(struct rcu_head *head) @@ -207,7 +206,7 @@ struct blk_queue_stats *blk_alloc_queue_stats(void) { struct blk_queue_stats *stats; - stats = kmalloc(sizeof(*stats), GFP_KERNEL); + stats = kmalloc_obj(*stats); if (!stats) return NULL; diff --git a/block/blk-stat.h b/block/blk-stat.h index 5d7f18ba436d..cc5b66e7ee60 100644 --- a/block/blk-stat.h +++ b/block/blk-stat.h @@ -17,7 +17,7 @@ * timer fires, @cpu_stat is flushed to @stat and @timer_fn is invoked. */ struct blk_stat_callback { - /* + /** * @list: RCU list of callbacks for a &struct request_queue. */ struct list_head list; @@ -50,7 +50,7 @@ struct blk_stat_callback { struct blk_rq_stat *stat; /** - * @fn: Callback function. + * @timer_fn: Callback function. */ void (*timer_fn)(struct blk_stat_callback *); @@ -59,6 +59,9 @@ struct blk_stat_callback { */ void *data; + /** + * @rcu: rcu list head + */ struct rcu_head rcu; }; @@ -126,6 +129,8 @@ void blk_stat_free_callback(struct blk_stat_callback *cb); * blk_stat_is_active() - Check if a block statistics callback is currently * gathering statistics. * @cb: The callback. + * + * Returns: %true iff the callback is active. */ static inline bool blk_stat_is_active(struct blk_stat_callback *cb) { @@ -148,7 +153,7 @@ static inline void blk_stat_activate_nsecs(struct blk_stat_callback *cb, static inline void blk_stat_deactivate(struct blk_stat_callback *cb) { - del_timer_sync(&cb->timer); + timer_delete_sync(&cb->timer); } /** diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index a2882751f0d2..520972676ab4 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -19,6 +19,7 @@ #include "blk-wbt.h" #include "blk-cgroup.h" #include "blk-throttle.h" +#include "error-injection.h" struct queue_sysfs_entry { struct attribute attr; @@ -64,10 +65,89 @@ static ssize_t queue_requests_show(struct gendisk *disk, char *page) static ssize_t queue_requests_store(struct gendisk *disk, const char *page, size_t count) { - unsigned long nr; - int ret, err; + struct request_queue *q = disk->queue; + struct blk_mq_tag_set *set = q->tag_set; + struct elevator_tags *et = NULL; unsigned int memflags; + unsigned long nr; + int ret; + + ret = queue_var_store(&nr, page, count); + if (ret < 0) + return ret; + + /* + * Serialize updating nr_requests with concurrent queue_requests_store() + * and switching elevator. + * + * Use trylock to avoid circular lock dependency with kernfs active + * reference during concurrent disk deletion: + * update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del) + * kn->active -> update_nr_hwq_lock (via this sysfs write path) + */ + if (!down_write_trylock(&set->update_nr_hwq_lock)) + return -EBUSY; + + if (nr == q->nr_requests) + goto unlock; + + if (nr < BLKDEV_MIN_RQ) + nr = BLKDEV_MIN_RQ; + + /* + * Switching elevator is protected by update_nr_hwq_lock: + * - read lock is held from elevator sysfs attribute; + * - write lock is held from updating nr_hw_queues; + * Hence it's safe to access q->elevator here with write lock held. + */ + if (nr <= set->reserved_tags || + (q->elevator && nr > MAX_SCHED_RQ) || + (!q->elevator && nr > set->queue_depth)) { + ret = -EINVAL; + goto unlock; + } + + if (!blk_mq_is_shared_tags(set->flags) && q->elevator && + nr > q->elevator->et->nr_requests) { + /* + * Tags will grow, allocate memory before freezing queue to + * prevent deadlock. + */ + et = blk_mq_alloc_sched_tags(set, q->nr_hw_queues, nr); + if (!et) { + ret = -ENOMEM; + goto unlock; + } + } + + memflags = blk_mq_freeze_queue(q); + mutex_lock(&q->elevator_lock); + et = blk_mq_update_nr_requests(q, et, nr); + mutex_unlock(&q->elevator_lock); + blk_mq_unfreeze_queue(q, memflags); + + if (et) + blk_mq_free_sched_tags(et, set); + +unlock: + up_write(&set->update_nr_hwq_lock); + return ret; +} + +static ssize_t queue_async_depth_show(struct gendisk *disk, char *page) +{ + guard(mutex)(&disk->queue->elevator_lock); + + return queue_var_show(disk->queue->async_depth, page); +} + +static ssize_t +queue_async_depth_store(struct gendisk *disk, const char *page, size_t count) +{ struct request_queue *q = disk->queue; + unsigned int memflags; + unsigned long nr; + int ret; if (!queue_is_mq(q)) return -EINVAL; @@ -76,16 +156,21 @@ queue_requests_store(struct gendisk *disk, const char *page, size_t count) if (ret < 0) return ret; - memflags = blk_mq_freeze_queue(q); - mutex_lock(&q->elevator_lock); - if (nr < BLKDEV_MIN_RQ) - nr = BLKDEV_MIN_RQ; + if (nr == 0) + return -EINVAL; - err = blk_mq_update_nr_requests(disk->queue, nr); - if (err) - ret = err; - mutex_unlock(&q->elevator_lock); + memflags = blk_mq_freeze_queue(q); + scoped_guard(mutex, &q->elevator_lock) { + if (q->elevator) { + q->async_depth = min(q->nr_requests, nr); + if (q->elevator->type->ops.depth_updated) + q->elevator->type->ops.depth_updated(q); + } else { + ret = -EINVAL; + } + } blk_mq_unfreeze_queue(q, memflags); + return ret; } @@ -105,21 +190,22 @@ queue_ra_store(struct gendisk *disk, const char *page, size_t count) { unsigned long ra_kb; ssize_t ret; - unsigned int memflags; struct request_queue *q = disk->queue; ret = queue_var_store(&ra_kb, page, count); if (ret < 0) return ret; /* - * ->ra_pages is protected by ->limits_lock because it is usually - * calculated from the queue limits by queue_limits_commit_update. + * The ->ra_pages change below is protected by ->limits_lock because it + * is usually calculated from the queue limits by + * queue_limits_commit_update(). + * + * bdi->ra_pages reads are not serialized against bdi->ra_pages writes. + * Use WRITE_ONCE() to write bdi->ra_pages once. */ mutex_lock(&q->limits_lock); - memflags = blk_mq_freeze_queue(q); - disk->bdi->ra_pages = ra_kb >> (PAGE_SHIFT - 10); + WRITE_ONCE(disk->bdi->ra_pages, ra_kb >> (PAGE_SHIFT - 10)); mutex_unlock(&q->limits_lock); - blk_mq_unfreeze_queue(q, memflags); return ret; } @@ -134,6 +220,8 @@ QUEUE_SYSFS_LIMIT_SHOW(max_segments) QUEUE_SYSFS_LIMIT_SHOW(max_discard_segments) QUEUE_SYSFS_LIMIT_SHOW(max_integrity_segments) QUEUE_SYSFS_LIMIT_SHOW(max_segment_size) +QUEUE_SYSFS_LIMIT_SHOW(max_write_streams) +QUEUE_SYSFS_LIMIT_SHOW(write_stream_granularity) QUEUE_SYSFS_LIMIT_SHOW(logical_block_size) QUEUE_SYSFS_LIMIT_SHOW(physical_block_size) QUEUE_SYSFS_LIMIT_SHOW(chunk_sectors) @@ -159,6 +247,8 @@ static ssize_t queue_##_field##_show(struct gendisk *disk, char *page) \ QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_discard_sectors) QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_hw_discard_sectors) QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_write_zeroes_sectors) +QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_hw_wzeroes_unmap_sectors) +QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_wzeroes_unmap_sectors) QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_max_sectors) QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_boundary_sectors) QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_zone_append_sectors) @@ -203,6 +293,24 @@ static int queue_max_discard_sectors_store(struct gendisk *disk, return 0; } +static int queue_max_wzeroes_unmap_sectors_store(struct gendisk *disk, + const char *page, size_t count, struct queue_limits *lim) +{ + unsigned long max_zeroes_bytes, max_hw_zeroes_bytes; + ssize_t ret; + + ret = queue_var_store(&max_zeroes_bytes, page, count); + if (ret < 0) + return ret; + + max_hw_zeroes_bytes = lim->max_hw_wzeroes_unmap_sectors << SECTOR_SHIFT; + if (max_zeroes_bytes != 0 && max_zeroes_bytes != max_hw_zeroes_bytes) + return -EINVAL; + + lim->max_user_wzeroes_unmap_sectors = max_zeroes_bytes >> SECTOR_SHIFT; + return 0; +} + static int queue_max_sectors_store(struct gendisk *disk, const char *page, size_t count, struct queue_limits *lim) @@ -283,6 +391,36 @@ static ssize_t queue_nr_zones_show(struct gendisk *disk, char *page) return queue_var_show(disk_nr_zones(disk), page); } +static ssize_t queue_zoned_qd1_writes_show(struct gendisk *disk, char *page) +{ + return queue_var_show(!!blk_queue_zoned_qd1_writes(disk->queue), + page); +} + +static ssize_t queue_zoned_qd1_writes_store(struct gendisk *disk, + const char *page, size_t count) +{ + struct request_queue *q = disk->queue; + unsigned long qd1_writes; + unsigned int memflags; + ssize_t ret; + + ret = queue_var_store(&qd1_writes, page, count); + if (ret < 0) + return ret; + + memflags = blk_mq_freeze_queue(q); + blk_mq_quiesce_queue(q); + if (qd1_writes) + blk_queue_flag_set(QUEUE_FLAG_ZONED_QD1_WRITES, q); + else + blk_queue_flag_clear(QUEUE_FLAG_ZONED_QD1_WRITES, q); + blk_mq_unquiesce_queue(q); + blk_mq_unfreeze_queue(q, memflags); + + return count; +} + static ssize_t queue_iostats_passthrough_show(struct gendisk *disk, char *page) { return queue_var_show(!!blk_queue_passthrough_stat(disk->queue), page); @@ -315,21 +453,18 @@ static ssize_t queue_nomerges_store(struct gendisk *disk, const char *page, size_t count) { unsigned long nm; - unsigned int memflags; struct request_queue *q = disk->queue; ssize_t ret = queue_var_store(&nm, page, count); if (ret < 0) return ret; - memflags = blk_mq_freeze_queue(q); blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, q); blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, q); if (nm == 2) blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q); else if (nm) blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q); - blk_mq_unfreeze_queue(q, memflags); return ret; } @@ -349,7 +484,6 @@ queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count) #ifdef CONFIG_SMP struct request_queue *q = disk->queue; unsigned long val; - unsigned int memflags; ret = queue_var_store(&val, page, count); if (ret < 0) @@ -361,7 +495,6 @@ queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count) * are accessed individually using atomic test_bit operation. So we * don't grab any lock while updating these flags. */ - memflags = blk_mq_freeze_queue(q); if (val == 2) { blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q); blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q); @@ -372,7 +505,6 @@ queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count) blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q); blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q); } - blk_mq_unfreeze_queue(q, memflags); #endif return ret; } @@ -386,11 +518,9 @@ static ssize_t queue_poll_delay_store(struct gendisk *disk, const char *page, static ssize_t queue_poll_store(struct gendisk *disk, const char *page, size_t count) { - unsigned int memflags; ssize_t ret = count; struct request_queue *q = disk->queue; - memflags = blk_mq_freeze_queue(q); if (!(q->limits.features & BLK_FEAT_POLL)) { ret = -EINVAL; goto out; @@ -399,7 +529,6 @@ static ssize_t queue_poll_store(struct gendisk *disk, const char *page, pr_info_ratelimited("writes to the poll attribute are ignored.\n"); pr_info_ratelimited("please use driver specific parameters instead.\n"); out: - blk_mq_unfreeze_queue(q, memflags); return ret; } @@ -412,7 +541,7 @@ static ssize_t queue_io_timeout_show(struct gendisk *disk, char *page) static ssize_t queue_io_timeout_store(struct gendisk *disk, const char *page, size_t count) { - unsigned int val, memflags; + unsigned int val; int err; struct request_queue *q = disk->queue; @@ -420,9 +549,7 @@ static ssize_t queue_io_timeout_store(struct gendisk *disk, const char *page, if (err || val == 0) return -EINVAL; - memflags = blk_mq_freeze_queue(q); blk_queue_rq_timeout(q, msecs_to_jiffies(val)); - blk_mq_unfreeze_queue(q, memflags); return count; } @@ -455,39 +582,42 @@ static int queue_wc_store(struct gendisk *disk, const char *page, return 0; } -#define QUEUE_RO_ENTRY(_prefix, _name) \ -static struct queue_sysfs_entry _prefix##_entry = { \ - .attr = { .name = _name, .mode = 0444 }, \ - .show = _prefix##_show, \ +#define QUEUE_RO_ENTRY(_prefix, _name) \ +static const struct queue_sysfs_entry _prefix##_entry = { \ + .attr = { .name = _name, .mode = 0444 }, \ + .show = _prefix##_show, \ }; -#define QUEUE_RW_ENTRY(_prefix, _name) \ -static struct queue_sysfs_entry _prefix##_entry = { \ - .attr = { .name = _name, .mode = 0644 }, \ - .show = _prefix##_show, \ - .store = _prefix##_store, \ +#define QUEUE_RW_ENTRY(_prefix, _name) \ +static const struct queue_sysfs_entry _prefix##_entry = { \ + .attr = { .name = _name, .mode = 0644 }, \ + .show = _prefix##_show, \ + .store = _prefix##_store, \ }; #define QUEUE_LIM_RO_ENTRY(_prefix, _name) \ -static struct queue_sysfs_entry _prefix##_entry = { \ +static const struct queue_sysfs_entry _prefix##_entry = { \ .attr = { .name = _name, .mode = 0444 }, \ .show_limit = _prefix##_show, \ } #define QUEUE_LIM_RW_ENTRY(_prefix, _name) \ -static struct queue_sysfs_entry _prefix##_entry = { \ +static const struct queue_sysfs_entry _prefix##_entry = { \ .attr = { .name = _name, .mode = 0644 }, \ .show_limit = _prefix##_show, \ .store_limit = _prefix##_store, \ } QUEUE_RW_ENTRY(queue_requests, "nr_requests"); +QUEUE_RW_ENTRY(queue_async_depth, "async_depth"); QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb"); QUEUE_LIM_RW_ENTRY(queue_max_sectors, "max_sectors_kb"); QUEUE_LIM_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb"); QUEUE_LIM_RO_ENTRY(queue_max_segments, "max_segments"); QUEUE_LIM_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments"); QUEUE_LIM_RO_ENTRY(queue_max_segment_size, "max_segment_size"); +QUEUE_LIM_RO_ENTRY(queue_max_write_streams, "max_write_streams"); +QUEUE_LIM_RO_ENTRY(queue_write_stream_granularity, "write_stream_granularity"); QUEUE_RW_ENTRY(elv_iosched, "scheduler"); QUEUE_LIM_RO_ENTRY(queue_logical_block_size, "logical_block_size"); @@ -510,10 +640,15 @@ QUEUE_LIM_RO_ENTRY(queue_atomic_write_unit_min, "atomic_write_unit_min_bytes"); QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes"); QUEUE_LIM_RO_ENTRY(queue_max_write_zeroes_sectors, "write_zeroes_max_bytes"); +QUEUE_LIM_RO_ENTRY(queue_max_hw_wzeroes_unmap_sectors, + "write_zeroes_unmap_max_hw_bytes"); +QUEUE_LIM_RW_ENTRY(queue_max_wzeroes_unmap_sectors, + "write_zeroes_unmap_max_bytes"); QUEUE_LIM_RO_ENTRY(queue_max_zone_append_sectors, "zone_append_max_bytes"); QUEUE_LIM_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity"); QUEUE_LIM_RO_ENTRY(queue_zoned, "zoned"); +QUEUE_RW_ENTRY(queue_zoned_qd1_writes, "zoned_qd1_writes"); QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones"); QUEUE_LIM_RO_ENTRY(queue_max_open_zones, "max_open_zones"); QUEUE_LIM_RO_ENTRY(queue_max_active_zones, "max_active_zones"); @@ -531,7 +666,7 @@ QUEUE_LIM_RO_ENTRY(queue_virt_boundary_mask, "virt_boundary_mask"); QUEUE_LIM_RO_ENTRY(queue_dma_alignment, "dma_alignment"); /* legacy alias for logical_block_size: */ -static struct queue_sysfs_entry queue_hw_sector_size_entry = { +static const struct queue_sysfs_entry queue_hw_sector_size_entry = { .attr = {.name = "hw_sector_size", .mode = 0444 }, .show_limit = queue_logical_block_size_show, }; @@ -560,7 +695,7 @@ static ssize_t queue_wb_lat_show(struct gendisk *disk, char *page) ssize_t ret; struct request_queue *q = disk->queue; - mutex_lock(&q->elevator_lock); + mutex_lock(&disk->rqos_state_mutex); if (!wbt_rq_qos(q)) { ret = -EINVAL; goto out; @@ -573,18 +708,15 @@ static ssize_t queue_wb_lat_show(struct gendisk *disk, char *page) ret = sysfs_emit(page, "%llu\n", div_u64(wbt_get_min_lat(q), 1000)); out: - mutex_unlock(&q->elevator_lock); + mutex_unlock(&disk->rqos_state_mutex); return ret; } static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page, size_t count) { - struct request_queue *q = disk->queue; - struct rq_qos *rqos; ssize_t ret; s64 val; - unsigned int memflags; ret = queue_var_store64(&val, page); if (ret < 0) @@ -592,47 +724,15 @@ static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page, if (val < -1) return -EINVAL; - memflags = blk_mq_freeze_queue(q); - mutex_lock(&q->elevator_lock); - - rqos = wbt_rq_qos(q); - if (!rqos) { - ret = wbt_init(disk); - if (ret) - goto out; - } - - ret = count; - if (val == -1) - val = wbt_default_latency_nsec(q); - else if (val >= 0) - val *= 1000ULL; - - if (wbt_get_min_lat(q) == val) - goto out; - - /* - * Ensure that the queue is idled, in case the latency update - * ends up either enabling or disabling wbt completely. We can't - * have IO inflight if that happens. - */ - blk_mq_quiesce_queue(q); - - wbt_set_min_lat(q, val); - - blk_mq_unquiesce_queue(q); -out: - mutex_unlock(&q->elevator_lock); - blk_mq_unfreeze_queue(q, memflags); - - return ret; + ret = wbt_set_lat(disk, val); + return ret ? ret : count; } QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec"); #endif /* Common attributes for bio-based and request-based queues. */ -static struct attribute *queue_attrs[] = { +static const struct attribute *const queue_attrs[] = { /* * Attributes which are protected with q->limits_lock. */ @@ -642,6 +742,8 @@ static struct attribute *queue_attrs[] = { &queue_max_discard_segments_entry.attr, &queue_max_integrity_segments_entry.attr, &queue_max_segment_size_entry.attr, + &queue_max_write_streams_entry.attr, + &queue_write_stream_granularity_entry.attr, &queue_hw_sector_size_entry.attr, &queue_logical_block_size_entry.attr, &queue_physical_block_size_entry.attr, @@ -656,6 +758,8 @@ static struct attribute *queue_attrs[] = { &queue_atomic_write_unit_min_entry.attr, &queue_atomic_write_unit_max_entry.attr, &queue_max_write_zeroes_sectors_entry.attr, + &queue_max_hw_wzeroes_unmap_sectors_entry.attr, + &queue_max_wzeroes_unmap_sectors_entry.attr, &queue_max_zone_append_sectors_entry.attr, &queue_zone_write_granularity_entry.attr, &queue_rotational_entry.attr, @@ -682,18 +786,20 @@ static struct attribute *queue_attrs[] = { &queue_nomerges_entry.attr, &queue_poll_entry.attr, &queue_poll_delay_entry.attr, + &queue_zoned_qd1_writes_entry.attr, NULL, }; /* Request-based queue attributes that are not relevant for bio-based queues. */ -static struct attribute *blk_mq_queue_attrs[] = { +static const struct attribute *const blk_mq_queue_attrs[] = { /* * Attributes which require some form of locking other than * q->sysfs_lock. */ &elv_iosched_entry.attr, &queue_requests_entry.attr, + &queue_async_depth_entry.attr, #ifdef CONFIG_BLK_WBT &queue_wb_lat_entry.attr, #endif @@ -706,14 +812,15 @@ static struct attribute *blk_mq_queue_attrs[] = { NULL, }; -static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr, +static umode_t queue_attr_visible(struct kobject *kobj, const struct attribute *attr, int n) { struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj); struct request_queue *q = disk->queue; if ((attr == &queue_max_open_zones_entry.attr || - attr == &queue_max_active_zones_entry.attr) && + attr == &queue_max_active_zones_entry.attr || + attr == &queue_zoned_qd1_writes_entry.attr) && !blk_queue_is_zoned(q)) return 0; @@ -721,7 +828,7 @@ static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr, } static umode_t blk_mq_queue_attr_visible(struct kobject *kobj, - struct attribute *attr, int n) + const struct attribute *attr, int n) { struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj); struct request_queue *q = disk->queue; @@ -735,17 +842,17 @@ static umode_t blk_mq_queue_attr_visible(struct kobject *kobj, return attr->mode; } -static struct attribute_group queue_attr_group = { - .attrs = queue_attrs, - .is_visible = queue_attr_visible, +static const struct attribute_group queue_attr_group = { + .attrs_const = queue_attrs, + .is_visible_const = queue_attr_visible, }; -static struct attribute_group blk_mq_queue_attr_group = { - .attrs = blk_mq_queue_attrs, - .is_visible = blk_mq_queue_attr_visible, +static const struct attribute_group blk_mq_queue_attr_group = { + .attrs_const = blk_mq_queue_attrs, + .is_visible_const = blk_mq_queue_attr_visible, }; -#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr) +#define to_queue(atr) container_of_const((atr), struct queue_sysfs_entry, attr) static ssize_t queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page) @@ -815,7 +922,7 @@ static void blk_queue_release(struct kobject *kobj) /* nothing to do here, all data is associated with the parent gendisk */ } -static const struct kobj_type blk_queue_ktype = { +const struct kobj_type blk_queue_ktype = { .default_groups = blk_queue_attr_groups, .sysfs_ops = &queue_sysfs_ops, .release = blk_queue_release, @@ -825,13 +932,15 @@ static void blk_debugfs_remove(struct gendisk *disk) { struct request_queue *q = disk->queue; - mutex_lock(&q->debugfs_mutex); + blk_debugfs_lock_nomemsave(q); blk_trace_shutdown(q); + if (IS_ENABLED(CONFIG_BLK_ERROR_INJECTION)) + blk_error_injection_exit(disk); debugfs_remove_recursive(q->debugfs_dir); q->debugfs_dir = NULL; q->sched_debugfs_dir = NULL; q->rqos_debugfs_dir = NULL; - mutex_unlock(&q->debugfs_mutex); + blk_debugfs_unlock_nomemrestore(q); } /** @@ -841,25 +950,35 @@ static void blk_debugfs_remove(struct gendisk *disk) int blk_register_queue(struct gendisk *disk) { struct request_queue *q = disk->queue; + unsigned int memflags; int ret; - kobject_init(&disk->queue_kobj, &blk_queue_ktype); ret = kobject_add(&disk->queue_kobj, &disk_to_dev(disk)->kobj, "queue"); if (ret < 0) - goto out_put_queue_kobj; + return ret; if (queue_is_mq(q)) { ret = blk_mq_sysfs_register(disk); if (ret) - goto out_put_queue_kobj; + goto out_del_queue_kobj; } mutex_lock(&q->sysfs_lock); - mutex_lock(&q->debugfs_mutex); + memflags = blk_debugfs_lock(q); q->debugfs_dir = debugfs_create_dir(disk->disk_name, blk_debugfs_root); + if (IS_ENABLED(CONFIG_BLK_ERROR_INJECTION)) + blk_error_injection_init(disk); if (queue_is_mq(q)) blk_mq_debugfs_register(q); - mutex_unlock(&q->debugfs_mutex); + blk_debugfs_unlock(q, memflags); + + /* + * For blk-mq rotational zoned devices, default to using QD=1 + * writes. For non-mq rotational zoned devices, the device driver can + * set an appropriate default. + */ + if (queue_is_mq(q) && blk_queue_rot(q) && blk_queue_is_zoned(q)) + blk_queue_flag_set(QUEUE_FLAG_ZONED_QD1_WRITES, q); ret = disk_register_independent_access_ranges(disk); if (ret) @@ -869,18 +988,11 @@ int blk_register_queue(struct gendisk *disk) if (ret) goto out_unregister_ia_ranges; - mutex_lock(&q->elevator_lock); - if (q->elevator) { - ret = elv_register_queue(q, false); - if (ret) { - mutex_unlock(&q->elevator_lock); - goto out_crypto_sysfs_unregister; - } - } - wbt_enable_default(disk); - mutex_unlock(&q->elevator_lock); + if (queue_is_mq(q)) + elevator_set_default(q); blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q); + wbt_init_enable_default(disk); /* Now everything is ready and send out KOBJ_ADD uevent */ kobject_uevent(&disk->queue_kobj, KOBJ_ADD); @@ -902,15 +1014,15 @@ int blk_register_queue(struct gendisk *disk) return ret; -out_crypto_sysfs_unregister: - blk_crypto_sysfs_unregister(disk); out_unregister_ia_ranges: disk_unregister_independent_access_ranges(disk); out_debugfs_remove: blk_debugfs_remove(disk); mutex_unlock(&q->sysfs_lock); -out_put_queue_kobj: - kobject_put(&disk->queue_kobj); + if (queue_is_mq(q)) + blk_mq_sysfs_unregister(disk); +out_del_queue_kobj: + kobject_del(&disk->queue_kobj); return ret; } @@ -949,10 +1061,6 @@ void blk_unregister_queue(struct gendisk *disk) blk_mq_sysfs_unregister(disk); blk_crypto_sysfs_unregister(disk); - mutex_lock(&q->elevator_lock); - elv_unregister_queue(q); - mutex_unlock(&q->elevator_lock); - mutex_lock(&q->sysfs_lock); disk_unregister_independent_access_ranges(disk); mutex_unlock(&q->sysfs_lock); @@ -961,5 +1069,8 @@ void blk_unregister_queue(struct gendisk *disk) kobject_uevent(&disk->queue_kobj, KOBJ_REMOVE); kobject_del(&disk->queue_kobj); + if (queue_is_mq(q)) + elevator_set_none(q); + blk_debugfs_remove(disk); } diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 91dab43c65ab..ffc3b70065d4 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -12,7 +12,6 @@ #include <linux/blktrace_api.h> #include "blk.h" #include "blk-cgroup-rwstat.h" -#include "blk-stat.h" #include "blk-throttle.h" /* Max dispatch from a group in 1 round */ @@ -22,9 +21,7 @@ #define THROTL_QUANTUM 32 /* Throttling is performed over a slice and after that slice is renewed */ -#define DFL_THROTL_SLICE_HD (HZ / 10) -#define DFL_THROTL_SLICE_SSD (HZ / 50) -#define MAX_THROTL_SLICE (HZ) +#define DFL_THROTL_SLICE (HZ / 10) /* A workqueue to queue throttle related work */ static struct workqueue_struct *kthrotld_workqueue; @@ -41,12 +38,8 @@ struct throtl_data /* Total Number of queued bios on READ and WRITE lists */ unsigned int nr_queued[2]; - unsigned int throtl_slice; - /* Work for dispatching throttled bios */ struct work_struct dispatch_work; - - bool track_bio_latency; }; static void throtl_pending_timer_fn(struct timer_list *t); @@ -143,7 +136,8 @@ static inline unsigned int throtl_bio_data_size(struct bio *bio) static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg) { INIT_LIST_HEAD(&qn->node); - bio_list_init(&qn->bios); + bio_list_init(&qn->bios_bps); + bio_list_init(&qn->bios_iops); qn->tg = tg; } @@ -151,18 +145,32 @@ static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg) * throtl_qnode_add_bio - add a bio to a throtl_qnode and activate it * @bio: bio being added * @qn: qnode to add bio to - * @queued: the service_queue->queued[] list @qn belongs to + * @sq: the service_queue @qn belongs to * - * Add @bio to @qn and put @qn on @queued if it's not already on. + * Add @bio to @qn and put @qn on @sq->queued if it's not already on. * @qn->tg's reference count is bumped when @qn is activated. See the * comment on top of throtl_qnode definition for details. */ static void throtl_qnode_add_bio(struct bio *bio, struct throtl_qnode *qn, - struct list_head *queued) + struct throtl_service_queue *sq) { - bio_list_add(&qn->bios, bio); + bool rw = bio_data_dir(bio); + + /* + * Split bios have already been throttled by bps, so they are + * directly queued into the iops path. + */ + if (bio_flagged(bio, BIO_TG_BPS_THROTTLED) || + bio_flagged(bio, BIO_BPS_THROTTLED)) { + bio_list_add(&qn->bios_iops, bio); + sq->nr_queued_iops[rw]++; + } else { + bio_list_add(&qn->bios_bps, bio); + sq->nr_queued_bps[rw]++; + } + if (list_empty(&qn->node)) { - list_add_tail(&qn->node, queued); + list_add_tail(&qn->node, &sq->queued[rw]); blkg_get(tg_to_blkg(qn->tg)); } } @@ -170,6 +178,10 @@ static void throtl_qnode_add_bio(struct bio *bio, struct throtl_qnode *qn, /** * throtl_peek_queued - peek the first bio on a qnode list * @queued: the qnode list to peek + * + * Always take a bio from the head of the iops queue first. If the queue is + * empty, we then take it from the bps queue to maintain the overall idea of + * fetching bios from the head. */ static struct bio *throtl_peek_queued(struct list_head *queued) { @@ -180,28 +192,33 @@ static struct bio *throtl_peek_queued(struct list_head *queued) return NULL; qn = list_first_entry(queued, struct throtl_qnode, node); - bio = bio_list_peek(&qn->bios); + bio = bio_list_peek(&qn->bios_iops); + if (!bio) + bio = bio_list_peek(&qn->bios_bps); WARN_ON_ONCE(!bio); return bio; } /** * throtl_pop_queued - pop the first bio form a qnode list - * @queued: the qnode list to pop a bio from + * @sq: the service_queue to pop a bio from * @tg_to_put: optional out argument for throtl_grp to put + * @rw: read/write * - * Pop the first bio from the qnode list @queued. After popping, the first - * qnode is removed from @queued if empty or moved to the end of @queued so - * that the popping order is round-robin. + * Pop the first bio from the qnode list @sq->queued. Note that we firstly + * focus on the iops list because bios are ultimately dispatched from it. + * After popping, the first qnode is removed from @sq->queued if empty or moved + * to the end of @sq->queued so that the popping order is round-robin. * * When the first qnode is removed, its associated throtl_grp should be put * too. If @tg_to_put is NULL, this function automatically puts it; * otherwise, *@tg_to_put is set to the throtl_grp to put and the caller is * responsible for putting it. */ -static struct bio *throtl_pop_queued(struct list_head *queued, - struct throtl_grp **tg_to_put) +static struct bio *throtl_pop_queued(struct throtl_service_queue *sq, + struct throtl_grp **tg_to_put, bool rw) { + struct list_head *queued = &sq->queued[rw]; struct throtl_qnode *qn; struct bio *bio; @@ -209,10 +226,17 @@ static struct bio *throtl_pop_queued(struct list_head *queued, return NULL; qn = list_first_entry(queued, struct throtl_qnode, node); - bio = bio_list_pop(&qn->bios); + bio = bio_list_pop(&qn->bios_iops); + if (bio) { + sq->nr_queued_iops[rw]--; + } else { + bio = bio_list_pop(&qn->bios_bps); + if (bio) + sq->nr_queued_bps[rw]--; + } WARN_ON_ONCE(!bio); - if (bio_list_empty(&qn->bios)) { + if (bio_list_empty(&qn->bios_bps) && bio_list_empty(&qn->bios_iops)) { list_del_init(&qn->node); if (tg_to_put) *tg_to_put = qn->tg; @@ -329,16 +353,25 @@ static void throtl_pd_online(struct blkg_policy_data *pd) tg_update_has_rules(tg); } -static void throtl_pd_free(struct blkg_policy_data *pd) +static void tg_release(struct rcu_head *rcu) { + struct blkg_policy_data *pd = + container_of(rcu, struct blkg_policy_data, rcu_head); struct throtl_grp *tg = pd_to_tg(pd); - del_timer_sync(&tg->service_queue.pending_timer); blkg_rwstat_exit(&tg->stat_bytes); blkg_rwstat_exit(&tg->stat_ios); kfree(tg); } +static void throtl_pd_free(struct blkg_policy_data *pd) +{ + struct throtl_grp *tg = pd_to_tg(pd); + + timer_delete_sync(&tg->service_queue.pending_timer); + call_rcu(&pd->rcu_head, tg_release); +} + static struct throtl_grp * throtl_rb_first(struct throtl_service_queue *parent_sq) { @@ -420,7 +453,7 @@ static void throtl_dequeue_tg(struct throtl_grp *tg) static void throtl_schedule_pending_timer(struct throtl_service_queue *sq, unsigned long expires) { - unsigned long max_expire = jiffies + 8 * sq_to_td(sq)->throtl_slice; + unsigned long max_expire = jiffies + 8 * DFL_THROTL_SLICE; /* * Since we are adjusting the throttle limit dynamically, the sleep @@ -488,7 +521,7 @@ static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg, if (time_after(start, tg->slice_start[rw])) tg->slice_start[rw] = start; - tg->slice_end[rw] = jiffies + tg->td->throtl_slice; + tg->slice_end[rw] = jiffies + DFL_THROTL_SLICE; throtl_log(&tg->service_queue, "[%c] new slice with credit start=%lu end=%lu jiffies=%lu", rw == READ ? 'R' : 'W', tg->slice_start[rw], @@ -503,7 +536,7 @@ static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw, tg->io_disp[rw] = 0; } tg->slice_start[rw] = jiffies; - tg->slice_end[rw] = jiffies + tg->td->throtl_slice; + tg->slice_end[rw] = jiffies + DFL_THROTL_SLICE; throtl_log(&tg->service_queue, "[%c] new slice start=%lu end=%lu jiffies=%lu", @@ -514,12 +547,15 @@ static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw, static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw, unsigned long jiffy_end) { - tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice); + tg->slice_end[rw] = roundup(jiffy_end, DFL_THROTL_SLICE); } static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw, unsigned long jiffy_end) { + if (!time_before(tg->slice_end[rw], jiffy_end)) + return; + throtl_set_slice_end(tg, rw, jiffy_end); throtl_log(&tg->service_queue, "[%c] extend slice start=%lu end=%lu jiffies=%lu", @@ -536,6 +572,11 @@ static bool throtl_slice_used(struct throtl_grp *tg, bool rw) return true; } +static unsigned int sq_queued(struct throtl_service_queue *sq, int type) +{ + return sq->nr_queued_bps[type] + sq->nr_queued_iops[type]; +} + static unsigned int calculate_io_allowed(u32 iops_limit, unsigned long jiffy_elapsed) { @@ -571,6 +612,48 @@ static u64 calculate_bytes_allowed(u64 bps_limit, unsigned long jiffy_elapsed) return mul_u64_u64_div_u64(bps_limit, (u64)jiffy_elapsed, (u64)HZ); } +static long long throtl_trim_bps(struct throtl_grp *tg, bool rw, + unsigned long time_elapsed) +{ + u64 bps_limit = tg_bps_limit(tg, rw); + long long bytes_trim; + + if (bps_limit == U64_MAX) + return 0; + + /* Need to consider the case of bytes_allowed overflow. */ + bytes_trim = calculate_bytes_allowed(bps_limit, time_elapsed); + if (bytes_trim <= 0 || tg->bytes_disp[rw] < bytes_trim) { + bytes_trim = tg->bytes_disp[rw]; + tg->bytes_disp[rw] = 0; + } else { + tg->bytes_disp[rw] -= bytes_trim; + } + + return bytes_trim; +} + +static int throtl_trim_iops(struct throtl_grp *tg, bool rw, + unsigned long time_elapsed) +{ + u32 iops_limit = tg_iops_limit(tg, rw); + int io_trim; + + if (iops_limit == UINT_MAX) + return 0; + + /* Need to consider the case of io_allowed overflow. */ + io_trim = calculate_io_allowed(iops_limit, time_elapsed); + if (io_trim <= 0 || tg->io_disp[rw] < io_trim) { + io_trim = tg->io_disp[rw]; + tg->io_disp[rw] = 0; + } else { + tg->io_disp[rw] -= io_trim; + } + + return io_trim; +} + /* Trim the used slices and adjust slice start accordingly */ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) { @@ -595,12 +678,12 @@ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) * sooner, then we need to reduce slice_end. A high bogus slice_end * is bad because it does not allow new slice to start. */ - throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice); + throtl_set_slice_end(tg, rw, jiffies + DFL_THROTL_SLICE); time_elapsed = rounddown(jiffies - tg->slice_start[rw], - tg->td->throtl_slice); + DFL_THROTL_SLICE); /* Don't trim slice until at least 2 slices are used */ - if (time_elapsed < tg->td->throtl_slice * 2) + if (time_elapsed < DFL_THROTL_SLICE * 2) return; /* @@ -611,28 +694,17 @@ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) * lower rate than expected. Therefore, other than the above rounddown, * one extra slice is preserved for deviation. */ - time_elapsed -= tg->td->throtl_slice; - bytes_trim = calculate_bytes_allowed(tg_bps_limit(tg, rw), - time_elapsed); - io_trim = calculate_io_allowed(tg_iops_limit(tg, rw), time_elapsed); - if (bytes_trim <= 0 && io_trim <= 0) + time_elapsed -= DFL_THROTL_SLICE; + bytes_trim = throtl_trim_bps(tg, rw, time_elapsed); + io_trim = throtl_trim_iops(tg, rw, time_elapsed); + if (!bytes_trim && !io_trim) return; - if ((long long)tg->bytes_disp[rw] >= bytes_trim) - tg->bytes_disp[rw] -= bytes_trim; - else - tg->bytes_disp[rw] = 0; - - if ((int)tg->io_disp[rw] >= io_trim) - tg->io_disp[rw] -= io_trim; - else - tg->io_disp[rw] = 0; - tg->slice_start[rw] += time_elapsed; throtl_log(&tg->service_queue, "[%c] trim slice nr=%lu bytes=%lld io=%d start=%lu end=%lu jiffies=%lu", - rw == READ ? 'R' : 'W', time_elapsed / tg->td->throtl_slice, + rw == READ ? 'R' : 'W', time_elapsed / DFL_THROTL_SLICE, bytes_trim, io_trim, tg->slice_start[rw], tg->slice_end[rw], jiffies); } @@ -643,21 +715,41 @@ static void __tg_update_carryover(struct throtl_grp *tg, bool rw, unsigned long jiffy_elapsed = jiffies - tg->slice_start[rw]; u64 bps_limit = tg_bps_limit(tg, rw); u32 iops_limit = tg_iops_limit(tg, rw); + long long bytes_allowed; + int io_allowed; + + /* + * If the queue is empty, carryover handling is not needed. In such cases, + * tg->[bytes/io]_disp should be reset to 0 to avoid impacting the dispatch + * of subsequent bios. The same handling applies when the previous BPS/IOPS + * limit was set to max. + */ + if (sq_queued(&tg->service_queue, rw) == 0) { + tg->bytes_disp[rw] = 0; + tg->io_disp[rw] = 0; + return; + } /* * If config is updated while bios are still throttled, calculate and - * accumulate how many bytes/ios are waited across changes. And - * carryover_bytes/ios will be used to calculate new wait time under new - * configuration. + * accumulate how many bytes/ios are waited across changes. And use the + * calculated carryover (@bytes/@ios) to update [bytes/io]_disp, which + * will be used to calculate new wait time under new configuration. + * And we need to consider the case of bytes/io_allowed overflow. */ - if (bps_limit != U64_MAX) - *bytes = calculate_bytes_allowed(bps_limit, jiffy_elapsed) - - tg->bytes_disp[rw]; - if (iops_limit != UINT_MAX) - *ios = calculate_io_allowed(iops_limit, jiffy_elapsed) - - tg->io_disp[rw]; - tg->bytes_disp[rw] -= *bytes; - tg->io_disp[rw] -= *ios; + if (bps_limit != U64_MAX) { + bytes_allowed = calculate_bytes_allowed(bps_limit, jiffy_elapsed); + if (bytes_allowed > 0) + *bytes = bytes_allowed - tg->bytes_disp[rw]; + } + if (iops_limit != UINT_MAX) { + io_allowed = calculate_io_allowed(iops_limit, jiffy_elapsed); + if (io_allowed > 0) + *ios = io_allowed - tg->io_disp[rw]; + } + + tg->bytes_disp[rw] = -*bytes; + tg->io_disp[rw] = -*ios; } static void tg_update_carryover(struct throtl_grp *tg) @@ -665,12 +757,10 @@ static void tg_update_carryover(struct throtl_grp *tg) long long bytes[2] = {0}; int ios[2] = {0}; - if (tg->service_queue.nr_queued[READ]) - __tg_update_carryover(tg, READ, &bytes[READ], &ios[READ]); - if (tg->service_queue.nr_queued[WRITE]) - __tg_update_carryover(tg, WRITE, &bytes[WRITE], &ios[WRITE]); + __tg_update_carryover(tg, READ, &bytes[READ], &ios[READ]); + __tg_update_carryover(tg, WRITE, &bytes[WRITE], &ios[WRITE]); - /* see comments in struct throtl_grp for meaning of these fields. */ + /* see comments in struct throtl_grp for meaning of carryover. */ throtl_log(&tg->service_queue, "%s: %lld %lld %d %d\n", __func__, bytes[READ], bytes[WRITE], ios[READ], ios[WRITE]); } @@ -682,14 +772,10 @@ static unsigned long tg_within_iops_limit(struct throtl_grp *tg, struct bio *bio int io_allowed; unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd; - if (iops_limit == UINT_MAX) { - return 0; - } - jiffy_elapsed = jiffies - tg->slice_start[rw]; /* Round up to the next throttle slice, wait time must be nonzero */ - jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, tg->td->throtl_slice); + jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, DFL_THROTL_SLICE); io_allowed = calculate_io_allowed(iops_limit, jiffy_elapsed_rnd); if (io_allowed > 0 && tg->io_disp[rw] + 1 <= io_allowed) return 0; @@ -711,20 +797,17 @@ static unsigned long tg_within_bps_limit(struct throtl_grp *tg, struct bio *bio, unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd; unsigned int bio_size = throtl_bio_data_size(bio); - /* no need to throttle if this bio's bytes have been accounted */ - if (bps_limit == U64_MAX || bio_flagged(bio, BIO_BPS_THROTTLED)) { - return 0; - } - jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw]; /* Slice has just started. Consider one slice interval */ if (!jiffy_elapsed) - jiffy_elapsed_rnd = tg->td->throtl_slice; + jiffy_elapsed_rnd = DFL_THROTL_SLICE; - jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice); + jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, DFL_THROTL_SLICE); bytes_allowed = calculate_bytes_allowed(bps_limit, jiffy_elapsed_rnd); - if (bytes_allowed > 0 && tg->bytes_disp[rw] + bio_size <= bytes_allowed) + /* Need to consider the case of bytes_allowed overflow. */ + if ((bytes_allowed > 0 && tg->bytes_disp[rw] + bio_size <= bytes_allowed) + || bytes_allowed < 0) return 0; /* Calc approx time to dispatch */ @@ -742,17 +825,82 @@ static unsigned long tg_within_bps_limit(struct throtl_grp *tg, struct bio *bio, return jiffy_wait; } +static void throtl_charge_bps_bio(struct throtl_grp *tg, struct bio *bio) +{ + unsigned int bio_size = throtl_bio_data_size(bio); + + /* Charge the bio to the group */ + if (!bio_flagged(bio, BIO_BPS_THROTTLED) && + !bio_flagged(bio, BIO_TG_BPS_THROTTLED)) { + bio_set_flag(bio, BIO_TG_BPS_THROTTLED); + tg->bytes_disp[bio_data_dir(bio)] += bio_size; + } +} + +static void throtl_charge_iops_bio(struct throtl_grp *tg, struct bio *bio) +{ + bio_clear_flag(bio, BIO_TG_BPS_THROTTLED); + tg->io_disp[bio_data_dir(bio)]++; +} + /* - * Returns whether one can dispatch a bio or not. Also returns approx number - * of jiffies to wait before this bio is with-in IO rate and can be dispatched + * If previous slice expired, start a new one otherwise renew/extend existing + * slice to make sure it is at least throtl_slice interval long since now. New + * slice is started only for empty throttle group. If there is queued bio, that + * means there should be an active slice and it should be extended instead. */ -static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, - unsigned long *wait) +static void tg_update_slice(struct throtl_grp *tg, bool rw) +{ + if (throtl_slice_used(tg, rw) && + sq_queued(&tg->service_queue, rw) == 0) + throtl_start_new_slice(tg, rw, true); + else + throtl_extend_slice(tg, rw, jiffies + DFL_THROTL_SLICE); +} + +static unsigned long tg_dispatch_bps_time(struct throtl_grp *tg, struct bio *bio) { bool rw = bio_data_dir(bio); - unsigned long bps_wait = 0, iops_wait = 0, max_wait = 0; u64 bps_limit = tg_bps_limit(tg, rw); + unsigned long bps_wait; + + /* no need to throttle if this bio's bytes have been accounted */ + if (bps_limit == U64_MAX || tg->flags & THROTL_TG_CANCELING || + bio_flagged(bio, BIO_BPS_THROTTLED) || + bio_flagged(bio, BIO_TG_BPS_THROTTLED)) + return 0; + + tg_update_slice(tg, rw); + bps_wait = tg_within_bps_limit(tg, bio, bps_limit); + throtl_extend_slice(tg, rw, jiffies + bps_wait); + + return bps_wait; +} + +static unsigned long tg_dispatch_iops_time(struct throtl_grp *tg, struct bio *bio) +{ + bool rw = bio_data_dir(bio); u32 iops_limit = tg_iops_limit(tg, rw); + unsigned long iops_wait; + + if (iops_limit == UINT_MAX || tg->flags & THROTL_TG_CANCELING) + return 0; + + tg_update_slice(tg, rw); + iops_wait = tg_within_iops_limit(tg, bio, iops_limit); + throtl_extend_slice(tg, rw, jiffies + iops_wait); + + return iops_wait; +} + +/* + * Returns approx number of jiffies to wait before this bio is with-in IO rate + * and can be moved to other queue or dispatched. + */ +static unsigned long tg_dispatch_time(struct throtl_grp *tg, struct bio *bio) +{ + bool rw = bio_data_dir(bio); + unsigned long wait; /* * Currently whole state machine of group depends on first bio @@ -760,62 +908,20 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, * this function with a different bio if there are other bios * queued. */ - BUG_ON(tg->service_queue.nr_queued[rw] && + BUG_ON(sq_queued(&tg->service_queue, rw) && bio != throtl_peek_queued(&tg->service_queue.queued[rw])); - /* If tg->bps = -1, then BW is unlimited */ - if ((bps_limit == U64_MAX && iops_limit == UINT_MAX) || - tg->flags & THROTL_TG_CANCELING) { - if (wait) - *wait = 0; - return true; - } + wait = tg_dispatch_bps_time(tg, bio); + if (wait != 0) + return wait; /* - * If previous slice expired, start a new one otherwise renew/extend - * existing slice to make sure it is at least throtl_slice interval - * long since now. New slice is started only for empty throttle group. - * If there is queued bio, that means there should be an active - * slice and it should be extended instead. + * Charge bps here because @bio will be directly placed into the + * iops queue afterward. */ - if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw])) - throtl_start_new_slice(tg, rw, true); - else { - if (time_before(tg->slice_end[rw], - jiffies + tg->td->throtl_slice)) - throtl_extend_slice(tg, rw, - jiffies + tg->td->throtl_slice); - } - - bps_wait = tg_within_bps_limit(tg, bio, bps_limit); - iops_wait = tg_within_iops_limit(tg, bio, iops_limit); - if (bps_wait + iops_wait == 0) { - if (wait) - *wait = 0; - return true; - } - - max_wait = max(bps_wait, iops_wait); + throtl_charge_bps_bio(tg, bio); - if (wait) - *wait = max_wait; - - if (time_before(tg->slice_end[rw], jiffies + max_wait)) - throtl_extend_slice(tg, rw, jiffies + max_wait); - - return false; -} - -static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio) -{ - bool rw = bio_data_dir(bio); - unsigned int bio_size = throtl_bio_data_size(bio); - - /* Charge the bio to the group */ - if (!bio_flagged(bio, BIO_BPS_THROTTLED)) - tg->bytes_disp[rw] += bio_size; - - tg->io_disp[rw]++; + return tg_dispatch_iops_time(tg, bio); } /** @@ -842,28 +948,36 @@ static void throtl_add_bio_tg(struct bio *bio, struct throtl_qnode *qn, * dispatched. Mark that @tg was empty. This is automatically * cleared on the next tg_update_disptime(). */ - if (!sq->nr_queued[rw]) + if (sq_queued(sq, rw) == 0) tg->flags |= THROTL_TG_WAS_EMPTY; - throtl_qnode_add_bio(bio, qn, &sq->queued[rw]); + throtl_qnode_add_bio(bio, qn, sq); + + /* + * Since we have split the queues, when the iops queue is + * previously empty and a new @bio is added into the first @qn, + * we also need to update the @tg->disptime. + */ + if (bio_flagged(bio, BIO_BPS_THROTTLED) && + bio == throtl_peek_queued(&sq->queued[rw])) + tg->flags |= THROTL_TG_IOPS_WAS_EMPTY; - sq->nr_queued[rw]++; throtl_enqueue_tg(tg); } static void tg_update_disptime(struct throtl_grp *tg) { struct throtl_service_queue *sq = &tg->service_queue; - unsigned long read_wait = -1, write_wait = -1, min_wait = -1, disptime; + unsigned long read_wait = -1, write_wait = -1, min_wait, disptime; struct bio *bio; bio = throtl_peek_queued(&sq->queued[READ]); if (bio) - tg_may_dispatch(tg, bio, &read_wait); + read_wait = tg_dispatch_time(tg, bio); bio = throtl_peek_queued(&sq->queued[WRITE]); if (bio) - tg_may_dispatch(tg, bio, &write_wait); + write_wait = tg_dispatch_time(tg, bio); min_wait = min(read_wait, write_wait); disptime = jiffies + min_wait; @@ -875,6 +989,7 @@ static void tg_update_disptime(struct throtl_grp *tg) /* see throtl_add_bio_tg() */ tg->flags &= ~THROTL_TG_WAS_EMPTY; + tg->flags &= ~THROTL_TG_IOPS_WAS_EMPTY; } static void start_parent_slice_with_credit(struct throtl_grp *child_tg, @@ -901,10 +1016,9 @@ static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw) * getting released prematurely. Remember the tg to put and put it * after @bio is transferred to @parent_sq. */ - bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put); - sq->nr_queued[rw]--; + bio = throtl_pop_queued(sq, &tg_to_put, rw); - throtl_charge_bio(tg, bio); + throtl_charge_iops_bio(tg, bio); /* * If our parent is another tg, we just need to transfer @bio to @@ -919,7 +1033,7 @@ static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw) } else { bio_set_flag(bio, BIO_BPS_THROTTLED); throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw], - &parent_sq->queued[rw]); + parent_sq); BUG_ON(tg->td->nr_queued[rw] <= 0); tg->td->nr_queued[rw]--; } @@ -941,7 +1055,7 @@ static int throtl_dispatch_tg(struct throtl_grp *tg) /* Try to dispatch 75% READS and 25% WRITES */ while ((bio = throtl_peek_queued(&sq->queued[READ])) && - tg_may_dispatch(tg, bio, NULL)) { + tg_dispatch_time(tg, bio) == 0) { tg_dispatch_one_bio(tg, READ); nr_reads++; @@ -951,7 +1065,7 @@ static int throtl_dispatch_tg(struct throtl_grp *tg) } while ((bio = throtl_peek_queued(&sq->queued[WRITE])) && - tg_may_dispatch(tg, bio, NULL)) { + tg_dispatch_time(tg, bio) == 0) { tg_dispatch_one_bio(tg, WRITE); nr_writes++; @@ -984,7 +1098,7 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq) nr_disp += throtl_dispatch_tg(tg); sq = &tg->service_queue; - if (sq->nr_queued[READ] || sq->nr_queued[WRITE]) + if (sq_queued(sq, READ) || sq_queued(sq, WRITE)) tg_update_disptime(tg); else throtl_dequeue_tg(tg); @@ -1013,7 +1127,8 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq) */ static void throtl_pending_timer_fn(struct timer_list *t) { - struct throtl_service_queue *sq = from_timer(sq, t, pending_timer); + struct throtl_service_queue *sq = timer_container_of(sq, t, + pending_timer); struct throtl_grp *tg = sq_to_tg(sq); struct throtl_data *td = sq_to_td(sq); struct throtl_service_queue *parent_sq; @@ -1037,9 +1152,11 @@ again: dispatched = false; while (true) { + unsigned int __maybe_unused bio_cnt_r = sq_queued(sq, READ); + unsigned int __maybe_unused bio_cnt_w = sq_queued(sq, WRITE); + throtl_log(sq, "dispatch nr_queued=%u read=%u write=%u", - sq->nr_queued[READ] + sq->nr_queued[WRITE], - sq->nr_queued[READ], sq->nr_queued[WRITE]); + bio_cnt_r + bio_cnt_w, bio_cnt_r, bio_cnt_w); ret = throtl_select_dispatch(sq); if (ret) { @@ -1061,7 +1178,8 @@ again: if (parent_sq) { /* @parent_sq is another throl_grp, propagate dispatch */ - if (tg->flags & THROTL_TG_WAS_EMPTY) { + if (tg->flags & THROTL_TG_WAS_EMPTY || + tg->flags & THROTL_TG_IOPS_WAS_EMPTY) { tg_update_disptime(tg); if (!throtl_schedule_next_dispatch(parent_sq, false)) { /* window is already open, repeat dispatching */ @@ -1101,14 +1219,14 @@ static void blk_throtl_dispatch_work_fn(struct work_struct *work) spin_lock_irq(&q->queue_lock); for (rw = READ; rw <= WRITE; rw++) - while ((bio = throtl_pop_queued(&td_sq->queued[rw], NULL))) + while ((bio = throtl_pop_queued(td_sq, NULL, rw))) bio_list_add(&bio_list_on_stack, bio); spin_unlock_irq(&q->queue_lock); if (!bio_list_empty(&bio_list_on_stack)) { blk_start_plug(&plug); while ((bio = bio_list_pop(&bio_list_on_stack))) - submit_bio_noacct_nocheck(bio); + submit_bio_noacct_nocheck(bio, false); blk_finish_plug(&plug); } } @@ -1211,33 +1329,19 @@ static int blk_throtl_init(struct gendisk *disk) INIT_WORK(&td->dispatch_work, blk_throtl_dispatch_work_fn); throtl_service_queue_init(&td->service_queue); - /* - * Freeze queue before activating policy, to synchronize with IO path, - * which is protected by 'q_usage_counter'. - */ memflags = blk_mq_freeze_queue(disk->queue); blk_mq_quiesce_queue(disk->queue); q->td = td; td->queue = q; - /* activate policy */ + /* activate policy, blk_throtl_activated() will return true */ ret = blkcg_activate_policy(disk, &blkcg_policy_throtl); if (ret) { q->td = NULL; kfree(td); - goto out; } - if (blk_queue_nonrot(q)) - td->throtl_slice = DFL_THROTL_SLICE_SSD; - else - td->throtl_slice = DFL_THROTL_SLICE_HD; - td->track_bio_latency = !queue_is_mq(q); - if (!td->track_bio_latency) - blk_stat_enable_accounting(q); - -out: blk_mq_unquiesce_queue(disk->queue); blk_mq_unfreeze_queue(disk->queue, memflags); @@ -1258,21 +1362,21 @@ static ssize_t tg_set_conf(struct kernfs_open_file *of, ret = blkg_conf_open_bdev(&ctx); if (ret) - goto out_finish; + return ret; if (!blk_throtl_activated(ctx.bdev->bd_queue)) { ret = blk_throtl_init(ctx.bdev->bd_disk); if (ret) - goto out_finish; + goto close_bdev; } ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, &ctx); if (ret) - goto out_finish; + goto close_bdev; ret = -EINVAL; if (sscanf(ctx.body, "%llu", &v) != 1) - goto out_finish; + goto unprep; if (!v) v = U64_MAX; @@ -1286,8 +1390,12 @@ static ssize_t tg_set_conf(struct kernfs_open_file *of, tg_conf_updated(tg, false); ret = 0; -out_finish: - blkg_conf_exit(&ctx); + +unprep: + blkg_conf_unprep(&ctx); + +close_bdev: + blkg_conf_close_bdev(&ctx); return ret ?: nbytes; } @@ -1442,17 +1550,17 @@ static ssize_t tg_set_limit(struct kernfs_open_file *of, ret = blkg_conf_open_bdev(&ctx); if (ret) - goto out_finish; + return ret; if (!blk_throtl_activated(ctx.bdev->bd_queue)) { ret = blk_throtl_init(ctx.bdev->bd_disk); if (ret) - goto out_finish; + goto close_bdev; } ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, &ctx); if (ret) - goto out_finish; + goto close_bdev; tg = blkg_to_tg(ctx.blkg); tg_update_carryover(tg); @@ -1478,11 +1586,11 @@ static ssize_t tg_set_limit(struct kernfs_open_file *of, p = tok; strsep(&p, "="); if (!p || (sscanf(p, "%llu", &val) != 1 && strcmp(p, "max"))) - goto out_finish; + goto unprep; ret = -ERANGE; if (!val) - goto out_finish; + goto unprep; ret = -EINVAL; if (!strcmp(tok, "rbps")) @@ -1494,7 +1602,7 @@ static ssize_t tg_set_limit(struct kernfs_open_file *of, else if (!strcmp(tok, "wiops")) v[3] = min_t(u64, val, UINT_MAX); else - goto out_finish; + goto unprep; } tg->bps[READ] = v[0]; @@ -1504,8 +1612,10 @@ static ssize_t tg_set_limit(struct kernfs_open_file *of, tg_conf_updated(tg, false); ret = 0; -out_finish: - blkg_conf_exit(&ctx); +unprep: + blkg_conf_unprep(&ctx); +close_bdev: + blkg_conf_close_bdev(&ctx); return ret ?: nbytes; } @@ -1554,7 +1664,7 @@ static void tg_flush_bios(struct throtl_grp *tg) */ tg_update_disptime(tg); - throtl_schedule_pending_timer(sq, jiffies + 1); + throtl_schedule_next_dispatch(sq->parent_sq, true); } static void throtl_pd_offline(struct blkg_policy_data *pd) @@ -1573,11 +1683,52 @@ struct blkcg_policy blkcg_policy_throtl = { .pd_free_fn = throtl_pd_free, }; +static void tg_cancel_writeback_bios(struct throtl_grp *tg, + struct bio_list *cancel_bios) +{ + struct throtl_service_queue *sq = &tg->service_queue; + struct throtl_data *td = sq_to_td(sq); + int rw; + + if (tg->flags & THROTL_TG_CANCELING) + return; + tg->flags |= THROTL_TG_CANCELING; + + for (rw = READ; rw <= WRITE; rw++) { + struct throtl_qnode *qn, *tmp; + unsigned int nr_bios = 0; + + list_for_each_entry_safe(qn, tmp, &sq->queued[rw], node) { + struct bio *bio; + + while ((bio = bio_list_pop(&qn->bios_iops))) { + sq->nr_queued_iops[rw]--; + bio_list_add(&cancel_bios[rw], bio); + nr_bios++; + } + while ((bio = bio_list_pop(&qn->bios_bps))) { + sq->nr_queued_bps[rw]--; + bio_list_add(&cancel_bios[rw], bio); + nr_bios++; + } + + list_del_init(&qn->node); + blkg_put(tg_to_blkg(qn->tg)); + } + + td->nr_queued[rw] -= nr_bios; + } + + throtl_dequeue_tg(tg); +} + void blk_throtl_cancel_bios(struct gendisk *disk) { struct request_queue *q = disk->queue; struct cgroup_subsys_state *pos_css; struct blkcg_gq *blkg; + struct bio_list cancel_bios[2] = { }; + int rw; if (!blk_throtl_activated(q)) return; @@ -1598,19 +1749,44 @@ void blk_throtl_cancel_bios(struct gendisk *disk) * Cancel bios here to ensure no bios are inflight after * del_gendisk. */ - tg_flush_bios(blkg_to_tg(blkg)); + tg_cancel_writeback_bios(blkg_to_tg(blkg), cancel_bios); } rcu_read_unlock(); spin_unlock_irq(&q->queue_lock); + + for (rw = READ; rw <= WRITE; rw++) { + struct bio *bio; + while ((bio = bio_list_pop(&cancel_bios[rw]))) + bio_io_error(bio); + } } static bool tg_within_limit(struct throtl_grp *tg, struct bio *bio, bool rw) { - /* throtl is FIFO - if bios are already queued, should queue */ - if (tg->service_queue.nr_queued[rw]) + struct throtl_service_queue *sq = &tg->service_queue; + + /* + * For a split bio, we need to specifically distinguish whether the + * iops queue is empty. + */ + if (bio_flagged(bio, BIO_BPS_THROTTLED)) + return sq->nr_queued_iops[rw] == 0 && + tg_dispatch_iops_time(tg, bio) == 0; + + /* + * Throtl is FIFO - if bios are already queued, should queue. + * If the bps queue is empty and @bio is within the bps limit, charge + * bps here for direct placement into the iops queue. + */ + if (sq_queued(&tg->service_queue, rw)) { + if (sq->nr_queued_bps[rw] == 0 && + tg_dispatch_bps_time(tg, bio) == 0) + throtl_charge_bps_bio(tg, bio); + return false; + } - return tg_may_dispatch(tg, bio, NULL); + return tg_dispatch_time(tg, bio) == 0; } bool __blk_throtl_bio(struct bio *bio) @@ -1631,7 +1807,7 @@ bool __blk_throtl_bio(struct bio *bio) while (true) { if (tg_within_limit(tg, bio, rw)) { /* within limits, let's charge and dispatch directly */ - throtl_charge_bio(tg, bio); + throtl_charge_iops_bio(tg, bio); /* * We need to trim slice even when bios are not being @@ -1654,7 +1830,8 @@ bool __blk_throtl_bio(struct bio *bio) * control algorithm is adaptive, and extra IO bytes * will be throttled for paying the debt */ - throtl_charge_bio(tg, bio); + throtl_charge_bps_bio(tg, bio); + throtl_charge_iops_bio(tg, bio); } else { /* if above limits, break to queue */ break; @@ -1680,7 +1857,7 @@ bool __blk_throtl_bio(struct bio *bio) tg->bytes_disp[rw], bio->bi_iter.bi_size, tg_bps_limit(tg, rw), tg->io_disp[rw], tg_iops_limit(tg, rw), - sq->nr_queued[READ], sq->nr_queued[WRITE]); + sq_queued(sq, READ), sq_queued(sq, WRITE)); td->nr_queued[rw]++; throtl_add_bio_tg(bio, qn, tg); @@ -1688,11 +1865,13 @@ bool __blk_throtl_bio(struct bio *bio) /* * Update @tg's dispatch time and force schedule dispatch if @tg - * was empty before @bio. The forced scheduling isn't likely to - * cause undue delay as @bio is likely to be dispatched directly if - * its @tg's disptime is not in the future. + * was empty before @bio, or the iops queue is empty and @bio will + * add to. The forced scheduling isn't likely to cause undue + * delay as @bio is likely to be dispatched directly if its @tg's + * disptime is not in the future. */ - if (tg->flags & THROTL_TG_WAS_EMPTY) { + if (tg->flags & THROTL_TG_WAS_EMPTY || + tg->flags & THROTL_TG_IOPS_WAS_EMPTY) { tg_update_disptime(tg); throtl_schedule_next_dispatch(tg->service_queue.parent_sq, true); } @@ -1708,18 +1887,21 @@ void blk_throtl_exit(struct gendisk *disk) { struct request_queue *q = disk->queue; - if (!blk_throtl_activated(q)) + /* + * blkg_destroy_all() already deactivate throtl policy, just check and + * free throtl data. + */ + if (!q->td) return; - del_timer_sync(&q->td->service_queue.pending_timer); + timer_delete_sync(&q->td->service_queue.pending_timer); throtl_shutdown_wq(q); - blkcg_deactivate_policy(disk, &blkcg_policy_throtl); kfree(q->td); } static int __init throtl_init(void) { - kthrotld_workqueue = alloc_workqueue("kthrotld", WQ_MEM_RECLAIM, 0); + kthrotld_workqueue = alloc_workqueue("kthrotld", WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!kthrotld_workqueue) panic("Failed to create kthrotld\n"); diff --git a/block/blk-throttle.h b/block/blk-throttle.h index 7964cc041e06..9d7a42c039a1 100644 --- a/block/blk-throttle.h +++ b/block/blk-throttle.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef BLK_THROTTLE_H #define BLK_THROTTLE_H @@ -28,7 +29,8 @@ */ struct throtl_qnode { struct list_head node; /* service_queue->queued[] */ - struct bio_list bios; /* queued bios */ + struct bio_list bios_bps; /* queued bios for bps limit */ + struct bio_list bios_iops; /* queued bios for iops limit */ struct throtl_grp *tg; /* tg this qnode belongs to */ }; @@ -40,7 +42,8 @@ struct throtl_service_queue { * children throtl_grp's. */ struct list_head queued[2]; /* throtl_qnode [READ/WRITE] */ - unsigned int nr_queued[2]; /* number of queued bios */ + unsigned int nr_queued_bps[2]; /* number of queued bps bios */ + unsigned int nr_queued_iops[2]; /* number of queued iops bios */ /* * RB tree of active children throtl_grp's, which are sorted by @@ -53,9 +56,14 @@ struct throtl_service_queue { }; enum tg_state_flags { - THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */ - THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */ - THROTL_TG_CANCELING = 1 << 2, /* starts to cancel bio */ + THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */ + THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */ + /* + * The sq's iops queue is empty, and a bio is about to be enqueued + * to the first qnode's bios_iops list. + */ + THROTL_TG_IOPS_WAS_EMPTY = 1 << 2, + THROTL_TG_CANCELING = 1 << 3, /* starts to cancel bio */ }; struct throtl_grp { @@ -101,19 +109,16 @@ struct throtl_grp { /* IOPS limits */ unsigned int iops[2]; - /* Number of bytes dispatched in current slice */ - int64_t bytes_disp[2]; - /* Number of bio's dispatched in current slice */ - int io_disp[2]; - /* - * The following two fields are updated when new configuration is - * submitted while some bios are still throttled, they record how many - * bytes/ios are waited already in previous configuration, and they will - * be used to calculate wait time under new configuration. + * Number of bytes/bio's dispatched in current slice. + * When new configuration is submitted while some bios are still throttled, + * first calculate the carryover: the amount of bytes/IOs already waited + * under the previous configuration. Then, [bytes/io]_disp are represented + * as the negative of the carryover, and they will be used to calculate the + * wait time under the new configuration. */ - long long carryover_bytes[2]; - int carryover_ios[2]; + int64_t bytes_disp[2]; + int io_disp[2]; unsigned long last_check_time; @@ -151,7 +156,13 @@ void blk_throtl_cancel_bios(struct gendisk *disk); static inline bool blk_throtl_activated(struct request_queue *q) { - return q->td != NULL; + /* + * q->td guarantees that the blk-throttle module is already loaded, + * and the plid of blk-throttle is assigned. + * blkcg_policy_enabled() guarantees that the policy is activated + * in the request_queue. + */ + return q->td != NULL && blkcg_policy_enabled(q, &blkcg_policy_throtl); } static inline bool blk_should_throtl(struct bio *bio) @@ -159,11 +170,6 @@ static inline bool blk_should_throtl(struct bio *bio) struct throtl_grp *tg; int rw = bio_data_dir(bio); - /* - * This is called under bio_queue_enter(), and it's synchronized with - * the activation of blk-throtl, which is protected by - * blk_mq_freeze_queue(). - */ if (!blk_throtl_activated(bio->bi_bdev->bd_queue)) return false; @@ -189,7 +195,10 @@ static inline bool blk_should_throtl(struct bio *bio) static inline bool blk_throtl_bio(struct bio *bio) { - + /* + * block throttling takes effect if the policy is activated + * in the bio's request_queue. + */ if (!blk_should_throtl(bio)) return false; diff --git a/block/blk-wbt.c b/block/blk-wbt.c index f1754d07f7e0..953d400fd013 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -37,7 +37,7 @@ enum wbt_flags { WBT_TRACKED = 1, /* write, tracked for throttling */ WBT_READ = 2, /* read */ - WBT_SWAP = 4, /* write, from swap_writepage() */ + WBT_SWAP = 4, /* write, from swap_writeout() */ WBT_DISCARD = 8, /* discard */ WBT_NR_BITS = 4, /* number of bits */ @@ -85,14 +85,16 @@ struct rq_wb { u64 sync_issue; void *sync_cookie; - unsigned long last_issue; /* last non-throttled issue */ - unsigned long last_comp; /* last non-throttled comp */ + unsigned long last_issue; /* issue time of last read rq */ + unsigned long last_comp; /* completion time of last read rq */ unsigned long min_lat_nsec; struct rq_qos rqos; struct rq_wait rq_wait[WBT_NUM_RWQ]; struct rq_depth rq_depth; }; +static int wbt_init(struct gendisk *disk, struct rq_wb *rwb); + static inline struct rq_wb *RQWB(struct rq_qos *rqos) { return container_of(rqos, struct rq_wb, rqos); @@ -248,13 +250,14 @@ static void wbt_done(struct rq_qos *rqos, struct request *rq) struct rq_wb *rwb = RQWB(rqos); if (!wbt_is_tracked(rq)) { - if (rwb->sync_cookie == rq) { - rwb->sync_issue = 0; - rwb->sync_cookie = NULL; - } + if (wbt_is_read(rq)) { + if (rwb->sync_cookie == rq) { + rwb->sync_issue = 0; + rwb->sync_cookie = NULL; + } - if (wbt_is_read(rq)) wb_timestamp(rwb, &rwb->last_comp); + } } else { WARN_ON_ONCE(rq == rwb->sync_cookie); __wbt_done(rqos, wbt_flags(rq)); @@ -505,7 +508,7 @@ u64 wbt_get_min_lat(struct request_queue *q) return RQWB(rqos)->min_lat_nsec; } -void wbt_set_min_lat(struct request_queue *q, u64 val) +static void wbt_set_min_lat(struct request_queue *q, u64 val) { struct rq_qos *rqos = wbt_rq_qos(q); if (!rqos) @@ -695,17 +698,53 @@ static void wbt_requeue(struct rq_qos *rqos, struct request *rq) } } +static int wbt_data_dir(const struct request *rq) +{ + const enum req_op op = req_op(rq); + + if (op == REQ_OP_READ) + return READ; + else if (op_is_write(op)) + return WRITE; + + /* don't account */ + return -1; +} + +static struct rq_wb *wbt_alloc(void) +{ + struct rq_wb *rwb = kzalloc_obj(*rwb); + + if (!rwb) + return NULL; + + rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb); + if (!rwb->cb) { + kfree(rwb); + return NULL; + } + + return rwb; +} + +static void wbt_free(struct rq_wb *rwb) +{ + blk_stat_free_callback(rwb->cb); + kfree(rwb); +} + /* * Enable wbt if defaults are configured that way */ -void wbt_enable_default(struct gendisk *disk) +static bool __wbt_enable_default(struct gendisk *disk) { struct request_queue *q = disk->queue; struct rq_qos *rqos; bool enable = IS_ENABLED(CONFIG_BLK_WBT_MQ); - if (q->elevator && - test_bit(ELEVATOR_FLAG_DISABLE_WBT, &q->elevator->flags)) + mutex_lock(&disk->rqos_state_mutex); + + if (blk_queue_disable_wbt(q)) enable = false; /* Throttling already enabled? */ @@ -713,41 +752,59 @@ void wbt_enable_default(struct gendisk *disk) if (rqos) { if (enable && RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT) RQWB(rqos)->enable_state = WBT_STATE_ON_DEFAULT; - return; + mutex_unlock(&disk->rqos_state_mutex); + return false; } + mutex_unlock(&disk->rqos_state_mutex); /* Queue not registered? Maybe shutting down... */ if (!blk_queue_registered(q)) - return; + return false; if (queue_is_mq(q) && enable) - wbt_init(disk); + return true; + return false; +} + +void wbt_enable_default(struct gendisk *disk) +{ + __wbt_enable_default(disk); } EXPORT_SYMBOL_GPL(wbt_enable_default); -u64 wbt_default_latency_nsec(struct request_queue *q) +void wbt_init_enable_default(struct gendisk *disk) +{ + struct request_queue *q = disk->queue; + struct rq_wb *rwb; + unsigned int memflags; + + if (!__wbt_enable_default(disk)) + return; + + rwb = wbt_alloc(); + if (!rwb) + return; + + if (wbt_init(disk, rwb)) { + pr_warn("%s: failed to enable wbt\n", disk->disk_name); + wbt_free(rwb); + return; + } + + memflags = blk_debugfs_lock(q); + blk_mq_debugfs_register_rq_qos(q); + blk_debugfs_unlock(q, memflags); +} + +static u64 wbt_default_latency_nsec(struct request_queue *q) { /* * We default to 2msec for non-rotational storage, and 75msec * for rotational storage. */ - if (blk_queue_nonrot(q)) - return 2000000ULL; - else + if (blk_queue_rot(q)) return 75000000ULL; -} - -static int wbt_data_dir(const struct request *rq) -{ - const enum req_op op = req_op(rq); - - if (op == REQ_OP_READ) - return READ; - else if (op_is_write(op)) - return WRITE; - - /* don't account */ - return -1; + return 2000000ULL; } static void wbt_queue_depth_changed(struct rq_qos *rqos) @@ -756,13 +813,27 @@ static void wbt_queue_depth_changed(struct rq_qos *rqos) wbt_update_limits(RQWB(rqos)); } +static bool wbt_set_lat_changed(struct request_queue *q, u64 val) +{ + struct rq_qos *rqos = wbt_rq_qos(q); + struct rq_wb *rwb; + + if (!rqos) + return true; + + rwb = RQWB(rqos); + if (rwb->min_lat_nsec != val) + return true; + + return rwb_enabled(rwb) != !!val; +} + static void wbt_exit(struct rq_qos *rqos) { struct rq_wb *rwb = RQWB(rqos); blk_stat_remove_callback(rqos->disk->queue, rwb->cb); - blk_stat_free_callback(rwb->cb); - kfree(rwb); + wbt_free(rwb); } /* @@ -774,11 +845,13 @@ void wbt_disable_default(struct gendisk *disk) struct rq_wb *rwb; if (!rqos) return; + mutex_lock(&disk->rqos_state_mutex); rwb = RQWB(rqos); if (rwb->enable_state == WBT_STATE_ON_DEFAULT) { blk_stat_deactivate(rwb->cb); rwb->enable_state = WBT_STATE_OFF_DEFAULT; } + mutex_unlock(&disk->rqos_state_mutex); } EXPORT_SYMBOL_GPL(wbt_disable_default); @@ -884,22 +957,11 @@ static const struct rq_qos_ops wbt_rqos_ops = { #endif }; -int wbt_init(struct gendisk *disk) +static int wbt_init(struct gendisk *disk, struct rq_wb *rwb) { struct request_queue *q = disk->queue; - struct rq_wb *rwb; - int i; int ret; - - rwb = kzalloc(sizeof(*rwb), GFP_KERNEL); - if (!rwb) - return -ENOMEM; - - rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb); - if (!rwb->cb) { - kfree(rwb); - return -ENOMEM; - } + int i; for (i = 0; i < WBT_NUM_RWQ; i++) rq_wait_init(&rwb->rq_wait[i]); @@ -919,15 +981,65 @@ int wbt_init(struct gendisk *disk) ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops); mutex_unlock(&q->rq_qos_mutex); if (ret) - goto err_free; + return ret; blk_stat_add_callback(q, rwb->cb); - return 0; +} -err_free: - blk_stat_free_callback(rwb->cb); - kfree(rwb); - return ret; +int wbt_set_lat(struct gendisk *disk, s64 val) +{ + struct request_queue *q = disk->queue; + struct rq_qos *rqos = wbt_rq_qos(q); + struct rq_wb *rwb = NULL; + unsigned int memflags; + int ret = 0; + + if (!rqos) { + rwb = wbt_alloc(); + if (!rwb) + return -ENOMEM; + } + + /* + * Ensure that the queue is idled, in case the latency update + * ends up either enabling or disabling wbt completely. We can't + * have IO inflight if that happens. + */ + memflags = blk_mq_freeze_queue(q); + if (!rqos) { + ret = wbt_init(disk, rwb); + if (ret) { + wbt_free(rwb); + goto out; + } + } + + if (val == -1) + val = wbt_default_latency_nsec(q); + else if (val >= 0) + val *= 1000ULL; + mutex_lock(&disk->rqos_state_mutex); + if (!wbt_set_lat_changed(q, val)) { + mutex_unlock(&disk->rqos_state_mutex); + goto out; + } + mutex_unlock(&disk->rqos_state_mutex); + + blk_mq_quiesce_queue(q); + + mutex_lock(&disk->rqos_state_mutex); + wbt_set_min_lat(q, val); + mutex_unlock(&disk->rqos_state_mutex); + + blk_mq_unquiesce_queue(q); +out: + blk_mq_unfreeze_queue(q, memflags); + + memflags = blk_debugfs_lock(q); + blk_mq_debugfs_register_rq_qos(q); + blk_debugfs_unlock(q, memflags); + + return ret; } diff --git a/block/blk-wbt.h b/block/blk-wbt.h index e5fc653b9b76..6e39da17218b 100644 --- a/block/blk-wbt.h +++ b/block/blk-wbt.h @@ -4,18 +4,20 @@ #ifdef CONFIG_BLK_WBT -int wbt_init(struct gendisk *disk); +void wbt_init_enable_default(struct gendisk *disk); void wbt_disable_default(struct gendisk *disk); void wbt_enable_default(struct gendisk *disk); u64 wbt_get_min_lat(struct request_queue *q); -void wbt_set_min_lat(struct request_queue *q, u64 val); -bool wbt_disabled(struct request_queue *); - -u64 wbt_default_latency_nsec(struct request_queue *); +bool wbt_disabled(struct request_queue *q); +int wbt_set_lat(struct gendisk *disk, s64 val); #else +static inline void wbt_init_enable_default(struct gendisk *disk) +{ +} + static inline void wbt_disable_default(struct gendisk *disk) { } diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 0c77244a35c9..ca30caec838e 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -16,6 +16,10 @@ #include <linux/spinlock.h> #include <linux/refcount.h> #include <linux/mempool.h> +#include <linux/kthread.h> +#include <linux/freezer.h> + +#include <trace/events/block.h> #include "blk.h" #include "blk-mq-sched.h" @@ -31,12 +35,20 @@ static const char *const zone_cond_name[] = { ZONE_COND_NAME(READONLY), ZONE_COND_NAME(FULL), ZONE_COND_NAME(OFFLINE), + ZONE_COND_NAME(ACTIVE), }; #undef ZONE_COND_NAME /* * Per-zone write plug. * @node: hlist_node structure for managing the plug using a hash table. + * @entry: list_head structure for listing the plug in the disk list of active + * zone write plugs. + * @bio_list: The list of BIOs that are currently plugged. + * @bio_work: Work struct to handle issuing of plugged BIOs + * @rcu_head: RCU head to free zone write plugs with an RCU grace period. + * @disk: The gendisk the plug belongs to. + * @lock: Spinlock to atomically manipulate the plug. * @ref: Zone write plug reference counter. A zone write plug reference is * always at least 1 when the plug is hashed in the disk plug hash table. * The reference is incremented whenever a new BIO needing plugging is @@ -46,29 +58,45 @@ static const char *const zone_cond_name[] = { * reference is dropped whenever the zone of the zone write plug is reset, * finished and when the zone becomes full (last write BIO to the zone * completes). - * @lock: Spinlock to atomically manipulate the plug. * @flags: Flags indicating the plug state. * @zone_no: The number of the zone the plug is managing. * @wp_offset: The zone write pointer location relative to the start of the zone * as a number of 512B sectors. - * @bio_list: The list of BIOs that are currently plugged. - * @bio_work: Work struct to handle issuing of plugged BIOs - * @rcu_head: RCU head to free zone write plugs with an RCU grace period. - * @disk: The gendisk the plug belongs to. + * @cond: Condition of the zone */ struct blk_zone_wplug { struct hlist_node node; - refcount_t ref; - spinlock_t lock; - unsigned int flags; - unsigned int zone_no; - unsigned int wp_offset; + struct list_head entry; struct bio_list bio_list; struct work_struct bio_work; struct rcu_head rcu_head; struct gendisk *disk; + spinlock_t lock; + refcount_t ref; + unsigned int flags; + unsigned int zone_no; + unsigned int wp_offset; + enum blk_zone_cond cond; }; +static inline bool disk_need_zone_resources(struct gendisk *disk) +{ + /* + * All request-based zoned devices need zone resources so that the + * block layer can automatically handle write BIO plugging. BIO-based + * device drivers (e.g. DM devices) are normally responsible for + * handling zone write ordering and do not need zone resources, unless + * the driver requires zone append emulation. + */ + return queue_is_mq(disk->queue) || + queue_emulates_zone_append(disk->queue); +} + +static inline unsigned int disk_zone_wplugs_hash_size(struct gendisk *disk) +{ + return 1U << disk->zone_wplugs_hash_bits; +} + /* * Zone write plug flags bits: * - BLK_ZONE_WPLUG_PLUGGED: Indicates that the zone write plug is plugged, @@ -76,25 +104,25 @@ struct blk_zone_wplug { * being executed or the zone write plug bio list is not empty. * - BLK_ZONE_WPLUG_NEED_WP_UPDATE: Indicates that we lost track of a zone * write pointer offset and need to update it. - * - BLK_ZONE_WPLUG_UNHASHED: Indicates that the zone write plug was removed - * from the disk hash table and that the initial reference to the zone - * write plug set when the plug was first added to the hash table has been - * dropped. This flag is set when a zone is reset, finished or become full, - * to prevent new references to the zone write plug to be taken for - * newly incoming BIOs. A zone write plug flagged with this flag will be - * freed once all remaining references from BIOs or functions are dropped. + * - BLK_ZONE_WPLUG_DEAD: Indicates that the zone write plug will be + * removed from the disk hash table of zone write plugs when the last + * reference on the zone write plug is dropped. If set, this flag also + * indicates that the initial extra reference on the zone write plug was + * dropped, meaning that the reference count indicates the current number of + * active users (code context or BIOs and requests in flight). This flag is + * set when a zone is reset, finished or becomes full. */ #define BLK_ZONE_WPLUG_PLUGGED (1U << 0) #define BLK_ZONE_WPLUG_NEED_WP_UPDATE (1U << 1) -#define BLK_ZONE_WPLUG_UNHASHED (1U << 2) +#define BLK_ZONE_WPLUG_DEAD (1U << 2) /** - * blk_zone_cond_str - Return string XXX in BLK_ZONE_COND_XXX. - * @zone_cond: BLK_ZONE_COND_XXX. + * blk_zone_cond_str - Return a zone condition name string + * @zone_cond: a zone condition BLK_ZONE_COND_name * - * Description: Centralize block layer function to convert BLK_ZONE_COND_XXX - * into string format. Useful in the debugging and tracing zone conditions. For - * invalid BLK_ZONE_COND_XXX it returns string "UNKNOWN". + * Convert a BLK_ZONE_COND_name zone condition into the string "name". Useful + * for the debugging and tracing zone conditions. For an invalid zone + * conditions, the string "UNKNOWN" is returned. */ const char *blk_zone_cond_str(enum blk_zone_cond zone_cond) { @@ -107,28 +135,108 @@ const char *blk_zone_cond_str(enum blk_zone_cond zone_cond) } EXPORT_SYMBOL_GPL(blk_zone_cond_str); -struct disk_report_zones_cb_args { - struct gendisk *disk; - report_zones_cb user_cb; - void *user_data; -}; +static void blk_zone_set_cond(u8 *zones_cond, unsigned int zno, + enum blk_zone_cond cond) +{ + if (!zones_cond) + return; + + switch (cond) { + case BLK_ZONE_COND_IMP_OPEN: + case BLK_ZONE_COND_EXP_OPEN: + case BLK_ZONE_COND_CLOSED: + zones_cond[zno] = BLK_ZONE_COND_ACTIVE; + return; + case BLK_ZONE_COND_NOT_WP: + case BLK_ZONE_COND_EMPTY: + case BLK_ZONE_COND_FULL: + case BLK_ZONE_COND_OFFLINE: + case BLK_ZONE_COND_READONLY: + default: + zones_cond[zno] = cond; + return; + } +} + +static void disk_zone_set_cond(struct gendisk *disk, sector_t sector, + enum blk_zone_cond cond) +{ + u8 *zones_cond; -static void disk_zone_wplug_sync_wp_offset(struct gendisk *disk, - struct blk_zone *zone); + rcu_read_lock(); + zones_cond = rcu_dereference(disk->zones_cond); + if (zones_cond) { + unsigned int zno = disk_zone_no(disk, sector); -static int disk_report_zones_cb(struct blk_zone *zone, unsigned int idx, - void *data) + /* + * The condition of a conventional, readonly and offline zones + * never changes, so do nothing if the target zone is in one of + * these conditions. + */ + switch (zones_cond[zno]) { + case BLK_ZONE_COND_NOT_WP: + case BLK_ZONE_COND_READONLY: + case BLK_ZONE_COND_OFFLINE: + break; + default: + blk_zone_set_cond(zones_cond, zno, cond); + break; + } + } + rcu_read_unlock(); +} + +/** + * bdev_zone_is_seq - check if a sector belongs to a sequential write zone + * @bdev: block device to check + * @sector: sector number + * + * Check if @sector on @bdev is contained in a sequential write required zone. + */ +bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector) { - struct disk_report_zones_cb_args *args = data; - struct gendisk *disk = args->disk; + struct gendisk *disk = bdev->bd_disk; + unsigned int zno = disk_zone_no(disk, sector); + bool is_seq = false; + u8 *zones_cond; - if (disk->zone_wplugs_hash) - disk_zone_wplug_sync_wp_offset(disk, zone); + if (!bdev_is_zoned(bdev)) + return false; - if (!args->user_cb) + rcu_read_lock(); + zones_cond = rcu_dereference(disk->zones_cond); + if (zones_cond && zno < disk->nr_zones) + is_seq = zones_cond[zno] != BLK_ZONE_COND_NOT_WP; + rcu_read_unlock(); + + return is_seq; +} +EXPORT_SYMBOL_GPL(bdev_zone_is_seq); + +/* + * Zone report arguments for block device drivers report_zones operation. + * @cb: report_zones_cb callback for each reported zone. + * @data: Private data passed to report_zones_cb. + */ +struct blk_report_zones_args { + report_zones_cb cb; + void *data; + bool report_active; +}; + +static int blkdev_do_report_zones(struct block_device *bdev, sector_t sector, + unsigned int nr_zones, + struct blk_report_zones_args *args) +{ + struct gendisk *disk = bdev->bd_disk; + + if (!bdev_is_zoned(bdev) || WARN_ON_ONCE(!disk->fops->report_zones)) + return -EOPNOTSUPP; + + if (!nr_zones || sector >= get_capacity(disk)) return 0; - return args->user_cb(zone, idx, args->user_data); + return disk->fops->report_zones(disk, sector, nr_zones, args); } /** @@ -153,22 +261,12 @@ static int disk_report_zones_cb(struct blk_zone *zone, unsigned int idx, int blkdev_report_zones(struct block_device *bdev, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data) { - struct gendisk *disk = bdev->bd_disk; - sector_t capacity = get_capacity(disk); - struct disk_report_zones_cb_args args = { - .disk = disk, - .user_cb = cb, - .user_data = data, + struct blk_report_zones_args args = { + .cb = cb, + .data = data, }; - if (!bdev_is_zoned(bdev) || WARN_ON_ONCE(!disk->fops->report_zones)) - return -EOPNOTSUPP; - - if (!nr_zones || sector >= capacity) - return 0; - - return disk->fops->report_zones(disk, sector, nr_zones, - disk_report_zones_cb, &args); + return blkdev_do_report_zones(bdev, sector, nr_zones, &args); } EXPORT_SYMBOL_GPL(blkdev_report_zones); @@ -177,6 +275,7 @@ static int blkdev_zone_reset_all(struct block_device *bdev) struct bio bio; bio_init(&bio, bdev, NULL, 0, REQ_OP_ZONE_RESET_ALL | REQ_SYNC); + trace_blkdev_zone_mgmt(&bio, 0); return submit_bio_wait(&bio); } @@ -240,6 +339,7 @@ int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, cond_resched(); } + trace_blkdev_zone_mgmt(bio, nr_sectors); ret = submit_bio_wait(bio); bio_put(bio); @@ -262,7 +362,12 @@ static int blkdev_copy_zone_to_user(struct blk_zone *zone, unsigned int idx, } /* - * BLKREPORTZONE ioctl processing. + * Mask of valid input flags for BLKREPORTZONEV2 ioctl. + */ +#define BLK_ZONE_REPV2_INPUT_FLAGS BLK_ZONE_REP_CACHED + +/* + * BLKREPORTZONE and BLKREPORTZONEV2 ioctl processing. * Called from blkdev_ioctl. */ int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd, @@ -286,8 +391,22 @@ int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd, return -EINVAL; args.zones = argp + sizeof(struct blk_zone_report); - ret = blkdev_report_zones(bdev, rep.sector, rep.nr_zones, - blkdev_copy_zone_to_user, &args); + + switch (cmd) { + case BLKREPORTZONE: + ret = blkdev_report_zones(bdev, rep.sector, rep.nr_zones, + blkdev_copy_zone_to_user, &args); + break; + case BLKREPORTZONEV2: + if (rep.flags & ~BLK_ZONE_REPV2_INPUT_FLAGS) + return -EINVAL; + ret = blkdev_report_zones_cached(bdev, rep.sector, rep.nr_zones, + blkdev_copy_zone_to_user, &args); + break; + default: + return -EINVAL; + } + if (ret < 0) return ret; @@ -298,20 +417,32 @@ int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd, return 0; } -static int blkdev_truncate_zone_range(struct block_device *bdev, - blk_mode_t mode, const struct blk_zone_range *zrange) +static int blkdev_reset_zone(struct block_device *bdev, blk_mode_t mode, + struct blk_zone_range *zrange) { loff_t start, end; + int ret = -EINVAL; + inode_lock(bdev->bd_mapping->host); + filemap_invalidate_lock(bdev->bd_mapping); if (zrange->sector + zrange->nr_sectors <= zrange->sector || zrange->sector + zrange->nr_sectors > get_capacity(bdev->bd_disk)) /* Out of range */ - return -EINVAL; + goto out_unlock; start = zrange->sector << SECTOR_SHIFT; end = ((zrange->sector + zrange->nr_sectors) << SECTOR_SHIFT) - 1; - return truncate_bdev_range(bdev, mode, start, end); + ret = truncate_bdev_range(bdev, mode, start, end); + if (ret) + goto out_unlock; + + ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, zrange->sector, + zrange->nr_sectors); +out_unlock: + filemap_invalidate_unlock(bdev->bd_mapping); + inode_unlock(bdev->bd_mapping->host); + return ret; } /* @@ -324,7 +455,6 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode, void __user *argp = (void __user *)arg; struct blk_zone_range zrange; enum req_op op; - int ret; if (!argp) return -EINVAL; @@ -340,14 +470,7 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode, switch (cmd) { case BLKRESETZONE: - op = REQ_OP_ZONE_RESET; - - /* Invalidate the page cache, including dirty pages. */ - filemap_invalidate_lock(bdev->bd_mapping); - ret = blkdev_truncate_zone_range(bdev, mode, &zrange); - if (ret) - goto fail; - break; + return blkdev_reset_zone(bdev, mode, &zrange); case BLKOPENZONE: op = REQ_OP_ZONE_OPEN; break; @@ -361,13 +484,7 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode, return -ENOTTY; } - ret = blkdev_zone_mgmt(bdev, op, zrange.sector, zrange.nr_sectors); - -fail: - if (cmd == BLKRESETZONE) - filemap_invalidate_unlock(bdev->bd_mapping); - - return ret; + return blkdev_zone_mgmt(bdev, op, zrange.sector, zrange.nr_sectors); } static bool disk_zone_is_last(struct gendisk *disk, struct blk_zone *zone) @@ -375,18 +492,12 @@ static bool disk_zone_is_last(struct gendisk *disk, struct blk_zone *zone) return zone->start + zone->len >= get_capacity(disk); } -static bool disk_zone_is_full(struct gendisk *disk, - unsigned int zno, unsigned int offset_in_zone) -{ - if (zno < disk->nr_zones - 1) - return offset_in_zone >= disk->zone_capacity; - return offset_in_zone >= disk->last_zone_capacity; -} - static bool disk_zone_wplug_is_full(struct gendisk *disk, struct blk_zone_wplug *zwplug) { - return disk_zone_is_full(disk, zwplug->zone_no, zwplug->wp_offset); + if (zwplug->zone_no < disk->nr_zones - 1) + return zwplug->wp_offset >= disk->zone_capacity; + return zwplug->wp_offset >= disk->last_zone_capacity; } static bool disk_insert_zone_wplug(struct gendisk *disk, @@ -394,6 +505,7 @@ static bool disk_insert_zone_wplug(struct gendisk *disk, { struct blk_zone_wplug *zwplg; unsigned long flags; + u8 *zones_cond; unsigned int idx = hash_32(zwplug->zone_no, disk->zone_wplugs_hash_bits); @@ -402,16 +514,31 @@ static bool disk_insert_zone_wplug(struct gendisk *disk, * are racing with other submission context, so we may already have a * zone write plug for the same zone. */ - spin_lock_irqsave(&disk->zone_wplugs_lock, flags); + spin_lock_irqsave(&disk->zone_wplugs_hash_lock, flags); hlist_for_each_entry_rcu(zwplg, &disk->zone_wplugs_hash[idx], node) { if (zwplg->zone_no == zwplug->zone_no) { - spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags); + spin_unlock_irqrestore(&disk->zone_wplugs_hash_lock, + flags); return false; } } + + /* + * Set the zone condition: if we do not yet have a zones_cond array + * attached to the disk, then this is a zone write plug insert from the + * first call to blk_revalidate_disk_zones(), in which case the zone is + * necessarilly in the active condition. + */ + zones_cond = rcu_dereference_check(disk->zones_cond, + lockdep_is_held(&disk->zone_wplugs_hash_lock)); + if (zones_cond) + zwplug->cond = zones_cond[zwplug->zone_no]; + else + zwplug->cond = BLK_ZONE_COND_ACTIVE; + hlist_add_head_rcu(&zwplug->node, &disk->zone_wplugs_hash[idx]); atomic_inc(&disk->nr_zone_wplugs); - spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags); + spin_unlock_irqrestore(&disk->zone_wplugs_hash_lock, flags); return true; } @@ -455,100 +582,98 @@ static void disk_free_zone_wplug_rcu(struct rcu_head *rcu_head) mempool_free(zwplug, zwplug->disk->zone_wplugs_pool); } -static inline void disk_put_zone_wplug(struct blk_zone_wplug *zwplug) +static void disk_free_zone_wplug(struct blk_zone_wplug *zwplug) { - if (refcount_dec_and_test(&zwplug->ref)) { - WARN_ON_ONCE(!bio_list_empty(&zwplug->bio_list)); - WARN_ON_ONCE(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED); - WARN_ON_ONCE(!(zwplug->flags & BLK_ZONE_WPLUG_UNHASHED)); + struct gendisk *disk = zwplug->disk; + unsigned long flags; - call_rcu(&zwplug->rcu_head, disk_free_zone_wplug_rcu); - } + WARN_ON_ONCE(!(zwplug->flags & BLK_ZONE_WPLUG_DEAD)); + WARN_ON_ONCE(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED); + WARN_ON_ONCE(!bio_list_empty(&zwplug->bio_list)); + + spin_lock_irqsave(&disk->zone_wplugs_hash_lock, flags); + blk_zone_set_cond(rcu_dereference_check(disk->zones_cond, + lockdep_is_held(&disk->zone_wplugs_hash_lock)), + zwplug->zone_no, zwplug->cond); + hlist_del_init_rcu(&zwplug->node); + atomic_dec(&disk->nr_zone_wplugs); + spin_unlock_irqrestore(&disk->zone_wplugs_hash_lock, flags); + + call_rcu(&zwplug->rcu_head, disk_free_zone_wplug_rcu); +} + +static inline void disk_put_zone_wplug(struct blk_zone_wplug *zwplug) +{ + if (refcount_dec_and_test(&zwplug->ref)) + disk_free_zone_wplug(zwplug); } -static inline bool disk_should_remove_zone_wplug(struct gendisk *disk, - struct blk_zone_wplug *zwplug) +/* + * Flag the zone write plug as dead and drop the initial reference we got when + * the zone write plug was added to the hash table. The zone write plug will be + * unhashed when its last reference is dropped. + */ +static void disk_mark_zone_wplug_dead(struct blk_zone_wplug *zwplug) { lockdep_assert_held(&zwplug->lock); - /* If the zone write plug was already removed, we are done. */ - if (zwplug->flags & BLK_ZONE_WPLUG_UNHASHED) - return false; + if (!(zwplug->flags & BLK_ZONE_WPLUG_DEAD)) { + zwplug->flags |= BLK_ZONE_WPLUG_DEAD; + disk_put_zone_wplug(zwplug); + } +} - /* If the zone write plug is still plugged, it cannot be removed. */ - if (zwplug->flags & BLK_ZONE_WPLUG_PLUGGED) +static inline bool disk_check_zone_wplug_dead(struct blk_zone_wplug *zwplug) +{ + if (!(zwplug->flags & BLK_ZONE_WPLUG_DEAD)) return false; /* - * Completions of BIOs with blk_zone_write_plug_bio_endio() may - * happen after handling a request completion with - * blk_zone_write_plug_finish_request() (e.g. with split BIOs - * that are chained). In such case, disk_zone_wplug_unplug_bio() - * should not attempt to remove the zone write plug until all BIO - * completions are seen. Check by looking at the zone write plug - * reference count, which is 2 when the plug is unused (one reference - * taken when the plug was allocated and another reference taken by the - * caller context). + * If a new write is received right after a zone reset completes and + * while the disk_zone_wplugs_worker() thread has not yet released the + * reference on the zone write plug after processing the last write to + * the zone, then the new write BIO will see the zone write plug marked + * as dead. This case is however a false positive and a perfectly valid + * pattern. In such case, restore the zone write plug to a live one. */ - if (refcount_read(&zwplug->ref) > 2) + if (!zwplug->wp_offset && bio_list_empty(&zwplug->bio_list)) { + zwplug->flags &= ~BLK_ZONE_WPLUG_DEAD; + refcount_inc(&zwplug->ref); return false; + } - /* We can remove zone write plugs for zones that are empty or full. */ - return !zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug); + return true; } -static void disk_remove_zone_wplug(struct gendisk *disk, - struct blk_zone_wplug *zwplug) +static bool disk_zone_wplug_submit_bio(struct gendisk *disk, + struct blk_zone_wplug *zwplug); + +static void blk_zone_wplug_bio_work(struct work_struct *work) { - unsigned long flags; + struct blk_zone_wplug *zwplug = + container_of(work, struct blk_zone_wplug, bio_work); - /* If the zone write plug was already removed, we have nothing to do. */ - if (zwplug->flags & BLK_ZONE_WPLUG_UNHASHED) - return; + disk_zone_wplug_submit_bio(zwplug->disk, zwplug); - /* - * Mark the zone write plug as unhashed and drop the extra reference we - * took when the plug was inserted in the hash table. - */ - zwplug->flags |= BLK_ZONE_WPLUG_UNHASHED; - spin_lock_irqsave(&disk->zone_wplugs_lock, flags); - hlist_del_init_rcu(&zwplug->node); - atomic_dec(&disk->nr_zone_wplugs); - spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags); + /* Drop the reference we took in disk_zone_wplug_schedule_work(). */ disk_put_zone_wplug(zwplug); } -static void blk_zone_wplug_bio_work(struct work_struct *work); - /* - * Get a reference on the write plug for the zone containing @sector. - * If the plug does not exist, it is allocated and hashed. - * Return a pointer to the zone write plug with the plug spinlock held. + * Get a zone write plug for the zone containing @sector. + * If the plug does not exist, it is allocated and inserted in the disk hash + * table. */ -static struct blk_zone_wplug *disk_get_and_lock_zone_wplug(struct gendisk *disk, - sector_t sector, gfp_t gfp_mask, - unsigned long *flags) +static struct blk_zone_wplug *disk_get_or_alloc_zone_wplug(struct gendisk *disk, + sector_t sector, gfp_t gfp_mask) { unsigned int zno = disk_zone_no(disk, sector); struct blk_zone_wplug *zwplug; again: zwplug = disk_get_zone_wplug(disk, sector); - if (zwplug) { - /* - * Check that a BIO completion or a zone reset or finish - * operation has not already removed the zone write plug from - * the hash table and dropped its reference count. In such case, - * we need to get a new plug so start over from the beginning. - */ - spin_lock_irqsave(&zwplug->lock, *flags); - if (zwplug->flags & BLK_ZONE_WPLUG_UNHASHED) { - spin_unlock_irqrestore(&zwplug->lock, *flags); - disk_put_zone_wplug(zwplug); - goto again; - } + if (zwplug) return zwplug; - } /* * Allocate and initialize a zone write plug with an extra reference @@ -567,17 +692,15 @@ again: zwplug->wp_offset = bdev_offset_from_zone_start(disk->part0, sector); bio_list_init(&zwplug->bio_list); INIT_WORK(&zwplug->bio_work, blk_zone_wplug_bio_work); + INIT_LIST_HEAD(&zwplug->entry); zwplug->disk = disk; - spin_lock_irqsave(&zwplug->lock, *flags); - /* * Insert the new zone write plug in the hash table. This can fail only * if another context already inserted a plug. Retry from the beginning * in such case. */ if (!disk_insert_zone_wplug(disk, zwplug)) { - spin_unlock_irqrestore(&zwplug->lock, *flags); mempool_free(zwplug, disk->zone_wplugs_pool); goto again; } @@ -593,7 +716,7 @@ static inline void blk_zone_wplug_bio_io_error(struct blk_zone_wplug *zwplug, bio_clear_flag(bio, BIO_ZONE_WRITE_PLUGGING); bio_io_error(bio); disk_put_zone_wplug(zwplug); - /* Drop the reference taken by disk_zone_wplug_add_bio(() */ + /* Drop the reference taken by disk_zone_wplug_add_bio(). */ blk_queue_exit(q); } @@ -602,8 +725,11 @@ static inline void blk_zone_wplug_bio_io_error(struct blk_zone_wplug *zwplug, */ static void disk_zone_wplug_abort(struct blk_zone_wplug *zwplug) { + struct gendisk *disk = zwplug->disk; struct bio *bio; + lockdep_assert_held(&zwplug->lock); + if (bio_list_empty(&zwplug->bio_list)) return; @@ -611,6 +737,38 @@ static void disk_zone_wplug_abort(struct blk_zone_wplug *zwplug) zwplug->disk->disk_name, zwplug->zone_no); while ((bio = bio_list_pop(&zwplug->bio_list))) blk_zone_wplug_bio_io_error(zwplug, bio); + + zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED; + + /* + * If we are using the per disk zone write plugs worker thread, remove + * the zone write plug from the work list and drop the reference we + * took when the zone write plug was added to that list. + */ + if (blk_queue_zoned_qd1_writes(disk->queue)) { + spin_lock(&disk->zone_wplugs_list_lock); + if (!list_empty(&zwplug->entry)) { + list_del_init(&zwplug->entry); + disk_put_zone_wplug(zwplug); + } + spin_unlock(&disk->zone_wplugs_list_lock); + } +} + +/* + * Update a zone write plug condition based on the write pointer offset. + */ +static void disk_zone_wplug_update_cond(struct gendisk *disk, + struct blk_zone_wplug *zwplug) +{ + lockdep_assert_held(&zwplug->lock); + + if (disk_zone_wplug_is_full(disk, zwplug)) + zwplug->cond = BLK_ZONE_COND_FULL; + else if (!zwplug->wp_offset) + zwplug->cond = BLK_ZONE_COND_EMPTY; + else + zwplug->cond = BLK_ZONE_COND_ACTIVE; } /* @@ -628,15 +786,11 @@ static void disk_zone_wplug_set_wp_offset(struct gendisk *disk, /* Update the zone write pointer and abort all plugged BIOs. */ zwplug->flags &= ~BLK_ZONE_WPLUG_NEED_WP_UPDATE; zwplug->wp_offset = wp_offset; - disk_zone_wplug_abort(zwplug); + disk_zone_wplug_update_cond(disk, zwplug); - /* - * The zone write plug now has no BIO plugged: remove it from the - * hash table so that it cannot be seen. The plug will be freed - * when the last reference is dropped. - */ - if (disk_should_remove_zone_wplug(disk, zwplug)) - disk_remove_zone_wplug(disk, zwplug); + disk_zone_wplug_abort(zwplug); + if (!zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug)) + disk_mark_zone_wplug_dead(zwplug); } static unsigned int blk_zone_wp_offset(struct blk_zone *zone) @@ -645,138 +799,418 @@ static unsigned int blk_zone_wp_offset(struct blk_zone *zone) case BLK_ZONE_COND_IMP_OPEN: case BLK_ZONE_COND_EXP_OPEN: case BLK_ZONE_COND_CLOSED: + case BLK_ZONE_COND_ACTIVE: return zone->wp - zone->start; - case BLK_ZONE_COND_FULL: - return zone->len; case BLK_ZONE_COND_EMPTY: return 0; + case BLK_ZONE_COND_FULL: case BLK_ZONE_COND_NOT_WP: case BLK_ZONE_COND_OFFLINE: case BLK_ZONE_COND_READONLY: default: /* - * Conventional, offline and read-only zones do not have a valid - * write pointer. + * Conventional, full, offline and read-only zones do not have + * a valid write pointer. */ return UINT_MAX; } } -static void disk_zone_wplug_sync_wp_offset(struct gendisk *disk, - struct blk_zone *zone) +static unsigned int disk_zone_wplug_sync_wp_offset(struct gendisk *disk, + struct blk_zone *zone) { struct blk_zone_wplug *zwplug; - unsigned long flags; + unsigned int wp_offset = blk_zone_wp_offset(zone); zwplug = disk_get_zone_wplug(disk, zone->start); - if (!zwplug) - return; + if (zwplug) { + unsigned long flags; - spin_lock_irqsave(&zwplug->lock, flags); - if (zwplug->flags & BLK_ZONE_WPLUG_NEED_WP_UPDATE) - disk_zone_wplug_set_wp_offset(disk, zwplug, - blk_zone_wp_offset(zone)); - spin_unlock_irqrestore(&zwplug->lock, flags); + spin_lock_irqsave(&zwplug->lock, flags); + if (zwplug->flags & BLK_ZONE_WPLUG_NEED_WP_UPDATE) + disk_zone_wplug_set_wp_offset(disk, zwplug, wp_offset); + spin_unlock_irqrestore(&zwplug->lock, flags); + disk_put_zone_wplug(zwplug); + } - disk_put_zone_wplug(zwplug); + return wp_offset; } -static int disk_zone_sync_wp_offset(struct gendisk *disk, sector_t sector) +/** + * disk_report_zone - Report one zone + * @disk: Target disk + * @zone: The zone to report + * @idx: The index of the zone in the overall zone report + * @args: report zones callback and data + * + * Description: + * Helper function for block device drivers to report one zone of a zone + * report initiated with blkdev_report_zones(). The zone being reported is + * specified by @zone and used to update, if necessary, the zone write plug + * information for the zone. If @args specifies a user callback function, + * this callback is executed. + */ +int disk_report_zone(struct gendisk *disk, struct blk_zone *zone, + unsigned int idx, struct blk_report_zones_args *args) { - struct disk_report_zones_cb_args args = { - .disk = disk, + if (args && args->report_active) { + /* + * If we come here, then this is a report zones as a fallback + * for a cached report. So collapse the implicit open, explicit + * open and closed conditions into the active zone condition. + */ + switch (zone->cond) { + case BLK_ZONE_COND_IMP_OPEN: + case BLK_ZONE_COND_EXP_OPEN: + case BLK_ZONE_COND_CLOSED: + zone->cond = BLK_ZONE_COND_ACTIVE; + break; + default: + break; + } + } + + if (disk->zone_wplugs_hash) + disk_zone_wplug_sync_wp_offset(disk, zone); + + if (args && args->cb) + return args->cb(zone, idx, args->data); + + return 0; +} +EXPORT_SYMBOL_GPL(disk_report_zone); + +static int blkdev_report_zone_cb(struct blk_zone *zone, unsigned int idx, + void *data) +{ + memcpy(data, zone, sizeof(struct blk_zone)); + return 0; +} + +static int blkdev_report_zone_fallback(struct block_device *bdev, + sector_t sector, struct blk_zone *zone) +{ + struct blk_report_zones_args args = { + .cb = blkdev_report_zone_cb, + .data = zone, + .report_active = true, }; + int error; - return disk->fops->report_zones(disk, sector, 1, - disk_report_zones_cb, &args); + error = blkdev_do_report_zones(bdev, sector, 1, &args); + if (error < 0) + return error; + if (error == 0) + return -EIO; + return 0; } -static bool blk_zone_wplug_handle_reset_or_finish(struct bio *bio, - unsigned int wp_offset) +/* + * For devices that natively support zone append operations, we do not use zone + * write plugging for zone append writes, which makes the zone condition + * tracking invalid once zone append was used. In that case fall back to a + * regular report zones to get correct information. + */ +static inline bool blkdev_has_cached_report_zones(struct block_device *bdev) { - struct gendisk *disk = bio->bi_bdev->bd_disk; - sector_t sector = bio->bi_iter.bi_sector; + return disk_need_zone_resources(bdev->bd_disk) && + (bdev_emulates_zone_append(bdev) || + !test_bit(GD_ZONE_APPEND_USED, &bdev->bd_disk->state)); +} + +/** + * blkdev_get_zone_info - Get a single zone information from cached data + * @bdev: Target block device + * @sector: Sector contained by the target zone + * @zone: zone structure to return the zone information + * + * Description: + * Get the zone information for the zone containing @sector using the zone + * write plug of the target zone, if one exist, or the disk zone condition + * array otherwise. The zone condition may be reported as being + * the BLK_ZONE_COND_ACTIVE condition for a zone that is in the implicit + * open, explicit open or closed condition. + * + * Returns 0 on success and a negative error code on failure. + */ +int blkdev_get_zone_info(struct block_device *bdev, sector_t sector, + struct blk_zone *zone) +{ + struct gendisk *disk = bdev->bd_disk; + sector_t zone_sectors = bdev_zone_sectors(bdev); struct blk_zone_wplug *zwplug; unsigned long flags; + u8 *zones_cond; - /* Conventional zones cannot be reset nor finished. */ - if (!bdev_zone_is_seq(bio->bi_bdev, sector)) { - bio_io_error(bio); - return true; + if (!bdev_is_zoned(bdev)) + return -EOPNOTSUPP; + + if (sector >= get_capacity(disk)) + return -EINVAL; + + memset(zone, 0, sizeof(*zone)); + sector = bdev_zone_start(bdev, sector); + + if (!blkdev_has_cached_report_zones(bdev)) + return blkdev_report_zone_fallback(bdev, sector, zone); + + rcu_read_lock(); + zones_cond = rcu_dereference(disk->zones_cond); + if (!disk->zone_wplugs_hash || !zones_cond) { + rcu_read_unlock(); + return blkdev_report_zone_fallback(bdev, sector, zone); } + zone->cond = zones_cond[disk_zone_no(disk, sector)]; + rcu_read_unlock(); + + zone->start = sector; + zone->len = zone_sectors; /* - * No-wait reset or finish BIOs do not make much sense as the callers - * issue these as blocking operations in most cases. To avoid issues - * the BIO execution potentially failing with BLK_STS_AGAIN, warn about - * REQ_NOWAIT being set and ignore that flag. + * If this is a conventional zone, we do not have a zone write plug and + * can report the zone immediately. */ - if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) - bio->bi_opf &= ~REQ_NOWAIT; + if (zone->cond == BLK_ZONE_COND_NOT_WP) { + zone->type = BLK_ZONE_TYPE_CONVENTIONAL; + zone->capacity = zone_sectors; + zone->wp = ULLONG_MAX; + return 0; + } /* - * If we have a zone write plug, set its write pointer offset to 0 - * (reset case) or to the zone size (finish case). This will abort all - * BIOs plugged for the target zone. It is fine as resetting or - * finishing zones while writes are still in-flight will result in the + * This is a sequential write required zone. If the zone is read-only or + * offline, only set the zone write pointer to an invalid value and + * report the zone. + */ + zone->type = BLK_ZONE_TYPE_SEQWRITE_REQ; + if (disk_zone_is_last(disk, zone)) + zone->capacity = disk->last_zone_capacity; + else + zone->capacity = disk->zone_capacity; + + if (zone->cond == BLK_ZONE_COND_READONLY || + zone->cond == BLK_ZONE_COND_OFFLINE) { + zone->wp = ULLONG_MAX; + return 0; + } + + /* + * If the zone does not have a zone write plug, it is either full or + * empty, as we otherwise would have a zone write plug for it. In this + * case, set the write pointer accordingly and report the zone. + * Otherwise, if we have a zone write plug, use it. + */ + zwplug = disk_get_zone_wplug(disk, sector); + if (!zwplug) { + if (zone->cond == BLK_ZONE_COND_FULL) + zone->wp = ULLONG_MAX; + else + zone->wp = sector; + return 0; + } + + spin_lock_irqsave(&zwplug->lock, flags); + if (zwplug->flags & BLK_ZONE_WPLUG_NEED_WP_UPDATE) { + spin_unlock_irqrestore(&zwplug->lock, flags); + disk_put_zone_wplug(zwplug); + return blkdev_report_zone_fallback(bdev, sector, zone); + } + zone->cond = zwplug->cond; + zone->wp = sector + zwplug->wp_offset; + spin_unlock_irqrestore(&zwplug->lock, flags); + + disk_put_zone_wplug(zwplug); + + return 0; +} +EXPORT_SYMBOL_GPL(blkdev_get_zone_info); + +/** + * blkdev_report_zones_cached - Get cached zones information + * @bdev: Target block device + * @sector: Sector from which to report zones + * @nr_zones: Maximum number of zones to report + * @cb: Callback function called for each reported zone + * @data: Private data for the callback function + * + * Description: + * Similar to blkdev_report_zones() but instead of calling into the low level + * device driver to get the zone report from the device, use + * blkdev_get_zone_info() to generate the report from the disk zone write + * plugs and zones condition array. Since calling this function without a + * callback does not make sense, @cb must be specified. + */ +int blkdev_report_zones_cached(struct block_device *bdev, sector_t sector, + unsigned int nr_zones, report_zones_cb cb, void *data) +{ + struct gendisk *disk = bdev->bd_disk; + sector_t capacity = get_capacity(disk); + sector_t zone_sectors = bdev_zone_sectors(bdev); + unsigned int idx = 0; + struct blk_zone zone; + int ret; + + if (!cb || !bdev_is_zoned(bdev) || + WARN_ON_ONCE(!disk->fops->report_zones)) + return -EOPNOTSUPP; + + if (!nr_zones || sector >= capacity) + return 0; + + if (!blkdev_has_cached_report_zones(bdev)) { + struct blk_report_zones_args args = { + .cb = cb, + .data = data, + .report_active = true, + }; + + return blkdev_do_report_zones(bdev, sector, nr_zones, &args); + } + + for (sector = bdev_zone_start(bdev, sector); + sector < capacity && idx < nr_zones; + sector += zone_sectors, idx++) { + ret = blkdev_get_zone_info(bdev, sector, &zone); + if (ret) + return ret; + + ret = cb(&zone, idx, data); + if (ret) + return ret; + } + + return idx; +} +EXPORT_SYMBOL_GPL(blkdev_report_zones_cached); + +static void blk_zone_reset_bio_endio(struct bio *bio) +{ + struct gendisk *disk = bio->bi_bdev->bd_disk; + sector_t sector = bio->bi_iter.bi_sector; + struct blk_zone_wplug *zwplug; + + /* + * If we have a zone write plug, set its write pointer offset to 0. + * This will abort all BIOs plugged for the target zone. It is fine as + * resetting zones while writes are still in-flight will result in the * writes failing anyway. */ zwplug = disk_get_zone_wplug(disk, sector); if (zwplug) { + unsigned long flags; + spin_lock_irqsave(&zwplug->lock, flags); - disk_zone_wplug_set_wp_offset(disk, zwplug, wp_offset); + disk_zone_wplug_set_wp_offset(disk, zwplug, 0); spin_unlock_irqrestore(&zwplug->lock, flags); disk_put_zone_wplug(zwplug); + } else { + disk_zone_set_cond(disk, sector, BLK_ZONE_COND_EMPTY); } - - return false; } -static bool blk_zone_wplug_handle_reset_all(struct bio *bio) +static void blk_zone_reset_all_bio_endio(struct bio *bio) { struct gendisk *disk = bio->bi_bdev->bd_disk; + sector_t capacity = get_capacity(disk); struct blk_zone_wplug *zwplug; unsigned long flags; sector_t sector; + unsigned int i; + + if (atomic_read(&disk->nr_zone_wplugs)) { + /* Update the condition of all zone write plugs. */ + rcu_read_lock(); + for (i = 0; i < disk_zone_wplugs_hash_size(disk); i++) { + hlist_for_each_entry_rcu(zwplug, + &disk->zone_wplugs_hash[i], + node) { + spin_lock_irqsave(&zwplug->lock, flags); + disk_zone_wplug_set_wp_offset(disk, zwplug, 0); + spin_unlock_irqrestore(&zwplug->lock, flags); + } + } + rcu_read_unlock(); + } + + /* Update the cached zone conditions. */ + for (sector = 0; sector < capacity; + sector += bdev_zone_sectors(bio->bi_bdev)) + disk_zone_set_cond(disk, sector, BLK_ZONE_COND_EMPTY); + clear_bit(GD_ZONE_APPEND_USED, &disk->state); +} + +static void blk_zone_finish_bio_endio(struct bio *bio) +{ + struct block_device *bdev = bio->bi_bdev; + struct gendisk *disk = bdev->bd_disk; + sector_t sector = bio->bi_iter.bi_sector; + struct blk_zone_wplug *zwplug; /* - * Set the write pointer offset of all zone write plugs to 0. This will - * abort all plugged BIOs. It is fine as resetting zones while writes - * are still in-flight will result in the writes failing anyway. + * If we have a zone write plug, set its write pointer offset to the + * zone size. This will abort all BIOs plugged for the target zone. It + * is fine as resetting zones while writes are still in-flight will + * result in the writes failing anyway. */ - for (sector = 0; sector < get_capacity(disk); - sector += disk->queue->limits.chunk_sectors) { - zwplug = disk_get_zone_wplug(disk, sector); - if (zwplug) { - spin_lock_irqsave(&zwplug->lock, flags); - disk_zone_wplug_set_wp_offset(disk, zwplug, 0); - spin_unlock_irqrestore(&zwplug->lock, flags); - disk_put_zone_wplug(zwplug); - } + zwplug = disk_get_zone_wplug(disk, sector); + if (zwplug) { + unsigned long flags; + + spin_lock_irqsave(&zwplug->lock, flags); + disk_zone_wplug_set_wp_offset(disk, zwplug, + bdev_zone_sectors(bdev)); + spin_unlock_irqrestore(&zwplug->lock, flags); + disk_put_zone_wplug(zwplug); + } else { + disk_zone_set_cond(disk, sector, BLK_ZONE_COND_FULL); } +} - return false; +void blk_zone_mgmt_bio_endio(struct bio *bio) +{ + /* If the BIO failed, we have nothing to do. */ + if (bio->bi_status != BLK_STS_OK) + return; + + switch (bio_op(bio)) { + case REQ_OP_ZONE_RESET: + blk_zone_reset_bio_endio(bio); + return; + case REQ_OP_ZONE_RESET_ALL: + blk_zone_reset_all_bio_endio(bio); + return; + case REQ_OP_ZONE_FINISH: + blk_zone_finish_bio_endio(bio); + return; + default: + return; + } } -static void disk_zone_wplug_schedule_bio_work(struct gendisk *disk, - struct blk_zone_wplug *zwplug) +static void disk_zone_wplug_schedule_work(struct gendisk *disk, + struct blk_zone_wplug *zwplug) { + lockdep_assert_held(&zwplug->lock); + /* - * Take a reference on the zone write plug and schedule the submission - * of the next plugged BIO. blk_zone_wplug_bio_work() will release the - * reference we take here. + * Schedule the submission of the next plugged BIO. Taking a reference + * to the zone write plug is required as the bio_work belongs to the + * plug, and thus we must ensure that the write plug does not go away + * while the work is being scheduled but has not run yet. + * blk_zone_wplug_bio_work() will release the reference we take here, + * and we also drop this reference if the work is already scheduled. */ WARN_ON_ONCE(!(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED)); + WARN_ON_ONCE(blk_queue_zoned_qd1_writes(disk->queue)); refcount_inc(&zwplug->ref); - queue_work(disk->zone_wplugs_wq, &zwplug->bio_work); + if (!queue_work(disk->zone_wplugs_wq, &zwplug->bio_work)) + disk_put_zone_wplug(zwplug); } static inline void disk_zone_wplug_add_bio(struct gendisk *disk, struct blk_zone_wplug *zwplug, struct bio *bio, unsigned int nr_segs) { - bool schedule_bio_work = false; - /* * Grab an extra reference on the BIO request queue usage counter. * This reference will be reused to submit a request for the BIO for @@ -793,16 +1227,6 @@ static inline void disk_zone_wplug_add_bio(struct gendisk *disk, bio_clear_polled(bio); /* - * REQ_NOWAIT BIOs are always handled using the zone write plug BIO - * work, which can block. So clear the REQ_NOWAIT flag and schedule the - * work if this is the first BIO we are plugging. - */ - if (bio->bi_opf & REQ_NOWAIT) { - schedule_bio_work = !(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED); - bio->bi_opf &= ~REQ_NOWAIT; - } - - /* * Reuse the poll cookie field to store the number of segments when * split to the hardware limits. */ @@ -815,11 +1239,24 @@ static inline void disk_zone_wplug_add_bio(struct gendisk *disk, * at the tail of the list to preserve the sequential write order. */ bio_list_add(&zwplug->bio_list, bio); + trace_disk_zone_wplug_add_bio(zwplug->disk->queue, zwplug->zone_no, + bio->bi_iter.bi_sector, bio_sectors(bio)); - zwplug->flags |= BLK_ZONE_WPLUG_PLUGGED; - - if (schedule_bio_work) - disk_zone_wplug_schedule_bio_work(disk, zwplug); + /* + * If we are using the disk zone write plugs worker instead of the per + * zone write plug BIO work, add the zone write plug to the work list + * if it is not already there. Make sure to also get an extra reference + * on the zone write plug so that it does not go away until it is + * removed from the work list. + */ + if (blk_queue_zoned_qd1_writes(disk->queue)) { + spin_lock(&disk->zone_wplugs_list_lock); + if (list_empty(&zwplug->entry)) { + list_add_tail(&zwplug->entry, &disk->zone_wplugs_list); + refcount_inc(&zwplug->ref); + } + spin_unlock(&disk->zone_wplugs_list_lock); + } } /* @@ -827,6 +1264,7 @@ static inline void disk_zone_wplug_add_bio(struct gendisk *disk, */ void blk_zone_write_plug_bio_merged(struct bio *bio) { + struct gendisk *disk = bio->bi_bdev->bd_disk; struct blk_zone_wplug *zwplug; unsigned long flags; @@ -848,13 +1286,13 @@ void blk_zone_write_plug_bio_merged(struct bio *bio) * have at least one request and one BIO referencing the zone write * plug. So this should not fail. */ - zwplug = disk_get_zone_wplug(bio->bi_bdev->bd_disk, - bio->bi_iter.bi_sector); + zwplug = disk_get_zone_wplug(disk, bio->bi_iter.bi_sector); if (WARN_ON_ONCE(!zwplug)) return; spin_lock_irqsave(&zwplug->lock, flags); zwplug->wp_offset += bio_sectors(bio); + disk_zone_wplug_update_cond(disk, zwplug); spin_unlock_irqrestore(&zwplug->lock, flags); } @@ -913,6 +1351,7 @@ void blk_zone_write_plug_init_request(struct request *req) /* Drop the reference taken by disk_zone_wplug_add_bio(). */ blk_queue_exit(q); zwplug->wp_offset += bio_sectors(bio); + disk_zone_wplug_update_cond(disk, zwplug); req_back_sector += bio_sectors(bio); } @@ -976,6 +1415,7 @@ static bool blk_zone_wplug_prepare_bio(struct blk_zone_wplug *zwplug, /* Advance the zone write pointer offset. */ zwplug->wp_offset += bio_sectors(bio); + disk_zone_wplug_update_cond(disk, zwplug); return true; } @@ -1014,7 +1454,7 @@ static bool blk_zone_wplug_handle_write(struct bio *bio, unsigned int nr_segs) if (bio->bi_opf & REQ_NOWAIT) gfp_mask = GFP_NOWAIT; - zwplug = disk_get_and_lock_zone_wplug(disk, sector, gfp_mask, &flags); + zwplug = disk_get_or_alloc_zone_wplug(disk, sector, gfp_mask); if (!zwplug) { if (bio->bi_opf & REQ_NOWAIT) bio_wouldblock_error(bio); @@ -1023,18 +1463,43 @@ static bool blk_zone_wplug_handle_write(struct bio *bio, unsigned int nr_segs) return true; } + spin_lock_irqsave(&zwplug->lock, flags); + + /* + * Check if we got a zone write plug marked as dead. If yes, then the + * user is likely issuing writes to a full zone, or without + * synchronizing with zone reset or zone finish operations. In such + * case, fail the BIO to signal this invalid usage. + */ + if (disk_check_zone_wplug_dead(zwplug)) { + spin_unlock_irqrestore(&zwplug->lock, flags); + disk_put_zone_wplug(zwplug); + bio_io_error(bio); + return true; + } + /* Indicate that this BIO is being handled using zone write plugging. */ bio_set_flag(bio, BIO_ZONE_WRITE_PLUGGING); /* - * If the zone is already plugged, add the BIO to the plug BIO list. - * Do the same for REQ_NOWAIT BIOs to ensure that we will not see a - * BLK_STS_AGAIN failure if we let the BIO execute. - * Otherwise, plug and let the BIO execute. + * Add REQ_NOWAIT BIOs to the plug list to ensure that we will not see a + * BLK_STS_AGAIN failure if we let the caller submit the BIO. + */ + if (bio->bi_opf & REQ_NOWAIT) { + bio->bi_opf &= ~REQ_NOWAIT; + goto queue_bio; + } + + /* + * For rotational devices, we will use the gendisk zone write plugs + * work instead of the per zone write plug BIO work, so queue the BIO. */ - if ((zwplug->flags & BLK_ZONE_WPLUG_PLUGGED) || - (bio->bi_opf & REQ_NOWAIT)) - goto plug; + if (blk_queue_zoned_qd1_writes(disk->queue)) + goto queue_bio; + + /* If the zone is already plugged, add the BIO to the BIO plug list. */ + if (zwplug->flags & BLK_ZONE_WPLUG_PLUGGED) + goto queue_bio; if (!blk_zone_wplug_prepare_bio(zwplug, bio)) { spin_unlock_irqrestore(&zwplug->lock, flags); @@ -1042,15 +1507,24 @@ static bool blk_zone_wplug_handle_write(struct bio *bio, unsigned int nr_segs) return true; } + /* Otherwise, plug and let the caller submit the BIO. */ zwplug->flags |= BLK_ZONE_WPLUG_PLUGGED; spin_unlock_irqrestore(&zwplug->lock, flags); return false; -plug: +queue_bio: disk_zone_wplug_add_bio(disk, zwplug, bio, nr_segs); + if (!(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED)) { + zwplug->flags |= BLK_ZONE_WPLUG_PLUGGED; + if (blk_queue_zoned_qd1_writes(disk->queue)) + wake_up_process(disk->zone_wplugs_worker); + else + disk_zone_wplug_schedule_work(disk, zwplug); + } + spin_unlock_irqrestore(&zwplug->lock, flags); return true; @@ -1062,6 +1536,9 @@ static void blk_zone_wplug_handle_native_zone_append(struct bio *bio) struct blk_zone_wplug *zwplug; unsigned long flags; + if (!test_bit(GD_ZONE_APPEND_USED, &disk->state)) + set_bit(GD_ZONE_APPEND_USED, &disk->state); + /* * We have native support for zone append operations, so we are not * going to handle @bio through plugging. However, we may already have a @@ -1091,12 +1568,36 @@ static void blk_zone_wplug_handle_native_zone_append(struct bio *bio) disk->disk_name, zwplug->zone_no); disk_zone_wplug_abort(zwplug); } - disk_remove_zone_wplug(disk, zwplug); + disk_mark_zone_wplug_dead(zwplug); spin_unlock_irqrestore(&zwplug->lock, flags); disk_put_zone_wplug(zwplug); } +static bool blk_zone_wplug_handle_zone_mgmt(struct bio *bio) +{ + if (bio_op(bio) != REQ_OP_ZONE_RESET_ALL && + !bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector)) { + /* + * Zone reset and zone finish operations do not apply to + * conventional zones. + */ + bio_io_error(bio); + return true; + } + + /* + * No-wait zone management BIOs do not make much sense as the callers + * issue these as blocking operations in most cases. To avoid issues + * with the BIO execution potentially failing with BLK_STS_AGAIN, warn + * about REQ_NOWAIT being set and ignore that flag. + */ + if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) + bio->bi_opf &= ~REQ_NOWAIT; + + return false; +} + /** * blk_zone_plug_bio - Handle a zone write BIO with zone write plugging * @bio: The BIO being submitted @@ -1113,25 +1614,7 @@ bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs) { struct block_device *bdev = bio->bi_bdev; - if (!bdev->bd_disk->zone_wplugs_hash) - return false; - - /* - * If the BIO already has the plugging flag set, then it was already - * handled through this path and this is a submission from the zone - * plug bio submit work. - */ - if (bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING)) - return false; - - /* - * We do not need to do anything special for empty flush BIOs, e.g - * BIOs such as issued by blkdev_issue_flush(). The is because it is - * the responsibility of the user to first wait for the completion of - * write operations for flush to have any effect on the persistence of - * the written data. - */ - if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) + if (WARN_ON_ONCE(!bdev->bd_disk->zone_wplugs_hash)) return false; /* @@ -1162,12 +1645,9 @@ bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs) case REQ_OP_WRITE_ZEROES: return blk_zone_wplug_handle_write(bio, nr_segs); case REQ_OP_ZONE_RESET: - return blk_zone_wplug_handle_reset_or_finish(bio, 0); case REQ_OP_ZONE_FINISH: - return blk_zone_wplug_handle_reset_or_finish(bio, - bdev_zone_sectors(bdev)); case REQ_OP_ZONE_RESET_ALL: - return blk_zone_wplug_handle_reset_all(bio); + return blk_zone_wplug_handle_zone_mgmt(bio); default: return false; } @@ -1183,25 +1663,39 @@ static void disk_zone_wplug_unplug_bio(struct gendisk *disk, spin_lock_irqsave(&zwplug->lock, flags); - /* Schedule submission of the next plugged BIO if we have one. */ - if (!bio_list_empty(&zwplug->bio_list)) { - disk_zone_wplug_schedule_bio_work(disk, zwplug); - spin_unlock_irqrestore(&zwplug->lock, flags); - return; - } - - zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED; - /* - * If the zone is full (it was fully written or finished, or empty - * (it was reset), remove its zone write plug from the hash table. + * For rotational devices, signal the BIO completion to the zone write + * plug work. Otherwise, schedule submission of the next plugged BIO + * if we have one. */ - if (disk_should_remove_zone_wplug(disk, zwplug)) - disk_remove_zone_wplug(disk, zwplug); + if (bio_list_empty(&zwplug->bio_list)) + zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED; + + if (blk_queue_zoned_qd1_writes(disk->queue)) + complete(&disk->zone_wplugs_worker_bio_done); + else if (!bio_list_empty(&zwplug->bio_list)) + disk_zone_wplug_schedule_work(disk, zwplug); + + if (!zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug)) + disk_mark_zone_wplug_dead(zwplug); spin_unlock_irqrestore(&zwplug->lock, flags); } +void blk_zone_append_update_request_bio(struct request *rq, struct bio *bio) +{ + /* + * For zone append requests, the request sector indicates the location + * at which the BIO data was written. Return this value to the BIO + * issuer through the BIO iter sector. + * For plugged zone writes, which include emulated zone append, we need + * the original BIO sector so that blk_zone_write_plug_bio_endio() can + * lookup the zone write plug. + */ + bio->bi_iter.bi_sector = rq->__sector; + trace_blk_zone_append_update_request_bio(rq); +} + void blk_zone_write_plug_bio_endio(struct bio *bio) { struct gendisk *disk = bio->bi_bdev->bd_disk; @@ -1222,6 +1716,7 @@ void blk_zone_write_plug_bio_endio(struct bio *bio) if (bio_flagged(bio, BIO_EMULATES_ZONE_APPEND)) { bio->bi_opf &= ~REQ_OP_MASK; bio->bi_opf |= REQ_OP_ZONE_APPEND; + bio_clear_flag(bio, BIO_EMULATES_ZONE_APPEND); } /* @@ -1273,37 +1768,37 @@ void blk_zone_write_plug_finish_request(struct request *req) disk_put_zone_wplug(zwplug); } -static void blk_zone_wplug_bio_work(struct work_struct *work) +static bool disk_zone_wplug_submit_bio(struct gendisk *disk, + struct blk_zone_wplug *zwplug) { - struct blk_zone_wplug *zwplug = - container_of(work, struct blk_zone_wplug, bio_work); struct block_device *bdev; unsigned long flags; struct bio *bio; + bool prepared; /* * Submit the next plugged BIO. If we do not have any, clear * the plugged flag. */ - spin_lock_irqsave(&zwplug->lock, flags); - again: + spin_lock_irqsave(&zwplug->lock, flags); bio = bio_list_pop(&zwplug->bio_list); if (!bio) { zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED; spin_unlock_irqrestore(&zwplug->lock, flags); - goto put_zwplug; + return false; } - if (!blk_zone_wplug_prepare_bio(zwplug, bio)) { - blk_zone_wplug_bio_io_error(zwplug, bio); - goto again; - } + trace_blk_zone_wplug_bio(zwplug->disk->queue, zwplug->zone_no, + bio->bi_iter.bi_sector, bio_sectors(bio)); + prepared = blk_zone_wplug_prepare_bio(zwplug, bio); spin_unlock_irqrestore(&zwplug->lock, flags); - bdev = bio->bi_bdev; - submit_bio_noacct_nocheck(bio); + if (!prepared) { + blk_zone_wplug_bio_io_error(zwplug, bio); + goto again; + } /* * blk-mq devices will reuse the extra reference on the request queue @@ -1311,22 +1806,88 @@ again: * path for BIO-based devices will not do that. So drop this extra * reference here. */ - if (bdev_test_flag(bdev, BD_HAS_SUBMIT_BIO)) + if (blk_queue_zoned_qd1_writes(disk->queue)) + reinit_completion(&disk->zone_wplugs_worker_bio_done); + bdev = bio->bi_bdev; + if (bdev_test_flag(bdev, BD_HAS_SUBMIT_BIO)) { + bdev->bd_disk->fops->submit_bio(bio); blk_queue_exit(bdev->bd_disk->queue); + } else { + blk_mq_submit_bio(bio); + } -put_zwplug: - /* Drop the reference we took in disk_zone_wplug_schedule_bio_work(). */ - disk_put_zone_wplug(zwplug); + return true; } -static inline unsigned int disk_zone_wplugs_hash_size(struct gendisk *disk) +static struct blk_zone_wplug *disk_get_zone_wplugs_work(struct gendisk *disk) { - return 1U << disk->zone_wplugs_hash_bits; + struct blk_zone_wplug *zwplug; + + spin_lock_irq(&disk->zone_wplugs_list_lock); + zwplug = list_first_entry_or_null(&disk->zone_wplugs_list, + struct blk_zone_wplug, entry); + if (zwplug) + list_del_init(&zwplug->entry); + spin_unlock_irq(&disk->zone_wplugs_list_lock); + + return zwplug; +} + +static int disk_zone_wplugs_worker(void *data) +{ + struct gendisk *disk = data; + struct blk_zone_wplug *zwplug; + unsigned int noio_flag; + + noio_flag = memalloc_noio_save(); + set_user_nice(current, MIN_NICE); + set_freezable(); + + for (;;) { + set_current_state(TASK_INTERRUPTIBLE | TASK_FREEZABLE); + + zwplug = disk_get_zone_wplugs_work(disk); + if (zwplug) { + /* + * Process all BIOs of this zone write plug and then + * drop the reference we took when adding the zone write + * plug to the active list. + */ + set_current_state(TASK_RUNNING); + while (disk_zone_wplug_submit_bio(disk, zwplug)) + blk_wait_io(&disk->zone_wplugs_worker_bio_done); + disk_put_zone_wplug(zwplug); + continue; + } + + /* + * Only sleep if nothing sets the state to running. Else check + * for zone write plugs work again as a newly submitted BIO + * might have added a zone write plug to the work list. + */ + if (get_current_state() == TASK_RUNNING) { + try_to_freeze(); + } else { + if (kthread_should_stop()) { + set_current_state(TASK_RUNNING); + break; + } + schedule(); + } + } + + WARN_ON_ONCE(!list_empty(&disk->zone_wplugs_list)); + memalloc_noio_restore(noio_flag); + + return 0; } void disk_init_zone_resources(struct gendisk *disk) { - spin_lock_init(&disk->zone_wplugs_lock); + spin_lock_init(&disk->zone_wplugs_hash_lock); + spin_lock_init(&disk->zone_wplugs_list_lock); + INIT_LIST_HEAD(&disk->zone_wplugs_list); + init_completion(&disk->zone_wplugs_worker_bio_done); } /* @@ -1342,14 +1903,15 @@ static int disk_alloc_zone_resources(struct gendisk *disk, unsigned int pool_size) { unsigned int i; + int ret = -ENOMEM; atomic_set(&disk->nr_zone_wplugs, 0); disk->zone_wplugs_hash_bits = min(ilog2(pool_size) + 1, BLK_ZONE_WPLUG_MAX_HASH_BITS); disk->zone_wplugs_hash = - kcalloc(disk_zone_wplugs_hash_size(disk), - sizeof(struct hlist_head), GFP_KERNEL); + kzalloc_objs(struct hlist_head, + disk_zone_wplugs_hash_size(disk)); if (!disk->zone_wplugs_hash) return -ENOMEM; @@ -1361,11 +1923,30 @@ static int disk_alloc_zone_resources(struct gendisk *disk, if (!disk->zone_wplugs_pool) goto free_hash; - disk->zone_wplugs_wq = - alloc_workqueue("%s_zwplugs", WQ_MEM_RECLAIM | WQ_HIGHPRI, - pool_size, disk->disk_name); - if (!disk->zone_wplugs_wq) + /* + * We may already have a zone write plug workqueue as this function may + * be called after disk_free_zone_resources(), which does not destroy + * the workqueue (the zone write plugs workqueue is destroyed at + * disk_release() time). + */ + if (!disk->zone_wplugs_wq) { + disk->zone_wplugs_wq = + alloc_workqueue("%s_zwplugs", + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, + pool_size, disk->disk_name); + if (!disk->zone_wplugs_wq) + goto destroy_pool; + } + + disk->zone_wplugs_worker = + kthread_create(disk_zone_wplugs_worker, disk, + "%s_zwplugs_worker", disk->disk_name); + if (IS_ERR(disk->zone_wplugs_worker)) { + ret = PTR_ERR(disk->zone_wplugs_worker); + disk->zone_wplugs_worker = NULL; goto destroy_pool; + } + wake_up_process(disk->zone_wplugs_worker); return 0; @@ -1376,7 +1957,7 @@ free_hash: kfree(disk->zone_wplugs_hash); disk->zone_wplugs_hash = NULL; disk->zone_wplugs_hash_bits = 0; - return -ENOMEM; + return ret; } static void disk_destroy_zone_wplugs_hash_table(struct gendisk *disk) @@ -1392,9 +1973,9 @@ static void disk_destroy_zone_wplugs_hash_table(struct gendisk *disk) while (!hlist_empty(&disk->zone_wplugs_hash[i])) { zwplug = hlist_entry(disk->zone_wplugs_hash[i].first, struct blk_zone_wplug, node); - refcount_inc(&zwplug->ref); - disk_remove_zone_wplug(disk, zwplug); - disk_put_zone_wplug(zwplug); + spin_lock_irq(&zwplug->lock); + disk_mark_zone_wplug_dead(zwplug); + spin_unlock_irq(&zwplug->lock); } } @@ -1402,71 +1983,82 @@ static void disk_destroy_zone_wplugs_hash_table(struct gendisk *disk) kfree(disk->zone_wplugs_hash); disk->zone_wplugs_hash = NULL; disk->zone_wplugs_hash_bits = 0; + + /* + * Wait for the zone write plugs to be RCU-freed before destroying the + * mempool. + */ + rcu_barrier(); + mempool_destroy(disk->zone_wplugs_pool); + disk->zone_wplugs_pool = NULL; } -static unsigned int disk_set_conv_zones_bitmap(struct gendisk *disk, - unsigned long *bitmap) +static void disk_set_zones_cond_array(struct gendisk *disk, u8 *zones_cond) { - unsigned int nr_conv_zones = 0; unsigned long flags; - spin_lock_irqsave(&disk->zone_wplugs_lock, flags); - if (bitmap) - nr_conv_zones = bitmap_weight(bitmap, disk->nr_zones); - bitmap = rcu_replace_pointer(disk->conv_zones_bitmap, bitmap, - lockdep_is_held(&disk->zone_wplugs_lock)); - spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags); - - kfree_rcu_mightsleep(bitmap); + spin_lock_irqsave(&disk->zone_wplugs_hash_lock, flags); + zones_cond = rcu_replace_pointer(disk->zones_cond, zones_cond, + lockdep_is_held(&disk->zone_wplugs_hash_lock)); + spin_unlock_irqrestore(&disk->zone_wplugs_hash_lock, flags); - return nr_conv_zones; + kfree_rcu_mightsleep(zones_cond); } -void disk_free_zone_resources(struct gendisk *disk) +static void disk_free_zone_resources(struct gendisk *disk) { - if (!disk->zone_wplugs_pool) - return; - - if (disk->zone_wplugs_wq) { - destroy_workqueue(disk->zone_wplugs_wq); - disk->zone_wplugs_wq = NULL; + if (disk->zone_wplugs_worker) { + kthread_stop(disk->zone_wplugs_worker); + disk->zone_wplugs_worker = NULL; } + WARN_ON_ONCE(!list_empty(&disk->zone_wplugs_list)); - disk_destroy_zone_wplugs_hash_table(disk); + if (disk->zone_wplugs_wq) + drain_workqueue(disk->zone_wplugs_wq); - /* - * Wait for the zone write plugs to be RCU-freed before - * destorying the mempool. - */ - rcu_barrier(); - - mempool_destroy(disk->zone_wplugs_pool); - disk->zone_wplugs_pool = NULL; + disk_destroy_zone_wplugs_hash_table(disk); - disk_set_conv_zones_bitmap(disk, NULL); + disk_set_zones_cond_array(disk, NULL); disk->zone_capacity = 0; disk->last_zone_capacity = 0; disk->nr_zones = 0; } -static inline bool disk_need_zone_resources(struct gendisk *disk) +void disk_release_zone_resources(struct gendisk *disk) { - /* - * All mq zoned devices need zone resources so that the block layer - * can automatically handle write BIO plugging. BIO-based device drivers - * (e.g. DM devices) are normally responsible for handling zone write - * ordering and do not need zone resources, unless the driver requires - * zone append emulation. - */ - return queue_is_mq(disk->queue) || - queue_emulates_zone_append(disk->queue); + if (disk->zone_wplugs_wq) { + destroy_workqueue(disk->zone_wplugs_wq); + disk->zone_wplugs_wq = NULL; + } + + disk_free_zone_resources(disk); } +struct blk_revalidate_zone_args { + struct gendisk *disk; + u8 *zones_cond; + unsigned int nr_zones; + unsigned int nr_conv_zones; + unsigned int zone_capacity; + unsigned int last_zone_capacity; + sector_t sector; +}; + static int disk_revalidate_zone_resources(struct gendisk *disk, - unsigned int nr_zones) + struct blk_revalidate_zone_args *args) { struct queue_limits *lim = &disk->queue->limits; unsigned int pool_size; + int ret = 0; + + args->disk = disk; + args->nr_zones = + DIV_ROUND_UP_ULL(get_capacity(disk), lim->chunk_sectors); + + /* Cached zone conditions: 1 byte per zone */ + args->zones_cond = kzalloc(args->nr_zones, GFP_NOIO); + if (!args->zones_cond) + return -ENOMEM; if (!disk_need_zone_resources(disk)) return 0; @@ -1477,23 +2069,18 @@ static int disk_revalidate_zone_resources(struct gendisk *disk, */ pool_size = max(lim->max_open_zones, lim->max_active_zones); if (!pool_size) - pool_size = min(BLK_ZONE_WPLUG_DEFAULT_POOL_SIZE, nr_zones); + pool_size = + min(BLK_ZONE_WPLUG_DEFAULT_POOL_SIZE, args->nr_zones); - if (!disk->zone_wplugs_hash) - return disk_alloc_zone_resources(disk, pool_size); + if (!disk->zone_wplugs_hash) { + ret = disk_alloc_zone_resources(disk, pool_size); + if (ret) + kfree(args->zones_cond); + } - return 0; + return ret; } -struct blk_revalidate_zone_args { - struct gendisk *disk; - unsigned long *conv_zones_bitmap; - unsigned int nr_zones; - unsigned int zone_capacity; - unsigned int last_zone_capacity; - sector_t sector; -}; - /* * Update the disk zone resources information and device queue limits. * The disk queue is frozen when this is executed. @@ -1502,30 +2089,36 @@ static int disk_update_zone_resources(struct gendisk *disk, struct blk_revalidate_zone_args *args) { struct request_queue *q = disk->queue; - unsigned int nr_seq_zones, nr_conv_zones; - unsigned int pool_size; + unsigned int nr_seq_zones; + unsigned int pool_size, memflags; struct queue_limits lim; + int ret = 0; + + lim = queue_limits_start_update(q); + + memflags = blk_mq_freeze_queue(q); disk->nr_zones = args->nr_zones; - disk->zone_capacity = args->zone_capacity; - disk->last_zone_capacity = args->last_zone_capacity; - nr_conv_zones = - disk_set_conv_zones_bitmap(disk, args->conv_zones_bitmap); - if (nr_conv_zones >= disk->nr_zones) { + if (args->nr_conv_zones >= disk->nr_zones) { + queue_limits_cancel_update(q); pr_warn("%s: Invalid number of conventional zones %u / %u\n", - disk->disk_name, nr_conv_zones, disk->nr_zones); - return -ENODEV; + disk->disk_name, args->nr_conv_zones, disk->nr_zones); + ret = -ENODEV; + goto unfreeze; } - lim = queue_limits_start_update(q); + disk->zone_capacity = args->zone_capacity; + disk->last_zone_capacity = args->last_zone_capacity; + disk_set_zones_cond_array(disk, args->zones_cond); + args->zones_cond = NULL; /* - * Some devices can advertize zone resource limits that are larger than + * Some devices can advertise zone resource limits that are larger than * the number of sequential zones of the zoned block device, e.g. a * small ZNS namespace. For such case, assume that the zoned device has * no zone resource limits. */ - nr_seq_zones = disk->nr_zones - nr_conv_zones; + nr_seq_zones = disk->nr_zones - args->nr_conv_zones; if (lim.max_open_zones >= nr_seq_zones) lim.max_open_zones = 0; if (lim.max_active_zones >= nr_seq_zones) @@ -1555,7 +2148,50 @@ static int disk_update_zone_resources(struct gendisk *disk, } commit: - return queue_limits_commit_update_frozen(q, &lim); + ret = queue_limits_commit_update(q, &lim); + +unfreeze: + blk_mq_unfreeze_queue(q, memflags); + + return ret; +} + +static int blk_revalidate_zone_cond(struct blk_zone *zone, unsigned int idx, + struct blk_revalidate_zone_args *args) +{ + enum blk_zone_cond cond = zone->cond; + + /* Check that the zone condition is consistent with the zone type. */ + switch (cond) { + case BLK_ZONE_COND_NOT_WP: + if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) + goto invalid_condition; + break; + case BLK_ZONE_COND_IMP_OPEN: + case BLK_ZONE_COND_EXP_OPEN: + case BLK_ZONE_COND_CLOSED: + case BLK_ZONE_COND_EMPTY: + case BLK_ZONE_COND_FULL: + case BLK_ZONE_COND_OFFLINE: + case BLK_ZONE_COND_READONLY: + if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ) + goto invalid_condition; + break; + default: + pr_warn("%s: Invalid zone condition 0x%X\n", + args->disk->disk_name, cond); + return -ENODEV; + } + + blk_zone_set_cond(args->zones_cond, idx, cond); + + return 0; + +invalid_condition: + pr_warn("%s: Invalid zone condition 0x%x for type 0x%x\n", + args->disk->disk_name, cond, zone->type); + + return -ENODEV; } static int blk_revalidate_conv_zone(struct blk_zone *zone, unsigned int idx, @@ -1572,17 +2208,7 @@ static int blk_revalidate_conv_zone(struct blk_zone *zone, unsigned int idx, if (disk_zone_is_last(disk, zone)) args->last_zone_capacity = zone->capacity; - if (!disk_need_zone_resources(disk)) - return 0; - - if (!args->conv_zones_bitmap) { - args->conv_zones_bitmap = - bitmap_zalloc(args->nr_zones, GFP_NOIO); - if (!args->conv_zones_bitmap) - return -ENOMEM; - } - - set_bit(idx, args->conv_zones_bitmap); + args->nr_conv_zones++; return 0; } @@ -1593,7 +2219,6 @@ static int blk_revalidate_seq_zone(struct blk_zone *zone, unsigned int idx, struct gendisk *disk = args->disk; struct blk_zone_wplug *zwplug; unsigned int wp_offset; - unsigned long flags; /* * Remember the capacity of the first sequential zone and check @@ -1616,19 +2241,16 @@ static int blk_revalidate_seq_zone(struct blk_zone *zone, unsigned int idx, * we have a zone write plug for such zone if the device has a zone * write plug hash table. */ - if (!queue_emulates_zone_append(disk->queue) || !disk->zone_wplugs_hash) + if (!disk->zone_wplugs_hash) return 0; - disk_zone_wplug_sync_wp_offset(disk, zone); - - wp_offset = blk_zone_wp_offset(zone); + wp_offset = disk_zone_wplug_sync_wp_offset(disk, zone); if (!wp_offset || wp_offset >= zone->capacity) return 0; - zwplug = disk_get_and_lock_zone_wplug(disk, zone->wp, GFP_NOIO, &flags); + zwplug = disk_get_or_alloc_zone_wplug(disk, zone->wp, GFP_NOIO); if (!zwplug) return -ENOMEM; - spin_unlock_irqrestore(&zwplug->lock, flags); disk_put_zone_wplug(zwplug); return 0; @@ -1680,6 +2302,11 @@ static int blk_revalidate_zone_cb(struct blk_zone *zone, unsigned int idx, return -ENODEV; } + /* Check zone condition */ + ret = blk_revalidate_zone_cond(zone, idx, args); + if (ret) + return ret; + /* Check zone type */ switch (zone->type) { case BLK_ZONE_TYPE_CONVENTIONAL: @@ -1720,7 +2347,11 @@ int blk_revalidate_disk_zones(struct gendisk *disk) sector_t zone_sectors = q->limits.chunk_sectors; sector_t capacity = get_capacity(disk); struct blk_revalidate_zone_args args = { }; - unsigned int noio_flag; + unsigned int memflags, noio_flag; + struct blk_report_zones_args rep_args = { + .cb = blk_revalidate_zone_cb, + .data = &args, + }; int ret = -ENOMEM; if (WARN_ON_ONCE(!blk_queue_is_zoned(q))) @@ -1743,47 +2374,47 @@ int blk_revalidate_disk_zones(struct gendisk *disk) * Ensure that all memory allocations in this context are done as if * GFP_NOIO was specified. */ - args.disk = disk; - args.nr_zones = (capacity + zone_sectors - 1) >> ilog2(zone_sectors); noio_flag = memalloc_noio_save(); - ret = disk_revalidate_zone_resources(disk, args.nr_zones); + ret = disk_revalidate_zone_resources(disk, &args); if (ret) { memalloc_noio_restore(noio_flag); return ret; } - ret = disk->fops->report_zones(disk, 0, UINT_MAX, - blk_revalidate_zone_cb, &args); + ret = disk->fops->report_zones(disk, 0, UINT_MAX, &rep_args); if (!ret) { pr_warn("%s: No zones reported\n", disk->disk_name); ret = -ENODEV; } memalloc_noio_restore(noio_flag); + if (ret <= 0) + goto free_resources; + /* * If zones where reported, make sure that the entire disk capacity * has been checked. */ - if (ret > 0 && args.sector != capacity) { + if (args.sector != capacity) { pr_warn("%s: Missing zones from sector %llu\n", disk->disk_name, args.sector); ret = -ENODEV; + goto free_resources; } - /* - * Set the new disk zone parameters only once the queue is frozen and - * all I/Os are completed. - */ - if (ret > 0) - ret = disk_update_zone_resources(disk, &args); - else - pr_warn("%s: failed to revalidate zones\n", disk->disk_name); - if (ret) { - unsigned int memflags = blk_mq_freeze_queue(q); + ret = disk_update_zone_resources(disk, &args); + if (ret) + goto free_resources; - disk_free_zone_resources(disk); - blk_mq_unfreeze_queue(q, memflags); - } + return 0; + +free_resources: + pr_warn("%s: failed to revalidate zones\n", disk->disk_name); + + kfree(args.zones_cond); + memflags = blk_mq_freeze_queue(q); + disk_free_zone_resources(disk); + blk_mq_unfreeze_queue(q, memflags); return ret; } @@ -1804,6 +2435,7 @@ EXPORT_SYMBOL_GPL(blk_revalidate_disk_zones); int blk_zone_issue_zeroout(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask) { + struct gendisk *disk = bdev->bd_disk; int ret; if (WARN_ON_ONCE(!bdev_is_zoned(bdev))) @@ -1819,7 +2451,7 @@ int blk_zone_issue_zeroout(struct block_device *bdev, sector_t sector, * pointer. Undo this using a report zone to update the zone write * pointer to the correct current value. */ - ret = disk_zone_sync_wp_offset(bdev->bd_disk, sector); + ret = disk->fops->report_zones(disk, sector, 1, NULL); if (ret != 1) return ret < 0 ? ret : -EIO; @@ -1838,18 +2470,22 @@ static void queue_zone_wplug_show(struct blk_zone_wplug *zwplug, unsigned int zwp_wp_offset, zwp_flags; unsigned int zwp_zone_no, zwp_ref; unsigned int zwp_bio_list_size; + enum blk_zone_cond zwp_cond; unsigned long flags; spin_lock_irqsave(&zwplug->lock, flags); zwp_zone_no = zwplug->zone_no; zwp_flags = zwplug->flags; zwp_ref = refcount_read(&zwplug->ref); + zwp_cond = zwplug->cond; zwp_wp_offset = zwplug->wp_offset; zwp_bio_list_size = bio_list_size(&zwplug->bio_list); spin_unlock_irqrestore(&zwplug->lock, flags); - seq_printf(m, "%u 0x%x %u %u %u\n", zwp_zone_no, zwp_flags, zwp_ref, - zwp_wp_offset, zwp_bio_list_size); + seq_printf(m, + "Zone no: %u, flags: 0x%x, ref: %u, cond: %s, wp ofst: %u, pending BIO: %u\n", + zwp_zone_no, zwp_flags, zwp_ref, blk_zone_cond_str(zwp_cond), + zwp_wp_offset, zwp_bio_list_size); } int queue_zone_wplugs_show(void *data, struct seq_file *m) diff --git a/block/blk.h b/block/blk.h index 006e3be433d2..eaac05815cb0 100644 --- a/block/blk.h +++ b/block/blk.h @@ -4,6 +4,7 @@ #include <linux/bio-integrity.h> #include <linux/blk-crypto.h> +#include <linux/part_stat.h> #include <linux/lockdep.h> #include <linux/memblock.h> /* for max_pfn/max_low_pfn */ #include <linux/sched/sysctl.h> @@ -11,7 +12,16 @@ #include <xen/xen.h> #include "blk-crypto-internal.h" -struct elevator_type; +struct elv_change_ctx; + +/* + * Default upper limit for the software max_sectors limit used for regular I/Os. + * This can be increased through sysfs. + * + * This should not be confused with the max_hw_sector limit that is entirely + * controlled by the block device driver, usually based on hardware limits. + */ +#define BLK_DEF_MAX_SECTORS_CAP (SZ_4M >> SECTOR_SHIFT) #define BLK_DEV_MAX_SECTORS (LLONG_MAX >> 9) #define BLK_MIN_SEGMENT_SIZE 4096 @@ -19,6 +29,7 @@ struct elevator_type; /* Max future timer expiry for timeouts */ #define BLK_MAX_TIMEOUT (5 * HZ) +extern const struct kobj_type blk_queue_ktype; extern struct dentry *blk_debugfs_root; struct blk_flush_queue { @@ -30,6 +41,7 @@ struct blk_flush_queue { struct list_head flush_queue[2]; unsigned long flush_data_in_flight; struct request *flush_rq; + struct rcu_head rcu_head; }; bool is_flush_rq(struct request *req); @@ -38,13 +50,18 @@ struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size, gfp_t flags); void blk_free_flush_queue(struct blk_flush_queue *q); +const char *blk_status_to_str(blk_status_t status); +const char *blk_status_to_tag(blk_status_t status); +blk_status_t tag_to_blk_status(const char *tag); +enum req_op str_to_blk_op(const char *op); + bool __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic); bool blk_queue_start_drain(struct request_queue *q); bool __blk_freeze_queue_start(struct request_queue *q, struct task_struct *owner); int __bio_queue_enter(struct request_queue *q, struct bio *bio); -void submit_bio_noacct_nocheck(struct bio *bio); -void bio_await_chain(struct bio *bio); +void submit_bio_noacct_nocheck(struct bio *bio, bool split); +int bio_submit_or_kill(struct bio *bio, unsigned int flags); static inline bool blk_try_enter_queue(struct request_queue *q, bool pm) { @@ -94,14 +111,11 @@ static inline void blk_wait_io(struct completion *done) wait_for_completion_io(done); } -#define BIO_INLINE_VECS 4 -struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, - gfp_t gfp_mask); -void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs); +struct block_device *blkdev_get_no_open(dev_t dev, bool autoload); +void blkdev_put_no_open(struct block_device *bdev); bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv, - struct page *page, unsigned len, unsigned offset, - bool *same_page); + struct page *page, unsigned len, unsigned offset); static inline bool biovec_phys_mergeable(struct request_queue *q, struct bio_vec *vec1, struct bio_vec *vec2) @@ -119,6 +133,8 @@ static inline bool biovec_phys_mergeable(struct request_queue *q, if (addr1 + vec1->bv_len != addr2) return false; + if (!zone_device_pages_have_same_pgmap(vec1->bv_page, vec2->bv_page)) + return false; if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) return false; if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) @@ -126,6 +142,25 @@ static inline bool biovec_phys_mergeable(struct request_queue *q, return true; } +/* + * Check if two pages from potentially different zone device pgmaps can + * coexist as separate bvec entries in the same bio. + * + * The block DMA iterator (blk_dma_map_iter_start) caches the P2PDMA mapping + * state from the first segment and applies it to all subsequent segments, so + * P2PDMA pages from different pgmaps must not be mixed in the same bio. + * + * Other zone device types (FS_DAX, GENERIC) use the same dma_map_phys() path + * as normal RAM. PRIVATE and COHERENT pages never appear in bios. + */ +static inline bool zone_device_pages_compatible(const struct page *a, + const struct page *b) +{ + if (is_pci_p2pdma_page(a) || is_pci_p2pdma_page(b)) + return zone_device_pages_have_same_pgmap(a, b); + return true; +} + static inline bool __bvec_gap_to_prev(const struct queue_limits *lim, struct bio_vec *bprv, unsigned int offset) { @@ -195,10 +230,14 @@ static inline unsigned int blk_queue_get_max_sectors(struct request *rq) struct request_queue *q = rq->q; enum req_op op = req_op(rq); - if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)) + if (unlikely(op == REQ_OP_DISCARD)) return min(q->limits.max_discard_sectors, UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_SECURE_ERASE)) + return min(q->limits.max_secure_erase_sectors, + UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_WRITE_ZEROES)) return q->limits.max_write_zeroes_sectors; @@ -319,11 +358,10 @@ bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, bool blk_insert_flush(struct request *rq); -int elevator_switch(struct request_queue *q, struct elevator_type *new_e); -void elevator_disable(struct request_queue *q); -void elevator_exit(struct request_queue *q); -int elv_register_queue(struct request_queue *q, bool uevent); -void elv_unregister_queue(struct request_queue *q); +void elv_update_nr_hw_queues(struct request_queue *q, + struct elv_change_ctx *ctx); +void elevator_set_default(struct request_queue *q); +void elevator_set_none(struct request_queue *q); ssize_t part_size_show(struct device *dev, struct device_attribute *attr, char *buf); @@ -359,12 +397,20 @@ struct bio *bio_split_zone_append(struct bio *bio, static inline bool bio_may_need_split(struct bio *bio, const struct queue_limits *lim) { + const struct bio_vec *bv; + if (lim->chunk_sectors) return true; - if (bio->bi_vcnt != 1) + + if (!bio->bi_io_vec) + return true; + + bv = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); + if (bio->bi_iter.bi_size > bv->bv_len - bio->bi_iter.bi_bvec_done) + return true; + if ((bv->bv_offset | bv->bv_len) & lim->dma_alignment) return true; - return bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > - lim->min_segment_size; + return bv->bv_len + bv->bv_offset > lim->max_fast_segment_size; } /** @@ -404,6 +450,27 @@ static inline struct bio *__bio_split_to_limits(struct bio *bio, } } +/** + * get_max_segment_size() - maximum number of bytes to add as a single segment + * @lim: Request queue limits. + * @paddr: address of the range to add + * @len: maximum length available to add at @paddr + * + * Returns the maximum number of bytes of the range starting at @paddr that can + * be added to a single segment. + */ +static inline unsigned get_max_segment_size(const struct queue_limits *lim, + phys_addr_t paddr, unsigned int len) +{ + /* + * Prevent an overflow if mask = ULONG_MAX and offset = 0 by adding 1 + * after having calculated the minimum. + */ + return min_t(unsigned long, len, + min(lim->seg_boundary_mask - (lim->seg_boundary_mask & paddr), + (unsigned long)lim->max_segment_size - 1) + 1); +} + int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs); bool blk_attempt_req_merge(struct request_queue *q, struct request *rq, @@ -418,7 +485,6 @@ void blk_apply_bdi_limits(struct backing_dev_info *bdi, int blk_dev_init(void); void update_io_ticks(struct block_device *part, unsigned long now, bool end); -unsigned int part_in_flight(struct block_device *part); static inline void req_set_nomerge(struct request_queue *q, struct request *req) { @@ -427,6 +493,26 @@ static inline void req_set_nomerge(struct request_queue *q, struct request *req) q->last_merge = NULL; } +static inline void bdev_inc_in_flight(struct block_device *bdev, + enum req_op op) +{ + bool rw = op_is_write(op); + + part_stat_local_inc(bdev, in_flight[rw]); + if (bdev_is_partition(bdev)) + part_stat_local_inc(bdev_whole(bdev), in_flight[rw]); +} + +static inline void bdev_dec_in_flight(struct block_device *bdev, + enum req_op op) +{ + bool rw = op_is_write(op); + + part_stat_local_dec(bdev, in_flight[rw]); + if (bdev_is_partition(bdev)) + part_stat_local_dec(bdev_whole(bdev), in_flight[rw]); +} + /* * Internal io_context interface */ @@ -440,50 +526,40 @@ static inline void ioc_clear_queue(struct request_queue *q) } #endif /* CONFIG_BLK_ICQ */ -struct bio *__blk_queue_bounce(struct bio *bio, struct request_queue *q); - -static inline bool blk_queue_may_bounce(struct request_queue *q) -{ - return IS_ENABLED(CONFIG_BOUNCE) && - (q->limits.features & BLK_FEAT_BOUNCE_HIGH) && - max_low_pfn >= max_pfn; -} - -static inline struct bio *blk_queue_bounce(struct bio *bio, - struct request_queue *q) -{ - if (unlikely(blk_queue_may_bounce(q) && bio_has_data(bio))) - return __blk_queue_bounce(bio, q); - return bio; -} - #ifdef CONFIG_BLK_DEV_ZONED void disk_init_zone_resources(struct gendisk *disk); -void disk_free_zone_resources(struct gendisk *disk); +void disk_release_zone_resources(struct gendisk *disk); static inline bool bio_zone_write_plugging(struct bio *bio) { return bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING); } -void blk_zone_write_plug_bio_merged(struct bio *bio); -void blk_zone_write_plug_init_request(struct request *rq); -static inline void blk_zone_update_request_bio(struct request *rq, - struct bio *bio) +static inline bool blk_req_bio_is_zone_append(struct request *rq, + struct bio *bio) { - /* - * For zone append requests, the request sector indicates the location - * at which the BIO data was written. Return this value to the BIO - * issuer through the BIO iter sector. - * For plugged zone writes, which include emulated zone append, we need - * the original BIO sector so that blk_zone_write_plug_bio_endio() can - * lookup the zone write plug. - */ - if (req_op(rq) == REQ_OP_ZONE_APPEND || bio_zone_write_plugging(bio)) - bio->bi_iter.bi_sector = rq->__sector; + return req_op(rq) == REQ_OP_ZONE_APPEND || + bio_flagged(bio, BIO_EMULATES_ZONE_APPEND); } +void blk_zone_write_plug_bio_merged(struct bio *bio); +void blk_zone_write_plug_init_request(struct request *rq); +void blk_zone_append_update_request_bio(struct request *rq, struct bio *bio); +void blk_zone_mgmt_bio_endio(struct bio *bio); void blk_zone_write_plug_bio_endio(struct bio *bio); static inline void blk_zone_bio_endio(struct bio *bio) { /* + * Zone management BIOs may impact zone write plugs (e.g. a zone reset + * changes a zone write plug zone write pointer offset), but these + * operation do not go through zone write plugging as they may operate + * on zones that do not have a zone write + * plug. blk_zone_mgmt_bio_endio() handles the potential changes to zone + * write plugs that are present. + */ + if (op_is_zone_mgmt(bio_op(bio))) { + blk_zone_mgmt_bio_endio(bio); + return; + } + + /* * For write BIOs to zoned devices, signal the completion of the BIO so * that the next write BIO can be submitted by zone write plugging. */ @@ -505,21 +581,26 @@ int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode, static inline void disk_init_zone_resources(struct gendisk *disk) { } -static inline void disk_free_zone_resources(struct gendisk *disk) +static inline void disk_release_zone_resources(struct gendisk *disk) { } static inline bool bio_zone_write_plugging(struct bio *bio) { return false; } +static inline bool blk_req_bio_is_zone_append(struct request *req, + struct bio *bio) +{ + return false; +} static inline void blk_zone_write_plug_bio_merged(struct bio *bio) { } static inline void blk_zone_write_plug_init_request(struct request *rq) { } -static inline void blk_zone_update_request_bio(struct request *rq, - struct bio *bio) +static inline void blk_zone_append_update_request_bio(struct request *rq, + struct bio *bio) { } static inline void blk_zone_bio_endio(struct bio *bio) @@ -562,17 +643,6 @@ void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors); struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id, struct lock_class_key *lkclass); - -/* - * Clean up a page appropriately, where the page may be pinned, may have a - * ref taken on it or neither. - */ -static inline void bio_release_page(struct bio *bio, struct page *page) -{ - if (bio_flagged(bio, BIO_PAGE_PINNED)) - unpin_user_page(page); -} - struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id); int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode); @@ -602,6 +672,7 @@ extern const struct address_space_operations def_blk_aops; int disk_register_independent_access_ranges(struct gendisk *disk); void disk_unregister_independent_access_ranges(struct gendisk *disk); +int should_fail_bio(struct bio *bio); #ifdef CONFIG_FAIL_MAKE_REQUEST bool should_fail_request(struct block_device *part, unsigned int bytes); #else /* CONFIG_FAIL_MAKE_REQUEST */ @@ -646,6 +717,7 @@ static inline int req_ref_read(struct request *req) static inline u64 blk_time_get_ns(void) { struct blk_plug *plug = current->plug; + u64 now; if (!plug || !in_task()) return ktime_get_ns(); @@ -654,12 +726,18 @@ static inline u64 blk_time_get_ns(void) * 0 could very well be a valid time, but rather than flag "this is * a valid timestamp" separately, just accept that we'll do an extra * ktime_get_ns() if we just happen to get 0 as the current time. + * + * cur_ktime can be zeroed by pre-emption the moment PF_BLOCK_TS is set. */ - if (!plug->cur_ktime) { - plug->cur_ktime = ktime_get_ns(); + now = READ_ONCE(plug->cur_ktime); + if (!now) { + now = ktime_get_ns(); + WRITE_ONCE(plug->cur_ktime, now); + /* Ensure PF_BLOCK_TS is set after cur_ktime. */ + barrier(); current->flags |= PF_BLOCK_TS; } - return plug->cur_ktime; + return now; } static inline ktime_t blk_time_get(void) @@ -667,55 +745,15 @@ static inline ktime_t blk_time_get(void) return ns_to_ktime(blk_time_get_ns()); } -/* - * From most significant bit: - * 1 bit: reserved for other usage, see below - * 12 bits: original size of bio - * 51 bits: issue time of bio - */ -#define BIO_ISSUE_RES_BITS 1 -#define BIO_ISSUE_SIZE_BITS 12 -#define BIO_ISSUE_RES_SHIFT (64 - BIO_ISSUE_RES_BITS) -#define BIO_ISSUE_SIZE_SHIFT (BIO_ISSUE_RES_SHIFT - BIO_ISSUE_SIZE_BITS) -#define BIO_ISSUE_TIME_MASK ((1ULL << BIO_ISSUE_SIZE_SHIFT) - 1) -#define BIO_ISSUE_SIZE_MASK \ - (((1ULL << BIO_ISSUE_SIZE_BITS) - 1) << BIO_ISSUE_SIZE_SHIFT) -#define BIO_ISSUE_RES_MASK (~((1ULL << BIO_ISSUE_RES_SHIFT) - 1)) - -/* Reserved bit for blk-throtl */ -#define BIO_ISSUE_THROTL_SKIP_LATENCY (1ULL << 63) - -static inline u64 __bio_issue_time(u64 time) -{ - return time & BIO_ISSUE_TIME_MASK; -} - -static inline u64 bio_issue_time(struct bio_issue *issue) -{ - return __bio_issue_time(issue->value); -} - -static inline sector_t bio_issue_size(struct bio_issue *issue) -{ - return ((issue->value & BIO_ISSUE_SIZE_MASK) >> BIO_ISSUE_SIZE_SHIFT); -} - -static inline void bio_issue_init(struct bio_issue *issue, - sector_t size) -{ - size &= (1ULL << BIO_ISSUE_SIZE_BITS) - 1; - issue->value = ((issue->value & BIO_ISSUE_RES_MASK) | - (blk_time_get_ns() & BIO_ISSUE_TIME_MASK) | - ((u64)size << BIO_ISSUE_SIZE_SHIFT)); -} - void bdev_release(struct file *bdev_file); int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder, const struct blk_holder_ops *hops, struct file *bdev_file); int bdev_permission(dev_t dev, blk_mode_t mode, void *holder); -void blk_integrity_generate(struct bio *bio); -void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter); +void bio_integrity_generate(struct bio *bio); +blk_status_t bio_integrity_verify(struct bio *bio, + struct bvec_iter *saved_iter); + void blk_integrity_prepare(struct request *rq); void blk_integrity_complete(struct request *rq, unsigned int nr_bytes); @@ -744,4 +782,39 @@ static inline void blk_unfreeze_release_lock(struct request_queue *q) } #endif +/* + * debugfs directory and file creation can trigger fs reclaim, which can enter + * back into the block layer request_queue. This can cause deadlock if the + * queue is frozen. Use NOIO context together with debugfs_mutex to prevent fs + * reclaim from triggering block I/O. + */ +static inline void blk_debugfs_lock_nomemsave(struct request_queue *q) + __acquires(&q->debugfs_mutex) +{ + mutex_lock(&q->debugfs_mutex); +} + +static inline void blk_debugfs_unlock_nomemrestore(struct request_queue *q) + __releases(&q->debugfs_mutex) +{ + mutex_unlock(&q->debugfs_mutex); +} + +static inline unsigned int __must_check blk_debugfs_lock(struct request_queue *q) + __acquires(&q->debugfs_mutex) +{ + unsigned int memflags = memalloc_noio_save(); + + blk_debugfs_lock_nomemsave(q); + return memflags; +} + +static inline void blk_debugfs_unlock(struct request_queue *q, + unsigned int memflags) + __releases(&q->debugfs_mutex) +{ + blk_debugfs_unlock_nomemrestore(q); + memalloc_noio_restore(memflags); +} + #endif /* BLK_INTERNAL_H */ diff --git a/block/bounce.c b/block/bounce.c deleted file mode 100644 index 09a9616cf209..000000000000 --- a/block/bounce.c +++ /dev/null @@ -1,267 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* bounce buffer handling for block devices - * - * - Split from highmem.c - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/mm.h> -#include <linux/export.h> -#include <linux/swap.h> -#include <linux/gfp.h> -#include <linux/bio-integrity.h> -#include <linux/pagemap.h> -#include <linux/mempool.h> -#include <linux/blkdev.h> -#include <linux/backing-dev.h> -#include <linux/init.h> -#include <linux/hash.h> -#include <linux/highmem.h> -#include <linux/printk.h> -#include <asm/tlbflush.h> - -#include <trace/events/block.h> -#include "blk.h" -#include "blk-cgroup.h" - -#define POOL_SIZE 64 -#define ISA_POOL_SIZE 16 - -static struct bio_set bounce_bio_set, bounce_bio_split; -static mempool_t page_pool; - -static void init_bounce_bioset(void) -{ - static bool bounce_bs_setup; - int ret; - - if (bounce_bs_setup) - return; - - ret = bioset_init(&bounce_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); - BUG_ON(ret); - - ret = bioset_init(&bounce_bio_split, BIO_POOL_SIZE, 0, 0); - BUG_ON(ret); - bounce_bs_setup = true; -} - -static __init int init_emergency_pool(void) -{ - int ret; - -#ifndef CONFIG_MEMORY_HOTPLUG - if (max_pfn <= max_low_pfn) - return 0; -#endif - - ret = mempool_init_page_pool(&page_pool, POOL_SIZE, 0); - BUG_ON(ret); - pr_info("pool size: %d pages\n", POOL_SIZE); - - init_bounce_bioset(); - return 0; -} - -__initcall(init_emergency_pool); - -/* - * Simple bounce buffer support for highmem pages. Depending on the - * queue gfp mask set, *to may or may not be a highmem page. kmap it - * always, it will do the Right Thing - */ -static void copy_to_high_bio_irq(struct bio *to, struct bio *from) -{ - struct bio_vec tovec, fromvec; - struct bvec_iter iter; - /* - * The bio of @from is created by bounce, so we can iterate - * its bvec from start to end, but the @from->bi_iter can't be - * trusted because it might be changed by splitting. - */ - struct bvec_iter from_iter = BVEC_ITER_ALL_INIT; - - bio_for_each_segment(tovec, to, iter) { - fromvec = bio_iter_iovec(from, from_iter); - if (tovec.bv_page != fromvec.bv_page) { - /* - * fromvec->bv_offset and fromvec->bv_len might have - * been modified by the block layer, so use the original - * copy, bounce_copy_vec already uses tovec->bv_len - */ - memcpy_to_bvec(&tovec, page_address(fromvec.bv_page) + - tovec.bv_offset); - } - bio_advance_iter(from, &from_iter, tovec.bv_len); - } -} - -static void bounce_end_io(struct bio *bio) -{ - struct bio *bio_orig = bio->bi_private; - struct bio_vec *bvec, orig_vec; - struct bvec_iter orig_iter = bio_orig->bi_iter; - struct bvec_iter_all iter_all; - - /* - * free up bounce indirect pages used - */ - bio_for_each_segment_all(bvec, bio, iter_all) { - orig_vec = bio_iter_iovec(bio_orig, orig_iter); - if (bvec->bv_page != orig_vec.bv_page) { - dec_zone_page_state(bvec->bv_page, NR_BOUNCE); - mempool_free(bvec->bv_page, &page_pool); - } - bio_advance_iter(bio_orig, &orig_iter, orig_vec.bv_len); - } - - bio_orig->bi_status = bio->bi_status; - bio_endio(bio_orig); - bio_put(bio); -} - -static void bounce_end_io_write(struct bio *bio) -{ - bounce_end_io(bio); -} - -static void bounce_end_io_read(struct bio *bio) -{ - struct bio *bio_orig = bio->bi_private; - - if (!bio->bi_status) - copy_to_high_bio_irq(bio_orig, bio); - - bounce_end_io(bio); -} - -static struct bio *bounce_clone_bio(struct bio *bio_src) -{ - struct bvec_iter iter; - struct bio_vec bv; - struct bio *bio; - - /* - * Pre immutable biovecs, __bio_clone() used to just do a memcpy from - * bio_src->bi_io_vec to bio->bi_io_vec. - * - * We can't do that anymore, because: - * - * - The point of cloning the biovec is to produce a bio with a biovec - * the caller can modify: bi_idx and bi_bvec_done should be 0. - * - * - The original bio could've had more than BIO_MAX_VECS biovecs; if - * we tried to clone the whole thing bio_alloc_bioset() would fail. - * But the clone should succeed as long as the number of biovecs we - * actually need to allocate is fewer than BIO_MAX_VECS. - * - * - Lastly, bi_vcnt should not be looked at or relied upon by code - * that does not own the bio - reason being drivers don't use it for - * iterating over the biovec anymore, so expecting it to be kept up - * to date (i.e. for clones that share the parent biovec) is just - * asking for trouble and would force extra work. - */ - bio = bio_alloc_bioset(bio_src->bi_bdev, bio_segments(bio_src), - bio_src->bi_opf, GFP_NOIO, &bounce_bio_set); - if (bio_flagged(bio_src, BIO_REMAPPED)) - bio_set_flag(bio, BIO_REMAPPED); - bio->bi_ioprio = bio_src->bi_ioprio; - bio->bi_write_hint = bio_src->bi_write_hint; - bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; - bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; - - switch (bio_op(bio)) { - case REQ_OP_DISCARD: - case REQ_OP_SECURE_ERASE: - case REQ_OP_WRITE_ZEROES: - break; - default: - bio_for_each_segment(bv, bio_src, iter) - bio->bi_io_vec[bio->bi_vcnt++] = bv; - break; - } - - if (bio_crypt_clone(bio, bio_src, GFP_NOIO) < 0) - goto err_put; - - if (bio_integrity(bio_src) && - bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) - goto err_put; - - bio_clone_blkg_association(bio, bio_src); - - return bio; - -err_put: - bio_put(bio); - return NULL; -} - -struct bio *__blk_queue_bounce(struct bio *bio_orig, struct request_queue *q) -{ - struct bio *bio; - int rw = bio_data_dir(bio_orig); - struct bio_vec *to, from; - struct bvec_iter iter; - unsigned i = 0, bytes = 0; - bool bounce = false; - int sectors; - - bio_for_each_segment(from, bio_orig, iter) { - if (i++ < BIO_MAX_VECS) - bytes += from.bv_len; - if (PageHighMem(from.bv_page)) - bounce = true; - } - if (!bounce) - return bio_orig; - - /* - * Individual bvecs might not be logical block aligned. Round down - * the split size so that each bio is properly block size aligned, - * even if we do not use the full hardware limits. - */ - sectors = ALIGN_DOWN(bytes, queue_logical_block_size(q)) >> - SECTOR_SHIFT; - if (sectors < bio_sectors(bio_orig)) { - bio = bio_split(bio_orig, sectors, GFP_NOIO, &bounce_bio_split); - bio_chain(bio, bio_orig); - submit_bio_noacct(bio_orig); - bio_orig = bio; - } - bio = bounce_clone_bio(bio_orig); - - /* - * Bvec table can't be updated by bio_for_each_segment_all(), - * so retrieve bvec from the table directly. This way is safe - * because the 'bio' is single-page bvec. - */ - for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) { - struct page *bounce_page; - - if (!PageHighMem(to->bv_page)) - continue; - - bounce_page = mempool_alloc(&page_pool, GFP_NOIO); - inc_zone_page_state(bounce_page, NR_BOUNCE); - - if (rw == WRITE) { - flush_dcache_page(to->bv_page); - memcpy_from_bvec(page_address(bounce_page), to); - } - to->bv_page = bounce_page; - } - - trace_block_bio_bounce(bio_orig); - - bio->bi_flags |= (1 << BIO_BOUNCED); - - if (rw == READ) - bio->bi_end_io = bounce_end_io_read; - else - bio->bi_end_io = bounce_end_io_write; - - bio->bi_private = bio_orig; - return bio; -} diff --git a/block/bsg-lib.c b/block/bsg-lib.c index 9ceb5d0832f5..895db30a7033 100644 --- a/block/bsg-lib.c +++ b/block/bsg-lib.c @@ -299,7 +299,7 @@ out: /* called right after the request is allocated for the request_queue */ static int bsg_init_rq(struct blk_mq_tag_set *set, struct request *req, - unsigned int hctx_idx, unsigned int numa_node) + unsigned int hctx_idx, int numa_node) { struct bsg_job *job = blk_mq_rq_to_pdu(req); @@ -368,7 +368,7 @@ struct request_queue *bsg_setup_queue(struct device *dev, const char *name, struct request_queue *q; int ret = -ENOMEM; - bset = kzalloc(sizeof(*bset), GFP_KERNEL); + bset = kzalloc_obj(*bset); if (!bset) return ERR_PTR(-ENOMEM); @@ -393,7 +393,7 @@ struct request_queue *bsg_setup_queue(struct device *dev, const char *name, blk_queue_rq_timeout(q, BLK_DEFAULT_SG_TIMEOUT); - bset->bd = bsg_register_queue(q, dev, name, bsg_transport_sg_io_fn); + bset->bd = bsg_register_queue(q, dev, name, bsg_transport_sg_io_fn, NULL); if (IS_ERR(bset->bd)) { ret = PTR_ERR(bset->bd); goto out_cleanup_queue; diff --git a/block/bsg.c b/block/bsg.c index 72157a59b788..82aaf3cee582 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -12,6 +12,7 @@ #include <linux/idr.h> #include <linux/bsg.h> #include <linux/slab.h> +#include <linux/io_uring/cmd.h> #include <scsi/scsi.h> #include <scsi/scsi_ioctl.h> @@ -28,6 +29,7 @@ struct bsg_device { unsigned int timeout; unsigned int reserved_size; bsg_sg_io_fn *sg_io_fn; + bsg_uring_cmd_fn *uring_cmd_fn; }; static inline struct bsg_device *to_bsg_device(struct inode *inode) @@ -158,11 +160,38 @@ static long bsg_ioctl(struct file *file, unsigned int cmd, unsigned long arg) } } +static int bsg_check_uring_features(unsigned int issue_flags) +{ + /* BSG passthrough requires big SQE/CQE support */ + if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) != + (IO_URING_F_SQE128|IO_URING_F_CQE32)) + return -EOPNOTSUPP; + return 0; +} + +static int bsg_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) +{ + struct bsg_device *bd = to_bsg_device(file_inode(ioucmd->file)); + bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE; + struct request_queue *q = bd->queue; + int ret; + + ret = bsg_check_uring_features(issue_flags); + if (ret) + return ret; + + if (!bd->uring_cmd_fn) + return -EOPNOTSUPP; + + return bd->uring_cmd_fn(q, ioucmd, issue_flags, open_for_write); +} + static const struct file_operations bsg_fops = { .open = bsg_open, .release = bsg_release, .unlocked_ioctl = bsg_ioctl, .compat_ioctl = compat_ptr_ioctl, + .uring_cmd = bsg_uring_cmd, .owner = THIS_MODULE, .llseek = default_llseek, }; @@ -187,18 +216,20 @@ void bsg_unregister_queue(struct bsg_device *bd) EXPORT_SYMBOL_GPL(bsg_unregister_queue); struct bsg_device *bsg_register_queue(struct request_queue *q, - struct device *parent, const char *name, bsg_sg_io_fn *sg_io_fn) + struct device *parent, const char *name, bsg_sg_io_fn *sg_io_fn, + bsg_uring_cmd_fn *uring_cmd_fn) { struct bsg_device *bd; int ret; - bd = kzalloc(sizeof(*bd), GFP_KERNEL); + bd = kzalloc_obj(*bd); if (!bd) return ERR_PTR(-ENOMEM); bd->max_queue = BSG_DEFAULT_CMDS; bd->reserved_size = INT_MAX; bd->queue = q; bd->sg_io_fn = sg_io_fn; + bd->uring_cmd_fn = uring_cmd_fn; ret = ida_alloc_max(&bsg_minor_ida, BSG_MAX_DEVS - 1, GFP_KERNEL); if (ret < 0) { diff --git a/block/disk-events.c b/block/disk-events.c index 2f697224386a..074731ecc3d2 100644 --- a/block/disk-events.c +++ b/block/disk-events.c @@ -290,13 +290,14 @@ EXPORT_SYMBOL(disk_check_media_change); * Should be called when the media changes for @disk. Generates a uevent * and attempts to free all dentries and inodes and invalidates all block * device page cache entries in that case. + * + * Callers that need a partition re-scan should arrange for one explicitly. */ void disk_force_media_change(struct gendisk *disk) { disk_event_uevent(disk, DISK_EVENT_MEDIA_CHANGE); inc_diskseq(disk); bdev_mark_dead(disk->part0, true); - set_bit(GD_NEED_PART_SCAN, &disk->state); } EXPORT_SYMBOL_GPL(disk_force_media_change); @@ -436,7 +437,7 @@ int disk_alloc_events(struct gendisk *disk) if (!disk->fops->check_events || !disk->events) return 0; - ev = kzalloc(sizeof(*ev), GFP_KERNEL); + ev = kzalloc_obj(*ev); if (!ev) { pr_warn("%s: failed to initialize events\n", disk->disk_name); return -ENOMEM; diff --git a/block/elevator.c b/block/elevator.c index b4d08026b02c..2161b6eea680 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -121,7 +121,7 @@ static struct elevator_type *elevator_find_get(const char *name) static const struct kobj_type elv_ktype; struct elevator_queue *elevator_alloc(struct request_queue *q, - struct elevator_type *e) + struct elevator_type *e, struct elevator_resources *res) { struct elevator_queue *eq; @@ -134,10 +134,11 @@ struct elevator_queue *elevator_alloc(struct request_queue *q, kobject_init(&eq->kobj, &elv_ktype); mutex_init(&eq->sysfs_lock); hash_init(eq->hash); + eq->et = res->et; + eq->elevator_data = res->data; return eq; } -EXPORT_SYMBOL(elevator_alloc); static void elevator_release(struct kobject *kobj) { @@ -148,18 +149,17 @@ static void elevator_release(struct kobject *kobj) kfree(e); } -void elevator_exit(struct request_queue *q) +static void elevator_exit(struct request_queue *q) { struct elevator_queue *e = q->elevator; + lockdep_assert_held(&q->elevator_lock); + ioc_clear_queue(q); - blk_mq_sched_free_rqs(q); mutex_lock(&e->sysfs_lock); blk_mq_exit_sched(q, e); mutex_unlock(&e->sysfs_lock); - - kobject_put(&e->kobj); } static inline void __elv_rqhash_del(struct request *rq) @@ -412,14 +412,15 @@ elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page) { const struct elv_fs_entry *entry = to_elv(attr); struct elevator_queue *e; - ssize_t error; + ssize_t error = -ENODEV; if (!entry->show) return -EIO; e = container_of(kobj, struct elevator_queue, kobj); mutex_lock(&e->sysfs_lock); - error = e->type ? entry->show(e, page) : -ENOENT; + if (!test_bit(ELEVATOR_FLAG_DYING, &e->flags)) + error = entry->show(e, page); mutex_unlock(&e->sysfs_lock); return error; } @@ -430,14 +431,15 @@ elv_attr_store(struct kobject *kobj, struct attribute *attr, { const struct elv_fs_entry *entry = to_elv(attr); struct elevator_queue *e; - ssize_t error; + ssize_t error = -ENODEV; if (!entry->store) return -EIO; e = container_of(kobj, struct elevator_queue, kobj); mutex_lock(&e->sysfs_lock); - error = e->type ? entry->store(e, page, length) : -ENOENT; + if (!test_bit(ELEVATOR_FLAG_DYING, &e->flags)) + error = entry->store(e, page, length); mutex_unlock(&e->sysfs_lock); return error; } @@ -452,13 +454,12 @@ static const struct kobj_type elv_ktype = { .release = elevator_release, }; -int elv_register_queue(struct request_queue *q, bool uevent) +static int elv_register_queue(struct request_queue *q, + struct elevator_queue *e, + bool uevent) { - struct elevator_queue *e = q->elevator; int error; - lockdep_assert_held(&q->elevator_lock); - error = kobject_add(&e->kobj, &q->disk->queue_kobj, "iosched"); if (!error) { const struct elv_fs_entry *attr = e->type->elevator_attrs; @@ -472,20 +473,25 @@ int elv_register_queue(struct request_queue *q, bool uevent) if (uevent) kobject_uevent(&e->kobj, KOBJ_ADD); + /* + * Sched is initialized, it is ready to export it via + * debugfs + */ + blk_mq_sched_reg_debugfs(q); set_bit(ELEVATOR_FLAG_REGISTERED, &e->flags); } return error; } -void elv_unregister_queue(struct request_queue *q) +static void elv_unregister_queue(struct request_queue *q, + struct elevator_queue *e) { - struct elevator_queue *e = q->elevator; - - lockdep_assert_held(&q->elevator_lock); - if (e && test_and_clear_bit(ELEVATOR_FLAG_REGISTERED, &e->flags)) { kobject_uevent(&e->kobj, KOBJ_REMOVE); kobject_del(&e->kobj); + + /* unexport via debugfs before exiting sched */ + blk_mq_sched_unreg_debugfs(q); } } @@ -548,159 +554,224 @@ void elv_unregister(struct elevator_type *e) EXPORT_SYMBOL_GPL(elv_unregister); /* - * For single queue devices, default to using mq-deadline. If we have multiple - * queues or mq-deadline is not available, default to "none". - */ -static struct elevator_type *elevator_get_default(struct request_queue *q) -{ - if (q->tag_set->flags & BLK_MQ_F_NO_SCHED_BY_DEFAULT) - return NULL; - - if (q->nr_hw_queues != 1 && - !blk_mq_is_shared_tags(q->tag_set->flags)) - return NULL; - - return elevator_find_get("mq-deadline"); -} - -/* - * Use the default elevator settings. If the chosen elevator initialization - * fails, fall back to the "none" elevator (no elevator). - */ -void elevator_init_mq(struct request_queue *q) -{ - struct elevator_type *e; - unsigned int memflags; - int err; - - WARN_ON_ONCE(blk_queue_registered(q)); - - if (unlikely(q->elevator)) - return; - - e = elevator_get_default(q); - if (!e) - return; - - /* - * We are called before adding disk, when there isn't any FS I/O, - * so freezing queue plus canceling dispatch work is enough to - * drain any dispatch activities originated from passthrough - * requests, then no need to quiesce queue which may add long boot - * latency, especially when lots of disks are involved. - * - * Disk isn't added yet, so verifying queue lock only manually. - */ - memflags = blk_mq_freeze_queue(q); - - blk_mq_cancel_work_sync(q); - - err = blk_mq_init_sched(q, e); - - blk_mq_unfreeze_queue(q, memflags); - - if (err) { - pr_warn("\"%s\" elevator initialization failed, " - "falling back to \"none\"\n", e->elevator_name); - } - - elevator_put(e); -} - -/* * Switch to new_e io scheduler. * * If switching fails, we are most likely running out of memory and not able * to restore the old io scheduler, so leaving the io scheduler being none. */ -int elevator_switch(struct request_queue *q, struct elevator_type *new_e) +static int elevator_switch(struct request_queue *q, struct elv_change_ctx *ctx) { - unsigned int memflags; - int ret; + struct elevator_type *new_e = NULL; + int ret = 0; + WARN_ON_ONCE(q->mq_freeze_depth == 0); lockdep_assert_held(&q->elevator_lock); - memflags = blk_mq_freeze_queue(q); + if (strncmp(ctx->name, "none", 4)) { + new_e = elevator_find_get(ctx->name); + if (!new_e) + return -EINVAL; + } + blk_mq_quiesce_queue(q); if (q->elevator) { - elv_unregister_queue(q); + ctx->old = q->elevator; elevator_exit(q); } - ret = blk_mq_init_sched(q, new_e); - if (ret) - goto out_unfreeze; - - ret = elv_register_queue(q, true); - if (ret) { - elevator_exit(q); - goto out_unfreeze; + if (new_e) { + ret = blk_mq_init_sched(q, new_e, &ctx->res); + if (ret) + goto out_unfreeze; + ctx->new = q->elevator; + } else { + blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q); + q->elevator = NULL; + q->nr_requests = q->tag_set->queue_depth; + q->async_depth = q->tag_set->queue_depth; } - blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name); + blk_add_trace_msg(q, "elv switch: %s", ctx->name); out_unfreeze: blk_mq_unquiesce_queue(q); - blk_mq_unfreeze_queue(q, memflags); if (ret) { pr_warn("elv: switch to \"%s\" failed, falling back to \"none\"\n", new_e->elevator_name); } + if (new_e) + elevator_put(new_e); return ret; } -void elevator_disable(struct request_queue *q) +static void elv_exit_and_release(struct elv_change_ctx *ctx, + struct request_queue *q) { - unsigned int memflags; - - lockdep_assert_held(&q->elevator_lock); + struct elevator_queue *e; + unsigned memflags; memflags = blk_mq_freeze_queue(q); - blk_mq_quiesce_queue(q); - - elv_unregister_queue(q); + mutex_lock(&q->elevator_lock); + e = q->elevator; elevator_exit(q); - blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q); - q->elevator = NULL; - q->nr_requests = q->tag_set->queue_depth; - blk_add_trace_msg(q, "elv switch: none"); - - blk_mq_unquiesce_queue(q); + mutex_unlock(&q->elevator_lock); blk_mq_unfreeze_queue(q, memflags); + if (e) { + blk_mq_free_sched_res(&ctx->res, ctx->type, q->tag_set); + kobject_put(&e->kobj); + } +} + +static int elevator_change_done(struct request_queue *q, + struct elv_change_ctx *ctx) +{ + int ret = 0; + + if (ctx->old) { + struct elevator_resources res = { + .et = ctx->old->et, + .data = ctx->old->elevator_data + }; + + elv_unregister_queue(q, ctx->old); + blk_mq_free_sched_res(&res, ctx->old->type, q->tag_set); + kobject_put(&ctx->old->kobj); + } + if (ctx->new) { + ret = elv_register_queue(q, ctx->new, !ctx->no_uevent); + if (ret) + elv_exit_and_release(ctx, q); + } + return ret; } /* * Switch this queue to the given IO scheduler. */ -static int elevator_change(struct request_queue *q, const char *elevator_name) +static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx) { - struct elevator_type *e; - int ret; + unsigned int memflags; + struct blk_mq_tag_set *set = q->tag_set; + int ret = 0; - /* Make sure queue is not in the middle of being removed */ - if (!blk_queue_registered(q)) - return -ENOENT; + lockdep_assert_held(&set->update_nr_hwq_lock); - if (!strncmp(elevator_name, "none", 4)) { - if (q->elevator) - elevator_disable(q); - return 0; + if (strncmp(ctx->name, "none", 4)) { + ret = blk_mq_alloc_sched_res(q, ctx->type, &ctx->res, + set->nr_hw_queues); + if (ret) + return ret; } - if (q->elevator && elevator_match(q->elevator->type, elevator_name)) - return 0; + memflags = blk_mq_freeze_queue(q); + /* + * May be called before adding disk, when there isn't any FS I/O, + * so freezing queue plus canceling dispatch work is enough to + * drain any dispatch activities originated from passthrough + * requests, then no need to quiesce queue which may add long boot + * latency, especially when lots of disks are involved. + * + * Disk isn't added yet, so verifying queue lock only manually. + */ + blk_mq_cancel_work_sync(q); + mutex_lock(&q->elevator_lock); + if (!(q->elevator && elevator_match(q->elevator->type, ctx->name))) + ret = elevator_switch(q, ctx); + mutex_unlock(&q->elevator_lock); + blk_mq_unfreeze_queue(q, memflags); + if (!ret) + ret = elevator_change_done(q, ctx); + + /* + * Free sched resource if it's allocated but we couldn't switch elevator. + */ + if (!ctx->new) + blk_mq_free_sched_res(&ctx->res, ctx->type, set); - e = elevator_find_get(elevator_name); - if (!e) - return -EINVAL; - ret = elevator_switch(q, e); - elevator_put(e); return ret; } -static void elv_iosched_load_module(char *elevator_name) +/* + * The I/O scheduler depends on the number of hardware queues, this forces a + * reattachment when nr_hw_queues changes. + */ +void elv_update_nr_hw_queues(struct request_queue *q, + struct elv_change_ctx *ctx) +{ + struct blk_mq_tag_set *set = q->tag_set; + int ret = -ENODEV; + + WARN_ON_ONCE(q->mq_freeze_depth == 0); + + if (ctx->type && !blk_queue_dying(q) && blk_queue_registered(q)) { + mutex_lock(&q->elevator_lock); + /* force to reattach elevator after nr_hw_queue is updated */ + ret = elevator_switch(q, ctx); + mutex_unlock(&q->elevator_lock); + } + blk_mq_unfreeze_queue_nomemrestore(q); + if (!ret) + WARN_ON_ONCE(elevator_change_done(q, ctx)); + + /* + * Free sched resource if it's allocated but we couldn't switch elevator. + */ + if (!ctx->new) + blk_mq_free_sched_res(&ctx->res, ctx->type, set); +} + +/* + * Use the default elevator settings. If the chosen elevator initialization + * fails, fall back to the "none" elevator (no elevator). + */ +void elevator_set_default(struct request_queue *q) +{ + struct elv_change_ctx ctx = { + .name = "mq-deadline", + .no_uevent = true, + }; + int err; + + /* now we allow to switch elevator */ + blk_queue_flag_clear(QUEUE_FLAG_NO_ELV_SWITCH, q); + + if (q->tag_set->flags & BLK_MQ_F_NO_SCHED_BY_DEFAULT) + return; + + /* + * For single queue devices, default to using mq-deadline. If we + * have multiple queues or mq-deadline is not available, default + * to "none". + */ + ctx.type = elevator_find_get(ctx.name); + if (!ctx.type) + return; + + if ((q->nr_hw_queues == 1 || + blk_mq_is_shared_tags(q->tag_set->flags))) { + err = elevator_change(q, &ctx); + if (err < 0) + pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n", + ctx.name, err); + } + elevator_put(ctx.type); +} + +void elevator_set_none(struct request_queue *q) +{ + struct elv_change_ctx ctx = { + .name = "none", + }; + int err; + + err = elevator_change(q, &ctx); + if (err < 0) + pr_warn("%s: set none elevator failed %d\n", __func__, err); +} + +static void elv_iosched_load_module(const char *elevator_name) { struct elevator_type *found; @@ -716,10 +787,14 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf, size_t count) { char elevator_name[ELV_NAME_MAX]; - char *name; + struct elv_change_ctx ctx = {}; int ret; - unsigned int memflags; struct request_queue *q = disk->queue; + struct blk_mq_tag_set *set = q->tag_set; + + /* Make sure queue is not in the middle of being removed */ + if (!blk_queue_registered(q)) + return -ENOENT; /* * If the attribute needs to load a module, do it before freezing the @@ -727,24 +802,44 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf, * queue is the one for the device storing the module file. */ strscpy(elevator_name, buf, sizeof(elevator_name)); - name = strstrip(elevator_name); + ctx.name = strstrip(elevator_name); - elv_iosched_load_module(name); + elv_iosched_load_module(ctx.name); + ctx.type = elevator_find_get(ctx.name); - memflags = blk_mq_freeze_queue(q); - mutex_lock(&q->elevator_lock); - ret = elevator_change(q, name); - if (!ret) - ret = count; - mutex_unlock(&q->elevator_lock); - blk_mq_unfreeze_queue(q, memflags); + /* + * Use trylock to avoid circular lock dependency with kernfs active + * reference during concurrent disk deletion: + * update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del) + * kn->active -> update_nr_hwq_lock (via this sysfs write path) + * + * Use the writer lock instead of the reader lock of update_nr_hwq_lock + * to serialize the two-stage elevator switch steps in + * elevator_change(): the core switch step under the elevator lock and + * the elevator_change_done() step outside the elevator lock. + */ + if (!down_write_trylock(&set->update_nr_hwq_lock)) { + ret = -EBUSY; + goto out; + } + if (!blk_queue_no_elv_switch(q)) { + ret = elevator_change(q, &ctx); + if (!ret) + ret = count; + } else { + ret = -ENOENT; + } + up_write(&set->update_nr_hwq_lock); + +out: + if (ctx.type) + elevator_put(ctx.type); return ret; } ssize_t elv_iosched_show(struct gendisk *disk, char *name) { struct request_queue *q = disk->queue; - struct elevator_queue *eq = q->elevator; struct elevator_type *cur = NULL, *e; int len = 0; @@ -753,7 +848,7 @@ ssize_t elv_iosched_show(struct gendisk *disk, char *name) len += sprintf(name+len, "[none] "); } else { len += sprintf(name+len, "none "); - cur = eq->type; + cur = q->elevator->type; } spin_lock(&elv_list_lock); diff --git a/block/elevator.h b/block/elevator.h index e4e44dfac503..3eb32516be0b 100644 --- a/block/elevator.h +++ b/block/elevator.h @@ -23,12 +23,45 @@ enum elv_merge { struct blk_mq_alloc_data; struct blk_mq_hw_ctx; +struct elevator_tags { + /* num. of hardware queues for which tags are allocated */ + unsigned int nr_hw_queues; + /* depth used while allocating tags */ + unsigned int nr_requests; + /* shared tag is stored at index 0 */ + struct blk_mq_tags *tags[]; +}; + +struct elevator_resources { + /* holds elevator data */ + void *data; + /* holds elevator tags */ + struct elevator_tags *et; +}; + +/* Holding context data for changing elevator */ +struct elv_change_ctx { + const char *name; + bool no_uevent; + + /* for unregistering old elevator */ + struct elevator_queue *old; + /* for registering new elevator */ + struct elevator_queue *new; + /* store elevator type */ + struct elevator_type *type; + /* store elevator resources */ + struct elevator_resources res; +}; + struct elevator_mq_ops { - int (*init_sched)(struct request_queue *, struct elevator_type *); + int (*init_sched)(struct request_queue *, struct elevator_queue *); void (*exit_sched)(struct elevator_queue *); int (*init_hctx)(struct blk_mq_hw_ctx *, unsigned int); void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); - void (*depth_updated)(struct blk_mq_hw_ctx *); + void (*depth_updated)(struct request_queue *); + void *(*alloc_sched_data)(struct request_queue *); + void (*free_sched_data)(void *); bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); bool (*bio_merge)(struct request_queue *, struct bio *, unsigned int); @@ -113,6 +146,7 @@ struct request *elv_rqhash_find(struct request_queue *q, sector_t offset); struct elevator_queue { struct elevator_type *type; + struct elevator_tags *et; void *elevator_data; struct kobject kobj; struct mutex sysfs_lock; @@ -121,7 +155,7 @@ struct elevator_queue }; #define ELEVATOR_FLAG_REGISTERED 0 -#define ELEVATOR_FLAG_DISABLE_WBT 1 +#define ELEVATOR_FLAG_DYING 1 /* * block elevator interface @@ -136,7 +170,6 @@ extern bool elv_attempt_insert_merge(struct request_queue *, struct request *, struct list_head *); extern struct request *elv_former_request(struct request_queue *, struct request *); extern struct request *elv_latter_request(struct request_queue *, struct request *); -void elevator_init_mq(struct request_queue *q); /* * io scheduler registration @@ -151,8 +184,8 @@ ssize_t elv_iosched_show(struct gendisk *disk, char *page); ssize_t elv_iosched_store(struct gendisk *disk, const char *page, size_t count); extern bool elv_bio_merge_ok(struct request *, struct bio *); -extern struct elevator_queue *elevator_alloc(struct request_queue *, - struct elevator_type *); +struct elevator_queue *elevator_alloc(struct request_queue *, + struct elevator_type *, struct elevator_resources *); /* * Helper functions. @@ -182,4 +215,7 @@ extern struct request *elv_rb_find(struct rb_root *, sector_t); #define rq_entry_fifo(ptr) list_entry((ptr), struct request, queuelist) #define rq_fifo_clear(rq) list_del_init(&(rq)->queuelist) +void blk_mq_sched_reg_debugfs(struct request_queue *q); +void blk_mq_sched_unreg_debugfs(struct request_queue *q); + #endif /* _ELEVATOR_H */ diff --git a/block/error-injection.c b/block/error-injection.c new file mode 100644 index 000000000000..e14bc4b723ef --- /dev/null +++ b/block/error-injection.c @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Christoph Hellwig. + */ +#include <linux/debugfs.h> +#include <linux/blkdev.h> +#include <linux/parser.h> +#include <linux/seq_file.h> +#include "blk.h" +#include "error-injection.h" + +struct blk_error_inject { + struct list_head entry; + sector_t start; + sector_t end; + enum req_op op; + blk_status_t status; + + /* only inject every 1 / chance times */ + unsigned int chance; +}; + +DEFINE_STATIC_KEY_FALSE(blk_error_injection_enabled); + +bool __blk_error_inject(struct bio *bio) +{ + struct gendisk *disk = bio->bi_bdev->bd_disk; + struct blk_error_inject *inj; + + rcu_read_lock(); + list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) { + if (bio_op(bio) != inj->op) + continue; + /* + * This never matches 0-sized bios like empty WRITEs with + * REQ_PREFLUSH or ZONE_RESET_ALL. While adding a special case + * for them would be trivial, that means any WRITE rule would + * trigger for flushes. So before we can make this work + * properly, we'll need to start using REQ_OP_FLUSH for pure + * flushes at the bio level like we already do in blk-mq. + */ + if (bio->bi_iter.bi_sector > inj->end || + bio_end_sector(bio) <= inj->start) + continue; + if (inj->chance > 1 && (get_random_u32() % inj->chance) != 0) + continue; + + pr_info_ratelimited("%pg: injecting %s error for %s at sector %llu:%u\n", + disk->part0, blk_status_to_str(inj->status), + blk_op_str(inj->op), bio->bi_iter.bi_sector, + bio_sectors(bio)); + bio->bi_status = inj->status; + rcu_read_unlock(); + bio_endio(bio); + return true; + } + rcu_read_unlock(); + return false; +} + +static int error_inject_add(struct gendisk *disk, enum req_op op, + sector_t start, u64 nr_sectors, blk_status_t status, + unsigned int chance) +{ + struct blk_error_inject *inj; + int error = -EINVAL; + + if (op == REQ_OP_LAST) + return -EINVAL; + if (status == BLK_STS_OK) + return -EINVAL; + + inj = kzalloc_obj(*inj); + if (!inj) + return -ENOMEM; + + if (nr_sectors) { + if (U64_MAX - nr_sectors < start) + goto out_free_inj; + inj->end = start + nr_sectors - 1; + } else { + inj->end = U64_MAX; + } + + inj->op = op; + inj->start = start; + inj->status = status; + inj->chance = chance; + + pr_debug_ratelimited("%pg: adding %s injection for %s at sector %llu:%llu\n", + disk->part0, blk_status_to_str(status), + blk_op_str(op), + start, nr_sectors); + + /* + * Add to the front of the list so that newer entries can partially + * override other entries. This also intentionally allows duplicate + * entries as there is no real reason to reject them. + */ + mutex_lock(&disk->error_injection_lock); + if (!disk_live(disk)) { + mutex_unlock(&disk->error_injection_lock); + error = -ENODEV; + goto out_free_inj; + } + if (list_empty(&disk->error_injection_list)) + static_branch_inc(&blk_error_injection_enabled); + list_add_rcu(&inj->entry, &disk->error_injection_list); + set_bit(GD_ERROR_INJECT, &disk->state); + mutex_unlock(&disk->error_injection_lock); + return 0; + +out_free_inj: + kfree(inj); + return error; +} + +static void error_inject_removeall(struct gendisk *disk) +{ + struct blk_error_inject *inj; + + mutex_lock(&disk->error_injection_lock); + if (test_and_clear_bit(GD_ERROR_INJECT, &disk->state)) + static_branch_dec(&blk_error_injection_enabled); + while ((inj = list_first_entry_or_null(&disk->error_injection_list, + struct blk_error_inject, entry))) { + list_del_rcu(&inj->entry); + kfree_rcu_mightsleep(inj); + } + mutex_unlock(&disk->error_injection_lock); +} + +enum options { + Opt_add = (1u << 0), + Opt_removeall = (1u << 1), + + Opt_op = (1u << 16), + Opt_start = (1u << 17), + Opt_nr_sectors = (1u << 18), + Opt_status = (1u << 19), + Opt_chance = (1u << 20), + + Opt_invalid, +}; + +static const match_table_t opt_tokens = { + { Opt_add, "add", }, + { Opt_removeall, "removeall", }, + { Opt_op, "op=%s", }, + { Opt_start, "start=%u" }, + { Opt_nr_sectors, "nr_sectors=%u" }, + { Opt_status, "status=%s" }, + { Opt_chance, "chance=%u" }, + { Opt_invalid, NULL, }, +}; + +static int match_op(substring_t *args, enum req_op *op) +{ + const char *tag; + + tag = match_strdup(args); + if (!tag) + return -ENOMEM; + *op = str_to_blk_op(tag); + if (*op == REQ_OP_LAST) + pr_warn("invalid op '%s'\n", tag); + kfree(tag); + return 0; +} + +static int match_status(substring_t *args, blk_status_t *status) +{ + const char *tag; + + tag = match_strdup(args); + if (!tag) + return -ENOMEM; + *status = tag_to_blk_status(tag); + if (!*status) + pr_warn("invalid status '%s'\n", tag); + kfree(tag); + return 0; +} + +static ssize_t blk_error_injection_parse_options(struct gendisk *disk, + char *options) +{ + enum { Unset, Add, Removeall } action = Unset; + unsigned int option_mask = 0, chance = 1; + enum req_op op = REQ_OP_LAST; + u64 start = 0, nr_sectors = 0; + blk_status_t status = BLK_STS_OK; + substring_t args[MAX_OPT_ARGS]; + char *p; + + while ((p = strsep(&options, ",\n")) != NULL) { + int error = 0; + ssize_t token; + + if (!*p) + continue; + token = match_token(p, opt_tokens, args); + option_mask |= token; + switch (token) { + case Opt_add: + if (action != Unset) + return -EINVAL; + action = Add; + break; + case Opt_removeall: + if (action != Unset) + return -EINVAL; + action = Removeall; + break; + case Opt_op: + error = match_op(args, &op); + break; + case Opt_start: + error = match_u64(args, &start); + break; + case Opt_nr_sectors: + error = match_u64(args, &nr_sectors); + break; + case Opt_status: + error = match_status(args, &status); + break; + case Opt_chance: + error = match_uint(args, &chance); + if (!error && chance == 0) + error = -EINVAL; + break; + default: + pr_warn("unknown parameter or missing value '%s'\n", p); + error = -EINVAL; + } + if (error) + return error; + } + + switch (action) { + case Add: + return error_inject_add(disk, op, start, nr_sectors, status, + chance); + case Removeall: + if (option_mask & ~Opt_removeall) + return -EINVAL; + error_inject_removeall(disk); + return 0; + default: + return -EINVAL; + } +} + +static ssize_t blk_error_injection_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *pos) +{ + struct gendisk *disk = file_inode(file)->i_private; + char *options; + int error; + + options = memdup_user_nul(ubuf, count); + if (IS_ERR(options)) + return PTR_ERR(options); + error = blk_error_injection_parse_options(disk, options); + kfree(options); + + if (error) + return error; + return count; +} + +static int blk_error_injection_show(struct seq_file *s, void *private) +{ + struct gendisk *disk = s->private; + struct blk_error_inject *inj; + + rcu_read_lock(); + list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) { + seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u", + inj->start, inj->end, + blk_op_str(inj->op), + blk_status_to_tag(inj->status), inj->chance); + seq_putc(s, '\n'); + } + rcu_read_unlock(); + return 0; +} + +static int blk_error_injection_open(struct inode *inode, struct file *file) +{ + return single_open(file, blk_error_injection_show, inode->i_private); +} + +static int blk_error_injection_release(struct inode *inode, struct file *file) +{ + return single_release(inode, file); +} + +static const struct file_operations blk_error_injection_fops = { + .owner = THIS_MODULE, + .write = blk_error_injection_write, + .read = seq_read, + .open = blk_error_injection_open, + .release = blk_error_injection_release, +}; + +void blk_error_injection_init(struct gendisk *disk) +{ + debugfs_create_file("error_injection", 0600, disk->queue->debugfs_dir, + disk, &blk_error_injection_fops); +} + +void blk_error_injection_exit(struct gendisk *disk) +{ + error_inject_removeall(disk); +} diff --git a/block/error-injection.h b/block/error-injection.h new file mode 100644 index 000000000000..9821d773abab --- /dev/null +++ b/block/error-injection.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BLK_ERROR_INJECTION_H +#define _BLK_ERROR_INJECTION_H 1 + +#include <linux/jump_label.h> + +DECLARE_STATIC_KEY_FALSE(blk_error_injection_enabled); + +void blk_error_injection_init(struct gendisk *disk); +void blk_error_injection_exit(struct gendisk *disk); +bool __blk_error_inject(struct bio *bio); +static inline bool blk_error_inject(struct bio *bio) +{ + if (IS_ENABLED(CONFIG_BLK_ERROR_INJECTION) && + static_branch_unlikely(&blk_error_injection_enabled) && + test_bit(GD_ERROR_INJECT, &bio->bi_bdev->bd_disk->state)) + return __blk_error_inject(bio); + return false; +} + +#endif /* _BLK_ERROR_INJECTION_H */ diff --git a/block/fops.c b/block/fops.c index be9f1dbea9ce..15783a6180de 100644 --- a/block/fops.c +++ b/block/fops.c @@ -7,6 +7,7 @@ #include <linux/init.h> #include <linux/mm.h> #include <linux/blkdev.h> +#include <linux/blk-integrity.h> #include <linux/buffer_head.h> #include <linux/mpage.h> #include <linux/uio.h> @@ -38,8 +39,15 @@ static blk_opf_t dio_bio_write_op(struct kiocb *iocb) static bool blkdev_dio_invalid(struct block_device *bdev, struct kiocb *iocb, struct iov_iter *iter) { - return iocb->ki_pos & (bdev_logical_block_size(bdev) - 1) || - !bdev_iter_is_aligned(bdev, iter); + return (iocb->ki_pos | iov_iter_count(iter)) & + (bdev_logical_block_size(bdev) - 1); +} + +static inline int blkdev_iov_iter_get_pages(struct bio *bio, + struct iov_iter *iter, struct block_device *bdev) +{ + return bio_iov_iter_get_pages(bio, iter, + bdev_logical_block_size(bdev) - 1); } #define DIO_INLINE_BIO_VECS 4 @@ -54,12 +62,10 @@ static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb, struct bio bio; ssize_t ret; - WARN_ON_ONCE(iocb->ki_flags & IOCB_HAS_METADATA); if (nr_pages <= DIO_INLINE_BIO_VECS) vecs = inline_vecs; else { - vecs = kmalloc_array(nr_pages, sizeof(struct bio_vec), - GFP_KERNEL); + vecs = kmalloc_objs(struct bio_vec, nr_pages); if (!vecs) return -ENOMEM; } @@ -73,11 +79,12 @@ static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb, } bio.bi_iter.bi_sector = pos >> SECTOR_SHIFT; bio.bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint; + bio.bi_write_stream = iocb->ki_write_stream; bio.bi_ioprio = iocb->ki_ioprio; if (iocb->ki_flags & IOCB_ATOMIC) bio.bi_opf |= REQ_ATOMIC; - ret = bio_iov_iter_get_pages(&bio, iter); + ret = blkdev_iov_iter_get_pages(&bio, iter, bdev); if (unlikely(ret)) goto out; ret = bio.bi_iter.bi_size; @@ -130,7 +137,7 @@ static void blkdev_bio_end_io(struct bio *bio) if (bio->bi_status && !dio->bio.bi_status) dio->bio.bi_status = bio->bi_status; - if (!is_sync && (dio->iocb->ki_flags & IOCB_HAS_METADATA)) + if (bio_integrity(bio)) bio_integrity_unmap_user(bio); if (atomic_dec_and_test(&dio->ref)) { @@ -176,8 +183,6 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, loff_t pos = iocb->ki_pos; int ret = 0; - if (iocb->ki_flags & IOCB_ALLOC_CACHE) - opf |= REQ_ALLOC_CACHE; bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL, &blkdev_dio_pool); dio = container_of(bio, struct blkdev_dio, bio); @@ -206,14 +211,14 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, for (;;) { bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT; bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint; + bio->bi_write_stream = iocb->ki_write_stream; bio->bi_private = dio; bio->bi_end_io = blkdev_bio_end_io; bio->bi_ioprio = iocb->ki_ioprio; - ret = bio_iov_iter_get_pages(bio, iter); + ret = blkdev_iov_iter_get_pages(bio, iter, bdev); if (unlikely(ret)) { - bio->bi_status = BLK_STS_IOERR; - bio_endio(bio); + bio_endio_status(bio, BLK_STS_IOERR); break; } if (iocb->ki_flags & IOCB_NOWAIT) { @@ -231,7 +236,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, } bio->bi_opf |= REQ_NOWAIT; } - if (!is_sync && (iocb->ki_flags & IOCB_HAS_METADATA)) { + if (iocb->ki_flags & IOCB_HAS_METADATA) { ret = bio_integrity_map_iter(bio, iocb->private); if (unlikely(ret)) goto fail; @@ -299,7 +304,7 @@ static void blkdev_bio_end_io_async(struct bio *bio) ret = blk_status_to_errno(bio->bi_status); } - if (iocb->ki_flags & IOCB_HAS_METADATA) + if (bio_integrity(bio)) bio_integrity_unmap_user(bio); iocb->ki_complete(iocb, ret); @@ -324,8 +329,6 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, loff_t pos = iocb->ki_pos; int ret = 0; - if (iocb->ki_flags & IOCB_ALLOC_CACHE) - opf |= REQ_ALLOC_CACHE; bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL, &blkdev_dio_pool); dio = container_of(bio, struct blkdev_dio, bio); @@ -333,6 +336,7 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, dio->iocb = iocb; bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT; bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint; + bio->bi_write_stream = iocb->ki_write_stream; bio->bi_end_io = blkdev_bio_end_io_async; bio->bi_ioprio = iocb->ki_ioprio; @@ -345,7 +349,7 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, */ bio_iov_bvec_set(bio, iter); } else { - ret = bio_iov_iter_get_pages(bio, iter); + ret = blkdev_iov_iter_get_pages(bio, iter, bdev); if (unlikely(ret)) goto out_bio_put; } @@ -398,8 +402,29 @@ static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (blkdev_dio_invalid(bdev, iocb, iter)) return -EINVAL; + if (iov_iter_rw(iter) == WRITE) { + u16 max_write_streams = bdev_max_write_streams(bdev); + + if (iocb->ki_write_stream) { + if (iocb->ki_write_stream > max_write_streams) + return -EINVAL; + } else if (max_write_streams) { + enum rw_hint write_hint = + file_inode(iocb->ki_filp)->i_write_hint; + + /* + * Just use the write hint as write stream for block + * device writes. This assumes no file system is + * mounted that would use the streams differently. + */ + if (write_hint <= max_write_streams) + iocb->ki_write_stream = write_hint; + } + } + nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS + 1); - if (likely(nr_pages <= BIO_MAX_VECS)) { + if (likely(nr_pages <= BIO_MAX_VECS && + !(iocb->ki_flags & IOCB_HAS_METADATA))) { if (is_sync_kiocb(iocb)) return __blkdev_direct_IO_simple(iocb, iter, bdev, nr_pages); @@ -451,12 +476,13 @@ static int blkdev_get_block(struct inode *inode, sector_t iblock, static int blkdev_writepages(struct address_space *mapping, struct writeback_control *wbc) { + struct folio *folio = NULL; struct blk_plug plug; int err; blk_start_plug(&plug); - err = write_cache_pages(mapping, wbc, block_write_full_folio, - blkdev_get_block); + while ((folio = writeback_iter(mapping, wbc, folio, &err))) + err = block_write_full_folio(folio, wbc, blkdev_get_block); blk_finish_plug(&plug); return err; @@ -472,71 +498,63 @@ static void blkdev_readahead(struct readahead_control *rac) mpage_readahead(rac, blkdev_get_block); } -static int blkdev_write_begin(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, struct folio **foliop, void **fsdata) -{ - return block_write_begin(mapping, pos, len, foliop, blkdev_get_block); -} - -static int blkdev_write_end(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, unsigned copied, struct folio *folio, - void *fsdata) -{ - int ret; - ret = block_write_end(file, mapping, pos, len, copied, folio, fsdata); - - folio_unlock(folio); - folio_put(folio); - - return ret; -} - const struct address_space_operations def_blk_aops = { .dirty_folio = block_dirty_folio, .invalidate_folio = block_invalidate_folio, .read_folio = blkdev_read_folio, .readahead = blkdev_readahead, .writepages = blkdev_writepages, - .write_begin = blkdev_write_begin, - .write_end = blkdev_write_end, .migrate_folio = buffer_migrate_folio_norefs, .is_dirty_writeback = buffer_check_dirty_writeback, }; #else /* CONFIG_BUFFER_HEAD */ static int blkdev_read_folio(struct file *file, struct folio *folio) { - return iomap_read_folio(folio, &blkdev_iomap_ops); + iomap_bio_read_folio(folio, &blkdev_iomap_ops); + return 0; } static void blkdev_readahead(struct readahead_control *rac) { - iomap_readahead(rac, &blkdev_iomap_ops); + iomap_bio_readahead(rac, &blkdev_iomap_ops); } -static int blkdev_map_blocks(struct iomap_writepage_ctx *wpc, - struct inode *inode, loff_t offset, unsigned int len) +static ssize_t blkdev_writeback_range(struct iomap_writepage_ctx *wpc, + struct folio *folio, u64 offset, unsigned int len, u64 end_pos) { - loff_t isize = i_size_read(inode); + loff_t isize = i_size_read(wpc->inode); if (WARN_ON_ONCE(offset >= isize)) return -EIO; - if (offset >= wpc->iomap.offset && - offset < wpc->iomap.offset + wpc->iomap.length) - return 0; - return blkdev_iomap_begin(inode, offset, isize - offset, - IOMAP_WRITE, &wpc->iomap, NULL); + + if (offset < wpc->iomap.offset || + offset >= wpc->iomap.offset + wpc->iomap.length) { + int error; + + error = blkdev_iomap_begin(wpc->inode, offset, isize - offset, + IOMAP_WRITE, &wpc->iomap, NULL); + if (error) + return error; + } + + return iomap_add_to_ioend(wpc, folio, offset, end_pos, len); } static const struct iomap_writeback_ops blkdev_writeback_ops = { - .map_blocks = blkdev_map_blocks, + .writeback_range = blkdev_writeback_range, + .writeback_submit = iomap_ioend_writeback_submit, }; static int blkdev_writepages(struct address_space *mapping, struct writeback_control *wbc) { - struct iomap_writepage_ctx wpc = { }; + struct iomap_writepage_ctx wpc = { + .inode = mapping->host, + .wbc = wbc, + .ops = &blkdev_writeback_ops + }; - return iomap_writepages(mapping, wbc, &wpc, &blkdev_writeback_ops); + return iomap_writepages(&wpc); } const struct address_space_operations def_blk_aops = { @@ -642,12 +660,14 @@ static int blkdev_open(struct inode *inode, struct file *filp) if (ret) return ret; - bdev = blkdev_get_no_open(inode->i_rdev); + bdev = blkdev_get_no_open(inode->i_rdev, true); if (!bdev) return -ENXIO; if (bdev_can_atomic_write(bdev)) filp->f_mode |= FMODE_CAN_ATOMIC_WRITE; + if (blk_get_integrity(bdev->bd_disk)) + filp->f_mode |= FMODE_HAS_METADATA; ret = bdev_open(bdev, mode, filp->private_data, NULL, filp); if (ret) @@ -687,7 +707,8 @@ blkdev_direct_write(struct kiocb *iocb, struct iov_iter *from) static ssize_t blkdev_buffered_write(struct kiocb *iocb, struct iov_iter *from) { - return iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops, NULL); + return iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops, NULL, + NULL); } /* @@ -746,7 +767,14 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) ret = direct_write_fallback(iocb, from, ret, blkdev_buffered_write(iocb, from)); } else { + /* + * Take i_rwsem and invalidate_lock to avoid racing with + * set_blocksize changing i_blkbits/folio order and punching + * out the pagecache. + */ + inode_lock_shared(bd_inode); ret = blkdev_buffered_write(iocb, from); + inode_unlock_shared(bd_inode); } if (ret > 0) @@ -757,6 +785,7 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) { + struct inode *bd_inode = bdev_file_inode(iocb->ki_filp); struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host); loff_t size = bdev_nr_bytes(bdev); loff_t pos = iocb->ki_pos; @@ -793,7 +822,13 @@ static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) goto reexpand; } + /* + * Take i_rwsem and invalidate_lock to avoid racing with set_blocksize + * changing i_blkbits/folio order and punching out the pagecache. + */ + inode_lock_shared(bd_inode); ret = filemap_read(iocb, to, ret); + inode_unlock_shared(bd_inode); reexpand: if (unlikely(shorted)) @@ -803,7 +838,7 @@ reexpand: #define BLKDEV_FALLOC_FL_SUPPORTED \ (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \ - FALLOC_FL_ZERO_RANGE) + FALLOC_FL_ZERO_RANGE | FALLOC_FL_WRITE_ZEROES) static long blkdev_fallocate(struct file *file, int mode, loff_t start, loff_t len) @@ -812,11 +847,19 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start, struct block_device *bdev = I_BDEV(inode); loff_t end = start + len - 1; loff_t isize; + unsigned int flags; int error; /* Fail if we don't recognize the flags. */ if (mode & ~BLKDEV_FALLOC_FL_SUPPORTED) return -EOPNOTSUPP; + /* + * Don't allow writing zeroes if the device does not enable the + * unmap write zeroes operation. + */ + if ((mode & FALLOC_FL_WRITE_ZEROES) && + !bdev_write_zeroes_unmap_sectors(bdev)) + return -EOPNOTSUPP; /* Don't go off the end of the device. */ isize = bdev_nr_bytes(bdev); @@ -836,49 +879,49 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start, if ((start | len) & (bdev_logical_block_size(bdev) - 1)) return -EINVAL; + inode_lock(inode); filemap_invalidate_lock(inode->i_mapping); - /* - * Invalidate the page cache, including dirty pages, for valid - * de-allocate mode calls to fallocate(). - */ switch (mode) { case FALLOC_FL_ZERO_RANGE: case FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE: - error = truncate_bdev_range(bdev, file_to_blk_mode(file), start, end); - if (error) - goto fail; - - error = blkdev_issue_zeroout(bdev, start >> SECTOR_SHIFT, - len >> SECTOR_SHIFT, GFP_KERNEL, - BLKDEV_ZERO_NOUNMAP); + flags = BLKDEV_ZERO_NOUNMAP; break; case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE: - error = truncate_bdev_range(bdev, file_to_blk_mode(file), start, end); - if (error) - goto fail; - - error = blkdev_issue_zeroout(bdev, start >> SECTOR_SHIFT, - len >> SECTOR_SHIFT, GFP_KERNEL, - BLKDEV_ZERO_NOFALLBACK); + flags = BLKDEV_ZERO_NOFALLBACK; + break; + case FALLOC_FL_WRITE_ZEROES: + flags = 0; break; default: error = -EOPNOTSUPP; + goto fail; } + /* + * Invalidate the page cache, including dirty pages, for valid + * de-allocate mode calls to fallocate(). + */ + error = truncate_bdev_range(bdev, file_to_blk_mode(file), start, end); + if (error) + goto fail; + + error = blkdev_issue_zeroout(bdev, start >> SECTOR_SHIFT, + len >> SECTOR_SHIFT, GFP_KERNEL, flags); fail: filemap_invalidate_unlock(inode->i_mapping); + inode_unlock(inode); return error; } -static int blkdev_mmap(struct file *file, struct vm_area_struct *vma) +static int blkdev_mmap_prepare(struct vm_area_desc *desc) { - struct inode *bd_inode = bdev_file_inode(file); + struct file *file = desc->file; - if (bdev_read_only(I_BDEV(bd_inode))) - return generic_file_readonly_mmap(file, vma); + if (bdev_read_only(I_BDEV(bdev_file_inode(file)))) + return generic_file_readonly_mmap_prepare(desc); - return generic_file_mmap(file, vma); + return generic_file_mmap_prepare(desc); } const struct file_operations def_blk_fops = { @@ -888,7 +931,7 @@ const struct file_operations def_blk_fops = { .read_iter = blkdev_read_iter, .write_iter = blkdev_write_iter, .iopoll = iocb_bio_iopoll, - .mmap = blkdev_mmap, + .mmap_prepare = blkdev_mmap_prepare, .fsync = blkdev_fsync, .unlocked_ioctl = blkdev_ioctl, #ifdef CONFIG_COMPAT diff --git a/block/genhd.c b/block/genhd.c index c2bd86cd09de..df2c3c69b467 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -90,7 +90,7 @@ bool set_capacity_and_notify(struct gendisk *disk, sector_t size) (disk->flags & GENHD_FL_HIDDEN)) return false; - pr_info("%s: detected capacity change from %lld to %lld\n", + pr_info_ratelimited("%s: detected capacity change from %lld to %lld\n", disk->disk_name, capacity, size); /* @@ -125,37 +125,50 @@ static void part_stat_read_all(struct block_device *part, } } -unsigned int part_in_flight(struct block_device *part) +static void bdev_count_inflight_rw(struct block_device *part, + unsigned int inflight[2], bool mq_driver) { - unsigned int inflight = 0; + int write = 0; + int read = 0; int cpu; + if (mq_driver) { + blk_mq_in_driver_rw(part, inflight); + return; + } + for_each_possible_cpu(cpu) { - inflight += part_stat_local_read_cpu(part, in_flight[0], cpu) + - part_stat_local_read_cpu(part, in_flight[1], cpu); + read += part_stat_local_read_cpu(part, in_flight[READ], cpu); + write += part_stat_local_read_cpu(part, in_flight[WRITE], cpu); } - if ((int)inflight < 0) - inflight = 0; - return inflight; + /* + * While iterating all CPUs, some IOs may be issued from a CPU already + * traversed and complete on a CPU that has not yet been traversed, + * causing the inflight number to be negative. + */ + inflight[READ] = read > 0 ? read : 0; + inflight[WRITE] = write > 0 ? write : 0; } -static void part_in_flight_rw(struct block_device *part, - unsigned int inflight[2]) +/** + * bdev_count_inflight - get the number of inflight IOs for a block device. + * + * @part: the block device. + * + * Inflight here means started IO accounting, from bdev_start_io_acct() for + * bio-based block device, and from blk_account_io_start() for rq-based block + * device. + */ +unsigned int bdev_count_inflight(struct block_device *part) { - int cpu; + unsigned int inflight[2] = {0}; - inflight[0] = 0; - inflight[1] = 0; - for_each_possible_cpu(cpu) { - inflight[0] += part_stat_local_read_cpu(part, in_flight[0], cpu); - inflight[1] += part_stat_local_read_cpu(part, in_flight[1], cpu); - } - if ((int)inflight[0] < 0) - inflight[0] = 0; - if ((int)inflight[1] < 0) - inflight[1] = 0; + bdev_count_inflight_rw(part, inflight, false); + + return inflight[READ] + inflight[WRITE]; } +EXPORT_SYMBOL_GPL(bdev_count_inflight); /* * Can be deleted altogether. Later. @@ -251,7 +264,7 @@ int __register_blkdev(unsigned int major, const char *name, goto out; } - p = kmalloc(sizeof(struct blk_major_name), GFP_KERNEL); + p = kmalloc_obj(struct blk_major_name); if (p == NULL) { ret = -ENOMEM; goto out; @@ -389,19 +402,31 @@ int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode) return ret; } -/** - * add_disk_fwnode - add disk information to kernel list with fwnode - * @parent: parent device for the disk - * @disk: per-device partitioning information - * @groups: Additional per-device sysfs groups - * @fwnode: attached disk fwnode - * - * This function registers the partitioning information in @disk - * with the kernel. Also attach a fwnode to the disk device. - */ -int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, - const struct attribute_group **groups, - struct fwnode_handle *fwnode) +static void add_disk_final(struct gendisk *disk) +{ + struct device *ddev = disk_to_dev(disk); + + if (!(disk->flags & GENHD_FL_HIDDEN)) { + bdev_add(disk->part0, ddev->devt); + if (get_capacity(disk)) + disk_scan_partitions(disk, BLK_OPEN_READ); + + /* + * Announce the disk and partitions after all partitions are + * created. (for hidden disks uevents remain suppressed forever) + */ + dev_set_uevent_suppress(ddev, 0); + disk_uevent(disk, KOBJ_ADD); + } + + blk_apply_bdi_limits(disk->bdi, &disk->queue->limits); + disk_add_events(disk); + set_bit(GD_ADDED, &disk->state); +} + +static int __add_disk(struct device *parent, struct gendisk *disk, + const struct attribute_group **groups, + struct fwnode_handle *fwnode) { struct device *ddev = disk_to_dev(disk); @@ -416,12 +441,6 @@ int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, */ if (disk->fops->submit_bio || disk->fops->poll_bio) return -EINVAL; - - /* - * Initialize the I/O scheduler code and pick a default one if - * needed. - */ - elevator_init_mq(disk->queue); } else { if (!disk->fops->submit_bio) return -EINVAL; @@ -438,7 +457,7 @@ int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, ret = -EINVAL; if (disk->major) { if (WARN_ON(!disk->minors)) - goto out_exit_elevator; + goto out; if (disk->minors > DISK_MAX_PARTS) { pr_err("block: can't allocate more than %d partitions\n", @@ -448,14 +467,14 @@ int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, if (disk->first_minor > MINORMASK || disk->minors > MINORMASK + 1 || disk->first_minor + disk->minors > MINORMASK + 1) - goto out_exit_elevator; + goto out; } else { if (WARN_ON(disk->minors)) - goto out_exit_elevator; + goto out; ret = blk_alloc_ext_minor(); if (ret < 0) - goto out_exit_elevator; + goto out; disk->major = BLOCK_EXT_MAJOR; disk->first_minor = ret; } @@ -516,21 +535,6 @@ int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, &disk->bdi->dev->kobj, "bdi"); if (ret) goto out_unregister_bdi; - - /* Make sure the first partition scan will be proceed */ - if (get_capacity(disk) && disk_has_partscan(disk)) - set_bit(GD_NEED_PART_SCAN, &disk->state); - - bdev_add(disk->part0, ddev->devt); - if (get_capacity(disk)) - disk_scan_partitions(disk, BLK_OPEN_READ); - - /* - * Announce the disk and partitions after all partitions are - * created. (for hidden disks uevents remain suppressed forever) - */ - dev_set_uevent_suppress(ddev, 0); - disk_uevent(disk, KOBJ_ADD); } else { /* * Even if the block_device for a hidden gendisk is not @@ -539,10 +543,6 @@ int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, */ disk->part0->bd_dev = MKDEV(disk->major, disk->first_minor); } - - blk_apply_bdi_limits(disk->bdi, &disk->queue->limits); - disk_add_events(disk); - set_bit(GD_ADDED, &disk->state); return 0; out_unregister_bdi: @@ -564,12 +564,46 @@ out_device_del: out_free_ext_minor: if (disk->major == BLOCK_EXT_MAJOR) blk_free_ext_minor(disk->first_minor); -out_exit_elevator: - if (disk->queue->elevator) { - mutex_lock(&disk->queue->elevator_lock); - elevator_exit(disk->queue); - mutex_unlock(&disk->queue->elevator_lock); +out: + return ret; +} + +/** + * add_disk_fwnode - add disk information to kernel list with fwnode + * @parent: parent device for the disk + * @disk: per-device partitioning information + * @groups: Additional per-device sysfs groups + * @fwnode: attached disk fwnode + * + * This function registers the partitioning information in @disk + * with the kernel. Also attach a fwnode to the disk device. + */ +int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk, + const struct attribute_group **groups, + struct fwnode_handle *fwnode) +{ + struct blk_mq_tag_set *set; + unsigned int memflags; + int ret; + + if (queue_is_mq(disk->queue)) { + set = disk->queue->tag_set; + memflags = memalloc_noio_save(); + down_read(&set->update_nr_hwq_lock); + ret = __add_disk(parent, disk, groups, fwnode); + up_read(&set->update_nr_hwq_lock); + memalloc_noio_restore(memflags); + } else { + ret = __add_disk(parent, disk, groups, fwnode); } + + /* + * add_disk_final() needn't to read `nr_hw_queues`, so move it out + * of read lock `set->update_nr_hwq_lock` for avoiding unnecessary + * lock dependency on `disk->open_mutex` from scanning partition. + */ + if (!ret) + add_disk_final(disk); return ret; } EXPORT_SYMBOL_GPL(add_disk_fwnode); @@ -652,26 +686,7 @@ void blk_mark_disk_dead(struct gendisk *disk) } EXPORT_SYMBOL_GPL(blk_mark_disk_dead); -/** - * del_gendisk - remove the gendisk - * @disk: the struct gendisk to remove - * - * Removes the gendisk and all its associated resources. This deletes the - * partitions associated with the gendisk, and unregisters the associated - * request_queue. - * - * This is the counter to the respective __device_add_disk() call. - * - * The final removal of the struct gendisk happens when its refcount reaches 0 - * with put_disk(), which should be called after del_gendisk(), if - * __device_add_disk() was used. - * - * Drivers exist which depend on the release of the gendisk to be synchronous, - * it should not be deferred. - * - * Context: can sleep - */ -void del_gendisk(struct gendisk *disk) +static void __del_gendisk(struct gendisk *disk) { struct request_queue *q = disk->queue; struct block_device *part; @@ -743,14 +758,7 @@ void del_gendisk(struct gendisk *disk) if (queue_is_mq(q)) blk_mq_cancel_work_sync(q); - blk_mq_quiesce_queue(q); - if (q->elevator) { - mutex_lock(&q->elevator_lock); - elevator_exit(q); - mutex_unlock(&q->elevator_lock); - } rq_qos_exit(q); - blk_mq_unquiesce_queue(q); /* * If the disk does not own the queue, allow using passthrough requests @@ -764,6 +772,55 @@ void del_gendisk(struct gendisk *disk) if (start_drain) blk_unfreeze_release_lock(q); } + +static void disable_elv_switch(struct request_queue *q) +{ + struct blk_mq_tag_set *set = q->tag_set; + WARN_ON_ONCE(!queue_is_mq(q)); + + down_write(&set->update_nr_hwq_lock); + blk_queue_flag_set(QUEUE_FLAG_NO_ELV_SWITCH, q); + up_write(&set->update_nr_hwq_lock); +} + +/** + * del_gendisk - remove the gendisk + * @disk: the struct gendisk to remove + * + * Removes the gendisk and all its associated resources. This deletes the + * partitions associated with the gendisk, and unregisters the associated + * request_queue. + * + * This is the counter to the respective device_add_disk() call. + * + * The final removal of the struct gendisk happens when its refcount reaches 0 + * with put_disk(), which should be called after del_gendisk(), if + * device_add_disk() was used. + * + * Drivers exist which depend on the release of the gendisk to be synchronous, + * it should not be deferred. + * + * Context: can sleep + */ +void del_gendisk(struct gendisk *disk) +{ + struct blk_mq_tag_set *set; + unsigned int memflags; + + if (!queue_is_mq(disk->queue)) { + __del_gendisk(disk); + } else { + set = disk->queue->tag_set; + + disable_elv_switch(disk->queue); + + memflags = memalloc_noio_save(); + down_read(&set->update_nr_hwq_lock); + __del_gendisk(disk); + up_read(&set->update_nr_hwq_lock); + memalloc_noio_restore(memflags); + } +} EXPORT_SYMBOL(del_gendisk); /** @@ -853,7 +910,7 @@ static void *disk_seqf_start(struct seq_file *seqf, loff_t *pos) struct class_dev_iter *iter; struct device *dev; - iter = kmalloc(sizeof(*iter), GFP_KERNEL); + iter = kmalloc_obj(*iter); if (!iter) return ERR_PTR(-ENOMEM); @@ -1005,7 +1062,7 @@ ssize_t part_stat_show(struct device *dev, struct disk_stats stat; unsigned int inflight; - inflight = part_in_flight(bdev); + inflight = bdev_count_inflight(bdev); if (inflight) { part_stat_lock(); update_io_ticks(bdev, jiffies, true); @@ -1042,19 +1099,21 @@ ssize_t part_stat_show(struct device *dev, (unsigned int)div_u64(stat.nsecs[STAT_FLUSH], NSEC_PER_MSEC)); } +/* + * Show the number of IOs issued to driver. + * For bio-based device, started from bdev_start_io_acct(); + * For rq-based device, started from blk_mq_start_request(); + */ ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr, char *buf) { struct block_device *bdev = dev_to_bdev(dev); struct request_queue *q = bdev_get_queue(bdev); - unsigned int inflight[2]; + unsigned int inflight[2] = {0}; - if (queue_is_mq(q)) - blk_mq_in_flight_rw(q, bdev, inflight); - else - part_in_flight_rw(bdev, inflight); + bdev_count_inflight_rw(bdev, inflight, queue_is_mq(q)); - return sysfs_emit(buf, "%8u %8u\n", inflight[0], inflight[1]); + return sysfs_emit(buf, "%8u %8u\n", inflight[READ], inflight[WRITE]); } static ssize_t disk_capability_show(struct device *dev, @@ -1202,7 +1261,7 @@ static const struct attribute_group *disk_attr_groups[] = { * * This function releases all allocated resources of the gendisk. * - * Drivers which used __device_add_disk() have a gendisk with a request_queue + * Drivers which used device_add_disk() have a gendisk with a request_queue * assigned. Since the request_queue sits on top of the gendisk for these * drivers we also call blk_put_queue() for them, and we expect the * request_queue refcount to reach 0 at this point, and so the request_queue @@ -1237,9 +1296,10 @@ static void disk_release(struct device *dev) disk_release_events(disk); kfree(disk->random); - disk_free_zone_resources(disk); + disk_release_zone_resources(disk); xa_destroy(&disk->part_tbl); + kobject_put(&disk->queue_kobj); disk->queue->disk = NULL; blk_put_queue(disk->queue); @@ -1307,7 +1367,7 @@ static int diskstats_show(struct seq_file *seqf, void *v) if (bdev_is_partition(hd) && !bdev_nr_sectors(hd)) continue; - inflight = part_in_flight(hd); + inflight = bdev_count_inflight(hd); if (inflight) { part_stat_lock(); update_io_ticks(hd, jiffies, true); @@ -1422,6 +1482,12 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id, #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED INIT_LIST_HEAD(&disk->slave_bdevs); #endif +#ifdef CONFIG_BLK_ERROR_INJECTION + mutex_init(&disk->error_injection_lock); + INIT_LIST_HEAD(&disk->error_injection_list); +#endif + mutex_init(&disk->rqos_state_mutex); + kobject_init(&disk->queue_kobj, &blk_queue_ktype); return disk; out_erase_part0: diff --git a/block/holder.c b/block/holder.c index 791091a7eac2..267beb3dc7a7 100644 --- a/block/holder.c +++ b/block/holder.c @@ -92,7 +92,7 @@ int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk) goto out_unlock; } - holder = kzalloc(sizeof(*holder), GFP_KERNEL); + holder = kzalloc_obj(*holder); if (!holder) { ret = -ENOMEM; goto out_unlock; diff --git a/block/ioctl.c b/block/ioctl.c index faa40f383e27..3d4ea1537457 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -13,6 +13,7 @@ #include <linux/uaccess.h> #include <linux/pagemap.h> #include <linux/io_uring/cmd.h> +#include <linux/blk-integrity.h> #include <uapi/linux/blkdev.h> #include "blk.h" #include "blk-crypto-internal.h" @@ -142,6 +143,7 @@ static int blk_ioctl_discard(struct block_device *bdev, blk_mode_t mode, if (err) return err; + inode_lock(bdev->bd_mapping->host); filemap_invalidate_lock(bdev->bd_mapping); err = truncate_bdev_range(bdev, mode, start, start + len - 1); if (err) @@ -151,13 +153,7 @@ static int blk_ioctl_discard(struct block_device *bdev, blk_mode_t mode, nr_sects = len >> SECTOR_SHIFT; blk_start_plug(&plug); - while (1) { - if (fatal_signal_pending(current)) { - if (prev) - bio_await_chain(prev); - err = -EINTR; - goto out_unplug; - } + while (!fatal_signal_pending(current)) { bio = blk_alloc_discard_bio(bdev, §or, &nr_sects, GFP_KERNEL); if (!bio) @@ -165,15 +161,15 @@ static int blk_ioctl_discard(struct block_device *bdev, blk_mode_t mode, prev = bio_chain_and_submit(prev, bio); } if (prev) { - err = submit_bio_wait(prev); + err = bio_submit_or_kill(prev, BLKDEV_ZERO_KILLABLE); if (err == -EOPNOTSUPP) err = 0; bio_put(prev); } -out_unplug: blk_finish_plug(&plug); fail: filemap_invalidate_unlock(bdev->bd_mapping); + inode_unlock(bdev->bd_mapping->host); return err; } @@ -199,12 +195,14 @@ static int blk_ioctl_secure_erase(struct block_device *bdev, blk_mode_t mode, end > bdev_nr_bytes(bdev)) return -EINVAL; + inode_lock(bdev->bd_mapping->host); filemap_invalidate_lock(bdev->bd_mapping); err = truncate_bdev_range(bdev, mode, start, end - 1); if (!err) err = blkdev_issue_secure_erase(bdev, start >> 9, len >> 9, GFP_KERNEL); filemap_invalidate_unlock(bdev->bd_mapping); + inode_unlock(bdev->bd_mapping->host); return err; } @@ -236,6 +234,7 @@ static int blk_ioctl_zeroout(struct block_device *bdev, blk_mode_t mode, return -EINVAL; /* Invalidate the page cache, including dirty pages */ + inode_lock(bdev->bd_mapping->host); filemap_invalidate_lock(bdev->bd_mapping); err = truncate_bdev_range(bdev, mode, start, end); if (err) @@ -246,6 +245,7 @@ static int blk_ioctl_zeroout(struct block_device *bdev, blk_mode_t mode, fail: filemap_invalidate_unlock(bdev->bd_mapping); + inode_unlock(bdev->bd_mapping->host); return err; } @@ -311,7 +311,13 @@ int blkdev_compat_ptr_ioctl(struct block_device *bdev, blk_mode_t mode, EXPORT_SYMBOL(blkdev_compat_ptr_ioctl); #endif -static bool blkdev_pr_allowed(struct block_device *bdev, blk_mode_t mode) +enum pr_direction { + PR_IN, /* read from device */ + PR_OUT, /* write to device */ +}; + +static bool blkdev_pr_allowed(struct block_device *bdev, blk_mode_t mode, + enum pr_direction dir) { /* no sense to make reservations for partitions */ if (bdev_is_partition(bdev)) @@ -319,11 +325,17 @@ static bool blkdev_pr_allowed(struct block_device *bdev, blk_mode_t mode) if (capable(CAP_SYS_ADMIN)) return true; + /* - * Only allow unprivileged reservations if the file descriptor is open - * for writing. + * Only allow unprivileged reservation _out_ commands if the file + * descriptor is open for writing. Allow reservation _in_ commands if + * the file descriptor is open for reading since they do not modify the + * device. */ - return mode & BLK_OPEN_WRITE; + if (dir == PR_IN) + return mode & BLK_OPEN_READ; + else + return mode & BLK_OPEN_WRITE; } static int blkdev_pr_register(struct block_device *bdev, blk_mode_t mode, @@ -332,7 +344,7 @@ static int blkdev_pr_register(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_registration reg; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_register) return -EOPNOTSUPP; @@ -350,7 +362,7 @@ static int blkdev_pr_reserve(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_reservation rsv; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_reserve) return -EOPNOTSUPP; @@ -368,7 +380,7 @@ static int blkdev_pr_release(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_reservation rsv; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_release) return -EOPNOTSUPP; @@ -386,7 +398,7 @@ static int blkdev_pr_preempt(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_preempt p; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_preempt) return -EOPNOTSUPP; @@ -404,7 +416,7 @@ static int blkdev_pr_clear(struct block_device *bdev, blk_mode_t mode, const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; struct pr_clear c; - if (!blkdev_pr_allowed(bdev, mode)) + if (!blkdev_pr_allowed(bdev, mode, PR_OUT)) return -EPERM; if (!ops || !ops->pr_clear) return -EOPNOTSUPP; @@ -416,6 +428,87 @@ static int blkdev_pr_clear(struct block_device *bdev, blk_mode_t mode, return ops->pr_clear(bdev, c.key); } +static int blkdev_pr_read_keys(struct block_device *bdev, blk_mode_t mode, + struct pr_read_keys __user *arg) +{ + const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; + struct pr_keys *keys_info; + struct pr_read_keys read_keys; + u64 __user *keys_ptr; + size_t keys_info_len; + size_t keys_copy_len; + int ret; + + if (!blkdev_pr_allowed(bdev, mode, PR_IN)) + return -EPERM; + if (!ops || !ops->pr_read_keys) + return -EOPNOTSUPP; + + if (copy_from_user(&read_keys, arg, sizeof(read_keys))) + return -EFAULT; + + if (read_keys.num_keys > PR_KEYS_MAX) + return -EINVAL; + + keys_info_len = struct_size(keys_info, keys, read_keys.num_keys); + + keys_info = kvzalloc(keys_info_len, GFP_KERNEL); + if (!keys_info) + return -ENOMEM; + + keys_info->num_keys = read_keys.num_keys; + + ret = ops->pr_read_keys(bdev, keys_info); + if (ret) + goto out; + + /* Copy out individual keys */ + keys_ptr = u64_to_user_ptr(read_keys.keys_ptr); + keys_copy_len = min(read_keys.num_keys, keys_info->num_keys) * + sizeof(keys_info->keys[0]); + + if (copy_to_user(keys_ptr, keys_info->keys, keys_copy_len)) { + ret = -EFAULT; + goto out; + } + + /* Copy out the arg struct */ + read_keys.generation = keys_info->generation; + read_keys.num_keys = keys_info->num_keys; + + if (copy_to_user(arg, &read_keys, sizeof(read_keys))) + ret = -EFAULT; +out: + kvfree(keys_info); + return ret; +} + +static int blkdev_pr_read_reservation(struct block_device *bdev, + blk_mode_t mode, struct pr_read_reservation __user *arg) +{ + const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; + struct pr_held_reservation rsv = {}; + struct pr_read_reservation out = {}; + int ret; + + if (!blkdev_pr_allowed(bdev, mode, PR_IN)) + return -EPERM; + if (!ops || !ops->pr_read_reservation) + return -EOPNOTSUPP; + + ret = ops->pr_read_reservation(bdev, &rsv); + if (ret) + return ret; + + out.key = rsv.key; + out.generation = rsv.generation; + out.type = rsv.type; + + if (copy_to_user(arg, &out, sizeof(out))) + return -EFAULT; + return 0; +} + static int blkdev_flushbuf(struct block_device *bdev, unsigned cmd, unsigned long arg) { @@ -474,7 +567,7 @@ static int blkdev_getgeo(struct block_device *bdev, */ memset(&geo, 0, sizeof(geo)); geo.start = get_start_sect(bdev); - ret = disk->fops->getgeo(bdev, &geo); + ret = disk->fops->getgeo(disk, &geo); if (ret) return ret; if (copy_to_user(argp, &geo, sizeof(geo))) @@ -508,7 +601,7 @@ static int compat_hdio_getgeo(struct block_device *bdev, * want to override it. */ geo.start = get_start_sect(bdev); - ret = disk->fops->getgeo(bdev, &geo); + ret = disk->fops->getgeo(disk, &geo); if (ret) return ret; @@ -574,6 +667,7 @@ static int blkdev_common_ioctl(struct block_device *bdev, blk_mode_t mode, case BLKGETDISKSEQ: return put_u64(argp, bdev->bd_disk->diskseq); case BLKREPORTZONE: + case BLKREPORTZONEV2: return blkdev_report_zones_ioctl(bdev, cmd, arg); case BLKRESETZONE: case BLKOPENZONE: @@ -603,7 +697,7 @@ static int blkdev_common_ioctl(struct block_device *bdev, blk_mode_t mode, queue_max_sectors(bdev_get_queue(bdev))); return put_ushort(argp, max_sectors); case BLKROTATIONAL: - return put_ushort(argp, !bdev_nonrot(bdev)); + return put_ushort(argp, bdev_rot(bdev)); case BLKRASET: case BLKFRASET: if(!capable(CAP_SYS_ADMIN)) @@ -637,8 +731,12 @@ static int blkdev_common_ioctl(struct block_device *bdev, blk_mode_t mode, return blkdev_pr_preempt(bdev, mode, argp, true); case IOC_PR_CLEAR: return blkdev_pr_clear(bdev, mode, argp); + case IOC_PR_READ_KEYS: + return blkdev_pr_read_keys(bdev, mode, argp); + case IOC_PR_READ_RESERVATION: + return blkdev_pr_read_reservation(bdev, mode, argp); default: - return -ENOIOCTLCMD; + return blk_get_meta_cap(bdev, cmd, argp); } } @@ -684,6 +782,7 @@ long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) /* Incompatible alignment on i386 */ case BLKTRACESETUP: + case BLKTRACESETUP2: return blk_trace_ioctl(bdev, cmd, argp); default: break; @@ -758,18 +857,22 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) #endif struct blk_iou_cmd { + u64 start; + u64 len; int res; bool nowait; }; -static void blk_cmd_complete(struct io_uring_cmd *cmd, unsigned int issue_flags) +static void blk_cmd_complete(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req); struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd); if (bic->res == -EAGAIN && bic->nowait) io_uring_cmd_issue_blocking(cmd); else - io_uring_cmd_done(cmd, bic->res, 0, issue_flags); + io_uring_cmd_done(cmd, bic->res, + IO_URING_CMD_TASK_WORK_ISSUE_FLAGS); } static void bio_cmd_bio_end_io(struct bio *bio) @@ -845,23 +948,27 @@ int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) { struct block_device *bdev = I_BDEV(cmd->file->f_mapping->host); struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd); - const struct io_uring_sqe *sqe = cmd->sqe; u32 cmd_op = cmd->cmd_op; - uint64_t start, len; - if (unlikely(sqe->ioprio || sqe->__pad1 || sqe->len || - sqe->rw_flags || sqe->file_index)) - return -EINVAL; + /* Read what we need from the SQE on the first issue */ + if (!(cmd->flags & IORING_URING_CMD_REISSUE)) { + const struct io_uring_sqe *sqe = cmd->sqe; + + if (unlikely(sqe->ioprio || sqe->__pad1 || sqe->len || + sqe->rw_flags || sqe->file_index)) + return -EINVAL; + + bic->start = READ_ONCE(sqe->addr); + bic->len = READ_ONCE(sqe->addr3); + } bic->res = 0; bic->nowait = issue_flags & IO_URING_F_NONBLOCK; - start = READ_ONCE(sqe->addr); - len = READ_ONCE(sqe->addr3); - switch (cmd_op) { case BLOCK_URING_CMD_DISCARD: - return blkdev_cmd_discard(cmd, bdev, start, len, bic->nowait); + return blkdev_cmd_discard(cmd, bdev, bic->start, bic->len, + bic->nowait); } return -EINVAL; } diff --git a/block/ioprio.c b/block/ioprio.c index 73301a261429..f0ee2798539c 100644 --- a/block/ioprio.c +++ b/block/ioprio.c @@ -46,12 +46,8 @@ int ioprio_check_cap(int ioprio) */ if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_NICE)) return -EPERM; - fallthrough; - /* rt has prio field too */ - case IOPRIO_CLASS_BE: - if (level >= IOPRIO_NR_LEVELS) - return -EINVAL; break; + case IOPRIO_CLASS_BE: case IOPRIO_CLASS_IDLE: break; case IOPRIO_CLASS_NONE: diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index 0f0f8452609a..971818bcdc9d 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -47,9 +47,8 @@ enum { * asynchronous requests, we reserve 25% of requests for synchronous * operations. */ - KYBER_ASYNC_PERCENT = 75, + KYBER_DEFAULT_ASYNC_PERCENT = 75, }; - /* * Maximum device-wide depth for each scheduling domain. * @@ -157,12 +156,6 @@ struct kyber_queue_data { */ struct sbitmap_queue domain_tokens[KYBER_NUM_DOMAINS]; - /* - * Async request percentage, converted to per-word depth for - * sbitmap_get_shallow(). - */ - unsigned int async_depth; - struct kyber_cpu_latency __percpu *cpu_latency; /* Timer for stats aggregation and adjusting domain tokens. */ @@ -276,7 +269,7 @@ static void kyber_resize_domain(struct kyber_queue_data *kqd, static void kyber_timer_fn(struct timer_list *t) { - struct kyber_queue_data *kqd = from_timer(kqd, t, timer); + struct kyber_queue_data *kqd = timer_container_of(kqd, t, timer); unsigned int sched_domain; int cpu; bool bad = false; @@ -402,38 +395,50 @@ err: return ERR_PTR(ret); } -static int kyber_init_sched(struct request_queue *q, struct elevator_type *e) +static void kyber_depth_updated(struct request_queue *q) { - struct kyber_queue_data *kqd; - struct elevator_queue *eq; - - eq = elevator_alloc(q, e); - if (!eq) - return -ENOMEM; - - kqd = kyber_queue_data_alloc(q); - if (IS_ERR(kqd)) { - kobject_put(&eq->kobj); - return PTR_ERR(kqd); - } + blk_mq_set_min_shallow_depth(q, q->async_depth); +} +static int kyber_init_sched(struct request_queue *q, struct elevator_queue *eq) +{ blk_stat_enable_accounting(q); blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q); - eq->elevator_data = kqd; q->elevator = eq; + q->async_depth = q->nr_requests * KYBER_DEFAULT_ASYNC_PERCENT / 100; + kyber_depth_updated(q); return 0; } +static void *kyber_alloc_sched_data(struct request_queue *q) +{ + struct kyber_queue_data *kqd; + + kqd = kyber_queue_data_alloc(q); + if (IS_ERR(kqd)) + return NULL; + + return kqd; +} + static void kyber_exit_sched(struct elevator_queue *e) { struct kyber_queue_data *kqd = e->elevator_data; - int i; timer_shutdown_sync(&kqd->timer); blk_stat_disable_accounting(kqd->q); +} + +static void kyber_free_sched_data(void *elv_data) +{ + struct kyber_queue_data *kqd = elv_data; + int i; + + if (!kqd) + return; for (i = 0; i < KYBER_NUM_DOMAINS; i++) sbitmap_queue_free(&kqd->domain_tokens[i]); @@ -450,17 +455,6 @@ static void kyber_ctx_queue_init(struct kyber_ctx_queue *kcq) INIT_LIST_HEAD(&kcq->rq_list[i]); } -static void kyber_depth_updated(struct blk_mq_hw_ctx *hctx) -{ - struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data; - struct blk_mq_tags *tags = hctx->sched_tags; - unsigned int shift = tags->bitmap_tags.sb.shift; - - kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U; - - sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, kqd->async_depth); -} - static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) { struct kyber_hctx_data *khd; @@ -505,7 +499,6 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) khd->batching = 0; hctx->sched_data = khd; - kyber_depth_updated(hctx); return 0; @@ -553,15 +546,8 @@ static void rq_clear_domain_token(struct kyber_queue_data *kqd, static void kyber_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) { - /* - * We use the scheduler tags as per-hardware queue queueing tokens. - * Async requests can be limited at this stage. - */ - if (!op_is_sync(opf)) { - struct kyber_queue_data *kqd = data->q->elevator->elevator_data; - - data->shallow_depth = kqd->async_depth; - } + if (!blk_mq_is_sync_read(opf)) + data->shallow_depth = data->q->async_depth; } static bool kyber_bio_merge(struct request_queue *q, struct bio *bio, @@ -896,6 +882,9 @@ static const struct elv_fs_entry kyber_sched_attrs[] = { }; #undef KYBER_LAT_ATTR +#define HCTX_FROM_SEQ_FILE(m) ((struct blk_mq_hw_ctx *)(m)->private) +#define KYBER_HCTX_DATA(hctx) ((struct kyber_hctx_data *)(hctx)->sched_data) + #ifdef CONFIG_BLK_DEBUG_FS #define KYBER_DEBUGFS_DOMAIN_ATTRS(domain, name) \ static int kyber_##name##_tokens_show(void *data, struct seq_file *m) \ @@ -908,7 +897,7 @@ static int kyber_##name##_tokens_show(void *data, struct seq_file *m) \ } \ \ static void *kyber_##name##_rqs_start(struct seq_file *m, loff_t *pos) \ - __acquires(&khd->lock) \ + __acquires(&KYBER_HCTX_DATA(HCTX_FROM_SEQ_FILE(m))->lock) \ { \ struct blk_mq_hw_ctx *hctx = m->private; \ struct kyber_hctx_data *khd = hctx->sched_data; \ @@ -927,7 +916,7 @@ static void *kyber_##name##_rqs_next(struct seq_file *m, void *v, \ } \ \ static void kyber_##name##_rqs_stop(struct seq_file *m, void *v) \ - __releases(&khd->lock) \ + __releases(&KYBER_HCTX_DATA(HCTX_FROM_SEQ_FILE(m))->lock) \ { \ struct blk_mq_hw_ctx *hctx = m->private; \ struct kyber_hctx_data *khd = hctx->sched_data; \ @@ -957,15 +946,6 @@ KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_DISCARD, discard) KYBER_DEBUGFS_DOMAIN_ATTRS(KYBER_OTHER, other) #undef KYBER_DEBUGFS_DOMAIN_ATTRS -static int kyber_async_depth_show(void *data, struct seq_file *m) -{ - struct request_queue *q = data; - struct kyber_queue_data *kqd = q->elevator->elevator_data; - - seq_printf(m, "%u\n", kqd->async_depth); - return 0; -} - static int kyber_cur_domain_show(void *data, struct seq_file *m) { struct blk_mq_hw_ctx *hctx = data; @@ -991,7 +971,6 @@ static const struct blk_mq_debugfs_attr kyber_queue_debugfs_attrs[] = { KYBER_QUEUE_DOMAIN_ATTRS(write), KYBER_QUEUE_DOMAIN_ATTRS(discard), KYBER_QUEUE_DOMAIN_ATTRS(other), - {"async_depth", 0400, kyber_async_depth_show}, {}, }; #undef KYBER_QUEUE_DOMAIN_ATTRS @@ -1017,6 +996,8 @@ static struct elevator_type kyber_sched = { .exit_sched = kyber_exit_sched, .init_hctx = kyber_init_hctx, .exit_hctx = kyber_exit_hctx, + .alloc_sched_data = kyber_alloc_sched_data, + .free_sched_data = kyber_free_sched_data, .limit_depth = kyber_limit_depth, .bio_merge = kyber_bio_merge, .prepare_request = kyber_prepare_request, diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 754f6b7415cd..824bfc17b2c6 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -71,7 +71,6 @@ struct io_stats_per_prio { * present on both sort_list[] and fifo_list[]. */ struct dd_per_prio { - struct list_head dispatch; struct rb_root sort_list[DD_DIR_COUNT]; struct list_head fifo_list[DD_DIR_COUNT]; /* Position of the most recently dispatched request. */ @@ -84,6 +83,7 @@ struct deadline_data { * run time data */ + struct list_head dispatch; struct dd_per_prio per_prio[DD_PRIO_COUNT]; /* Data direction of latest dispatched request. */ @@ -98,7 +98,6 @@ struct deadline_data { int fifo_batch; int writes_starved; int front_merges; - u32 async_depth; int prio_aging_expire; spinlock_t lock; @@ -136,10 +135,6 @@ static inline struct request *deadline_from_pos(struct dd_per_prio *per_prio, struct rb_node *node = per_prio->sort_list[data_dir].rb_node; struct request *rq, *res = NULL; - if (!node) - return NULL; - - rq = rb_entry_rq(node); while (node) { rq = rb_entry_rq(node); if (blk_rq_pos(rq) >= pos) { @@ -310,6 +305,19 @@ static bool started_after(struct deadline_data *dd, struct request *rq, return time_after(start_time, latest_start); } +static struct request *dd_start_request(struct deadline_data *dd, + enum dd_data_dir data_dir, + struct request *rq) +{ + u8 ioprio_class = dd_rq_ioclass(rq); + enum dd_prio prio = ioprio_class_to_prio[ioprio_class]; + + dd->per_prio[prio].latest_pos[data_dir] = blk_rq_pos(rq); + dd->per_prio[prio].stats.dispatched++; + rq->rq_flags |= RQF_STARTED; + return rq; +} + /* * deadline_dispatch_requests selects the best request according to * read/write expire, fifo_batch, etc and with a start time <= @latest_start. @@ -320,21 +328,9 @@ static struct request *__dd_dispatch_request(struct deadline_data *dd, { struct request *rq, *next_rq; enum dd_data_dir data_dir; - enum dd_prio prio; - u8 ioprio_class; lockdep_assert_held(&dd->lock); - if (!list_empty(&per_prio->dispatch)) { - rq = list_first_entry(&per_prio->dispatch, struct request, - queuelist); - if (started_after(dd, rq, latest_start)) - return NULL; - list_del_init(&rq->queuelist); - data_dir = rq_data_dir(rq); - goto done; - } - /* * batches are currently reads XOR writes */ @@ -414,13 +410,7 @@ dispatch_request: */ dd->batching++; deadline_move_request(dd, per_prio, rq); -done: - ioprio_class = dd_rq_ioclass(rq); - prio = ioprio_class_to_prio[ioprio_class]; - dd->per_prio[prio].latest_pos[data_dir] = blk_rq_pos(rq); - dd->per_prio[prio].stats.dispatched++; - rq->rq_flags |= RQF_STARTED; - return rq; + return dd_start_request(dd, data_dir, rq); } /* @@ -467,6 +457,14 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) enum dd_prio prio; spin_lock(&dd->lock); + + if (!list_empty(&dd->dispatch)) { + rq = list_first_entry(&dd->dispatch, struct request, queuelist); + list_del_init(&rq->queuelist); + dd_start_request(dd, rq_data_dir(rq), rq); + goto unlock; + } + rq = dd_dispatch_prio_aged_requests(dd, now); if (rq) goto unlock; @@ -487,56 +485,16 @@ unlock: return rq; } -/* - * 'depth' is a number in the range 1..INT_MAX representing a number of - * requests. Scale it with a factor (1 << bt->sb.shift) / q->nr_requests since - * 1..(1 << bt->sb.shift) is the range expected by sbitmap_get_shallow(). - * Values larger than q->nr_requests have the same effect as q->nr_requests. - */ -static int dd_to_word_depth(struct blk_mq_hw_ctx *hctx, unsigned int qdepth) -{ - struct sbitmap_queue *bt = &hctx->sched_tags->bitmap_tags; - const unsigned int nrr = hctx->queue->nr_requests; - - return ((qdepth << bt->sb.shift) + nrr - 1) / nrr; -} - -/* - * Called by __blk_mq_alloc_request(). The shallow_depth value set by this - * function is used by __blk_mq_get_tag(). - */ static void dd_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) { - struct deadline_data *dd = data->q->elevator->elevator_data; - - /* Do not throttle synchronous reads. */ - if (op_is_sync(opf) && !op_is_write(opf)) - return; - - /* - * Throttle asynchronous requests and writes such that these requests - * do not block the allocation of synchronous requests. - */ - data->shallow_depth = dd_to_word_depth(data->hctx, dd->async_depth); -} - -/* Called by blk_mq_update_nr_requests(). */ -static void dd_depth_updated(struct blk_mq_hw_ctx *hctx) -{ - struct request_queue *q = hctx->queue; - struct deadline_data *dd = q->elevator->elevator_data; - struct blk_mq_tags *tags = hctx->sched_tags; - - dd->async_depth = q->nr_requests; - - sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, 1); + if (!blk_mq_is_sync_read(opf)) + data->shallow_depth = data->q->async_depth; } -/* Called by blk_mq_init_hctx() and blk_mq_init_sched(). */ -static int dd_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) +/* Called by blk_mq_init_sched() and blk_mq_update_nr_requests(). */ +static void dd_depth_updated(struct request_queue *q) { - dd_depth_updated(hctx); - return 0; + blk_mq_set_min_shallow_depth(q, q->async_depth); } static void dd_exit_sched(struct elevator_queue *e) @@ -568,27 +526,21 @@ static void dd_exit_sched(struct elevator_queue *e) /* * initialize elevator private data (deadline_data). */ -static int dd_init_sched(struct request_queue *q, struct elevator_type *e) +static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq) { struct deadline_data *dd; - struct elevator_queue *eq; enum dd_prio prio; - int ret = -ENOMEM; - - eq = elevator_alloc(q, e); - if (!eq) - return ret; dd = kzalloc_node(sizeof(*dd), GFP_KERNEL, q->node); if (!dd) - goto put_eq; + return -ENOMEM; eq->elevator_data = dd; + INIT_LIST_HEAD(&dd->dispatch); for (prio = 0; prio <= DD_PRIO_MAX; prio++) { struct dd_per_prio *per_prio = &dd->per_prio[prio]; - INIT_LIST_HEAD(&per_prio->dispatch); INIT_LIST_HEAD(&per_prio->fifo_list[DD_READ]); INIT_LIST_HEAD(&per_prio->fifo_list[DD_WRITE]); per_prio->sort_list[DD_READ] = RB_ROOT; @@ -607,11 +559,9 @@ static int dd_init_sched(struct request_queue *q, struct elevator_type *e) blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q); q->elevator = eq; + q->async_depth = q->nr_requests; + dd_depth_updated(q); return 0; - -put_eq: - kobject_put(&eq->kobj); - return ret; } /* @@ -695,7 +645,7 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, trace_block_rq_insert(rq); if (flags & BLK_MQ_INSERT_AT_HEAD) { - list_add(&rq->queuelist, &per_prio->dispatch); + list_add(&rq->queuelist, &dd->dispatch); rq->fifo_time = jiffies; } else { deadline_add_rq_rb(per_prio, rq); @@ -715,7 +665,7 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, } /* - * Called from blk_mq_insert_request() or blk_mq_dispatch_plug_list(). + * Called from blk_mq_insert_request() or blk_mq_dispatch_list(). */ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, struct list_head *list, @@ -762,8 +712,7 @@ static void dd_finish_request(struct request *rq) static bool dd_has_work_for_prio(struct dd_per_prio *per_prio) { - return !list_empty_careful(&per_prio->dispatch) || - !list_empty_careful(&per_prio->fifo_list[DD_READ]) || + return !list_empty_careful(&per_prio->fifo_list[DD_READ]) || !list_empty_careful(&per_prio->fifo_list[DD_WRITE]); } @@ -772,6 +721,9 @@ static bool dd_has_work(struct blk_mq_hw_ctx *hctx) struct deadline_data *dd = hctx->queue->elevator->elevator_data; enum dd_prio prio; + if (!list_empty_careful(&dd->dispatch)) + return true; + for (prio = 0; prio <= DD_PRIO_MAX; prio++) if (dd_has_work_for_prio(&dd->per_prio[prio])) return true; @@ -795,7 +747,6 @@ SHOW_JIFFIES(deadline_write_expire_show, dd->fifo_expire[DD_WRITE]); SHOW_JIFFIES(deadline_prio_aging_expire_show, dd->prio_aging_expire); SHOW_INT(deadline_writes_starved_show, dd->writes_starved); SHOW_INT(deadline_front_merges_show, dd->front_merges); -SHOW_INT(deadline_async_depth_show, dd->async_depth); SHOW_INT(deadline_fifo_batch_show, dd->fifo_batch); #undef SHOW_INT #undef SHOW_JIFFIES @@ -825,7 +776,6 @@ STORE_JIFFIES(deadline_write_expire_store, &dd->fifo_expire[DD_WRITE], 0, INT_MA STORE_JIFFIES(deadline_prio_aging_expire_store, &dd->prio_aging_expire, 0, INT_MAX); STORE_INT(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX); STORE_INT(deadline_front_merges_store, &dd->front_merges, 0, 1); -STORE_INT(deadline_async_depth_store, &dd->async_depth, 1, INT_MAX); STORE_INT(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX); #undef STORE_FUNCTION #undef STORE_INT @@ -839,17 +789,20 @@ static const struct elv_fs_entry deadline_attrs[] = { DD_ATTR(write_expire), DD_ATTR(writes_starved), DD_ATTR(front_merges), - DD_ATTR(async_depth), DD_ATTR(fifo_batch), DD_ATTR(prio_aging_expire), __ATTR_NULL }; +#define RQ_FROM_SEQ_FILE(m) ((struct request_queue *)(m)->private) +#define DD_DATA_FROM_RQ(rq) \ + ((struct deadline_data *)(rq)->elevator->elevator_data) + #ifdef CONFIG_BLK_DEBUG_FS #define DEADLINE_DEBUGFS_DDIR_ATTRS(prio, data_dir, name) \ static void *deadline_##name##_fifo_start(struct seq_file *m, \ loff_t *pos) \ - __acquires(&dd->lock) \ + __acquires(&DD_DATA_FROM_RQ(RQ_FROM_SEQ_FILE(m))->lock) \ { \ struct request_queue *q = m->private; \ struct deadline_data *dd = q->elevator->elevator_data; \ @@ -870,7 +823,7 @@ static void *deadline_##name##_fifo_next(struct seq_file *m, void *v, \ } \ \ static void deadline_##name##_fifo_stop(struct seq_file *m, void *v) \ - __releases(&dd->lock) \ + __releases(&DD_DATA_FROM_RQ(RQ_FROM_SEQ_FILE(m))->lock) \ { \ struct request_queue *q = m->private; \ struct deadline_data *dd = q->elevator->elevator_data; \ @@ -926,15 +879,6 @@ static int deadline_starved_show(void *data, struct seq_file *m) return 0; } -static int dd_async_depth_show(void *data, struct seq_file *m) -{ - struct request_queue *q = data; - struct deadline_data *dd = q->elevator->elevator_data; - - seq_printf(m, "%u\n", dd->async_depth); - return 0; -} - static int dd_queued_show(void *data, struct seq_file *m) { struct request_queue *q = data; @@ -980,49 +924,39 @@ static int dd_owned_by_driver_show(void *data, struct seq_file *m) return 0; } -#define DEADLINE_DISPATCH_ATTR(prio) \ -static void *deadline_dispatch##prio##_start(struct seq_file *m, \ - loff_t *pos) \ - __acquires(&dd->lock) \ -{ \ - struct request_queue *q = m->private; \ - struct deadline_data *dd = q->elevator->elevator_data; \ - struct dd_per_prio *per_prio = &dd->per_prio[prio]; \ - \ - spin_lock(&dd->lock); \ - return seq_list_start(&per_prio->dispatch, *pos); \ -} \ - \ -static void *deadline_dispatch##prio##_next(struct seq_file *m, \ - void *v, loff_t *pos) \ -{ \ - struct request_queue *q = m->private; \ - struct deadline_data *dd = q->elevator->elevator_data; \ - struct dd_per_prio *per_prio = &dd->per_prio[prio]; \ - \ - return seq_list_next(v, &per_prio->dispatch, pos); \ -} \ - \ -static void deadline_dispatch##prio##_stop(struct seq_file *m, void *v) \ - __releases(&dd->lock) \ -{ \ - struct request_queue *q = m->private; \ - struct deadline_data *dd = q->elevator->elevator_data; \ - \ - spin_unlock(&dd->lock); \ -} \ - \ -static const struct seq_operations deadline_dispatch##prio##_seq_ops = { \ - .start = deadline_dispatch##prio##_start, \ - .next = deadline_dispatch##prio##_next, \ - .stop = deadline_dispatch##prio##_stop, \ - .show = blk_mq_debugfs_rq_show, \ +static void *deadline_dispatch_start(struct seq_file *m, loff_t *pos) + __acquires(&DD_DATA_FROM_RQ(RQ_FROM_SEQ_FILE(m))->lock) +{ + struct request_queue *q = m->private; + struct deadline_data *dd = q->elevator->elevator_data; + + spin_lock(&dd->lock); + return seq_list_start(&dd->dispatch, *pos); } -DEADLINE_DISPATCH_ATTR(0); -DEADLINE_DISPATCH_ATTR(1); -DEADLINE_DISPATCH_ATTR(2); -#undef DEADLINE_DISPATCH_ATTR +static void *deadline_dispatch_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct request_queue *q = m->private; + struct deadline_data *dd = q->elevator->elevator_data; + + return seq_list_next(v, &dd->dispatch, pos); +} + +static void deadline_dispatch_stop(struct seq_file *m, void *v) + __releases(&DD_DATA_FROM_RQ(RQ_FROM_SEQ_FILE(m))->lock) +{ + struct request_queue *q = m->private; + struct deadline_data *dd = q->elevator->elevator_data; + + spin_unlock(&dd->lock); +} + +static const struct seq_operations deadline_dispatch_seq_ops = { + .start = deadline_dispatch_start, + .next = deadline_dispatch_next, + .stop = deadline_dispatch_stop, + .show = blk_mq_debugfs_rq_show, +}; #define DEADLINE_QUEUE_DDIR_ATTRS(name) \ {#name "_fifo_list", 0400, \ @@ -1044,10 +978,7 @@ static const struct blk_mq_debugfs_attr deadline_queue_debugfs_attrs[] = { DEADLINE_NEXT_RQ_ATTR(write2), {"batching", 0400, deadline_batching_show}, {"starved", 0400, deadline_starved_show}, - {"async_depth", 0400, dd_async_depth_show}, - {"dispatch0", 0400, .seq_ops = &deadline_dispatch0_seq_ops}, - {"dispatch1", 0400, .seq_ops = &deadline_dispatch1_seq_ops}, - {"dispatch2", 0400, .seq_ops = &deadline_dispatch2_seq_ops}, + {"dispatch", 0400, .seq_ops = &deadline_dispatch_seq_ops}, {"owned_by_driver", 0400, dd_owned_by_driver_show}, {"queued", 0400, dd_queued_show}, {}, @@ -1072,7 +1003,6 @@ static struct elevator_type mq_deadline = { .has_work = dd_has_work, .init_sched = dd_init_sched, .exit_sched = dd_exit_sched, - .init_hctx = dd_init_hctx, }, #ifdef CONFIG_BLK_DEBUG_FS diff --git a/block/opal_proto.h b/block/opal_proto.h index d247a457bf6e..7c24247aa186 100644 --- a/block/opal_proto.h +++ b/block/opal_proto.h @@ -19,6 +19,7 @@ enum { TCG_SECP_00 = 0, TCG_SECP_01, + TCG_SECP_02, }; /* @@ -125,6 +126,7 @@ enum opal_uid { OPAL_LOCKING_INFO_TABLE, OPAL_ENTERPRISE_LOCKING_INFO_TABLE, OPAL_DATASTORE, + OPAL_LOCKING_TABLE, /* C_PIN_TABLE object ID's */ OPAL_C_PIN_MSID, OPAL_C_PIN_SID, @@ -154,6 +156,7 @@ enum opal_method { OPAL_AUTHENTICATE, OPAL_RANDOM, OPAL_ERASE, + OPAL_REACTIVATE, }; enum opal_token { @@ -224,6 +227,8 @@ enum opal_lockingstate { enum opal_parameter { OPAL_SUM_SET_LIST = 0x060000, + OPAL_SUM_RANGE_POLICY = 0x060001, + OPAL_SUM_ADMIN1_PIN = 0x060002, }; enum opal_revertlsp { @@ -269,6 +274,25 @@ struct opal_header { struct opal_data_subpacket subpkt; }; +/* + * TCG_Storage_Architecture_Core_Spec_v2.01_r1.00 + * Section: 3.3.4.7.5 STACK_RESET + */ +#define OPAL_STACK_RESET 0x0002 + +struct opal_stack_reset { + u8 extendedComID[4]; + __be32 request_code; +}; + +struct opal_stack_reset_response { + u8 extendedComID[4]; + __be32 request_code; + u8 reserved0[2]; + __be16 data_length; + __be32 response; +}; + #define FC_TPER 0x0001 #define FC_LOCKING 0x0002 #define FC_GEOMETRY 0x0003 diff --git a/block/partitions/acorn.c b/block/partitions/acorn.c index d2fc122d7426..067d6a27a3bd 100644 --- a/block/partitions/acorn.c +++ b/block/partitions/acorn.c @@ -9,6 +9,7 @@ */ #include <linux/buffer_head.h> #include <linux/adfs_fs.h> +#include <linux/minmax.h> #include "check.h" @@ -40,9 +41,7 @@ adfs_partition(struct parsed_partitions *state, char *name, char *data, (le32_to_cpu(dr->disc_size) >> 9); if (name) { - strlcat(state->pp_buf, " [", PAGE_SIZE); - strlcat(state->pp_buf, name, PAGE_SIZE); - strlcat(state->pp_buf, "]", PAGE_SIZE); + seq_buf_printf(&state->pp_buf, " [%s]", name); } put_partition(state, slot, first_sector, nr_sects); return dr; @@ -78,14 +77,14 @@ static int riscix_partition(struct parsed_partitions *state, if (!rr) return -1; - strlcat(state->pp_buf, " [RISCiX]", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " [RISCiX]"); if (rr->magic == RISCIX_MAGIC) { - unsigned long size = nr_sects > 2 ? 2 : nr_sects; + unsigned long size = min(nr_sects, 2); int part; - strlcat(state->pp_buf, " <", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " <"); put_partition(state, slot++, first_sect, size); for (part = 0; part < 8; part++) { @@ -94,13 +93,11 @@ static int riscix_partition(struct parsed_partitions *state, put_partition(state, slot++, le32_to_cpu(rr->part[part].start), le32_to_cpu(rr->part[part].length)); - strlcat(state->pp_buf, "(", PAGE_SIZE); - strlcat(state->pp_buf, rr->part[part].name, PAGE_SIZE); - strlcat(state->pp_buf, ")", PAGE_SIZE); + seq_buf_printf(&state->pp_buf, "(%s)", rr->part[part].name); } } - strlcat(state->pp_buf, " >\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " >\n"); } else { put_partition(state, slot++, first_sect, nr_sects); } @@ -128,9 +125,9 @@ static int linux_partition(struct parsed_partitions *state, { Sector sect; struct linux_part *linuxp; - unsigned long size = nr_sects > 2 ? 2 : nr_sects; + unsigned long size = min(nr_sects, 2); - strlcat(state->pp_buf, " [Linux]", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " [Linux]"); put_partition(state, slot++, first_sect, size); @@ -138,7 +135,7 @@ static int linux_partition(struct parsed_partitions *state, if (!linuxp) return -1; - strlcat(state->pp_buf, " <", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " <"); while (linuxp->magic == cpu_to_le32(LINUX_NATIVE_MAGIC) || linuxp->magic == cpu_to_le32(LINUX_SWAP_MAGIC)) { if (slot == state->limit) @@ -148,7 +145,7 @@ static int linux_partition(struct parsed_partitions *state, le32_to_cpu(linuxp->nr_sects)); linuxp ++; } - strlcat(state->pp_buf, " >", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " >"); put_dev_sector(sect); return slot; @@ -293,7 +290,7 @@ int adfspart_check_ADFS(struct parsed_partitions *state) break; } } - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } #endif @@ -366,7 +363,7 @@ int adfspart_check_ICS(struct parsed_partitions *state) return 0; } - strlcat(state->pp_buf, " [ICS]", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " [ICS]"); for (slot = 1, p = (const struct ics_part *)data; p->size; p++) { u32 start = le32_to_cpu(p->start); @@ -400,7 +397,7 @@ int adfspart_check_ICS(struct parsed_partitions *state) } put_dev_sector(sect); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } #endif @@ -460,7 +457,7 @@ int adfspart_check_POWERTEC(struct parsed_partitions *state) return 0; } - strlcat(state->pp_buf, " [POWERTEC]", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " [POWERTEC]"); for (i = 0, p = (const struct ptec_part *)data; i < 12; i++, p++) { u32 start = le32_to_cpu(p->start); @@ -471,7 +468,7 @@ int adfspart_check_POWERTEC(struct parsed_partitions *state) } put_dev_sector(sect); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } #endif @@ -542,7 +539,7 @@ int adfspart_check_EESOX(struct parsed_partitions *state) size = get_capacity(state->disk); put_partition(state, slot++, start, size - start); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); } return i ? 1 : 0; diff --git a/block/partitions/aix.c b/block/partitions/aix.c index 85f4b967565e..689837deba27 100644 --- a/block/partitions/aix.c +++ b/block/partitions/aix.c @@ -173,24 +173,22 @@ int aix_partition(struct parsed_partitions *state) if (d) { struct lvm_rec *p = (struct lvm_rec *)d; u16 lvm_version = be16_to_cpu(p->version); - char tmp[64]; if (lvm_version == 1) { int pp_size_log2 = be16_to_cpu(p->pp_size); pp_bytes_size = 1 << pp_size_log2; pp_blocks_size = pp_bytes_size / 512; - snprintf(tmp, sizeof(tmp), - " AIX LVM header version %u found\n", - lvm_version); + seq_buf_printf(&state->pp_buf, + " AIX LVM header version %u found\n", + lvm_version); vgda_len = be32_to_cpu(p->vgda_len); vgda_sector = be32_to_cpu(p->vgda_psn[0]); } else { - snprintf(tmp, sizeof(tmp), - " unsupported AIX LVM version %d found\n", - lvm_version); + seq_buf_printf(&state->pp_buf, + " unsupported AIX LVM version %d found\n", + lvm_version); } - strlcat(state->pp_buf, tmp, PAGE_SIZE); put_dev_sector(sect); } if (vgda_sector && (d = read_part_sector(state, vgda_sector, §))) { @@ -199,7 +197,7 @@ int aix_partition(struct parsed_partitions *state) numlvs = be16_to_cpu(p->numlvs); put_dev_sector(sect); } - lvip = kcalloc(state->limit, sizeof(struct lv_info), GFP_KERNEL); + lvip = kzalloc_objs(struct lv_info, state->limit); if (!lvip) return 0; if (numlvs && (d = read_part_sector(state, vgda_sector + 1, §))) { @@ -210,7 +208,14 @@ int aix_partition(struct parsed_partitions *state) if (n) { int foundlvs = 0; - for (i = 0; foundlvs < numlvs && i < state->limit; i += 1) { + /* + * The lvd array was read as a single sector; only the + * struct lvd entries that fit in it are valid. Bound the + * scan so an on-disk numlvs larger than that cannot walk + * the read buffer out of bounds. + */ + for (i = 0; foundlvs < numlvs && i < state->limit && + i < SECTOR_SIZE / (int)sizeof(struct lvd); i++) { lvip[i].pps_per_lv = be16_to_cpu(p[i].num_lps); if (lvip[i].pps_per_lv) foundlvs += 1; @@ -228,6 +233,15 @@ int aix_partition(struct parsed_partitions *state) int next_lp_ix = 1; int lp_ix; + /* + * pvd was read into a fixed-size struct pvd whose ppe[] array + * holds ARRAY_SIZE(pvd->ppe) entries. pp_count is an + * unvalidated on-disk __be16, so clamp the scan to the array + * size to avoid walking past the allocation. + */ + if (numpps > ARRAY_SIZE(pvd->ppe)) + numpps = ARRAY_SIZE(pvd->ppe); + for (i = 0; i < numpps; i += 1) { struct ppe *p = pvd->ppe + i; unsigned int lv_ix; @@ -251,14 +265,11 @@ int aix_partition(struct parsed_partitions *state) continue; } if (lp_ix == lvip[lv_ix].pps_per_lv) { - char tmp[70]; - put_partition(state, lv_ix + 1, (i + 1 - lp_ix) * pp_blocks_size + psn_part1, lvip[lv_ix].pps_per_lv * pp_blocks_size); - snprintf(tmp, sizeof(tmp), " <%s>\n", - n[lv_ix].name); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, " <%s>\n", + n[lv_ix].name); lvip[lv_ix].lv_is_contiguous = 1; ret = 1; next_lp_ix = 1; diff --git a/block/partitions/amiga.c b/block/partitions/amiga.c index 506921095412..8325046a14eb 100644 --- a/block/partitions/amiga.c +++ b/block/partitions/amiga.c @@ -81,13 +81,8 @@ int amiga_partition(struct parsed_partitions *state) /* blksize is blocks per 512 byte standard block */ blksize = be32_to_cpu( rdb->rdb_BlockBytes ) / 512; - { - char tmp[7 + 10 + 1 + 1]; - - /* Be more informative */ - snprintf(tmp, sizeof(tmp), " RDSK (%d)", blksize * 512); - strlcat(state->pp_buf, tmp, PAGE_SIZE); - } + /* Be more informative */ + seq_buf_printf(&state->pp_buf, " RDSK (%d)", blksize * 512); blk = be32_to_cpu(rdb->rdb_PartitionList); put_dev_sector(sect); for (part = 1; (s32) blk>0 && part<=16; part++, put_dev_sector(sect)) { @@ -179,27 +174,27 @@ int amiga_partition(struct parsed_partitions *state) { /* Be even more informative to aid mounting */ char dostype[4]; - char tmp[42]; __be32 *dt = (__be32 *)dostype; *dt = pb->pb_Environment[16]; if (dostype[3] < ' ') - snprintf(tmp, sizeof(tmp), " (%c%c%c^%c)", - dostype[0], dostype[1], - dostype[2], dostype[3] + '@' ); + seq_buf_printf(&state->pp_buf, + " (%c%c%c^%c)", + dostype[0], dostype[1], + dostype[2], + dostype[3] + '@'); else - snprintf(tmp, sizeof(tmp), " (%c%c%c%c)", - dostype[0], dostype[1], - dostype[2], dostype[3]); - strlcat(state->pp_buf, tmp, PAGE_SIZE); - snprintf(tmp, sizeof(tmp), "(res %d spb %d)", - be32_to_cpu(pb->pb_Environment[6]), - be32_to_cpu(pb->pb_Environment[4])); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, + " (%c%c%c%c)", + dostype[0], dostype[1], + dostype[2], dostype[3]); + seq_buf_printf(&state->pp_buf, "(res %d spb %d)", + be32_to_cpu(pb->pb_Environment[6]), + be32_to_cpu(pb->pb_Environment[4])); } res = 1; } - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); rdb_done: return res; diff --git a/block/partitions/atari.c b/block/partitions/atari.c index 9655c728262a..2438d1448f38 100644 --- a/block/partitions/atari.c +++ b/block/partitions/atari.c @@ -70,7 +70,7 @@ int atari_partition(struct parsed_partitions *state) } pi = &rs->part[0]; - strlcat(state->pp_buf, " AHDI", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " AHDI"); for (slot = 1; pi < &rs->part[4] && slot < state->limit; slot++, pi++) { struct rootsector *xrs; Sector sect2; @@ -89,7 +89,7 @@ int atari_partition(struct parsed_partitions *state) #ifdef ICD_PARTS part_fmt = 1; #endif - strlcat(state->pp_buf, " XGM<", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " XGM<"); partsect = extensect = be32_to_cpu(pi->st); while (1) { xrs = read_part_sector(state, partsect, §2); @@ -128,14 +128,14 @@ int atari_partition(struct parsed_partitions *state) break; } } - strlcat(state->pp_buf, " >", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " >"); } #ifdef ICD_PARTS if ( part_fmt!=1 ) { /* no extended partitions -> test ICD-format */ pi = &rs->icdpart[0]; /* sanity check: no ICD format if first partition invalid */ if (OK_id(pi->id)) { - strlcat(state->pp_buf, " ICD<", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " ICD<"); for (; pi < &rs->icdpart[8] && slot < state->limit; slot++, pi++) { /* accept only GEM,BGM,RAW,LNX,SWP partitions */ if (!((pi->flg & 1) && OK_id(pi->id))) @@ -144,13 +144,13 @@ int atari_partition(struct parsed_partitions *state) be32_to_cpu(pi->st), be32_to_cpu(pi->siz)); } - strlcat(state->pp_buf, " >", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " >"); } } #endif put_dev_sector(sect); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } diff --git a/block/partitions/check.h b/block/partitions/check.h index e5c1c61eb353..b0997467b61a 100644 --- a/block/partitions/check.h +++ b/block/partitions/check.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <linux/pagemap.h> #include <linux/blkdev.h> +#include <linux/seq_buf.h> #include "../blk.h" /* @@ -20,7 +21,7 @@ struct parsed_partitions { int next; int limit; bool access_beyond_eod; - char *pp_buf; + struct seq_buf pp_buf; }; typedef struct { @@ -37,12 +38,9 @@ static inline void put_partition(struct parsed_partitions *p, int n, sector_t from, sector_t size) { if (n < p->limit) { - char tmp[1 + BDEVNAME_SIZE + 10 + 1]; - p->parts[n].from = from; p->parts[n].size = size; - snprintf(tmp, sizeof(tmp), " %s%d", p->name, n); - strlcat(p->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&p->pp_buf, " %s%d", p->name, n); } } diff --git a/block/partitions/cmdline.c b/block/partitions/cmdline.c index da3e719d8e51..4fd52ed154b4 100644 --- a/block/partitions/cmdline.c +++ b/block/partitions/cmdline.c @@ -46,7 +46,7 @@ static int parse_subpart(struct cmdline_subpart **subpart, char *partdef) *subpart = NULL; - new_subpart = kzalloc(sizeof(struct cmdline_subpart), GFP_KERNEL); + new_subpart = kzalloc_obj(struct cmdline_subpart); if (!new_subpart) return -ENOMEM; @@ -122,7 +122,7 @@ static int parse_parts(struct cmdline_parts **parts, char *bdevdef) *parts = NULL; - newparts = kzalloc(sizeof(struct cmdline_parts), GFP_KERNEL); + newparts = kzalloc_obj(struct cmdline_parts); if (!newparts) return -ENOMEM; @@ -229,7 +229,6 @@ static int add_part(int slot, struct cmdline_subpart *subpart, struct parsed_partitions *state) { struct partition_meta_info *info; - char tmp[sizeof(info->volname) + 4]; if (slot >= state->limit) return 1; @@ -244,8 +243,7 @@ static int add_part(int slot, struct cmdline_subpart *subpart, strscpy(info->volname, subpart->name, sizeof(info->volname)); - snprintf(tmp, sizeof(tmp), "(%s)", info->volname); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, "(%s)", info->volname); state->parts[slot].has_info = true; @@ -379,7 +377,7 @@ int cmdline_partition(struct parsed_partitions *state) cmdline_parts_set(parts, disk_size, state); cmdline_parts_verifier(1, state); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } diff --git a/block/partitions/core.c b/block/partitions/core.c index 815ed33caa1b..b5c59b79ca7c 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -7,6 +7,8 @@ #include <linux/fs.h> #include <linux/major.h> #include <linux/slab.h> +#include <linux/string.h> +#include <linux/sysfs.h> #include <linux/ctype.h> #include <linux/vmalloc.h> #include <linux/raid/detect.h> @@ -93,7 +95,7 @@ static struct parsed_partitions *allocate_partitions(struct gendisk *hd) struct parsed_partitions *state; int nr = DISK_MAX_PARTS; - state = kzalloc(sizeof(*state), GFP_KERNEL); + state = kzalloc_obj(*state); if (!state) return NULL; @@ -122,16 +124,16 @@ static struct parsed_partitions *check_partition(struct gendisk *hd) state = allocate_partitions(hd); if (!state) return NULL; - state->pp_buf = (char *)__get_free_page(GFP_KERNEL); - if (!state->pp_buf) { + state->pp_buf.buffer = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!state->pp_buf.buffer) { free_partitions(state); return NULL; } - state->pp_buf[0] = '\0'; + seq_buf_init(&state->pp_buf, state->pp_buf.buffer, PAGE_SIZE); state->disk = hd; - snprintf(state->name, BDEVNAME_SIZE, "%s", hd->disk_name); - snprintf(state->pp_buf, PAGE_SIZE, " %s:", state->name); + strscpy(state->name, hd->disk_name); + seq_buf_printf(&state->pp_buf, " %s:", state->name); if (isdigit(state->name[strlen(state->name)-1])) sprintf(state->name, "p"); @@ -150,9 +152,9 @@ static struct parsed_partitions *check_partition(struct gendisk *hd) } if (res > 0) { - printk(KERN_INFO "%s", state->pp_buf); + printk(KERN_INFO "%s", seq_buf_str(&state->pp_buf)); - free_page((unsigned long)state->pp_buf); + kfree(state->pp_buf.buffer); return state; } if (state->access_beyond_eod) @@ -163,12 +165,12 @@ static struct parsed_partitions *check_partition(struct gendisk *hd) if (err) res = err; if (res) { - strlcat(state->pp_buf, - " unable to read partition table\n", PAGE_SIZE); - printk(KERN_INFO "%s", state->pp_buf); + seq_buf_puts(&state->pp_buf, + " unable to read partition table\n"); + printk(KERN_INFO "%s", seq_buf_str(&state->pp_buf)); } - free_page((unsigned long)state->pp_buf); + kfree(state->pp_buf.buffer); free_partitions(state); return ERR_PTR(res); } @@ -176,31 +178,31 @@ static struct parsed_partitions *check_partition(struct gendisk *hd) static ssize_t part_partition_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%d\n", bdev_partno(dev_to_bdev(dev))); + return sysfs_emit(buf, "%d\n", bdev_partno(dev_to_bdev(dev))); } static ssize_t part_start_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%llu\n", dev_to_bdev(dev)->bd_start_sect); + return sysfs_emit(buf, "%llu\n", dev_to_bdev(dev)->bd_start_sect); } static ssize_t part_ro_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%d\n", bdev_read_only(dev_to_bdev(dev))); + return sysfs_emit(buf, "%d\n", bdev_read_only(dev_to_bdev(dev))); } static ssize_t part_alignment_offset_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%u\n", bdev_alignment_offset(dev_to_bdev(dev))); + return sysfs_emit(buf, "%u\n", bdev_alignment_offset(dev_to_bdev(dev))); } static ssize_t part_discard_alignment_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%u\n", bdev_discard_alignment(dev_to_bdev(dev))); + return sysfs_emit(buf, "%u\n", bdev_discard_alignment(dev_to_bdev(dev))); } static DEVICE_ATTR(partition, 0444, part_partition_show, NULL); diff --git a/block/partitions/efi.c b/block/partitions/efi.c index 7acba66eed48..9865d59093fa 100644 --- a/block/partitions/efi.c +++ b/block/partitions/efi.c @@ -215,8 +215,7 @@ check_hybrid: sz = le32_to_cpu(mbr->partition_record[part].size_in_lba); if (sz != (uint32_t) total_sectors - 1 && sz != 0xFFFFFFFF) pr_debug("GPT: mbr size in lba (%u) different than whole disk (%u).\n", - sz, min_t(uint32_t, - total_sectors - 1, 0xFFFFFFFF)); + sz, (uint32_t)min(total_sectors - 1, 0xFFFFFFFF)); } done: return ret; @@ -596,7 +595,7 @@ static int find_valid_gpt(struct parsed_partitions *state, gpt_header **gpt, lastlba = last_lba(state->disk); if (!force_gpt) { /* This will be added to the EFI Spec. per Intel after v1.02. */ - legacymbr = kzalloc(sizeof(*legacymbr), GFP_KERNEL); + legacymbr = kzalloc_obj(*legacymbr); if (!legacymbr) goto fail; @@ -752,6 +751,6 @@ int efi_partition(struct parsed_partitions *state) } kfree(ptes); kfree(gpt); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } diff --git a/block/partitions/ibm.c b/block/partitions/ibm.c index 82d9c4c3fb41..54047e722a9d 100644 --- a/block/partitions/ibm.c +++ b/block/partitions/ibm.c @@ -173,15 +173,13 @@ static int find_vol1_partitions(struct parsed_partitions *state, { sector_t blk; int counter; - char tmp[64]; Sector sect; unsigned char *data; loff_t offset, size; struct vtoc_format1_label f1; int secperblk; - snprintf(tmp, sizeof(tmp), "VOL1/%8s:", name); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, "VOL1/%8s:", name); /* * get start of VTOC from the disk label and then search for format1 * and format8 labels @@ -219,7 +217,7 @@ static int find_vol1_partitions(struct parsed_partitions *state, blk++; data = read_part_sector(state, blk * secperblk, §); } - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); if (!data) return -1; @@ -237,11 +235,9 @@ static int find_lnx1_partitions(struct parsed_partitions *state, dasd_information2_t *info) { loff_t offset, geo_size, size; - char tmp[64]; int secperblk; - snprintf(tmp, sizeof(tmp), "LNX1/%8s:", name); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, "LNX1/%8s:", name); secperblk = blocksize >> 9; if (label->lnx.ldl_version == 0xf2) { size = label->lnx.formatted_blocks * secperblk; @@ -258,7 +254,7 @@ static int find_lnx1_partitions(struct parsed_partitions *state, size = nr_sectors; if (size != geo_size) { if (!info) { - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } if (!strcmp(info->type, "ECKD")) @@ -270,7 +266,7 @@ static int find_lnx1_partitions(struct parsed_partitions *state, /* first and only partition starts in the first block after the label */ offset = labelsect + secperblk; put_partition(state, 1, offset, size - offset); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } @@ -282,7 +278,6 @@ static int find_cms1_partitions(struct parsed_partitions *state, sector_t labelsect) { loff_t offset, size; - char tmp[64]; int secperblk; /* @@ -291,14 +286,12 @@ static int find_cms1_partitions(struct parsed_partitions *state, blocksize = label->cms.block_size; secperblk = blocksize >> 9; if (label->cms.disk_offset != 0) { - snprintf(tmp, sizeof(tmp), "CMS1/%8s(MDSK):", name); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, "CMS1/%8s(MDSK):", name); /* disk is reserved minidisk */ offset = label->cms.disk_offset * secperblk; size = (label->cms.block_count - 1) * secperblk; } else { - snprintf(tmp, sizeof(tmp), "CMS1/%8s:", name); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, "CMS1/%8s:", name); /* * Special case for FBA devices: * If an FBA device is CMS formatted with blocksize > 512 byte @@ -314,7 +307,7 @@ static int find_cms1_partitions(struct parsed_partitions *state, } put_partition(state, 1, offset, size-offset); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } @@ -347,18 +340,18 @@ int ibm_partition(struct parsed_partitions *state) nr_sectors = bdev_nr_sectors(bdev); if (nr_sectors == 0) goto out_symbol; - info = kmalloc(sizeof(dasd_information2_t), GFP_KERNEL); + info = kmalloc_obj(dasd_information2_t); if (info == NULL) goto out_symbol; - geo = kmalloc(sizeof(struct hd_geometry), GFP_KERNEL); + geo = kmalloc_obj(struct hd_geometry); if (geo == NULL) goto out_nogeo; - label = kmalloc(sizeof(union label_t), GFP_KERNEL); + label = kmalloc_obj(union label_t); if (label == NULL) goto out_nolab; /* set start if not filled by getgeo function e.g. virtblk */ geo->start = get_start_sect(bdev); - if (disk->fops->getgeo(bdev, geo)) + if (disk->fops->getgeo(disk, geo)) goto out_freeall; if (!fn || fn(disk, info)) { kfree(info); @@ -391,11 +384,11 @@ int ibm_partition(struct parsed_partitions *state) */ res = 1; if (info->format == DASD_FORMAT_LDL) { - strlcat(state->pp_buf, "(nonl)", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "(nonl)"); size = nr_sectors; offset = (info->label_block + 1) * (blocksize >> 9); put_partition(state, 1, offset, size-offset); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); } } else res = 0; diff --git a/block/partitions/karma.c b/block/partitions/karma.c index 4d93512f4bd4..a4e3c5050177 100644 --- a/block/partitions/karma.c +++ b/block/partitions/karma.c @@ -53,7 +53,7 @@ int karma_partition(struct parsed_partitions *state) } slot++; } - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); put_dev_sector(sect); return 1; } diff --git a/block/partitions/ldm.c b/block/partitions/ldm.c index 2bd42fedb907..c0bdcae58a3e 100644 --- a/block/partitions/ldm.c +++ b/block/partitions/ldm.c @@ -273,8 +273,8 @@ static bool ldm_validate_privheads(struct parsed_partitions *state, BUG_ON (!state || !ph1); - ph[1] = kmalloc (sizeof (*ph[1]), GFP_KERNEL); - ph[2] = kmalloc (sizeof (*ph[2]), GFP_KERNEL); + ph[1] = kmalloc_obj(*ph[1]); + ph[2] = kmalloc_obj(*ph[2]); if (!ph[1] || !ph[2]) { ldm_crit ("Out of memory."); goto out; @@ -362,7 +362,7 @@ static bool ldm_validate_tocblocks(struct parsed_partitions *state, BUG_ON(!state || !ldb); ph = &ldb->ph; tb[0] = &ldb->toc; - tb[1] = kmalloc_array(3, sizeof(*tb[1]), GFP_KERNEL); + tb[1] = kmalloc_objs(*tb[1], 3); if (!tb[1]) { ldm_crit("Out of memory."); goto err; @@ -582,7 +582,7 @@ static bool ldm_create_data_partitions (struct parsed_partitions *pp, return false; } - strlcat(pp->pp_buf, " [LDM]", PAGE_SIZE); + seq_buf_puts(&pp->pp_buf, " [LDM]"); /* Create the data partitions */ list_for_each (item, &ldb->v_part) { @@ -597,7 +597,7 @@ static bool ldm_create_data_partitions (struct parsed_partitions *pp, part_num++; } - strlcat(pp->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&pp->pp_buf, "\n"); return true; } @@ -1158,7 +1158,7 @@ static bool ldm_ldmdb_add (u8 *data, int len, struct ldmdb *ldb) BUG_ON (!data || !ldb); - vb = kmalloc (sizeof (*vb), GFP_KERNEL); + vb = kmalloc_obj(*vb); if (!vb) { ldm_crit ("Out of memory."); return false; @@ -1438,7 +1438,7 @@ int ldm_partition(struct parsed_partitions *state) if (!ldm_validate_partition_table(state)) return 0; - ldb = kmalloc (sizeof (*ldb), GFP_KERNEL); + ldb = kmalloc_obj(*ldb); if (!ldb) { ldm_crit ("Out of memory."); goto out; diff --git a/block/partitions/mac.c b/block/partitions/mac.c index b02530d98629..df03ca428e15 100644 --- a/block/partitions/mac.c +++ b/block/partitions/mac.c @@ -86,7 +86,7 @@ int mac_partition(struct parsed_partitions *state) if (blocks_in_map >= state->limit) blocks_in_map = state->limit - 1; - strlcat(state->pp_buf, " [mac]", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " [mac]"); for (slot = 1; slot <= blocks_in_map; ++slot) { int pos = slot * secsize; put_dev_sector(sect); @@ -152,6 +152,6 @@ int mac_partition(struct parsed_partitions *state) #endif put_dev_sector(sect); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } diff --git a/block/partitions/msdos.c b/block/partitions/msdos.c index 073be78ba0b0..200ea53ea6a2 100644 --- a/block/partitions/msdos.c +++ b/block/partitions/msdos.c @@ -263,18 +263,11 @@ static void parse_solaris_x86(struct parsed_partitions *state, put_dev_sector(sect); return; } - { - char tmp[1 + BDEVNAME_SIZE + 10 + 11 + 1]; - - snprintf(tmp, sizeof(tmp), " %s%d: <solaris:", state->name, origin); - strlcat(state->pp_buf, tmp, PAGE_SIZE); - } + seq_buf_printf(&state->pp_buf, " %s%d: <solaris:", state->name, origin); if (le32_to_cpu(v->v_version) != 1) { - char tmp[64]; - - snprintf(tmp, sizeof(tmp), " cannot handle version %d vtoc>\n", - le32_to_cpu(v->v_version)); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, + " cannot handle version %d vtoc>\n", + le32_to_cpu(v->v_version)); put_dev_sector(sect); return; } @@ -282,12 +275,10 @@ static void parse_solaris_x86(struct parsed_partitions *state, max_nparts = le16_to_cpu(v->v_nparts) > 8 ? SOLARIS_X86_NUMSLICE : 8; for (i = 0; i < max_nparts && state->next < state->limit; i++) { struct solaris_x86_slice *s = &v->v_slice[i]; - char tmp[3 + 10 + 1 + 1]; if (s->s_size == 0) continue; - snprintf(tmp, sizeof(tmp), " [s%d]", i); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, " [s%d]", i); /* solaris partitions are relative to current MS-DOS * one; must add the offset of the current partition */ put_partition(state, state->next++, @@ -295,7 +286,7 @@ static void parse_solaris_x86(struct parsed_partitions *state, le32_to_cpu(s->s_size)); } put_dev_sector(sect); - strlcat(state->pp_buf, " >\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " >\n"); #endif } @@ -359,7 +350,6 @@ static void parse_bsd(struct parsed_partitions *state, Sector sect; struct bsd_disklabel *l; struct bsd_partition *p; - char tmp[64]; l = read_part_sector(state, offset + 1, §); if (!l) @@ -369,8 +359,7 @@ static void parse_bsd(struct parsed_partitions *state, return; } - snprintf(tmp, sizeof(tmp), " %s%d: <%s:", state->name, origin, flavour); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, " %s%d: <%s:", state->name, origin, flavour); if (le16_to_cpu(l->d_npartitions) < max_partitions) max_partitions = le16_to_cpu(l->d_npartitions); @@ -391,18 +380,16 @@ static void parse_bsd(struct parsed_partitions *state, /* full parent partition, we have it already */ continue; if (offset > bsd_start || offset+size < bsd_start+bsd_size) { - strlcat(state->pp_buf, "bad subpartition - ignored\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "bad subpartition - ignored\n"); continue; } put_partition(state, state->next++, bsd_start, bsd_size); } put_dev_sector(sect); - if (le16_to_cpu(l->d_npartitions) > max_partitions) { - snprintf(tmp, sizeof(tmp), " (ignored %d more)", - le16_to_cpu(l->d_npartitions) - max_partitions); - strlcat(state->pp_buf, tmp, PAGE_SIZE); - } - strlcat(state->pp_buf, " >\n", PAGE_SIZE); + if (le16_to_cpu(l->d_npartitions) > max_partitions) + seq_buf_printf(&state->pp_buf, " (ignored %d more)", + le16_to_cpu(l->d_npartitions) - max_partitions); + seq_buf_puts(&state->pp_buf, " >\n"); } #endif @@ -496,12 +483,7 @@ static void parse_unixware(struct parsed_partitions *state, put_dev_sector(sect); return; } - { - char tmp[1 + BDEVNAME_SIZE + 10 + 12 + 1]; - - snprintf(tmp, sizeof(tmp), " %s%d: <unixware:", state->name, origin); - strlcat(state->pp_buf, tmp, PAGE_SIZE); - } + seq_buf_printf(&state->pp_buf, " %s%d: <unixware:", state->name, origin); p = &l->vtoc.v_slice[1]; /* I omit the 0th slice as it is the same as whole disk. */ while (p - &l->vtoc.v_slice[0] < UNIXWARE_NUMSLICE) { @@ -515,7 +497,7 @@ static void parse_unixware(struct parsed_partitions *state, p++; } put_dev_sector(sect); - strlcat(state->pp_buf, " >\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " >\n"); #endif } @@ -546,10 +528,7 @@ static void parse_minix(struct parsed_partitions *state, * the normal boot sector. */ if (msdos_magic_present(data + 510) && p->sys_ind == MINIX_PARTITION) { /* subpartition table present */ - char tmp[1 + BDEVNAME_SIZE + 10 + 9 + 1]; - - snprintf(tmp, sizeof(tmp), " %s%d: <minix:", state->name, origin); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, " %s%d: <minix:", state->name, origin); for (i = 0; i < MINIX_NR_SUBPARTITIONS; i++, p++) { if (state->next == state->limit) break; @@ -558,7 +537,7 @@ static void parse_minix(struct parsed_partitions *state, put_partition(state, state->next++, start_sect(p), nr_sects(p)); } - strlcat(state->pp_buf, " >\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " >\n"); } put_dev_sector(sect); #endif /* CONFIG_MINIX_SUBPARTITION */ @@ -602,7 +581,7 @@ int msdos_partition(struct parsed_partitions *state) #ifdef CONFIG_AIX_PARTITION return aix_partition(state); #else - strlcat(state->pp_buf, " [AIX]", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " [AIX]"); return 0; #endif } @@ -629,7 +608,7 @@ int msdos_partition(struct parsed_partitions *state) fb = (struct fat_boot_sector *) data; if (slot == 1 && fb->reserved && fb->fats && fat_valid_media(fb->media)) { - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); put_dev_sector(sect); return 1; } else { @@ -678,9 +657,9 @@ int msdos_partition(struct parsed_partitions *state) n = min(size, max(sector_size, n)); put_partition(state, slot, start, n); - strlcat(state->pp_buf, " <", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " <"); parse_extended(state, start, size, disksig); - strlcat(state->pp_buf, " >", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, " >"); continue; } put_partition(state, slot, start, size); @@ -688,12 +667,12 @@ int msdos_partition(struct parsed_partitions *state) if (p->sys_ind == LINUX_RAID_PARTITION) state->parts[slot].flags = ADDPART_FLAG_RAID; if (p->sys_ind == DM6_PARTITION) - strlcat(state->pp_buf, "[DM]", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "[DM]"); if (p->sys_ind == EZD_PARTITION) - strlcat(state->pp_buf, "[EZD]", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "[EZD]"); } - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); /* second pass - output for each on a separate line */ p = (struct msdos_partition *) (0x1be + data); diff --git a/block/partitions/of.c b/block/partitions/of.c index 4e760fdffb3f..53664ea06b65 100644 --- a/block/partitions/of.c +++ b/block/partitions/of.c @@ -36,7 +36,6 @@ static void add_of_partition(struct parsed_partitions *state, int slot, struct device_node *np) { struct partition_meta_info *info; - char tmp[sizeof(info->volname) + 4]; const char *partname; int len; @@ -63,8 +62,7 @@ static void add_of_partition(struct parsed_partitions *state, int slot, partname = of_get_property(np, "name", &len); strscpy(info->volname, partname, sizeof(info->volname)); - snprintf(tmp, sizeof(tmp), "(%s)", info->volname); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, "(%s)", info->volname); } int of_partition(struct parsed_partitions *state) @@ -76,8 +74,10 @@ int of_partition(struct parsed_partitions *state) struct device_node *partitions_np = of_node_get(ddev->of_node); if (!partitions_np || - !of_device_is_compatible(partitions_np, "fixed-partitions")) + !of_device_is_compatible(partitions_np, "fixed-partitions")) { + of_node_put(partitions_np); return 0; + } slot = 1; /* Validate parition offset and size */ @@ -104,7 +104,8 @@ int of_partition(struct parsed_partitions *state) slot++; } - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); + of_node_put(partitions_np); return 1; } diff --git a/block/partitions/osf.c b/block/partitions/osf.c index 84560d0765ed..2a692584dba9 100644 --- a/block/partitions/osf.c +++ b/block/partitions/osf.c @@ -81,7 +81,7 @@ int osf_partition(struct parsed_partitions *state) le32_to_cpu(partition->p_size)); slot++; } - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); put_dev_sector(sect); return 1; } diff --git a/block/partitions/sgi.c b/block/partitions/sgi.c index b5ecddd5181a..2383ca63cd66 100644 --- a/block/partitions/sgi.c +++ b/block/partitions/sgi.c @@ -79,7 +79,7 @@ int sgi_partition(struct parsed_partitions *state) } slot++; } - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); put_dev_sector(sect); return 1; } diff --git a/block/partitions/sun.c b/block/partitions/sun.c index 2419af76120f..92c645fcd2e0 100644 --- a/block/partitions/sun.c +++ b/block/partitions/sun.c @@ -121,7 +121,7 @@ int sun_partition(struct parsed_partitions *state) } slot++; } - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); put_dev_sector(sect); return 1; } diff --git a/block/partitions/sysv68.c b/block/partitions/sysv68.c index 6f6257fd4eb4..470e0f9de7be 100644 --- a/block/partitions/sysv68.c +++ b/block/partitions/sysv68.c @@ -54,7 +54,6 @@ int sysv68_partition(struct parsed_partitions *state) unsigned char *data; struct dkblk0 *b; struct slice *slice; - char tmp[64]; data = read_part_sector(state, 0, §); if (!data) @@ -74,8 +73,7 @@ int sysv68_partition(struct parsed_partitions *state) return -1; slices -= 1; /* last slice is the whole disk */ - snprintf(tmp, sizeof(tmp), "sysV68: %s(s%u)", state->name, slices); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, "sysV68: %s(s%u)", state->name, slices); slice = (struct slice *)data; for (i = 0; i < slices; i++, slice++) { if (slot == state->limit) @@ -84,12 +82,11 @@ int sysv68_partition(struct parsed_partitions *state) put_partition(state, slot, be32_to_cpu(slice->blkoff), be32_to_cpu(slice->nblocks)); - snprintf(tmp, sizeof(tmp), "(s%u)", i); - strlcat(state->pp_buf, tmp, PAGE_SIZE); + seq_buf_printf(&state->pp_buf, "(s%u)", i); } slot++; } - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); put_dev_sector(sect); return 1; } diff --git a/block/partitions/ultrix.c b/block/partitions/ultrix.c index 4aaa81043ca0..b4b9ddc57a5d 100644 --- a/block/partitions/ultrix.c +++ b/block/partitions/ultrix.c @@ -39,7 +39,7 @@ int ultrix_partition(struct parsed_partitions *state) label->pt_part[i].pi_blkoff, label->pt_part[i].pi_nblocks); put_dev_sector(sect); - strlcat(state->pp_buf, "\n", PAGE_SIZE); + seq_buf_puts(&state->pp_buf, "\n"); return 1; } else { put_dev_sector(sect); diff --git a/block/sed-opal.c b/block/sed-opal.c index 5a28f23f7f22..79b290d9458a 100644 --- a/block/sed-opal.c +++ b/block/sed-opal.c @@ -160,6 +160,8 @@ static const u8 opaluid[][OPAL_UID_LENGTH] = { { 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00 }, [OPAL_DATASTORE] = { 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00 }, + [OPAL_LOCKING_TABLE] = + { 0x00, 0x00, 0x08, 0x02, 0x00, 0x00, 0x00, 0x00 }, /* C_PIN_TABLE object ID's */ [OPAL_C_PIN_MSID] = @@ -218,6 +220,8 @@ static const u8 opalmethod[][OPAL_METHOD_LENGTH] = { { 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x06, 0x01 }, [OPAL_ERASE] = { 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x08, 0x03 }, + [OPAL_REACTIVATE] = + { 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x08, 0x01 }, }; static int end_opal_session_error(struct opal_dev *dev); @@ -1514,7 +1518,7 @@ static inline int enable_global_lr(struct opal_dev *dev, u8 *uid, return err; } -static int setup_locking_range(struct opal_dev *dev, void *data) +static int setup_enable_range(struct opal_dev *dev, void *data) { u8 uid[OPAL_UID_LENGTH]; struct opal_user_lr_setup *setup = data; @@ -1528,38 +1532,47 @@ static int setup_locking_range(struct opal_dev *dev, void *data) if (lr == 0) err = enable_global_lr(dev, uid, setup); - else { - err = cmd_start(dev, uid, opalmethod[OPAL_SET]); + else + err = generic_lr_enable_disable(dev, uid, !!setup->RLE, !!setup->WLE, 0, 0); + if (err) { + pr_debug("Failed to create enable lr command.\n"); + return err; + } - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, OPAL_VALUES); - add_token_u8(&err, dev, OPAL_STARTLIST); + return finalize_and_send(dev, parse_and_check_status); +} - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, OPAL_RANGESTART); - add_token_u64(&err, dev, setup->range_start); - add_token_u8(&err, dev, OPAL_ENDNAME); +static int setup_locking_range_start_length(struct opal_dev *dev, void *data) +{ + int err; + u8 uid[OPAL_UID_LENGTH]; + struct opal_user_lr_setup *setup = data; - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, OPAL_RANGELENGTH); - add_token_u64(&err, dev, setup->range_length); - add_token_u8(&err, dev, OPAL_ENDNAME); + err = build_locking_range(uid, sizeof(uid), setup->session.opal_key.lr); + if (err) + return err; - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, OPAL_READLOCKENABLED); - add_token_u64(&err, dev, !!setup->RLE); - add_token_u8(&err, dev, OPAL_ENDNAME); + err = cmd_start(dev, uid, opalmethod[OPAL_SET]); - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, OPAL_WRITELOCKENABLED); - add_token_u64(&err, dev, !!setup->WLE); - add_token_u8(&err, dev, OPAL_ENDNAME); + add_token_u8(&err, dev, OPAL_STARTNAME); + add_token_u8(&err, dev, OPAL_VALUES); + add_token_u8(&err, dev, OPAL_STARTLIST); + + add_token_u8(&err, dev, OPAL_STARTNAME); + add_token_u8(&err, dev, OPAL_RANGESTART); + add_token_u64(&err, dev, setup->range_start); + add_token_u8(&err, dev, OPAL_ENDNAME); + + add_token_u8(&err, dev, OPAL_STARTNAME); + add_token_u8(&err, dev, OPAL_RANGELENGTH); + add_token_u64(&err, dev, setup->range_length); + add_token_u8(&err, dev, OPAL_ENDNAME); + + add_token_u8(&err, dev, OPAL_ENDLIST); + add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); - add_token_u8(&err, dev, OPAL_ENDNAME); - } if (err) { - pr_debug("Error building Setup Locking range command.\n"); + pr_debug("Error building Setup Locking RangeStartLength command.\n"); return err; } @@ -1568,7 +1581,7 @@ static int setup_locking_range(struct opal_dev *dev, void *data) static int response_get_column(const struct parsed_resp *resp, int *iter, - u8 column, + u64 column, u64 *value) { const struct opal_resp_tok *tok; @@ -1586,7 +1599,7 @@ static int response_get_column(const struct parsed_resp *resp, n++; if (response_get_u64(resp, n) != column) { - pr_debug("Token %d does not match expected column %u.\n", + pr_debug("Token %d does not match expected column %llu.\n", n, column); return OPAL_INVAL_PARAM; } @@ -1744,6 +1757,12 @@ static int start_anybodyASP_opal_session(struct opal_dev *dev, void *data) OPAL_ADMINSP_UID, NULL, 0); } +static int start_anybodyLSP_opal_session(struct opal_dev *dev, void *data) +{ + return start_generic_opal_session(dev, OPAL_ANYBODY_UID, + OPAL_LOCKINGSP_UID, NULL, 0); +} + static int start_SIDASP_opal_session(struct opal_dev *dev, void *data) { int ret; @@ -2285,6 +2304,74 @@ static int activate_lsp(struct opal_dev *dev, void *data) return finalize_and_send(dev, parse_and_check_status); } +static int reactivate_lsp(struct opal_dev *dev, void *data) +{ + struct opal_lr_react *opal_react = data; + u8 user_lr[OPAL_UID_LENGTH]; + int err, i; + + err = cmd_start(dev, opaluid[OPAL_THISSP_UID], + opalmethod[OPAL_REACTIVATE]); + + if (err) { + pr_debug("Error building Reactivate LockingSP command.\n"); + return err; + } + + /* + * If neither 'entire_table' nor 'num_lrs' is set, the device + * gets reactivated with SUM disabled. Only Admin1PIN will change + * if set. + */ + if (opal_react->entire_table) { + /* Entire Locking table (all locking ranges) will be put in SUM. */ + add_token_u8(&err, dev, OPAL_STARTNAME); + add_token_u64(&err, dev, OPAL_SUM_SET_LIST); + add_token_bytestring(&err, dev, opaluid[OPAL_LOCKING_TABLE], OPAL_UID_LENGTH); + add_token_u8(&err, dev, OPAL_ENDNAME); + } else if (opal_react->num_lrs) { + /* Subset of Locking table (selected locking range(s)) to be put in SUM */ + err = build_locking_range(user_lr, sizeof(user_lr), + opal_react->lr[0]); + if (err) + return err; + + add_token_u8(&err, dev, OPAL_STARTNAME); + add_token_u64(&err, dev, OPAL_SUM_SET_LIST); + + add_token_u8(&err, dev, OPAL_STARTLIST); + add_token_bytestring(&err, dev, user_lr, OPAL_UID_LENGTH); + for (i = 1; i < opal_react->num_lrs; i++) { + user_lr[7] = opal_react->lr[i]; + add_token_bytestring(&err, dev, user_lr, OPAL_UID_LENGTH); + } + add_token_u8(&err, dev, OPAL_ENDLIST); + add_token_u8(&err, dev, OPAL_ENDNAME); + } + + /* Skipping the rangle policy parameter is same as setting its value to zero */ + if (opal_react->range_policy && (opal_react->num_lrs || opal_react->entire_table)) { + add_token_u8(&err, dev, OPAL_STARTNAME); + add_token_u64(&err, dev, OPAL_SUM_RANGE_POLICY); + add_token_u8(&err, dev, 1); + add_token_u8(&err, dev, OPAL_ENDNAME); + } + + /* + * Optional parameter. If set, it changes the Admin1 PIN even when SUM + * is being disabled. + */ + if (opal_react->new_admin_key.key_len) { + add_token_u8(&err, dev, OPAL_STARTNAME); + add_token_u64(&err, dev, OPAL_SUM_ADMIN1_PIN); + add_token_bytestring(&err, dev, opal_react->new_admin_key.key, + opal_react->new_admin_key.key_len); + add_token_u8(&err, dev, OPAL_ENDNAME); + } + + return finalize_and_send(dev, parse_and_check_status); +} + /* Determine if we're in the Manufactured Inactive or Active state */ static int get_lsp_lifecycle(struct opal_dev *dev, void *data) { @@ -2512,7 +2599,7 @@ struct opal_dev *init_opal_dev(void *data, sec_send_recv *send_recv) { struct opal_dev *dev; - dev = kmalloc(sizeof(*dev), GFP_KERNEL); + dev = kmalloc_obj(*dev); if (!dev) return NULL; @@ -2719,7 +2806,7 @@ static int opal_save(struct opal_dev *dev, struct opal_lock_unlock *lk_unlk) { struct opal_suspend_data *suspend; - suspend = kzalloc(sizeof(*suspend), GFP_KERNEL); + suspend = kzalloc_obj(*suspend); if (!suspend) return -ENOMEM; @@ -2940,7 +3027,8 @@ static int opal_activate_lsp(struct opal_dev *dev, }; int ret; - if (!opal_lr_act->num_lrs || opal_lr_act->num_lrs > OPAL_MAX_LRS) + if (opal_lr_act->sum && + (!opal_lr_act->num_lrs || opal_lr_act->num_lrs > OPAL_MAX_LRS)) return -EINVAL; ret = opal_get_key(dev, &opal_lr_act->key); @@ -2954,12 +3042,92 @@ static int opal_activate_lsp(struct opal_dev *dev, return ret; } +static int opal_reactivate_lsp(struct opal_dev *dev, + struct opal_lr_react *opal_lr_react) +{ + const struct opal_step active_steps[] = { + { start_admin1LSP_opal_session, &opal_lr_react->key }, + { reactivate_lsp, opal_lr_react }, + /* No end_opal_session. The controller terminates the session */ + }; + int ret; + + /* use either 'entire_table' parameter or set of locking ranges */ + if (opal_lr_react->num_lrs > OPAL_MAX_LRS || + (opal_lr_react->num_lrs && opal_lr_react->entire_table)) + return -EINVAL; + + ret = opal_get_key(dev, &opal_lr_react->key); + if (ret) + return ret; + mutex_lock(&dev->dev_lock); + setup_opal_dev(dev); + ret = execute_steps(dev, active_steps, ARRAY_SIZE(active_steps)); + mutex_unlock(&dev->dev_lock); + + return ret; +} + static int opal_setup_locking_range(struct opal_dev *dev, struct opal_user_lr_setup *opal_lrs) { const struct opal_step lr_steps[] = { { start_auth_opal_session, &opal_lrs->session }, - { setup_locking_range, opal_lrs }, + { setup_locking_range_start_length, opal_lrs }, + { setup_enable_range, opal_lrs }, + { end_opal_session, } + }, lr_global_steps[] = { + { start_auth_opal_session, &opal_lrs->session }, + { setup_enable_range, opal_lrs }, + { end_opal_session, } + }; + int ret; + + ret = opal_get_key(dev, &opal_lrs->session.opal_key); + if (ret) + return ret; + mutex_lock(&dev->dev_lock); + setup_opal_dev(dev); + if (opal_lrs->session.opal_key.lr == 0) + ret = execute_steps(dev, lr_global_steps, ARRAY_SIZE(lr_global_steps)); + else + ret = execute_steps(dev, lr_steps, ARRAY_SIZE(lr_steps)); + mutex_unlock(&dev->dev_lock); + + return ret; +} + +static int opal_setup_locking_range_start_length(struct opal_dev *dev, + struct opal_user_lr_setup *opal_lrs) +{ + const struct opal_step lr_steps[] = { + { start_auth_opal_session, &opal_lrs->session }, + { setup_locking_range_start_length, opal_lrs }, + { end_opal_session, } + }; + int ret; + + /* we can not set global locking range offset or length */ + if (opal_lrs->session.opal_key.lr == 0) + return -EINVAL; + + ret = opal_get_key(dev, &opal_lrs->session.opal_key); + if (ret) + return ret; + mutex_lock(&dev->dev_lock); + setup_opal_dev(dev); + ret = execute_steps(dev, lr_steps, ARRAY_SIZE(lr_steps)); + mutex_unlock(&dev->dev_lock); + + return ret; +} + +static int opal_enable_disable_range(struct opal_dev *dev, + struct opal_user_lr_setup *opal_lrs) +{ + const struct opal_step lr_steps[] = { + { start_auth_opal_session, &opal_lrs->session }, + { setup_enable_range, opal_lrs }, { end_opal_session, } }; int ret; @@ -3227,6 +3395,200 @@ static int opal_get_geometry(struct opal_dev *dev, void __user *data) return 0; } +static int get_sum_ranges(struct opal_dev *dev, void *data) +{ + const char *lr_uid; + size_t lr_uid_len; + u64 val; + const struct opal_resp_tok *tok; + int err, tok_n = 2; + struct opal_sum_ranges *sranges = data; + const __u8 lr_all[OPAL_MAX_LRS] = { 0, 1, 2, 3, 4, 5, 6, 7, 8 }; + + err = generic_get_columns(dev, opaluid[OPAL_LOCKING_INFO_TABLE], OPAL_SUM_SET_LIST, + OPAL_SUM_RANGE_POLICY); + if (err) { + pr_debug("Couldn't get locking info table columns %d to %d.\n", + OPAL_SUM_SET_LIST, OPAL_SUM_RANGE_POLICY); + return err; + } + + tok = response_get_token(&dev->parsed, tok_n); + if (IS_ERR(tok)) + return PTR_ERR(tok); + + if (!response_token_matches(tok, OPAL_STARTNAME)) { + pr_debug("Unexpected response token type %d.\n", tok_n); + return OPAL_INVAL_PARAM; + } + tok_n++; + + if (response_get_u64(&dev->parsed, tok_n) != OPAL_SUM_SET_LIST) { + pr_debug("Token %d does not match expected column %u.\n", + tok_n, OPAL_SUM_SET_LIST); + return OPAL_INVAL_PARAM; + } + tok_n++; + + tok = response_get_token(&dev->parsed, tok_n); + if (IS_ERR(tok)) + return PTR_ERR(tok); + + /* + * The OPAL_SUM_SET_LIST response contains two distinct values: + * + * - the list of individual locking ranges (UIDs) put in SUM. The list + * may also be empty signaling the SUM is disabled. + * + * - the Locking table UID if the entire Locking table is put in SUM. + */ + if (response_token_matches(tok, OPAL_STARTLIST)) { + sranges->num_lrs = 0; + + tok_n++; + tok = response_get_token(&dev->parsed, tok_n); + if (IS_ERR(tok)) + return PTR_ERR(tok); + + while (!response_token_matches(tok, OPAL_ENDLIST)) { + lr_uid_len = response_get_string(&dev->parsed, tok_n, &lr_uid); + if (lr_uid_len != OPAL_UID_LENGTH) { + pr_debug("Unexpected response token type %d.\n", tok_n); + return OPAL_INVAL_PARAM; + } + + if (memcmp(lr_uid, opaluid[OPAL_LOCKINGRANGE_GLOBAL], OPAL_UID_LENGTH)) { + if (lr_uid[5] != LOCKING_RANGE_NON_GLOBAL) { + pr_debug("Unexpected byte %d at LR UUID position 5.\n", + lr_uid[5]); + return OPAL_INVAL_PARAM; + } + sranges->lr[sranges->num_lrs++] = lr_uid[7]; + } else + sranges->lr[sranges->num_lrs++] = 0; + + tok_n++; + tok = response_get_token(&dev->parsed, tok_n); + if (IS_ERR(tok)) + return PTR_ERR(tok); + } + } else { + /* Only OPAL_LOCKING_TABLE UID is an alternative to OPAL_STARTLIST here. */ + lr_uid_len = response_get_string(&dev->parsed, tok_n, &lr_uid); + if (lr_uid_len != OPAL_UID_LENGTH) { + pr_debug("Unexpected response token type %d.\n", tok_n); + return OPAL_INVAL_PARAM; + } + + if (memcmp(lr_uid, opaluid[OPAL_LOCKING_TABLE], OPAL_UID_LENGTH)) { + pr_debug("Unexpected response UID.\n"); + return OPAL_INVAL_PARAM; + } + + /* sed-opal kernel API already provides following limit in Activate command */ + sranges->num_lrs = OPAL_MAX_LRS; + memcpy(sranges->lr, lr_all, OPAL_MAX_LRS); + } + tok_n++; + + tok = response_get_token(&dev->parsed, tok_n); + if (IS_ERR(tok)) + return PTR_ERR(tok); + + if (!response_token_matches(tok, OPAL_ENDNAME)) { + pr_debug("Unexpected response token type %d.\n", tok_n); + return OPAL_INVAL_PARAM; + } + tok_n++; + + err = response_get_column(&dev->parsed, &tok_n, OPAL_SUM_RANGE_POLICY, &val); + if (err) + return err; + + sranges->range_policy = val ? 1 : 0; + + return 0; +} + +static int opal_get_sum_ranges(struct opal_dev *dev, struct opal_sum_ranges *opal_sum_rngs, + void __user *data) +{ + const struct opal_step admin_steps[] = { + { start_admin1LSP_opal_session, &opal_sum_rngs->key }, + { get_sum_ranges, opal_sum_rngs }, + { end_opal_session, } + }, anybody_steps[] = { + { start_anybodyLSP_opal_session, NULL }, + { get_sum_ranges, opal_sum_rngs }, + { end_opal_session, } + }; + int ret; + + mutex_lock(&dev->dev_lock); + setup_opal_dev(dev); + if (opal_sum_rngs->key.key_len) + /* Use Admin1 session (authenticated by PIN) to retrieve LockingInfo columns */ + ret = execute_steps(dev, admin_steps, ARRAY_SIZE(admin_steps)); + else + /* Use Anybody session (no key) to retrieve LockingInfo columns */ + ret = execute_steps(dev, anybody_steps, ARRAY_SIZE(anybody_steps)); + mutex_unlock(&dev->dev_lock); + + /* skip session info when copying back to uspace */ + if (!ret && copy_to_user(data + offsetof(struct opal_sum_ranges, num_lrs), + (void *)opal_sum_rngs + offsetof(struct opal_sum_ranges, num_lrs), + sizeof(*opal_sum_rngs) - offsetof(struct opal_sum_ranges, num_lrs))) { + pr_debug("Error copying SUM ranges info to userspace\n"); + return -EFAULT; + } + + return ret; +} + +static int opal_stack_reset(struct opal_dev *dev) +{ + struct opal_stack_reset *req; + struct opal_stack_reset_response *resp; + int ret; + + mutex_lock(&dev->dev_lock); + + memset(dev->cmd, 0, IO_BUFFER_LENGTH); + req = (struct opal_stack_reset *)dev->cmd; + req->extendedComID[0] = dev->comid >> 8; + req->extendedComID[1] = dev->comid & 0xFF; + req->request_code = cpu_to_be32(OPAL_STACK_RESET); + + ret = dev->send_recv(dev->data, dev->comid, TCG_SECP_02, + dev->cmd, IO_BUFFER_LENGTH, true); + if (ret) { + pr_debug("Error sending stack reset: %d\n", ret); + goto out; + } + + memset(dev->resp, 0, IO_BUFFER_LENGTH); + ret = dev->send_recv(dev->data, dev->comid, TCG_SECP_02, + dev->resp, IO_BUFFER_LENGTH, false); + if (ret) { + pr_debug("Error receiving stack reset response: %d\n", ret); + goto out; + } + + resp = (struct opal_stack_reset_response *)dev->resp; + if (be16_to_cpu(resp->data_length) != 4) { + pr_debug("Stack reset pending\n"); + ret = -EBUSY; + goto out; + } + if (be32_to_cpu(resp->response) != 0) { + pr_debug("Stack reset failed: %u\n", be32_to_cpu(resp->response)); + ret = -EIO; + } +out: + mutex_unlock(&dev->dev_lock); + return ret; +} + int sed_ioctl(struct opal_dev *dev, unsigned int cmd, void __user *arg) { void *p; @@ -3312,6 +3674,21 @@ int sed_ioctl(struct opal_dev *dev, unsigned int cmd, void __user *arg) case IOC_OPAL_SET_SID_PW: ret = opal_set_new_sid_pw(dev, p); break; + case IOC_OPAL_REACTIVATE_LSP: + ret = opal_reactivate_lsp(dev, p); + break; + case IOC_OPAL_LR_SET_START_LEN: + ret = opal_setup_locking_range_start_length(dev, p); + break; + case IOC_OPAL_ENABLE_DISABLE_LR: + ret = opal_enable_disable_range(dev, p); + break; + case IOC_OPAL_GET_SUM_STATUS: + ret = opal_get_sum_ranges(dev, p, arg); + break; + case IOC_OPAL_STACK_RESET: + ret = opal_stack_reset(dev); + break; default: break; diff --git a/block/t10-pi.c b/block/t10-pi.c index 851db518ee5e..a19b4e102a83 100644 --- a/block/t10-pi.c +++ b/block/t10-pi.c @@ -12,462 +12,556 @@ #include <linux/unaligned.h> #include "blk.h" +#define APP_TAG_ESCAPE 0xffff +#define REF_TAG_ESCAPE 0xffffffff + +/* + * This union is used for onstack allocations when the pi field is split across + * segments. blk_validate_integrity_limits() guarantees pi_tuple_size matches + * the sizeof one of these two types. + */ +union pi_tuple { + struct crc64_pi_tuple crc64_pi; + struct t10_pi_tuple t10_pi; +}; + struct blk_integrity_iter { - void *prot_buf; - void *data_buf; - sector_t seed; - unsigned int data_size; - unsigned short interval; - const char *disk_name; + struct bio *bio; + struct bio_integrity_payload *bip; + struct blk_integrity *bi; + struct bvec_iter data_iter; + struct bvec_iter prot_iter; + unsigned int interval_remaining; + u64 seed; + u64 csum; }; -static __be16 t10_pi_csum(__be16 csum, void *data, unsigned int len, - unsigned char csum_type) +static void blk_calculate_guard(struct blk_integrity_iter *iter, void *data, + unsigned int len) { - if (csum_type == BLK_INTEGRITY_CSUM_IP) - return (__force __be16)ip_compute_csum(data, len); - return cpu_to_be16(crc_t10dif_update(be16_to_cpu(csum), data, len)); + switch (iter->bi->csum_type) { + case BLK_INTEGRITY_CSUM_CRC64: + iter->csum = crc64_nvme(iter->csum, data, len); + break; + case BLK_INTEGRITY_CSUM_CRC: + iter->csum = crc_t10dif_update(iter->csum, data, len); + break; + case BLK_INTEGRITY_CSUM_IP: + iter->csum = (__force u32)csum_partial(data, len, + (__force __wsum)iter->csum); + break; + default: + WARN_ON_ONCE(1); + iter->csum = U64_MAX; + break; + } +} + +static void blk_integrity_csum_finish(struct blk_integrity_iter *iter) +{ + switch (iter->bi->csum_type) { + case BLK_INTEGRITY_CSUM_IP: + iter->csum = (__force u16)csum_fold((__force __wsum)iter->csum); + break; + default: + break; + } } /* - * Type 1 and Type 2 protection use the same format: 16 bit guard tag, - * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref - * tag. + * Update the csum for formats that have metadata padding in front of the data + * integrity field */ -static void t10_pi_generate(struct blk_integrity_iter *iter, - struct blk_integrity *bi) +static void blk_integrity_csum_offset(struct blk_integrity_iter *iter) { - u8 offset = bi->pi_offset; - unsigned int i; + unsigned int offset = iter->bi->pi_offset; + struct bio_vec *bvec = iter->bip->bip_vec; + + while (offset > 0) { + struct bio_vec pbv = bvec_iter_bvec(bvec, iter->prot_iter); + unsigned int len = min(pbv.bv_len, offset); + void *prot_buf = bvec_kmap_local(&pbv); + + blk_calculate_guard(iter, prot_buf, len); + kunmap_local(prot_buf); + offset -= len; + bvec_iter_advance_single(bvec, &iter->prot_iter, len); + } + blk_integrity_csum_finish(iter); +} - for (i = 0 ; i < iter->data_size ; i += iter->interval) { - struct t10_pi_tuple *pi = iter->prot_buf + offset; +static void blk_integrity_copy_from_tuple(struct bio_integrity_payload *bip, + struct bvec_iter *iter, void *tuple, + unsigned int tuple_size) +{ + while (tuple_size) { + struct bio_vec pbv = bvec_iter_bvec(bip->bip_vec, *iter); + unsigned int len = min(tuple_size, pbv.bv_len); + void *prot_buf = bvec_kmap_local(&pbv); + + memcpy(prot_buf, tuple, len); + kunmap_local(prot_buf); + bvec_iter_advance_single(bip->bip_vec, iter, len); + tuple_size -= len; + tuple += len; + } +} - pi->guard_tag = t10_pi_csum(0, iter->data_buf, iter->interval, - bi->csum_type); - if (offset) - pi->guard_tag = t10_pi_csum(pi->guard_tag, - iter->prot_buf, offset, bi->csum_type); - pi->app_tag = 0; +static void blk_integrity_copy_to_tuple(struct bio_integrity_payload *bip, + struct bvec_iter *iter, void *tuple, + unsigned int tuple_size) +{ + while (tuple_size) { + struct bio_vec pbv = bvec_iter_bvec(bip->bip_vec, *iter); + unsigned int len = min(tuple_size, pbv.bv_len); + void *prot_buf = bvec_kmap_local(&pbv); + + memcpy(tuple, prot_buf, len); + kunmap_local(prot_buf); + bvec_iter_advance_single(bip->bip_vec, iter, len); + tuple_size -= len; + tuple += len; + } +} - if (bi->flags & BLK_INTEGRITY_REF_TAG) - pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed)); - else - pi->ref_tag = 0; +static bool ext_pi_ref_escape(const u8 ref_tag[6]) +{ + static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - iter->data_buf += iter->interval; - iter->prot_buf += bi->tuple_size; - iter->seed++; - } + return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0; } -static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter, - struct blk_integrity *bi) -{ - u8 offset = bi->pi_offset; - unsigned int i; - - for (i = 0 ; i < iter->data_size ; i += iter->interval) { - struct t10_pi_tuple *pi = iter->prot_buf + offset; - __be16 csum; - - if (bi->flags & BLK_INTEGRITY_REF_TAG) { - if (pi->app_tag == T10_PI_APP_ESCAPE) - goto next; - - if (be32_to_cpu(pi->ref_tag) != - lower_32_bits(iter->seed)) { - pr_err("%s: ref tag error at location %llu " \ - "(rcvd %u)\n", iter->disk_name, - (unsigned long long) - iter->seed, be32_to_cpu(pi->ref_tag)); - return BLK_STS_PROTECTION; - } - } else { - if (pi->app_tag == T10_PI_APP_ESCAPE && - pi->ref_tag == T10_PI_REF_ESCAPE) - goto next; +static blk_status_t blk_verify_ext_pi(struct blk_integrity_iter *iter, + struct crc64_pi_tuple *pi) +{ + u64 seed = lower_48_bits(iter->seed); + u64 guard = get_unaligned_be64(&pi->guard_tag); + u64 ref = get_unaligned_be48(pi->ref_tag); + u16 app = get_unaligned_be16(&pi->app_tag); + + if (iter->bi->flags & BLK_INTEGRITY_REF_TAG) { + if (app == APP_TAG_ESCAPE) + return BLK_STS_OK; + if (ref != seed) { + pr_err("%s: ref tag error at location %llu (rcvd %llu)\n", + iter->bio->bi_bdev->bd_disk->disk_name, seed, + ref); + return BLK_STS_PROTECTION; } + } else if (app == APP_TAG_ESCAPE && ext_pi_ref_escape(pi->ref_tag)) { + return BLK_STS_OK; + } + + if (guard != iter->csum) { + pr_err("%s: guard tag error at sector %llu (rcvd %016llx, want %016llx)\n", + iter->bio->bi_bdev->bd_disk->disk_name, iter->seed, + guard, iter->csum); + return BLK_STS_PROTECTION; + } + + return BLK_STS_OK; +} - csum = t10_pi_csum(0, iter->data_buf, iter->interval, - bi->csum_type); - if (offset) - csum = t10_pi_csum(csum, iter->prot_buf, offset, - bi->csum_type); - - if (pi->guard_tag != csum) { - pr_err("%s: guard tag error at sector %llu " \ - "(rcvd %04x, want %04x)\n", iter->disk_name, - (unsigned long long)iter->seed, - be16_to_cpu(pi->guard_tag), be16_to_cpu(csum)); +static blk_status_t blk_verify_pi(struct blk_integrity_iter *iter, + struct t10_pi_tuple *pi, u16 guard) +{ + u32 seed = lower_32_bits(iter->seed); + u32 ref = get_unaligned_be32(&pi->ref_tag); + u16 app = get_unaligned_be16(&pi->app_tag); + + if (iter->bi->flags & BLK_INTEGRITY_REF_TAG) { + if (app == APP_TAG_ESCAPE) + return BLK_STS_OK; + if (ref != seed) { + pr_err("%s: ref tag error at location %u (rcvd %u)\n", + iter->bio->bi_bdev->bd_disk->disk_name, seed, + ref); return BLK_STS_PROTECTION; } + } else if (app == APP_TAG_ESCAPE && ref == REF_TAG_ESCAPE) { + return BLK_STS_OK; + } -next: - iter->data_buf += iter->interval; - iter->prot_buf += bi->tuple_size; - iter->seed++; + if (guard != (u16)iter->csum) { + pr_err("%s: guard tag error at sector %llu (rcvd %04x, want %04x)\n", + iter->bio->bi_bdev->bd_disk->disk_name, iter->seed, + guard, (u16)iter->csum); + return BLK_STS_PROTECTION; } return BLK_STS_OK; } -/** - * t10_pi_type1_prepare - prepare PI prior submitting request to device - * @rq: request with PI that should be prepared - * - * For Type 1/Type 2, the virtual start sector is the one that was - * originally submitted by the block layer for the ref_tag usage. Due to - * partitioning, MD/DM cloning, etc. the actual physical start sector is - * likely to be different. Remap protection information to match the - * physical LBA. - */ -static void t10_pi_type1_prepare(struct request *rq) +static blk_status_t blk_verify_t10_pi(struct blk_integrity_iter *iter, + struct t10_pi_tuple *pi) { - struct blk_integrity *bi = &rq->q->limits.integrity; - const int tuple_sz = bi->tuple_size; - u32 ref_tag = t10_pi_ref_tag(rq); - u8 offset = bi->pi_offset; - struct bio *bio; + u16 guard = get_unaligned_be16(&pi->guard_tag); - __rq_for_each_bio(bio, rq) { - struct bio_integrity_payload *bip = bio_integrity(bio); - u32 virt = bip_get_seed(bip) & 0xffffffff; - struct bio_vec iv; - struct bvec_iter iter; + return blk_verify_pi(iter, pi, guard); +} - /* Already remapped? */ - if (bip->bip_flags & BIP_MAPPED_INTEGRITY) - break; +static blk_status_t blk_verify_ip_pi(struct blk_integrity_iter *iter, + struct t10_pi_tuple *pi) +{ + u16 guard = get_unaligned((u16 *)&pi->guard_tag); - bip_for_each_vec(iv, bip, iter) { - unsigned int j; - void *p; - - p = bvec_kmap_local(&iv); - for (j = 0; j < iv.bv_len; j += tuple_sz) { - struct t10_pi_tuple *pi = p + offset; - - if (be32_to_cpu(pi->ref_tag) == virt) - pi->ref_tag = cpu_to_be32(ref_tag); - virt++; - ref_tag++; - p += tuple_sz; - } - kunmap_local(p); - } + return blk_verify_pi(iter, pi, guard); +} - bip->bip_flags |= BIP_MAPPED_INTEGRITY; +static blk_status_t blk_integrity_verify(struct blk_integrity_iter *iter, + union pi_tuple *tuple) +{ + switch (iter->bi->csum_type) { + case BLK_INTEGRITY_CSUM_CRC64: + return blk_verify_ext_pi(iter, &tuple->crc64_pi); + case BLK_INTEGRITY_CSUM_CRC: + return blk_verify_t10_pi(iter, &tuple->t10_pi); + case BLK_INTEGRITY_CSUM_IP: + return blk_verify_ip_pi(iter, &tuple->t10_pi); + default: + return BLK_STS_OK; } } -/** - * t10_pi_type1_complete - prepare PI prior returning request to the blk layer - * @rq: request with PI that should be prepared - * @nr_bytes: total bytes to prepare - * - * For Type 1/Type 2, the virtual start sector is the one that was - * originally submitted by the block layer for the ref_tag usage. Due to - * partitioning, MD/DM cloning, etc. the actual physical start sector is - * likely to be different. Since the physical start sector was submitted - * to the device, we should remap it back to virtual values expected by the - * block layer. - */ -static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes) +static void blk_set_ext_pi(struct blk_integrity_iter *iter, + struct crc64_pi_tuple *pi) { - struct blk_integrity *bi = &rq->q->limits.integrity; - unsigned intervals = nr_bytes >> bi->interval_exp; - const int tuple_sz = bi->tuple_size; - u32 ref_tag = t10_pi_ref_tag(rq); - u8 offset = bi->pi_offset; - struct bio *bio; + put_unaligned_be64(iter->csum, &pi->guard_tag); + put_unaligned_be16(0, &pi->app_tag); + put_unaligned_be48(iter->seed, &pi->ref_tag); +} - __rq_for_each_bio(bio, rq) { - struct bio_integrity_payload *bip = bio_integrity(bio); - u32 virt = bip_get_seed(bip) & 0xffffffff; - struct bio_vec iv; - struct bvec_iter iter; - - bip_for_each_vec(iv, bip, iter) { - unsigned int j; - void *p; - - p = bvec_kmap_local(&iv); - for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) { - struct t10_pi_tuple *pi = p + offset; - - if (be32_to_cpu(pi->ref_tag) == ref_tag) - pi->ref_tag = cpu_to_be32(virt); - virt++; - ref_tag++; - intervals--; - p += tuple_sz; - } - kunmap_local(p); - } - } +static void blk_set_pi(struct blk_integrity_iter *iter, + struct t10_pi_tuple *pi, __be16 csum) +{ + put_unaligned(csum, &pi->guard_tag); + put_unaligned_be16(0, &pi->app_tag); + put_unaligned_be32(iter->seed, &pi->ref_tag); } -static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len) +static void blk_set_t10_pi(struct blk_integrity_iter *iter, + struct t10_pi_tuple *pi) { - return cpu_to_be64(crc64_nvme(crc, data, len)); + blk_set_pi(iter, pi, cpu_to_be16((u16)iter->csum)); } -static void ext_pi_crc64_generate(struct blk_integrity_iter *iter, - struct blk_integrity *bi) +static void blk_set_ip_pi(struct blk_integrity_iter *iter, + struct t10_pi_tuple *pi) { - u8 offset = bi->pi_offset; - unsigned int i; + blk_set_pi(iter, pi, (__force __be16)(u16)iter->csum); +} - for (i = 0 ; i < iter->data_size ; i += iter->interval) { - struct crc64_pi_tuple *pi = iter->prot_buf + offset; +static void blk_integrity_set(struct blk_integrity_iter *iter, + union pi_tuple *tuple) +{ + switch (iter->bi->csum_type) { + case BLK_INTEGRITY_CSUM_CRC64: + return blk_set_ext_pi(iter, &tuple->crc64_pi); + case BLK_INTEGRITY_CSUM_CRC: + return blk_set_t10_pi(iter, &tuple->t10_pi); + case BLK_INTEGRITY_CSUM_IP: + return blk_set_ip_pi(iter, &tuple->t10_pi); + default: + WARN_ON_ONCE(1); + return; + } +} - pi->guard_tag = ext_pi_crc64(0, iter->data_buf, iter->interval); - if (offset) - pi->guard_tag = ext_pi_crc64(be64_to_cpu(pi->guard_tag), - iter->prot_buf, offset); - pi->app_tag = 0; +static blk_status_t blk_integrity_interval(struct blk_integrity_iter *iter, + bool verify) +{ + blk_status_t ret = BLK_STS_OK; + union pi_tuple tuple; + void *ptuple = &tuple; + struct bio_vec pbv; + + blk_integrity_csum_offset(iter); + pbv = bvec_iter_bvec(iter->bip->bip_vec, iter->prot_iter); + if (pbv.bv_len >= iter->bi->pi_tuple_size) { + ptuple = bvec_kmap_local(&pbv); + bvec_iter_advance_single(iter->bip->bip_vec, &iter->prot_iter, + iter->bi->metadata_size - iter->bi->pi_offset); + } else if (verify) { + blk_integrity_copy_to_tuple(iter->bip, &iter->prot_iter, + ptuple, iter->bi->pi_tuple_size); + } - if (bi->flags & BLK_INTEGRITY_REF_TAG) - put_unaligned_be48(iter->seed, pi->ref_tag); - else - put_unaligned_be48(0ULL, pi->ref_tag); + if (verify) + ret = blk_integrity_verify(iter, ptuple); + else + blk_integrity_set(iter, ptuple); - iter->data_buf += iter->interval; - iter->prot_buf += bi->tuple_size; - iter->seed++; + if (likely(ptuple != &tuple)) { + kunmap_local(ptuple); + } else if (!verify) { + blk_integrity_copy_from_tuple(iter->bip, &iter->prot_iter, + ptuple, iter->bi->pi_tuple_size); } + + iter->interval_remaining = 1 << iter->bi->interval_exp; + iter->csum = 0; + iter->seed++; + return ret; } -static bool ext_pi_ref_escape(const u8 ref_tag[6]) +static blk_status_t blk_integrity_iterate(struct bio *bio, + struct bvec_iter *data_iter, + bool verify) { - static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); + struct bio_integrity_payload *bip = bio_integrity(bio); + struct blk_integrity_iter iter = { + .bio = bio, + .bip = bip, + .bi = bi, + .data_iter = *data_iter, + .prot_iter = bip->bip_iter, + .interval_remaining = 1 << bi->interval_exp, + .seed = data_iter->bi_sector, + .csum = 0, + }; + blk_status_t ret = BLK_STS_OK; + + while (iter.data_iter.bi_size && ret == BLK_STS_OK) { + struct bio_vec bv = bvec_iter_bvec(iter.bio->bi_io_vec, + iter.data_iter); + void *kaddr = bvec_kmap_local(&bv); + void *data = kaddr; + unsigned int len; + + bvec_iter_advance_single(iter.bio->bi_io_vec, &iter.data_iter, + bv.bv_len); + while (bv.bv_len && ret == BLK_STS_OK) { + len = min(iter.interval_remaining, bv.bv_len); + blk_calculate_guard(&iter, data, len); + bv.bv_len -= len; + data += len; + iter.interval_remaining -= len; + if (!iter.interval_remaining) + ret = blk_integrity_interval(&iter, verify); + } + kunmap_local(kaddr); + } - return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0; + return ret; } -static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter, - struct blk_integrity *bi) -{ - u8 offset = bi->pi_offset; - unsigned int i; - - for (i = 0; i < iter->data_size; i += iter->interval) { - struct crc64_pi_tuple *pi = iter->prot_buf + offset; - u64 ref, seed; - __be64 csum; - - if (bi->flags & BLK_INTEGRITY_REF_TAG) { - if (pi->app_tag == T10_PI_APP_ESCAPE) - goto next; - - ref = get_unaligned_be48(pi->ref_tag); - seed = lower_48_bits(iter->seed); - if (ref != seed) { - pr_err("%s: ref tag error at location %llu (rcvd %llu)\n", - iter->disk_name, seed, ref); - return BLK_STS_PROTECTION; - } - } else { - if (pi->app_tag == T10_PI_APP_ESCAPE && - ext_pi_ref_escape(pi->ref_tag)) - goto next; - } +void bio_integrity_generate(struct bio *bio) +{ + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); - csum = ext_pi_crc64(0, iter->data_buf, iter->interval); - if (offset) - csum = ext_pi_crc64(be64_to_cpu(csum), iter->prot_buf, - offset); + switch (bi->csum_type) { + case BLK_INTEGRITY_CSUM_CRC64: + case BLK_INTEGRITY_CSUM_CRC: + case BLK_INTEGRITY_CSUM_IP: + blk_integrity_iterate(bio, &bio->bi_iter, false); + break; + default: + break; + } +} - if (pi->guard_tag != csum) { - pr_err("%s: guard tag error at sector %llu " \ - "(rcvd %016llx, want %016llx)\n", - iter->disk_name, (unsigned long long)iter->seed, - be64_to_cpu(pi->guard_tag), be64_to_cpu(csum)); - return BLK_STS_PROTECTION; - } +blk_status_t bio_integrity_verify(struct bio *bio, struct bvec_iter *saved_iter) +{ + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); -next: - iter->data_buf += iter->interval; - iter->prot_buf += bi->tuple_size; - iter->seed++; + switch (bi->csum_type) { + case BLK_INTEGRITY_CSUM_CRC64: + case BLK_INTEGRITY_CSUM_CRC: + case BLK_INTEGRITY_CSUM_IP: + return blk_integrity_iterate(bio, saved_iter, true); + default: + break; } return BLK_STS_OK; } -static void ext_pi_type1_prepare(struct request *rq) +/* + * Advance @iter past the protection offset for protection formats that + * contain front padding on the metadata region. + */ +static void blk_pi_advance_offset(struct blk_integrity *bi, + struct bio_integrity_payload *bip, + struct bvec_iter *iter) { - struct blk_integrity *bi = &rq->q->limits.integrity; - const int tuple_sz = bi->tuple_size; - u64 ref_tag = ext_pi_ref_tag(rq); - u8 offset = bi->pi_offset; - struct bio *bio; + unsigned int offset = bi->pi_offset; - __rq_for_each_bio(bio, rq) { - struct bio_integrity_payload *bip = bio_integrity(bio); - u64 virt = lower_48_bits(bip_get_seed(bip)); - struct bio_vec iv; - struct bvec_iter iter; + while (offset > 0) { + struct bio_vec bv = mp_bvec_iter_bvec(bip->bip_vec, *iter); + unsigned int len = min(bv.bv_len, offset); - /* Already remapped? */ - if (bip->bip_flags & BIP_MAPPED_INTEGRITY) - break; + bvec_iter_advance_single(bip->bip_vec, iter, len); + offset -= len; + } +} - bip_for_each_vec(iv, bip, iter) { - unsigned int j; - void *p; - - p = bvec_kmap_local(&iv); - for (j = 0; j < iv.bv_len; j += tuple_sz) { - struct crc64_pi_tuple *pi = p + offset; - u64 ref = get_unaligned_be48(pi->ref_tag); - - if (ref == virt) - put_unaligned_be48(ref_tag, pi->ref_tag); - virt++; - ref_tag++; - p += tuple_sz; - } - kunmap_local(p); - } +static void *blk_tuple_remap_begin(union pi_tuple *tuple, + struct blk_integrity *bi, + struct bio_integrity_payload *bip, + struct bvec_iter *iter) +{ + struct bvec_iter titer; + struct bio_vec pbv; - bip->bip_flags |= BIP_MAPPED_INTEGRITY; + blk_pi_advance_offset(bi, bip, iter); + pbv = bvec_iter_bvec(bip->bip_vec, *iter); + if (likely(pbv.bv_len >= bi->pi_tuple_size)) + return bvec_kmap_local(&pbv); + + /* + * We need to preserve the state of the original iter for the + * copy_from_tuple at the end, so make a temp iter for here. + */ + titer = *iter; + blk_integrity_copy_to_tuple(bip, &titer, tuple, bi->pi_tuple_size); + return tuple; +} + +static void blk_tuple_remap_end(union pi_tuple *tuple, void *ptuple, + struct blk_integrity *bi, + struct bio_integrity_payload *bip, + struct bvec_iter *iter) +{ + unsigned int len = bi->metadata_size - bi->pi_offset; + + if (likely(ptuple != tuple)) { + kunmap_local(ptuple); + } else { + blk_integrity_copy_from_tuple(bip, iter, ptuple, + bi->pi_tuple_size); + len -= bi->pi_tuple_size; } + + bvec_iter_advance(bip->bip_vec, iter, len); } -static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes) +static void blk_set_ext_unmap_ref(struct crc64_pi_tuple *pi, u64 virt, + u64 ref_tag) { - struct blk_integrity *bi = &rq->q->limits.integrity; - unsigned intervals = nr_bytes >> bi->interval_exp; - const int tuple_sz = bi->tuple_size; - u64 ref_tag = ext_pi_ref_tag(rq); - u8 offset = bi->pi_offset; - struct bio *bio; + u64 ref = get_unaligned_be48(&pi->ref_tag); - __rq_for_each_bio(bio, rq) { - struct bio_integrity_payload *bip = bio_integrity(bio); - u64 virt = lower_48_bits(bip_get_seed(bip)); - struct bio_vec iv; - struct bvec_iter iter; - - bip_for_each_vec(iv, bip, iter) { - unsigned int j; - void *p; - - p = bvec_kmap_local(&iv); - for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) { - struct crc64_pi_tuple *pi = p + offset; - u64 ref = get_unaligned_be48(pi->ref_tag); - - if (ref == ref_tag) - put_unaligned_be48(virt, pi->ref_tag); - virt++; - ref_tag++; - intervals--; - p += tuple_sz; - } - kunmap_local(p); - } - } + if (ref == lower_48_bits(ref_tag) && ref != lower_48_bits(virt)) + put_unaligned_be48(virt, pi->ref_tag); } -void blk_integrity_generate(struct bio *bio) +static void blk_set_t10_unmap_ref(struct t10_pi_tuple *pi, u32 virt, + u32 ref_tag) { - struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); - struct bio_integrity_payload *bip = bio_integrity(bio); - struct blk_integrity_iter iter; - struct bvec_iter bviter; - struct bio_vec bv; - - iter.disk_name = bio->bi_bdev->bd_disk->disk_name; - iter.interval = 1 << bi->interval_exp; - iter.seed = bio->bi_iter.bi_sector; - iter.prot_buf = bvec_virt(bip->bip_vec); - bio_for_each_segment(bv, bio, bviter) { - void *kaddr = bvec_kmap_local(&bv); + u32 ref = get_unaligned_be32(&pi->ref_tag); - iter.data_buf = kaddr; - iter.data_size = bv.bv_len; - switch (bi->csum_type) { - case BLK_INTEGRITY_CSUM_CRC64: - ext_pi_crc64_generate(&iter, bi); - break; - case BLK_INTEGRITY_CSUM_CRC: - case BLK_INTEGRITY_CSUM_IP: - t10_pi_generate(&iter, bi); - break; - default: - break; - } - kunmap_local(kaddr); + if (ref == ref_tag && ref != virt) + put_unaligned_be32(virt, &pi->ref_tag); +} + +static void blk_reftag_remap_complete(struct blk_integrity *bi, + union pi_tuple *tuple, u64 virt, u64 ref) +{ + switch (bi->csum_type) { + case BLK_INTEGRITY_CSUM_CRC64: + blk_set_ext_unmap_ref(&tuple->crc64_pi, virt, ref); + break; + case BLK_INTEGRITY_CSUM_CRC: + case BLK_INTEGRITY_CSUM_IP: + blk_set_t10_unmap_ref(&tuple->t10_pi, virt, ref); + break; + default: + WARN_ON_ONCE(1); + break; } } -void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter) +static void blk_set_ext_map_ref(struct crc64_pi_tuple *pi, u64 virt, + u64 ref_tag) { - struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); - struct bio_integrity_payload *bip = bio_integrity(bio); - struct blk_integrity_iter iter; - struct bvec_iter bviter; - struct bio_vec bv; + u64 ref = get_unaligned_be48(&pi->ref_tag); - /* - * At the moment verify is called bi_iter has been advanced during split - * and completion, so use the copy created during submission here. - */ - iter.disk_name = bio->bi_bdev->bd_disk->disk_name; - iter.interval = 1 << bi->interval_exp; - iter.seed = saved_iter->bi_sector; - iter.prot_buf = bvec_virt(bip->bip_vec); - __bio_for_each_segment(bv, bio, bviter, *saved_iter) { - void *kaddr = bvec_kmap_local(&bv); - blk_status_t ret = BLK_STS_OK; + if (ref == lower_48_bits(virt) && ref != ref_tag) + put_unaligned_be48(ref_tag, pi->ref_tag); +} - iter.data_buf = kaddr; - iter.data_size = bv.bv_len; - switch (bi->csum_type) { - case BLK_INTEGRITY_CSUM_CRC64: - ret = ext_pi_crc64_verify(&iter, bi); - break; - case BLK_INTEGRITY_CSUM_CRC: - case BLK_INTEGRITY_CSUM_IP: - ret = t10_pi_verify(&iter, bi); - break; - default: - break; - } - kunmap_local(kaddr); +static void blk_set_t10_map_ref(struct t10_pi_tuple *pi, u32 virt, u32 ref_tag) +{ + u32 ref = get_unaligned_be32(&pi->ref_tag); - if (ret) { - bio->bi_status = ret; - return; - } + if (ref == virt && ref != ref_tag) + put_unaligned_be32(ref_tag, &pi->ref_tag); +} + +static void blk_reftag_remap_prepare(struct blk_integrity *bi, + union pi_tuple *tuple, + u64 virt, u64 ref) +{ + switch (bi->csum_type) { + case BLK_INTEGRITY_CSUM_CRC64: + blk_set_ext_map_ref(&tuple->crc64_pi, virt, ref); + break; + case BLK_INTEGRITY_CSUM_CRC: + case BLK_INTEGRITY_CSUM_IP: + blk_set_t10_map_ref(&tuple->t10_pi, virt, ref); + break; + default: + WARN_ON_ONCE(1); + break; } } -void blk_integrity_prepare(struct request *rq) +static void __blk_reftag_remap(struct bio *bio, struct blk_integrity *bi, + unsigned *intervals, u64 *ref, bool prep) { - struct blk_integrity *bi = &rq->q->limits.integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); + struct bvec_iter iter = bip->bip_iter; + u64 virt = bip_get_seed(bip); + union pi_tuple *ptuple; + union pi_tuple tuple; - if (!(bi->flags & BLK_INTEGRITY_REF_TAG)) + if (prep && bip->bip_flags & BIP_MAPPED_INTEGRITY) { + *ref += bio->bi_iter.bi_size >> bi->interval_exp; return; + } - if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64) - ext_pi_type1_prepare(rq); - else - t10_pi_type1_prepare(rq); + while (iter.bi_size && *intervals) { + ptuple = blk_tuple_remap_begin(&tuple, bi, bip, &iter); + + if (prep) + blk_reftag_remap_prepare(bi, ptuple, virt, *ref); + else + blk_reftag_remap_complete(bi, ptuple, virt, *ref); + + blk_tuple_remap_end(&tuple, ptuple, bi, bip, &iter); + (*intervals)--; + (*ref)++; + virt++; + } + + if (prep) + bip->bip_flags |= BIP_MAPPED_INTEGRITY; } -void blk_integrity_complete(struct request *rq, unsigned int nr_bytes) +static void blk_integrity_remap(struct request *rq, unsigned int nr_bytes, + bool prep) { struct blk_integrity *bi = &rq->q->limits.integrity; + u64 ref = blk_rq_pos(rq) >> (bi->interval_exp - SECTOR_SHIFT); + unsigned intervals = nr_bytes >> bi->interval_exp; + struct bio *bio; if (!(bi->flags & BLK_INTEGRITY_REF_TAG)) return; - if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64) - ext_pi_type1_complete(rq, nr_bytes); - else - t10_pi_type1_complete(rq, nr_bytes); + __rq_for_each_bio(bio, rq) { + __blk_reftag_remap(bio, bi, &intervals, &ref, prep); + if (!intervals) + break; + } +} + +void blk_integrity_prepare(struct request *rq) +{ + blk_integrity_remap(rq, blk_rq_bytes(rq), true); +} + +void blk_integrity_complete(struct request *rq, unsigned int nr_bytes) +{ + blk_integrity_remap(rq, nr_bytes, false); } |