diff options
Diffstat (limited to 'drivers/md')
34 files changed, 1457 insertions, 564 deletions
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig index a3fcdca7e6db..df27c7d066d2 100644 --- a/drivers/md/Kconfig +++ b/drivers/md/Kconfig @@ -315,6 +315,17 @@ config DM_CRYPT If unsure, say N. +config DM_INLINECRYPT + tristate "Inline encryption target support" + depends on BLK_DEV_DM + depends on (KEYS || KEYS=n) + depends on BLK_INLINE_ENCRYPTION + help + This device-mapper target is similar to dm-crypt, but it uses the + blk-crypto API instead of the regular crypto API. This allows it to + take advantage of inline encryption hardware such as that commonly + built into UFS host controllers. + config DM_SNAPSHOT tristate "Snapshot target" depends on BLK_DEV_DM diff --git a/drivers/md/Makefile b/drivers/md/Makefile index c338cc6fbe2e..517d1f7d8288 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -55,6 +55,7 @@ obj-$(CONFIG_DM_UNSTRIPED) += dm-unstripe.o obj-$(CONFIG_DM_BUFIO) += dm-bufio.o obj-$(CONFIG_DM_BIO_PRISON) += dm-bio-prison.o obj-$(CONFIG_DM_CRYPT) += dm-crypt.o +obj-$(CONFIG_DM_INLINECRYPT) += dm-inlinecrypt.o obj-$(CONFIG_DM_DELAY) += dm-delay.o obj-$(CONFIG_DM_DUST) += dm-dust.o obj-$(CONFIG_DM_FLAKEY) += dm-flakey.o diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 26fedf5883ef..a458b9fd2fcd 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -2238,7 +2238,9 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c struct dm_io_region io_reg = { .bdev = c->bdev, .sector = block_to_sector(c, block), - .count = block_to_sector(c, count), + .count = likely(c->sectors_per_block_bits >= 0) ? + count << c->sectors_per_block_bits : + count * (c->block_size >> SECTOR_SHIFT), }; if (WARN_ON_ONCE(dm_bufio_in_request())) diff --git a/drivers/md/dm-cache-policy-smq.c b/drivers/md/dm-cache-policy-smq.c index 1ae304c2f573..7411a88885b8 100644 --- a/drivers/md/dm-cache-policy-smq.c +++ b/drivers/md/dm-cache-policy-smq.c @@ -22,6 +22,16 @@ /*----------------------------------------------------------------*/ /* + * Maximum number of concurrent background work items (promotions, + * demotions, writebacks) that can be queued in the background tracker. + * Tuneable via the module parameter smq_max_background_work. + * Only affects newly created cache devices. + */ +static unsigned int smq_max_background_work = 4096; +module_param(smq_max_background_work, uint, 0644); +MODULE_PARM_DESC(smq_max_background_work, "Max concurrent background work items"); + +/* * Safe division functions that return zero on divide by zero. */ static unsigned int safe_div(unsigned int n, unsigned int d) @@ -1820,7 +1830,7 @@ __smq_create(dm_cblock_t cache_size, sector_t origin_size, sector_t cache_block_ mq->next_hotspot_period = jiffies; mq->next_cache_period = jiffies; - mq->bg_work = btracker_create(4096); /* FIXME: hard coded value */ + mq->bg_work = btracker_create(max(1u, smq_max_background_work)); if (!mq->bg_work) goto bad_btracker; diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c index 05285c04ff2c..7fe4d19ade4f 100644 --- a/drivers/md/dm-era-target.c +++ b/drivers/md/dm-era-target.c @@ -810,8 +810,10 @@ static struct era_metadata *metadata_open(struct block_device *bdev, int r; struct era_metadata *md = kzalloc_obj(*md); - if (!md) - return NULL; + if (!md) { + DMERR("could not allocate metadata struct"); + return ERR_PTR(-ENOMEM); + } md->bdev = bdev; md->block_size = block_size; @@ -1229,6 +1231,7 @@ static dm_block_t get_block(struct era *era, struct bio *bio) static void remap_to_origin(struct era *era, struct bio *bio) { bio_set_dev(bio, era->origin_dev->bdev); + bio->bi_iter.bi_sector = dm_target_offset(era->ti, bio->bi_iter.bi_sector); } /* @@ -1486,7 +1489,7 @@ static int era_ctr(struct dm_target *ti, unsigned int argc, char **argv) if (r) { ti->error = "Error opening metadata device"; era_destroy(era); - return -EINVAL; + return r; } r = dm_get_device(ti, argv[1], BLK_OPEN_READ | BLK_OPEN_WRITE, @@ -1494,7 +1497,7 @@ static int era_ctr(struct dm_target *ti, unsigned int argc, char **argv) if (r) { ti->error = "Error opening data device"; era_destroy(era); - return -EINVAL; + return r; } r = sscanf(argv[2], "%u%c", &era->sectors_per_block, &dummy); @@ -1508,7 +1511,7 @@ static int era_ctr(struct dm_target *ti, unsigned int argc, char **argv) if (r) { ti->error = "could not set max io len"; era_destroy(era); - return -EINVAL; + return r; } if (!valid_block_size(era->sectors_per_block)) { @@ -1560,7 +1563,7 @@ static void era_dtr(struct dm_target *ti) static int era_map(struct dm_target *ti, struct bio *bio) { struct era *era = ti->private; - dm_block_t block = get_block(era, bio); + dm_block_t block; /* * All bios get remapped to the origin device. We do this now, but @@ -1568,6 +1571,7 @@ static int era_map(struct dm_target *ti, struct bio *bio) * block is marked in this era. */ remap_to_origin(era, bio); + block = get_block(era, bio); /* * REQ_PREFLUSH bios carry no data, so we're not interested in them. diff --git a/drivers/md/dm-ima.c b/drivers/md/dm-ima.c index 9495ca035056..5e7232582feb 100644 --- a/drivers/md/dm-ima.c +++ b/drivers/md/dm-ima.c @@ -21,25 +21,32 @@ * character, so that they don't interfere with the construction of key-value pairs, * and clients can split the key1=val1,key2=val2,key3=val3; pairs properly. */ -static void fix_separator_chars(char **buf) +static void fix_separator_chars(char *buf) { - int l = strlen(*buf); + int l = strlen(buf); int i, j, sp = 0; for (i = 0; i < l; i++) - if ((*buf)[i] == '\\' || (*buf)[i] == ';' || (*buf)[i] == '=' || (*buf)[i] == ',') + if (buf[i] == '\\' || buf[i] == ';' || buf[i] == '=' || buf[i] == ',') sp++; if (!sp) return; + buf[l + sp] = '\0'; for (i = l-1, j = i+sp; i >= 0; i--) { - (*buf)[j--] = (*buf)[i]; - if ((*buf)[i] == '\\' || (*buf)[i] == ';' || (*buf)[i] == '=' || (*buf)[i] == ',') - (*buf)[j--] = '\\'; + buf[j--] = buf[i]; + if (buf[i] == '\\' || buf[i] == ';' || buf[i] == '=' || buf[i] == ',') + buf[j--] = '\\'; } } +static void fix_context_strings(struct dm_ima_context *context) +{ + fix_separator_chars(context->dev_name); + fix_separator_chars(context->dev_uuid); +} + /* * Internal function to allocate memory for IMA measurements. */ @@ -59,68 +66,85 @@ static void *dm_ima_alloc(size_t len, bool noio) return ptr; } -/* - * Internal function to allocate and copy name and uuid for IMA measurements. - */ -static int dm_ima_alloc_and_copy_name_uuid(struct mapped_device *md, char **dev_name, - char **dev_uuid, bool noio) +void dm_ima_init(struct mapped_device *md) { - int r; - *dev_name = dm_ima_alloc(DM_NAME_LEN*2, noio); - if (!(*dev_name)) { - r = -ENOMEM; - goto error; - } + md->ima.update_idx = 0; + md->ima.measure_idx = 0; + init_waitqueue_head(&md->ima.ima_wq); + spin_lock_init(&md->ima.ima_lock); +} - *dev_uuid = dm_ima_alloc(DM_UUID_LEN*2, noio); - if (!(*dev_uuid)) { - r = -ENOMEM; - goto error; - } +void dm_ima_alloc_context(struct dm_ima_context **context, bool noio) +{ + *context = dm_ima_alloc(sizeof(struct dm_ima_context), noio); +} - r = dm_copy_name_and_uuid(md, *dev_name, *dev_uuid); - if (r) - goto error; +void dm_ima_free_context(struct dm_ima_context *context) +{ + if (likely(context)) { + kfree(context->table.device_metadata); + kfree(context->table.hash); + kfree(context); + } +} - fix_separator_chars(dev_name); - fix_separator_chars(dev_uuid); +static void wait_to_measure(struct dm_ima_measurements *ima, + unsigned int update_idx) +{ + spin_lock_irq(&ima->ima_lock); + wait_event_lock_irq(ima->ima_wq, + ima->measure_idx == update_idx, + ima->ima_lock); + spin_unlock_irq(&ima->ima_lock); +} - return 0; -error: - kfree(*dev_name); - kfree(*dev_uuid); - *dev_name = NULL; - *dev_uuid = NULL; - return r; +static void wake_next_measure(struct dm_ima_measurements *ima) +{ + spin_lock_irq(&ima->ima_lock); + ima->measure_idx++; + spin_unlock_irq(&ima->ima_lock); + wake_up_all(&ima->ima_wq); } /* - * Internal function to allocate and copy device data for IMA measurements. + * Helper function for swapping the table, to make sure that the + * correct table metadata is saved and restored. */ -static int dm_ima_alloc_and_copy_device_data(struct mapped_device *md, char **device_data, - unsigned int num_targets, bool noio) +void dm_ima_context_table_op(struct mapped_device *md, + struct dm_ima_context *context, + enum dm_ima_table_op op) { - char *dev_name = NULL, *dev_uuid = NULL; - int r; + struct dm_ima_measurements *ima = &md->ima; - r = dm_ima_alloc_and_copy_name_uuid(md, &dev_name, &dev_uuid, noio); - if (r) - return r; + if (unlikely(!context)) + return; - *device_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio); - if (!(*device_data)) { - r = -ENOMEM; - goto error; + wait_to_measure(ima, context->update_idx); + + if (op == DM_IMA_TABLE_SAVE) { + context->table = ima->inactive_table; + memset(&ima->inactive_table, 0, sizeof(ima->inactive_table)); + } else { + ima->inactive_table = context->table; + memset(&context->table, 0, sizeof(context->table)); } - scnprintf(*device_data, DM_IMA_DEVICE_BUF_LEN, + wake_next_measure(ima); +} + +/* + * Internal function to copy device data for IMA measurements. + */ +static void dm_ima_copy_device_data(struct mapped_device *md, char *device_data, + struct dm_ima_context *context, + unsigned int num_targets) +{ + memset(device_data, 0, DM_IMA_DEVICE_BUF_LEN); + scnprintf(device_data, DM_IMA_DEVICE_BUF_LEN, "name=%s,uuid=%s,major=%d,minor=%d,minor_count=%d,num_targets=%u;", - dev_name, dev_uuid, md->disk->major, md->disk->first_minor, - md->disk->minors, num_targets); -error: - kfree(dev_name); - kfree(dev_uuid); - return r; + context->dev_name, context->dev_uuid, md->disk->major, + md->disk->first_minor, md->disk->minors, num_targets); + } /* @@ -141,42 +165,21 @@ static void dm_ima_measure_data(const char *event_name, const void *buf, size_t memalloc_noio_restore(noio_flag); } -/* - * Internal function to allocate and copy current device capacity for IMA measurements. - */ -static int dm_ima_alloc_and_copy_capacity_str(struct mapped_device *md, char **capacity_str, - bool noio) -{ - sector_t capacity; - - capacity = get_capacity(md->disk); - - *capacity_str = dm_ima_alloc(DM_IMA_DEVICE_CAPACITY_BUF_LEN, noio); - if (!(*capacity_str)) - return -ENOMEM; - - return scnprintf(*capacity_str, DM_IMA_DEVICE_BUF_LEN, "current_device_capacity=%llu;", - capacity); -} - -/* - * Initialize/reset the dm ima related data structure variables. - */ -void dm_ima_reset_data(struct mapped_device *md) +static sector_t dm_ima_capacity(struct mapped_device *md) { - memset(&(md->ima), 0, sizeof(md->ima)); - md->ima.dm_version_str_len = strlen(DM_IMA_VERSION_STR); + return (md->ima.active_table.device_metadata) ? + md->ima.active_table.capacity : get_capacity(md->disk); } /* * Build up the IMA data for each target, and finally measure. */ -void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_flags) +void dm_ima_measure_on_table_load(struct dm_table *table, + struct dm_ima_context *context) { size_t device_data_buf_len, target_metadata_buf_len, target_data_buf_len, l = 0; char *target_metadata_buf = NULL, *target_data_buf = NULL, *digest_buf = NULL; char *ima_buf = NULL, *device_data_buf = NULL; - int last_target_measured = -1; status_type_t type = STATUSTYPE_IMA; size_t cur_total_buf_len = 0; unsigned int num_targets, i; @@ -185,9 +188,14 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl bool noio = false; char table_load_event_name[] = "dm_table_load"; + if (unlikely(!context)) + return; + + wait_to_measure(&table->md->ima, context->update_idx); + ima_buf = dm_ima_alloc(DM_IMA_MEASUREMENT_BUF_LEN, noio); if (!ima_buf) - return; + goto error; target_metadata_buf = dm_ima_alloc(DM_IMA_TARGET_METADATA_BUF_LEN, noio); if (!target_metadata_buf) @@ -199,13 +207,18 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl num_targets = table->num_targets; - if (dm_ima_alloc_and_copy_device_data(table->md, &device_data_buf, num_targets, noio)) + device_data_buf = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio); + if (!device_data_buf) goto error; + fix_context_strings(context); + dm_ima_copy_device_data(table->md, device_data_buf, context, + num_targets); + sha256_init(&hash_ctx); - memcpy(ima_buf + l, DM_IMA_VERSION_STR, table->md->ima.dm_version_str_len); - l += table->md->ima.dm_version_str_len; + memcpy(ima_buf + l, DM_IMA_VERSION_STR, strlen(DM_IMA_VERSION_STR)); + l += strlen(DM_IMA_VERSION_STR); device_data_buf_len = strlen(device_data_buf); memcpy(ima_buf + l, device_data_buf, device_data_buf_len); @@ -214,8 +227,6 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl for (i = 0; i < num_targets; i++) { struct dm_target *ti = dm_table_get_target(table, i); - last_target_measured = 0; - /* * First retrieve the target metadata. */ @@ -229,7 +240,7 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl * Then retrieve the actual target data. */ if (ti->type->status) - ti->type->status(ti, type, status_flags, target_data_buf, + ti->type->status(ti, type, 0, target_data_buf, DM_IMA_TARGET_DATA_BUF_LEN); else target_data_buf[0] = '\0'; @@ -260,19 +271,11 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl * prefix, so that multiple records from the same "dm_table_load" for * a given device can be linked together. */ - memcpy(ima_buf + l, DM_IMA_VERSION_STR, table->md->ima.dm_version_str_len); - l += table->md->ima.dm_version_str_len; + memcpy(ima_buf + l, DM_IMA_VERSION_STR, strlen(DM_IMA_VERSION_STR)); + l += strlen(DM_IMA_VERSION_STR); memcpy(ima_buf + l, device_data_buf, device_data_buf_len); l += device_data_buf_len; - - /* - * If this iteration of the for loop turns out to be the last target - * in the table, dm_ima_measure_data("dm_table_load", ...) doesn't need - * to be called again, just the hash needs to be finalized. - * "last_target_measured" tracks this state. - */ - last_target_measured = 1; } /* @@ -286,11 +289,8 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl l += target_data_buf_len; } - if (!last_target_measured) { - dm_ima_measure_data(table_load_event_name, ima_buf, l, noio); - - sha256_update(&hash_ctx, (const u8 *)ima_buf, l); - } + dm_ima_measure_data(table_load_event_name, ima_buf, l, noio); + sha256_update(&hash_ctx, (const u8 *)ima_buf, l); /* * Finalize the table hash, and store it in table->md->ima.inactive_table.hash, @@ -304,17 +304,14 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl if (!digest_buf) goto error; - if (table->md->ima.active_table.hash != table->md->ima.inactive_table.hash) - kfree(table->md->ima.inactive_table.hash); - + kfree(table->md->ima.inactive_table.hash); table->md->ima.inactive_table.hash = digest_buf; table->md->ima.inactive_table.hash_len = strlen(digest_buf); table->md->ima.inactive_table.num_targets = num_targets; + table->md->ima.inactive_table.capacity = dm_table_get_size(table); - if (table->md->ima.active_table.device_metadata != - table->md->ima.inactive_table.device_metadata) - kfree(table->md->ima.inactive_table.device_metadata); + kfree(table->md->ima.inactive_table.device_metadata); table->md->ima.inactive_table.device_metadata = device_data_buf; table->md->ima.inactive_table.device_metadata_len = device_data_buf_len; @@ -326,66 +323,55 @@ exit: kfree(ima_buf); kfree(target_metadata_buf); kfree(target_data_buf); + + wake_next_measure(&table->md->ima); } /* * Measure IMA data on device resume. */ -void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap) +void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap, + struct dm_ima_context *context) { - char *device_table_data, *dev_name = NULL, *dev_uuid = NULL, *capacity_str = NULL; + char *device_table_data = NULL; char active[] = "active_table_hash="; unsigned int active_len = strlen(active); unsigned int l = 0; bool noio = true; bool nodata = true; - int capacity_len; - device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio); - if (!device_table_data) + if (unlikely(!context)) return; - capacity_len = dm_ima_alloc_and_copy_capacity_str(md, &capacity_str, noio); - if (capacity_len < 0) - goto error; - - memcpy(device_table_data + l, DM_IMA_VERSION_STR, md->ima.dm_version_str_len); - l += md->ima.dm_version_str_len; + wait_to_measure(&md->ima, context->update_idx); if (swap) { - if (md->ima.active_table.hash != md->ima.inactive_table.hash) - kfree(md->ima.active_table.hash); - - md->ima.active_table.hash = NULL; - md->ima.active_table.hash_len = 0; - - if (md->ima.active_table.device_metadata != - md->ima.inactive_table.device_metadata) - kfree(md->ima.active_table.device_metadata); - - md->ima.active_table.device_metadata = NULL; - md->ima.active_table.device_metadata_len = 0; - md->ima.active_table.num_targets = 0; - - if (md->ima.inactive_table.hash) { - md->ima.active_table.hash = md->ima.inactive_table.hash; - md->ima.active_table.hash_len = md->ima.inactive_table.hash_len; - md->ima.inactive_table.hash = NULL; - md->ima.inactive_table.hash_len = 0; - } - - if (md->ima.inactive_table.device_metadata) { - md->ima.active_table.device_metadata = - md->ima.inactive_table.device_metadata; - md->ima.active_table.device_metadata_len = - md->ima.inactive_table.device_metadata_len; - md->ima.active_table.num_targets = md->ima.inactive_table.num_targets; - md->ima.inactive_table.device_metadata = NULL; - md->ima.inactive_table.device_metadata_len = 0; - md->ima.inactive_table.num_targets = 0; + kfree(md->ima.active_table.hash); + kfree(md->ima.active_table.device_metadata); + md->ima.active_table = context->table; + memset(&context->table, 0, sizeof(context->table)); + if (md->ima.active_table.device_metadata) { + /* + * A rename could have happened while the swap was + * going on. In that case, the saved table info would + * still have the old name. Update the metadata to be + * sure that it has the current name + */ + struct dm_ima_device_table_metadata *table = &md->ima.active_table; + fix_context_strings(context); + dm_ima_copy_device_data(md, table->device_metadata, + context, table->num_targets); + table->device_metadata_len = strlen(table->device_metadata); } } + device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio); + if (!device_table_data) + goto error; + + memcpy(device_table_data + l, DM_IMA_VERSION_STR, strlen(DM_IMA_VERSION_STR)); + l += strlen(DM_IMA_VERSION_STR); + if (md->ima.active_table.device_metadata) { memcpy(device_table_data + l, md->ima.active_table.device_metadata, md->ima.active_table.device_metadata_len); @@ -409,59 +395,54 @@ void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap) } if (nodata) { - if (dm_ima_alloc_and_copy_name_uuid(md, &dev_name, &dev_uuid, noio)) - goto error; - + fix_context_strings(context); l = scnprintf(device_table_data, DM_IMA_DEVICE_BUF_LEN, "%sname=%s,uuid=%s;device_resume=no_data;", - DM_IMA_VERSION_STR, dev_name, dev_uuid); + DM_IMA_VERSION_STR, context->dev_name, + context->dev_uuid); } - - memcpy(device_table_data + l, capacity_str, capacity_len); - l += capacity_len; + l += scnprintf(device_table_data + l, DM_IMA_DEVICE_BUF_LEN - l, + "current_device_capacity=%llu;", dm_ima_capacity(md)); dm_ima_measure_data("dm_device_resume", device_table_data, l, noio); - kfree(dev_name); - kfree(dev_uuid); error: - kfree(capacity_str); kfree(device_table_data); + + wake_next_measure(&md->ima); } /* * Measure IMA data on remove. */ -void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all) +void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all, + struct dm_ima_context *context, + unsigned int idx) { - char *device_table_data, *dev_name = NULL, *dev_uuid = NULL, *capacity_str = NULL; + char *device_table_data; char active_table_str[] = "active_table_hash="; char inactive_table_str[] = "inactive_table_hash="; char device_active_str[] = "device_active_metadata="; char device_inactive_str[] = "device_inactive_metadata="; - char remove_all_str[] = "remove_all="; unsigned int active_table_len = strlen(active_table_str); unsigned int inactive_table_len = strlen(inactive_table_str); unsigned int device_active_len = strlen(device_active_str); unsigned int device_inactive_len = strlen(device_inactive_str); - unsigned int remove_all_len = strlen(remove_all_str); unsigned int l = 0; bool noio = true; bool nodata = true; - int capacity_len; - device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN*2, noio); - if (!device_table_data) + wait_to_measure(&md->ima, idx); + + if (unlikely(!context)) goto exit; - capacity_len = dm_ima_alloc_and_copy_capacity_str(md, &capacity_str, noio); - if (capacity_len < 0) { - kfree(device_table_data); + device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN*2, noio); + if (!device_table_data) goto exit; - } - memcpy(device_table_data + l, DM_IMA_VERSION_STR, md->ima.dm_version_str_len); - l += md->ima.dm_version_str_len; + memcpy(device_table_data + l, DM_IMA_VERSION_STR, strlen(DM_IMA_VERSION_STR)); + l += strlen(DM_IMA_VERSION_STR); if (md->ima.active_table.device_metadata) { memcpy(device_table_data + l, device_active_str, device_active_len); @@ -518,68 +499,57 @@ void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all) * in IMA measurements. */ if (nodata) { - if (dm_ima_alloc_and_copy_name_uuid(md, &dev_name, &dev_uuid, noio)) - goto error; - + fix_context_strings(context); l = scnprintf(device_table_data, DM_IMA_DEVICE_BUF_LEN, "%sname=%s,uuid=%s;device_remove=no_data;", - DM_IMA_VERSION_STR, dev_name, dev_uuid); + DM_IMA_VERSION_STR, context->dev_name, + context->dev_uuid); } - memcpy(device_table_data + l, remove_all_str, remove_all_len); - l += remove_all_len; - memcpy(device_table_data + l, remove_all ? "y;" : "n;", 2); - l += 2; - - memcpy(device_table_data + l, capacity_str, capacity_len); - l += capacity_len; + l += scnprintf(device_table_data + l, (DM_IMA_DEVICE_BUF_LEN * 2) - l, + "remove_all=%c;current_device_capacity=%llu;", + remove_all ? 'y' : 'n', dm_ima_capacity(md)); dm_ima_measure_data("dm_device_remove", device_table_data, l, noio); -error: kfree(device_table_data); - kfree(capacity_str); exit: kfree(md->ima.active_table.device_metadata); - - if (md->ima.active_table.device_metadata != - md->ima.inactive_table.device_metadata) - kfree(md->ima.inactive_table.device_metadata); + kfree(md->ima.inactive_table.device_metadata); kfree(md->ima.active_table.hash); + kfree(md->ima.inactive_table.hash); - if (md->ima.active_table.hash != md->ima.inactive_table.hash) - kfree(md->ima.inactive_table.hash); - - dm_ima_reset_data(md); + memset(&md->ima.active_table, 0, sizeof(md->ima.active_table)); + memset(&md->ima.inactive_table, 0, sizeof(md->ima.inactive_table)); - kfree(dev_name); - kfree(dev_uuid); + wake_next_measure(&md->ima); } /* * Measure ima data on table clear. */ -void dm_ima_measure_on_table_clear(struct mapped_device *md, bool new_map) +void dm_ima_measure_on_table_clear(struct mapped_device *md, + struct dm_ima_context *context) { unsigned int l = 0; - char *device_table_data = NULL, *dev_name = NULL, *dev_uuid = NULL, *capacity_str = NULL; + char *device_table_data = NULL; char inactive_str[] = "inactive_table_hash="; unsigned int inactive_len = strlen(inactive_str); bool noio = true; bool nodata = true; - int capacity_len; - device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio); - if (!device_table_data) + if (unlikely(!context)) return; - capacity_len = dm_ima_alloc_and_copy_capacity_str(md, &capacity_str, noio); - if (capacity_len < 0) - goto error1; + wait_to_measure(&md->ima, context->update_idx); - memcpy(device_table_data + l, DM_IMA_VERSION_STR, md->ima.dm_version_str_len); - l += md->ima.dm_version_str_len; + device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio); + if (!device_table_data) + goto error; + + memcpy(device_table_data + l, DM_IMA_VERSION_STR, strlen(DM_IMA_VERSION_STR)); + l += strlen(DM_IMA_VERSION_STR); if (md->ima.inactive_table.device_metadata_len && md->ima.inactive_table.hash_len) { @@ -602,101 +572,79 @@ void dm_ima_measure_on_table_clear(struct mapped_device *md, bool new_map) } if (nodata) { - if (dm_ima_alloc_and_copy_name_uuid(md, &dev_name, &dev_uuid, noio)) - goto error2; - + fix_context_strings(context); l = scnprintf(device_table_data, DM_IMA_DEVICE_BUF_LEN, "%sname=%s,uuid=%s;table_clear=no_data;", - DM_IMA_VERSION_STR, dev_name, dev_uuid); + DM_IMA_VERSION_STR, context->dev_name, + context->dev_uuid); } - memcpy(device_table_data + l, capacity_str, capacity_len); - l += capacity_len; + l += scnprintf(device_table_data + l, DM_IMA_DEVICE_BUF_LEN - l, + "current_device_capacity=%llu;", dm_ima_capacity(md)); dm_ima_measure_data("dm_table_clear", device_table_data, l, noio); - if (new_map) { - if (md->ima.inactive_table.hash && - md->ima.inactive_table.hash != md->ima.active_table.hash) - kfree(md->ima.inactive_table.hash); - - md->ima.inactive_table.hash = NULL; - md->ima.inactive_table.hash_len = 0; - - if (md->ima.inactive_table.device_metadata && - md->ima.inactive_table.device_metadata != md->ima.active_table.device_metadata) - kfree(md->ima.inactive_table.device_metadata); - - md->ima.inactive_table.device_metadata = NULL; - md->ima.inactive_table.device_metadata_len = 0; - md->ima.inactive_table.num_targets = 0; - - if (md->ima.active_table.hash) { - md->ima.inactive_table.hash = md->ima.active_table.hash; - md->ima.inactive_table.hash_len = md->ima.active_table.hash_len; - } - - if (md->ima.active_table.device_metadata) { - md->ima.inactive_table.device_metadata = - md->ima.active_table.device_metadata; - md->ima.inactive_table.device_metadata_len = - md->ima.active_table.device_metadata_len; - md->ima.inactive_table.num_targets = - md->ima.active_table.num_targets; - } - } +error: + kfree(md->ima.inactive_table.hash); + kfree(md->ima.inactive_table.device_metadata); + memset(&md->ima.inactive_table, 0, sizeof(md->ima.inactive_table)); - kfree(dev_name); - kfree(dev_uuid); -error2: - kfree(capacity_str); -error1: kfree(device_table_data); + + wake_next_measure(&md->ima); } /* * Measure IMA data on device rename. */ -void dm_ima_measure_on_device_rename(struct mapped_device *md) +void dm_ima_measure_on_device_rename(struct mapped_device *md, + struct dm_ima_context *context) { - char *old_device_data = NULL, *new_device_data = NULL, *combined_device_data = NULL; - char *new_dev_name = NULL, *new_dev_uuid = NULL, *capacity_str = NULL; + char *old_device_data = NULL; + char *combined_device_data = NULL; bool noio = true; int len; + struct dm_ima_device_table_metadata *table; - if (dm_ima_alloc_and_copy_device_data(md, &new_device_data, - md->ima.active_table.num_targets, noio)) + if (unlikely(!context)) return; - if (dm_ima_alloc_and_copy_name_uuid(md, &new_dev_name, &new_dev_uuid, noio)) - goto error; + wait_to_measure(&md->ima, context->update_idx); + + fix_context_strings(context); combined_device_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN * 2, noio); if (!combined_device_data) - goto error; - - if (dm_ima_alloc_and_copy_capacity_str(md, &capacity_str, noio) < 0) - goto error; - - old_device_data = md->ima.active_table.device_metadata; - - md->ima.active_table.device_metadata = new_device_data; - md->ima.active_table.device_metadata_len = strlen(new_device_data); + goto exit; + if (md->ima.active_table.device_metadata) + old_device_data = md->ima.active_table.device_metadata; + else if (md->ima.inactive_table.device_metadata) + old_device_data = md->ima.inactive_table.device_metadata; + else + old_device_data = "device_rename=no_data;"; len = scnprintf(combined_device_data, DM_IMA_DEVICE_BUF_LEN * 2, - "%s%snew_name=%s,new_uuid=%s;%s", DM_IMA_VERSION_STR, old_device_data, - new_dev_name, new_dev_uuid, capacity_str); + "%s%snew_name=%s,new_uuid=%s;current_device_capacity=%llu;", + DM_IMA_VERSION_STR, old_device_data, context->dev_name, + context->dev_uuid, dm_ima_capacity(md)); dm_ima_measure_data("dm_device_rename", combined_device_data, len, noio); + kfree(combined_device_data); - goto exit; - -error: - kfree(new_device_data); exit: - kfree(capacity_str); - kfree(combined_device_data); - kfree(old_device_data); - kfree(new_dev_name); - kfree(new_dev_uuid); + if (md->ima.active_table.device_metadata) { + table = &md->ima.active_table; + dm_ima_copy_device_data(md, table->device_metadata, context, + table->num_targets); + table->device_metadata_len = strlen(table->device_metadata); + } + + if (md->ima.inactive_table.device_metadata) { + table = &md->ima.inactive_table; + dm_ima_copy_device_data(md, table->device_metadata, context, + table->num_targets); + table->device_metadata_len = strlen(table->device_metadata); + } + + wake_next_measure(&md->ima); } diff --git a/drivers/md/dm-ima.h b/drivers/md/dm-ima.h index a403deca6093..0ec013d1545c 100644 --- a/drivers/md/dm-ima.h +++ b/drivers/md/dm-ima.h @@ -24,6 +24,11 @@ __dm_ima_str(DM_VERSION_MINOR) "." \ __dm_ima_str(DM_VERSION_PATCHLEVEL) ";" +enum dm_ima_table_op { + DM_IMA_TABLE_SAVE, + DM_IMA_TABLE_RESTORE, +}; + #ifdef CONFIG_IMA struct dm_ima_device_table_metadata { @@ -36,6 +41,7 @@ struct dm_ima_device_table_metadata { char *device_metadata; unsigned int device_metadata_len; unsigned int num_targets; + sector_t capacity; /* * Contains the sha256 hashes of the IMA measurements of the target @@ -45,31 +51,67 @@ struct dm_ima_device_table_metadata { unsigned int hash_len; }; +struct dm_ima_context { + struct dm_ima_device_table_metadata table; + unsigned int update_idx; + char dev_name[DM_NAME_LEN*2]; + char dev_uuid[DM_UUID_LEN*2]; +}; + /* * This structure contains device metadata, and table hash for * active and inactive tables for ima measurements. */ struct dm_ima_measurements { + unsigned int update_idx; + unsigned int measure_idx; + struct wait_queue_head ima_wq; + spinlock_t ima_lock; struct dm_ima_device_table_metadata active_table; struct dm_ima_device_table_metadata inactive_table; - unsigned int dm_version_str_len; }; -void dm_ima_reset_data(struct mapped_device *md); -void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_flags); -void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap); -void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all); -void dm_ima_measure_on_table_clear(struct mapped_device *md, bool new_map); -void dm_ima_measure_on_device_rename(struct mapped_device *md); +void dm_ima_init(struct mapped_device *md); +void dm_ima_alloc_context(struct dm_ima_context **context, bool noio); +void dm_ima_free_context(struct dm_ima_context *context); +void dm_ima_context_table_op(struct mapped_device *md, + struct dm_ima_context *context, + enum dm_ima_table_op op); +void dm_ima_measure_on_table_load(struct dm_table *table, + struct dm_ima_context *context); +void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap, + struct dm_ima_context *context); +void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all, + struct dm_ima_context *context, + unsigned int idx); +void dm_ima_measure_on_table_clear(struct mapped_device *md, + struct dm_ima_context *context); +void dm_ima_measure_on_device_rename(struct mapped_device *md, + struct dm_ima_context *context); #else -static inline void dm_ima_reset_data(struct mapped_device *md) {} -static inline void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_flags) {} -static inline void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap) {} -static inline void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all) {} -static inline void dm_ima_measure_on_table_clear(struct mapped_device *md, bool new_map) {} -static inline void dm_ima_measure_on_device_rename(struct mapped_device *md) {} +struct dm_ima_context; + +static inline void dm_ima_init(struct mapped_device *md) {} +static inline void dm_ima_alloc_context(struct dm_ima_context **context, bool noio) {} +static inline void dm_ima_free_context(struct dm_ima_context *context) {} +static inline void dm_ima_context_table_op(struct mapped_device *md, + struct dm_ima_context *context, + enum dm_ima_table_op op) {} +static inline void dm_ima_measure_on_table_load(struct dm_table *table, + struct dm_ima_context *context) {} +static inline void dm_ima_measure_on_device_resume(struct mapped_device *md, + bool swap, + struct dm_ima_context *context) {} +static inline void dm_ima_measure_on_device_remove(struct mapped_device *md, + bool remove_all, + struct dm_ima_context *context, + unsigned int idx) {} +static inline void dm_ima_measure_on_table_clear(struct mapped_device *md, + struct dm_ima_context *context) {} +static inline void dm_ima_measure_on_device_rename(struct mapped_device *md, + struct dm_ima_context *context) {} #endif /* CONFIG_IMA */ diff --git a/drivers/md/dm-inlinecrypt.c b/drivers/md/dm-inlinecrypt.c new file mode 100644 index 000000000000..41293c18d10f --- /dev/null +++ b/drivers/md/dm-inlinecrypt.c @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2024 Google LLC + */ + +#include <linux/blk-crypto.h> +#include <linux/ctype.h> +#include <linux/device-mapper.h> +#include <linux/hex.h> +#include <linux/module.h> +#include <keys/user-type.h> + +#define DM_MSG_PREFIX "inlinecrypt" + +static const struct dm_inlinecrypt_cipher { + const char *name; + enum blk_crypto_mode_num mode_num; +} dm_inlinecrypt_ciphers[] = { + { + .name = "aes-xts-plain64", + .mode_num = BLK_ENCRYPTION_MODE_AES_256_XTS, + }, +}; + +/** + * struct inlinecrypt_ctx - private data of an inlinecrypt target + * @dev: the underlying device + * @start: starting sector of the range of @dev which this target actually maps. + * For this purpose a "sector" is 512 bytes. + * @cipher_string: the name of the encryption algorithm being used + * @key_size: size of the encryption key in bytes + * @iv_offset: starting offset for IVs. IVs are generated as if the target were + * preceded by @iv_offset 512-byte sectors. + * @sector_size: crypto sector size in bytes (usually 4096) + * @sector_bits: log2(sector_size) + * @key_type: type of the key -- either raw or hardware-wrapped + * @key: the encryption key to use + * @max_dun: the maximum DUN that may be used (computed from other params) + */ +struct inlinecrypt_ctx { + struct dm_dev *dev; + sector_t start; + const char *cipher_string; + unsigned int key_size; + u64 iv_offset; + unsigned int sector_size; + unsigned int sector_bits; + enum blk_crypto_key_type key_type; + struct blk_crypto_key key; + u64 max_dun; +}; + +static const struct dm_inlinecrypt_cipher * +lookup_cipher(const char *cipher_string) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(dm_inlinecrypt_ciphers); i++) { + if (strcmp(cipher_string, dm_inlinecrypt_ciphers[i].name) == 0) + return &dm_inlinecrypt_ciphers[i]; + } + return NULL; +} + +static void inlinecrypt_dtr(struct dm_target *ti) +{ + struct inlinecrypt_ctx *ctx = ti->private; + + if (ctx->dev) { + if (ctx->key.size) + blk_crypto_evict_key(ctx->dev->bdev, &ctx->key); + dm_put_device(ti, ctx->dev); + } + kfree_sensitive(ctx->cipher_string); + kfree_sensitive(ctx); +} + +#ifdef CONFIG_KEYS + +static bool contains_whitespace(const char *str) +{ + while (*str) + if (isspace(*str++)) + return true; + return false; +} + +static int set_key_user(struct key *key, char *key_bytes, + const unsigned int key_bytes_size) +{ + const struct user_key_payload *ukp; + + ukp = user_key_payload_locked(key); + if (!ukp) + return -EKEYREVOKED; + + if (key_bytes_size != ukp->datalen) + return -EINVAL; + + memcpy(key_bytes, ukp->data, key_bytes_size); + + return 0; +} + +static int inlinecrypt_get_keyring_key(const char *key_string, u8 *key_bytes, + const unsigned int key_bytes_size) +{ + char *key_desc; + int ret; + struct key_type *type; + struct key *key; + int (*set_key)(struct key *key, char *key_bytes, + const unsigned int key_bytes_size); + + /* + * Reject key_string with whitespace. dm core currently lacks code for + * proper whitespace escaping in arguments on DM_TABLE_STATUS path. + */ + if (contains_whitespace(key_string)) { + DMERR("whitespace chars not allowed in key string"); + return -EINVAL; + } + + /* look for next ':' separating key_type from key_description */ + key_desc = strchr(key_string, ':'); + if (!key_desc || key_desc == key_string || !strlen(key_desc + 1)) + return -EINVAL; + + if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) { + type = &key_type_logon; + set_key = set_key_user; + } else { + return -EINVAL; + } + + key = request_key(type, key_desc + 1, NULL); + if (IS_ERR(key)) + return PTR_ERR(key); + + down_read(&key->sem); + + ret = set_key(key, (char *)key_bytes, key_bytes_size); + + up_read(&key->sem); + key_put(key); + + return ret; +} + +static int get_key_size(char **key_string) +{ + char *colon, dummy; + int ret; + + if (*key_string[0] != ':') { + ret = strlen(*key_string); + + if (ret > 2 * BLK_CRYPTO_MAX_ANY_KEY_SIZE + || ret % 2 + || !ret) { + DMERR("Invalid keysize"); + return -EINVAL; + } + return ret >> 1; + } + + /* look for next ':' in key string */ + colon = strpbrk(*key_string + 1, ":"); + if (!colon) + return -EINVAL; + + if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':') + return -EINVAL; + + /* remaining key string should be :<logon|user>:<key_desc> */ + *key_string = colon; + + return ret; +} + +#else + +static int inlinecrypt_get_keyring_key(const char *key_string, u8 *key_bytes, + const unsigned int key_bytes_size) +{ + return -EINVAL; +} + +static int get_key_size(char **key_string) +{ + int key_hex_size = strlen(*key_string); + + if (*key_string[0] == ':') + return -EINVAL; + + if (key_hex_size > 2 * BLK_CRYPTO_MAX_ANY_KEY_SIZE + || key_hex_size % 2 + || !key_hex_size) { + DMERR("Invalid keysize"); + return -EINVAL; + } + + return key_hex_size >> 1; +} + +#endif /* CONFIG_KEYS */ + +static int inlinecrypt_get_key(const char *key_string, + u8 key[BLK_CRYPTO_MAX_ANY_KEY_SIZE], + const unsigned int key_size) +{ + int ret = 0; + + if (key_size > BLK_CRYPTO_MAX_ANY_KEY_SIZE) { + DMERR("Invalid keysize"); + return -EINVAL; + } + + /* ':' means the key is in kernel keyring, short-circuit normal key processing */ + if (key_string[0] == ':') { + /* key string should be :<logon|user>:<key_desc> */ + ret = inlinecrypt_get_keyring_key(key_string + 1, key, key_size); + goto out; + } + + if (hex2bin(key, key_string, key_size) != 0) + ret = -EINVAL; + +out: + return ret; +} + +static int inlinecrypt_ctr_optional(struct dm_target *ti, + unsigned int argc, char **argv) +{ + struct inlinecrypt_ctx *ctx = ti->private; + struct dm_arg_set as; + static const struct dm_arg _args[] = { + {0, 4, "Invalid number of feature args"}, + }; + unsigned int opt_params; + const char *opt_string; + bool iv_large_sectors = false; + char dummy; + int err; + + as.argc = argc; + as.argv = argv; + + err = dm_read_arg_group(_args, &as, &opt_params, &ti->error); + if (err) + return err; + + while (opt_params--) { + opt_string = dm_shift_arg(&as); + if (!opt_string) { + ti->error = "Not enough feature arguments"; + return -EINVAL; + } + if (str_has_prefix(opt_string, "keytype:")) { + const char *val = opt_string + strlen("keytype:"); + + if (!*val) { + ti->error = "Invalid block key type"; + return -EINVAL; + } + + if (!strcmp(val, "raw")) { + ctx->key_type = BLK_CRYPTO_KEY_TYPE_RAW; + } else if (!strcmp(val, "hw-wrapped")) { + ctx->key_type = BLK_CRYPTO_KEY_TYPE_HW_WRAPPED; + } else { + ti->error = "Invalid block key type"; + return -EINVAL; + } + } else if (!strcmp(opt_string, "allow_discards")) { + ti->num_discard_bios = 1; + } else if (sscanf(opt_string, "sector_size:%u%c", + &ctx->sector_size, &dummy) == 1) { + if (ctx->sector_size < SECTOR_SIZE || + ctx->sector_size > 4096 || + !is_power_of_2(ctx->sector_size)) { + ti->error = "Invalid sector_size"; + return -EINVAL; + } + } else if (!strcmp(opt_string, "iv_large_sectors")) { + iv_large_sectors = true; + } else { + ti->error = "Invalid feature arguments"; + return -EINVAL; + } + } + + /* dm-inlinecrypt doesn't implement iv_large_sectors=false. */ + if (ctx->sector_size != SECTOR_SIZE && !iv_large_sectors) { + ti->error = "iv_large_sectors must be specified"; + return -EINVAL; + } + + return 0; +} + +/* + * Construct an inlinecrypt mapping: + * <cipher> [<key>|:<key_size>:<logon>:<key_description>] <iv_offset> <dev_path> <start> + * + * This syntax matches dm-crypt's, but the set of supported functionality has + * been stripped down. + */ +static int inlinecrypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) +{ + struct inlinecrypt_ctx *ctx; + const struct dm_inlinecrypt_cipher *cipher; + u8 key_bytes[BLK_CRYPTO_MAX_ANY_KEY_SIZE]; + unsigned int dun_bytes; + unsigned long long tmpll; + char dummy; + int err; + + if (argc < 5) { + ti->error = "Not enough arguments"; + return -EINVAL; + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) { + ti->error = "Out of memory"; + return -ENOMEM; + } + ti->private = ctx; + + /* <cipher> */ + ctx->cipher_string = kstrdup(argv[0], GFP_KERNEL); + if (!ctx->cipher_string) { + ti->error = "Out of memory"; + err = -ENOMEM; + goto bad; + } + cipher = lookup_cipher(ctx->cipher_string); + if (!cipher) { + ti->error = "Unsupported cipher"; + err = -EINVAL; + goto bad; + } + + /* <key> */ + err = get_key_size(&argv[1]); + if (err < 0) { + ti->error = "Cannot parse key size"; + err = -EINVAL; + goto bad; + } + ctx->key_size = err; + + err = inlinecrypt_get_key(argv[1], key_bytes, ctx->key_size); + if (err) { + ti->error = "Malformed key string"; + goto bad; + } + + /* <iv_offset> */ + if (sscanf(argv[2], "%llu%c", &ctx->iv_offset, &dummy) != 1) { + ti->error = "Invalid iv_offset sector"; + err = -EINVAL; + goto bad; + } + + /* <dev_path> */ + err = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), + &ctx->dev); + if (err) { + ti->error = "Device lookup failed"; + goto bad; + } + + /* <start> */ + if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 || + tmpll != (sector_t)tmpll) { + ti->error = "Invalid start sector"; + err = -EINVAL; + goto bad; + } + ctx->start = tmpll; + + /* optional arguments */ + ctx->sector_size = SECTOR_SIZE; + ctx->key_type = BLK_CRYPTO_KEY_TYPE_RAW; + if (argc > 5) { + err = inlinecrypt_ctr_optional(ti, argc - 5, &argv[5]); + if (err) + goto bad; + } + ctx->sector_bits = ilog2(ctx->sector_size); + if (ti->len & ((ctx->sector_size >> SECTOR_SHIFT) - 1)) { + ti->error = "Device size is not a multiple of sector_size"; + err = -EINVAL; + goto bad; + } + if (ctx->iv_offset & ((ctx->sector_size >> SECTOR_SHIFT) - 1)) { + ti->error = "Wrong alignment of iv_offset sector"; + err = -EINVAL; + goto bad; + } + + ctx->max_dun = (ctx->iv_offset + ti->len - 1) >> + (ctx->sector_bits - SECTOR_SHIFT); + dun_bytes = DIV_ROUND_UP(fls64(ctx->max_dun), 8); + + err = blk_crypto_init_key(&ctx->key, key_bytes, ctx->key_size, + ctx->key_type, cipher->mode_num, + dun_bytes, ctx->sector_size); + if (err) { + ti->error = "Error initializing blk-crypto key"; + goto bad; + } + + err = blk_crypto_start_using_key(ctx->dev->bdev, &ctx->key); + if (err) { + ti->error = "Error starting to use blk-crypto"; + goto bad; + } + + ti->num_flush_bios = 1; + + err = 0; + goto out; + +bad: + inlinecrypt_dtr(ti); +out: + memzero_explicit(key_bytes, sizeof(key_bytes)); + return err; +} + +static int inlinecrypt_map(struct dm_target *ti, struct bio *bio) +{ + const struct inlinecrypt_ctx *ctx = ti->private; + sector_t sector_in_target; + u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE] = {}; + + bio_set_dev(bio, ctx->dev->bdev); + + /* + * If the bio is a device-level request which doesn't target a specific + * sector, there's nothing more to do. + */ + if (bio_sectors(bio) == 0) + return DM_MAPIO_REMAPPED; + + /* + * The bio should never have an encryption context already, since + * dm-inlinecrypt doesn't pass through any inline encryption + * capabilities to the layer above it. + */ + if (WARN_ON_ONCE(bio_has_crypt_ctx(bio))) + return DM_MAPIO_KILL; + + /* Map the bio's sector to the underlying device. (512-byte sectors) */ + sector_in_target = dm_target_offset(ti, bio->bi_iter.bi_sector); + bio->bi_iter.bi_sector = ctx->start + sector_in_target; + /* + * If the bio doesn't have any data (e.g. if it's a DISCARD request), + * there's nothing more to do. + */ + if (!bio_has_data(bio)) + return DM_MAPIO_REMAPPED; + + /* Calculate the DUN and enforce data-unit (crypto sector) alignment. */ + dun[0] = ctx->iv_offset + sector_in_target; /* 512-byte sectors */ + if (dun[0] & ((ctx->sector_size >> SECTOR_SHIFT) - 1)) + return DM_MAPIO_KILL; + dun[0] >>= ctx->sector_bits - SECTOR_SHIFT; /* crypto sectors */ + + /* + * This check isn't necessary as we should have calculated max_dun + * correctly, but be safe. + */ + if (WARN_ON_ONCE(dun[0] > ctx->max_dun)) + return DM_MAPIO_KILL; + + bio_crypt_set_ctx(bio, &ctx->key, dun, GFP_NOIO); + + /* + * Since we've added an encryption context to the bio and + * blk-crypto-fallback may be needed to process it, it's necessary to + * use the fallback-aware bio submission code rather than + * unconditionally returning DM_MAPIO_REMAPPED. + * + * To get the correct accounting for a dm target in the case where + * __blk_crypto_submit_bio() doesn't take ownership of the bio (returns + * true), call __blk_crypto_submit_bio() directly and return + * DM_MAPIO_REMAPPED in that case, rather than relying on + * blk_crypto_submit_bio() which calls submit_bio() in that case. + * + * TODO: blk-crypto fallback write slow-path currently double-accounts + * IO in vmstat, as encrypted bios are submitted via submit_bio(). + * This does not affect data correctness. Consider fixing this if + * a cleaner accounting model for derived bios is introduced. + */ + if (__blk_crypto_submit_bio(bio)) + return DM_MAPIO_REMAPPED; + return DM_MAPIO_SUBMITTED; +} + +static void inlinecrypt_status(struct dm_target *ti, status_type_t type, + unsigned int status_flags, char *result, + unsigned int maxlen) +{ + const struct inlinecrypt_ctx *ctx = ti->private; + unsigned int sz = 0; + int num_feature_args = 0; + + switch (type) { + case STATUSTYPE_INFO: + case STATUSTYPE_IMA: + result[0] = '\0'; + break; + + case STATUSTYPE_TABLE: + /* + * Warning: like dm-crypt, dm-inlinecrypt includes the key in + * the returned table. Userspace is responsible for redacting + * the key when needed. + */ + DMEMIT("%s %*phN %u %llu %s %llu", ctx->cipher_string, + ctx->key.size, ctx->key.bytes, + ctx->key_type, ctx->iv_offset, + ctx->dev->name, ctx->start); + num_feature_args += !!ti->num_discard_bios; + if (ctx->sector_size != SECTOR_SIZE) + num_feature_args += 2; + if (num_feature_args != 0) { + DMEMIT(" %d", num_feature_args); + if (ti->num_discard_bios) + DMEMIT(" allow_discards"); + if (ctx->sector_size != SECTOR_SIZE) { + DMEMIT(" sector_size:%u", ctx->sector_size); + DMEMIT(" iv_large_sectors"); + } + } + break; + } +} + +static int inlinecrypt_prepare_ioctl(struct dm_target *ti, + struct block_device **bdev, unsigned int cmd, + unsigned long arg, bool *forward) +{ + const struct inlinecrypt_ctx *ctx = ti->private; + const struct dm_dev *dev = ctx->dev; + + *bdev = dev->bdev; + + /* Only pass ioctls through if the device sizes match exactly. */ + return ctx->start != 0 || ti->len != bdev_nr_sectors(dev->bdev); +} + +static int inlinecrypt_iterate_devices(struct dm_target *ti, + iterate_devices_callout_fn fn, + void *data) +{ + const struct inlinecrypt_ctx *ctx = ti->private; + + return fn(ti, ctx->dev, ctx->start, ti->len, data); +} + +#ifdef CONFIG_BLK_DEV_ZONED +static int inlinecrypt_report_zones(struct dm_target *ti, + struct dm_report_zones_args *args, + unsigned int nr_zones) +{ + const struct inlinecrypt_ctx *ctx = ti->private; + + return dm_report_zones(ctx->dev->bdev, ctx->start, + ctx->start + dm_target_offset(ti, args->next_sector), + args, nr_zones); +} +#else +#define inlinecrypt_report_zones NULL +#endif + +static void inlinecrypt_io_hints(struct dm_target *ti, + struct queue_limits *limits) +{ + const struct inlinecrypt_ctx *ctx = ti->private; + const unsigned int sector_size = ctx->sector_size; + + limits->logical_block_size = + max_t(unsigned int, limits->logical_block_size, sector_size); + limits->physical_block_size = + max_t(unsigned int, limits->physical_block_size, sector_size); + limits->io_min = max_t(unsigned int, limits->io_min, sector_size); + limits->dma_alignment = limits->logical_block_size - 1; +} + +static struct target_type inlinecrypt_target = { + .name = "inlinecrypt", + .version = {1, 0, 0}, + /* + * Do not set DM_TARGET_PASSES_CRYPTO, since dm-inlinecrypt consumes the + * crypto capability itself. + */ + .features = DM_TARGET_ZONED_HM, + .module = THIS_MODULE, + .ctr = inlinecrypt_ctr, + .dtr = inlinecrypt_dtr, + .map = inlinecrypt_map, + .status = inlinecrypt_status, + .prepare_ioctl = inlinecrypt_prepare_ioctl, + .iterate_devices = inlinecrypt_iterate_devices, + .report_zones = inlinecrypt_report_zones, + .io_hints = inlinecrypt_io_hints, +}; + +module_dm(inlinecrypt); + +MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>"); +MODULE_AUTHOR("Linlin Zhang <linlin.zhang@oss.qualcomm.com>"); +MODULE_DESCRIPTION(DM_NAME " target for inline encryption"); +MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 65c30dec8222..1f2593f113f6 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -1480,9 +1480,6 @@ thorough_test: *metadata_offset = 0; } - if (unlikely(!is_power_of_2(ic->tag_size))) - hash_offset = (hash_offset + to_copy) % ic->tag_size; - total_size -= to_copy; } while (unlikely(total_size)); @@ -2523,6 +2520,9 @@ static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map) if (unlikely((bio->bi_opf & REQ_PREFLUSH) != 0)) return DM_MAPIO_REMAPPED; + if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) + return DM_MAPIO_KILL; + retry: if (!dio->integrity_payload) { unsigned digest_size, extra_size; @@ -2587,10 +2587,6 @@ skip_spinlock: dio->bio_details.bi_iter = bio->bi_iter; - if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) { - return DM_MAPIO_KILL; - } - bio->bi_iter.bi_sector += ic->start + SB_SECTORS; bip = bio_integrity_alloc(bio, GFP_NOIO, 1); @@ -2606,7 +2602,7 @@ skip_spinlock: struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); const char *mem = integrity_kmap(ic, bv.bv_page); if (ic->tag_size < ic->tuple_size) - memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tuple_size); + memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tag_size); integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, dio->integrity_payload + pos); integrity_kunmap(ic, mem); pos += ic->tuple_size; @@ -5130,6 +5126,20 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv ti->error = "Journal mac mismatch"; goto bad; } + if (ic->fix_hmac && !(ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) && ic->journal_mac_alg.key_string) { + /* + * If this happens, it may be either because someone tampered + * with the device or it may be due to a bug in the + * integritysetup tool. + * + * In the latter case, upgrade to integritysetup 2.8.7 and use + * the argument --integrity-legacy-hmac when using the open + * command. + */ + r = -EINVAL; + ti->error = "fix_hmac is on the command line but not in the superblock"; + goto bad; + } get_provided_data_sectors(ic); if (!ic->provided_data_sectors) { diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index a529174c94cf..61af2a437a05 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -259,6 +259,80 @@ static void free_cell(struct hash_cell *hc) } } +#ifdef CONFIG_IMA + +/* + * Called while holding to _hash_lock, to guarantee the ordering of the + * following dm_ima_measure_on_* functions, which should be called + * right after dropping the _hash_lock + */ +static unsigned int dm_ima_init_context(struct hash_cell *hc, + struct dm_ima_context *context, + bool need_idx) +{ + lockdep_assert_held(&_hash_lock); + + if (unlikely(!context)) + return need_idx ? hc->md->ima.update_idx++ : 0; + + context->update_idx = hc->md->ima.update_idx++; + strcpy(context->dev_name, hc->name); + strcpy(context->dev_uuid, hc->uuid ? : ""); + + return context->update_idx; +} + +/* + * Called by do_resume() to guarantee correct ordering, since do_resume() + * does not grab the _hash_lock when the table is not getting swapped or + * when actually swapping the active table + */ +static bool dm_ima_need_measure(struct mapped_device *md, + struct dm_table *table, + struct dm_ima_context *context) +{ + int srcu_idx; + struct hash_cell *hc; + bool need_measure = false; + + if (unlikely(!context)) + return false; + + down_write(&_hash_lock); + /* Check if the device has been removed */ + hc = dm_get_mdptr(md); + if (hc) { + /* + * If we have a table, we need to make sure that it's the + * active table. Otherwise we raced with another process + * setting the active table and it will do the measurement + */ + if (!table || dm_get_live_table(md, &srcu_idx) == table) { + dm_ima_init_context(hc, context, false); + need_measure = true; + } + if (table) + dm_put_live_table(md, srcu_idx); + } + up_write(&_hash_lock); + + return need_measure; +} +#else +static inline unsigned int dm_ima_init_context(struct hash_cell *hc, + struct dm_ima_context *context, + bool neex_idx) +{ + return 0; +} +static inline bool dm_ima_need_measure(struct mapped_device *md, + struct dm_table *table, + struct dm_ima_context *context) +{ + return false; +} +#endif + /* * The kdev_t and uuid of a device can never change once it is * initially inserted. @@ -344,7 +418,10 @@ static int dm_hash_remove_all(unsigned flags) struct hash_cell *hc; struct mapped_device *md; struct dm_table *t; + struct dm_ima_context *ima_context = NULL; + unsigned int ima_idx; + dm_ima_alloc_context(&ima_context, true); retry: dev_skipped = 0; @@ -353,6 +430,7 @@ retry: for (n = rb_first(&name_rb_tree); n; n = rb_next(n)) { if (flags & DM_REMOVE_INTERRUPTIBLE && fatal_signal_pending(current)) { up_write(&_hash_lock); + dm_ima_free_context(ima_context); return -EINTR; } @@ -367,6 +445,7 @@ retry: continue; } + ima_idx = dm_ima_init_context(hc, ima_context, true); t = __hash_remove(hc); up_write(&_hash_lock); @@ -375,7 +454,7 @@ retry: dm_sync_table(md); dm_table_destroy(t); } - dm_ima_measure_on_device_remove(md, true); + dm_ima_measure_on_device_remove(md, true, ima_context, ima_idx); dm_put(md); if (likely(flags & DM_REMOVE_KEEP_OPEN_DEVICES)) dm_destroy(md); @@ -396,6 +475,7 @@ retry: if (dev_skipped && !(flags & DM_REMOVE_ONLY_DEFERRED)) DMWARN("remove_all left %d open device(s)", dev_skipped); + dm_ima_free_context(ima_context); return 0; } @@ -443,6 +523,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param, struct mapped_device *md; unsigned int change_uuid = (param->flags & DM_UUID_FLAG) ? 1 : 0; int srcu_idx; + struct dm_ima_context *ima_context = NULL; /* * duplicate new. @@ -451,6 +532,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param, if (!new_data) return ERR_PTR(-ENOMEM); + dm_ima_alloc_context(&ima_context, true); down_write(&_hash_lock); /* @@ -467,6 +549,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param, param->name, new); dm_put(hc->md); up_write(&_hash_lock); + dm_ima_free_context(ima_context); kfree(new_data); return ERR_PTR(-EBUSY); } @@ -479,6 +562,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param, DMERR("Unable to rename non-existent device, %s to %s%s", param->name, change_uuid ? "uuid " : "", new); up_write(&_hash_lock); + dm_ima_free_context(ima_context); kfree(new_data); return ERR_PTR(-ENXIO); } @@ -492,6 +576,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param, param->name, new, hc->uuid); dm_put(hc->md); up_write(&_hash_lock); + dm_ima_free_context(ima_context); kfree(new_data); return ERR_PTR(-EINVAL); } @@ -514,9 +599,11 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param, md = hc->md; - dm_ima_measure_on_device_rename(md); + dm_ima_init_context(hc, ima_context, false); up_write(&_hash_lock); + dm_ima_measure_on_device_rename(md, ima_context); + dm_ima_free_context(ima_context); kfree(old_name); return md; @@ -698,7 +785,7 @@ static void list_version_get_info(struct target_type *tt, void *param) struct vers_iter *info = param; /* Check space - it might have changed since the first iteration */ - if ((char *)info->vers + sizeof(tt->version) + strlen(tt->name) + 1 > info->end) { + if ((char *)info->vers + sizeof(struct dm_target_versions) + strlen(tt->name) + 1 > info->end) { info->flags = DM_BUFFER_FULL_FLAG; return; } @@ -995,13 +1082,17 @@ static int dev_remove(struct file *filp, struct dm_ioctl *param, size_t param_si struct mapped_device *md; int r; struct dm_table *t; + struct dm_ima_context *ima_context = NULL; + unsigned int ima_idx; + dm_ima_alloc_context(&ima_context, true); down_write(&_hash_lock); hc = __find_device_hash_cell(param); if (!hc) { DMDEBUG_LIMIT("device doesn't appear to be in the dev hash table."); up_write(&_hash_lock); + dm_ima_free_context(ima_context); return -ENXIO; } @@ -1015,14 +1106,17 @@ static int dev_remove(struct file *filp, struct dm_ioctl *param, size_t param_si if (r == -EBUSY && param->flags & DM_DEFERRED_REMOVE) { up_write(&_hash_lock); dm_put(md); + dm_ima_free_context(ima_context); return 0; } DMDEBUG_LIMIT("unable to remove open device %s", hc->name); up_write(&_hash_lock); dm_put(md); + dm_ima_free_context(ima_context); return r; } + ima_idx = dm_ima_init_context(hc, ima_context, true); t = __hash_remove(hc); up_write(&_hash_lock); @@ -1033,7 +1127,8 @@ static int dev_remove(struct file *filp, struct dm_ioctl *param, size_t param_si param->flags &= ~DM_DEFERRED_REMOVE; - dm_ima_measure_on_device_remove(md, false); + dm_ima_measure_on_device_remove(md, false, ima_context, ima_idx); + dm_ima_free_context(ima_context); if (!dm_kobject_uevent(md, KOBJ_REMOVE, param->event_nr, false)) param->flags |= DM_UEVENT_GENERATED_FLAG; @@ -1169,13 +1264,16 @@ static int do_resume(struct dm_ioctl *param) struct mapped_device *md; struct dm_table *new_map, *old_map = NULL; bool need_resize_uevent = false; + struct dm_ima_context *ima_context = NULL; + dm_ima_alloc_context(&ima_context, true); down_write(&_hash_lock); hc = __find_device_hash_cell(param); if (!hc) { DMDEBUG_LIMIT("device doesn't appear to be in the dev hash table."); up_write(&_hash_lock); + dm_ima_free_context(ima_context); return -ENXIO; } @@ -1184,13 +1282,15 @@ static int do_resume(struct dm_ioctl *param) new_map = hc->new_map; hc->new_map = NULL; param->flags &= ~DM_INACTIVE_PRESENT_FLAG; - + if (new_map) + dm_ima_init_context(hc, ima_context, false); up_write(&_hash_lock); /* Do we need to load a new map ? */ if (new_map) { sector_t old_size, new_size; + dm_ima_context_table_op(md, ima_context, DM_IMA_TABLE_SAVE); /* Suspend if it isn't already suspended */ if (param->flags & DM_SKIP_LOCKFS_FLAG) suspend_flags &= ~DM_SUSPEND_LOCKFS_FLAG; @@ -1204,6 +1304,8 @@ static int do_resume(struct dm_ioctl *param) if (hc && !hc->new_map) { hc->new_map = new_map; new_map = NULL; + dm_ima_init_context(hc, ima_context, + false); } else { r = -ENXIO; } @@ -1211,7 +1313,9 @@ static int do_resume(struct dm_ioctl *param) if (new_map) { dm_sync_table(md); dm_table_destroy(new_map); - } + } else + dm_ima_context_table_op(md, ima_context, DM_IMA_TABLE_RESTORE); + dm_ima_free_context(ima_context); dm_put(md); return r; } @@ -1222,9 +1326,12 @@ static int do_resume(struct dm_ioctl *param) if (IS_ERR(old_map)) { dm_sync_table(md); dm_table_destroy(new_map); + dm_ima_free_context(ima_context); dm_put(md); return PTR_ERR(old_map); } + if (dm_ima_need_measure(md, new_map, ima_context)) + dm_ima_measure_on_device_resume(md, true, ima_context); new_size = dm_get_size(md); if (old_size && new_size && old_size != new_size) need_resize_uevent = true; @@ -1238,7 +1345,10 @@ static int do_resume(struct dm_ioctl *param) if (dm_suspended_md(md)) { r = dm_resume(md); if (!r) { - dm_ima_measure_on_device_resume(md, new_map ? true : false); + if (!new_map && dm_ima_need_measure(md, NULL, + ima_context)) + dm_ima_measure_on_device_resume(md, false, + ima_context); if (!dm_kobject_uevent(md, KOBJ_CHANGE, param->event_nr, need_resize_uevent)) param->flags |= DM_UEVENT_GENERATED_FLAG; @@ -1255,6 +1365,7 @@ static int do_resume(struct dm_ioctl *param) if (!r) __dev_status(md, param); + dm_ima_free_context(ima_context); dm_put(md); return r; } @@ -1532,11 +1643,12 @@ static bool is_valid_type(enum dm_queue_mode cur, enum dm_queue_mode new) static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_size) { - int r; + int r, srcu_idx; struct hash_cell *hc; struct dm_table *t, *old_map = NULL; struct mapped_device *md; struct target_type *immutable_target_type; + struct dm_ima_context *ima_context = NULL; md = find_device(param); if (!md) @@ -1552,8 +1664,6 @@ static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_si if (r) goto err_unlock_md_type; - dm_ima_measure_on_table_load(t, STATUSTYPE_IMA); - immutable_target_type = dm_get_immutable_target_type(md); if (immutable_target_type && (immutable_target_type != dm_table_get_immutable_target_type(t)) && @@ -1580,12 +1690,14 @@ static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_si dm_unlock_md_type(md); + dm_ima_alloc_context(&ima_context, false); /* stage inactive table */ down_write(&_hash_lock); hc = dm_get_mdptr(md); if (!hc) { DMERR("device has been removed from the dev hash table."); up_write(&_hash_lock); + dm_ima_free_context(ima_context); r = -ENXIO; goto err_destroy_table; } @@ -1593,8 +1705,15 @@ static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_si if (hc->new_map) old_map = hc->new_map; hc->new_map = t; + dm_ima_init_context(hc, ima_context, false); + /* Make sure new_map doesn't get freed before we measure it*/ + dm_get_live_table(md, &srcu_idx); up_write(&_hash_lock); + dm_ima_measure_on_table_load(t, ima_context); + dm_ima_free_context(ima_context); + dm_put_live_table(md, srcu_idx); + param->flags |= DM_INACTIVE_PRESENT_FLAG; __dev_status(md, param); @@ -1622,25 +1741,29 @@ static int table_clear(struct file *filp, struct dm_ioctl *param, size_t param_s struct hash_cell *hc; struct mapped_device *md; struct dm_table *old_map = NULL; - bool has_new_map = false; + struct dm_ima_context *ima_context = NULL; + dm_ima_alloc_context(&ima_context, true); down_write(&_hash_lock); hc = __find_device_hash_cell(param); if (!hc) { DMDEBUG_LIMIT("device doesn't appear to be in the dev hash table."); up_write(&_hash_lock); + dm_ima_free_context(ima_context); return -ENXIO; } if (hc->new_map) { old_map = hc->new_map; hc->new_map = NULL; - has_new_map = true; } + dm_ima_init_context(hc, ima_context, false); md = hc->md; up_write(&_hash_lock); + dm_ima_measure_on_table_clear(md, ima_context); + dm_ima_free_context(ima_context); param->flags &= ~DM_INACTIVE_PRESENT_FLAG; __dev_status(md, param); @@ -1649,7 +1772,6 @@ static int table_clear(struct file *filp, struct dm_ioctl *param, size_t param_s dm_sync_table(md); dm_table_destroy(old_map); } - dm_ima_measure_on_table_clear(md, has_new_map); dm_put(md); return 0; @@ -1816,8 +1938,11 @@ static int target_message(struct file *filp, struct dm_ioctl *param, size_t para goto out_argv; table = dm_get_live_table(md, &srcu_idx); - if (!table) + if (!table) { + DMERR("The device has no table."); + r = -EINVAL; goto out_table; + } if (dm_deleting_md(md)) { r = -ENXIO; @@ -2348,7 +2473,7 @@ int __init dm_early_create(struct dm_ioctl *dmi, /* resume device */ r = dm_resume(md); if (r) - goto err_destroy_table; + goto err_hash_remove; DMINFO("%s (%s) is ready", md->disk->disk_name, dmi->name); dm_put(md); diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index d316757a328b..2ddeb4250c59 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -425,6 +425,9 @@ static int create_log_context(struct dm_dirty_log *log, struct dm_target *ti, */ bitset_size = dm_round_up(region_count, BITS_PER_LONG); bitset_size >>= BYTE_SHIFT; + /* Handle dm_round_up rollover on 32-bit systems */ + if (!bitset_size) + bitset_size = 1UL << (BITS_PER_LONG - BYTE_SHIFT); lc->bitset_uint32_count = bitset_size / sizeof(*lc->clean_bits); diff --git a/drivers/md/dm-pcache/dm_pcache.c b/drivers/md/dm-pcache/dm_pcache.c index 81c795c0400e..d5cfd162c063 100644 --- a/drivers/md/dm-pcache/dm_pcache.c +++ b/drivers/md/dm-pcache/dm_pcache.c @@ -168,6 +168,10 @@ static int parse_cache_opts(struct dm_pcache *pcache, struct dm_arg_set *as, argc--; if (!strcmp(arg, "cache_mode")) { + if (!argc) { + *error = "Missing value for cache_mode"; + return -EINVAL; + } arg = dm_shift_arg(as); if (!strcmp(arg, "writeback")) { opts->cache_mode = PCACHE_CACHE_MODE_WRITEBACK; @@ -177,6 +181,10 @@ static int parse_cache_opts(struct dm_pcache *pcache, struct dm_arg_set *as, } argc--; } else if (!strcmp(arg, "data_crc")) { + if (!argc) { + *error = "Missing value for data_crc"; + return -EINVAL; + } arg = dm_shift_arg(as); if (!strcmp(arg, "true")) { opts->data_crc = true; diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index c5dc083c7244..8f5a5e1342a9 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -3831,6 +3831,7 @@ static void raid_presuspend(struct dm_target *ti) * resume, raid_postsuspend() is too late. */ set_bit(RT_FLAG_RS_FROZEN, &rs->runtime_flags); + set_bit(MD_DM_SUSPENDING, &mddev->flags); if (!reshape_interrupted(mddev)) return; @@ -3847,13 +3848,16 @@ static void raid_presuspend(struct dm_target *ti) static void raid_presuspend_undo(struct dm_target *ti) { struct raid_set *rs = ti->private; + struct mddev *mddev = &rs->md; + clear_bit(MD_DM_SUSPENDING, &mddev->flags); clear_bit(RT_FLAG_RS_FROZEN, &rs->runtime_flags); } static void raid_postsuspend(struct dm_target *ti) { struct raid_set *rs = ti->private; + struct mddev *mddev = &rs->md; if (!test_and_set_bit(RT_FLAG_RS_SUSPENDED, &rs->runtime_flags)) { /* @@ -3864,6 +3868,8 @@ static void raid_postsuspend(struct dm_target *ti) mddev_suspend(&rs->md, false); rs->md.ro = MD_RDONLY; } + clear_bit(MD_DM_SUSPENDING, &mddev->flags); + } static void attempt_restore_of_faulty_devices(struct raid_set *rs) diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index 9703b3ae364e..9a386254d836 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -462,7 +462,7 @@ static void dm_start_request(struct mapped_device *md, struct request *orig) } static int dm_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, - unsigned int hctx_idx, unsigned int numa_node) + unsigned int hctx_idx, int numa_node) { struct mapped_device *md = set->driver_data; struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c index c53cf07ab7b0..5df710061a11 100644 --- a/drivers/md/dm-stats.c +++ b/drivers/md/dm-stats.c @@ -692,10 +692,8 @@ void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw, */ last = raw_cpu_ptr(stats->last); stats_aux->merged = - (bi_sector == (READ_ONCE(last->last_sector) && - ((bi_rw == WRITE) == - (READ_ONCE(last->last_rw) == WRITE)) - )); + bi_sector == READ_ONCE(last->last_sector) && + (bi_rw == WRITE) == (READ_ONCE(last->last_rw) == WRITE); WRITE_ONCE(last->last_sector, end_sector); WRITE_ONCE(last->last_rw, bi_rw); } else @@ -842,10 +840,10 @@ static unsigned long long dm_jiffies_to_msec64(struct dm_stat *s, unsigned long result = jiffies_to_msecs(j & 0x3fffff); if (j >= 1 << 22) { mult = jiffies_to_msecs(1 << 22); - result += (unsigned long long)mult * (unsigned long long)jiffies_to_msecs((j >> 22) & 0x3fffff); + result += (unsigned long long)mult * ((j >> 22) & 0x3fffff); } if (j >= 1ULL << 44) - result += (unsigned long long)mult * (unsigned long long)mult * (unsigned long long)jiffies_to_msecs(j >> 44); + result += (unsigned long long)mult * (unsigned long long)(1 << 22) * (j >> 44); return result; } diff --git a/drivers/md/dm-thin-metadata.c b/drivers/md/dm-thin-metadata.c index b6a2d2081a24..e60e1326376a 100644 --- a/drivers/md/dm-thin-metadata.c +++ b/drivers/md/dm-thin-metadata.c @@ -186,6 +186,7 @@ struct dm_pool_metadata { uint32_t time; dm_block_t root; dm_block_t details_root; + dm_block_t held_root; struct list_head thin_devices; uint64_t trans_id; unsigned long flags; @@ -748,6 +749,7 @@ static int __open_metadata(struct dm_pool_metadata *pmd) */ pmd->root = le64_to_cpu(disk_super->data_mapping_root); pmd->details_root = le64_to_cpu(disk_super->device_details_root); + pmd->held_root = le64_to_cpu(disk_super->held_root); __setup_btree_details(pmd); dm_bm_unlock(sblock); @@ -838,6 +840,7 @@ static int __begin_transaction(struct dm_pool_metadata *pmd) pmd->time = le32_to_cpu(disk_super->time); pmd->root = le64_to_cpu(disk_super->data_mapping_root); pmd->details_root = le64_to_cpu(disk_super->device_details_root); + pmd->held_root = le64_to_cpu(disk_super->held_root); pmd->trans_id = le64_to_cpu(disk_super->trans_id); pmd->flags = le32_to_cpu(disk_super->flags); pmd->data_block_size = le32_to_cpu(disk_super->data_block_size); @@ -928,6 +931,7 @@ static int __commit_transaction(struct dm_pool_metadata *pmd) disk_super->time = cpu_to_le32(pmd->time); disk_super->data_mapping_root = cpu_to_le64(pmd->root); disk_super->device_details_root = cpu_to_le64(pmd->details_root); + disk_super->held_root = cpu_to_le64(pmd->held_root); disk_super->trans_id = cpu_to_le64(pmd->trans_id); disk_super->flags = cpu_to_le32(pmd->flags); @@ -1333,9 +1337,14 @@ static int __reserve_metadata_snap(struct dm_pool_metadata *pmd) { int r, inc; struct thin_disk_superblock *disk_super; - struct dm_block *copy, *sblock; + struct dm_block *copy; dm_block_t held_root; + if (pmd->held_root) { + DMWARN("Pool metadata snapshot already exists: release this before taking another."); + return -EBUSY; + } + /* * We commit to ensure the btree roots which we increment in a * moment are up to date. @@ -1353,22 +1362,16 @@ static int __reserve_metadata_snap(struct dm_pool_metadata *pmd) dm_sm_inc_block(pmd->metadata_sm, THIN_SUPERBLOCK_LOCATION); r = dm_tm_shadow_block(pmd->tm, THIN_SUPERBLOCK_LOCATION, &sb_validator, ©, &inc); - if (r) + if (r) { + dm_sm_dec_block(pmd->metadata_sm, THIN_SUPERBLOCK_LOCATION); return r; + } BUG_ON(!inc); held_root = dm_block_location(copy); disk_super = dm_block_data(copy); - if (le64_to_cpu(disk_super->held_root)) { - DMWARN("Pool metadata snapshot already exists: release this before taking another."); - - dm_tm_dec(pmd->tm, held_root); - dm_tm_unlock(pmd->tm, copy); - return -EBUSY; - } - /* * Wipe the spacemap since we're not publishing this. */ @@ -1384,18 +1387,8 @@ static int __reserve_metadata_snap(struct dm_pool_metadata *pmd) dm_tm_inc(pmd->tm, le64_to_cpu(disk_super->device_details_root)); dm_tm_unlock(pmd->tm, copy); - /* - * Write the held root into the superblock. - */ - r = superblock_lock(pmd, &sblock); - if (r) { - dm_tm_dec(pmd->tm, held_root); - return r; - } + pmd->held_root = held_root; - disk_super = dm_block_data(sblock); - disk_super->held_root = cpu_to_le64(held_root); - dm_bm_unlock(sblock); return 0; } @@ -1415,18 +1408,10 @@ static int __release_metadata_snap(struct dm_pool_metadata *pmd) { int r; struct thin_disk_superblock *disk_super; - struct dm_block *sblock, *copy; + struct dm_block *copy; dm_block_t held_root; - r = superblock_lock(pmd, &sblock); - if (r) - return r; - - disk_super = dm_block_data(sblock); - held_root = le64_to_cpu(disk_super->held_root); - disk_super->held_root = cpu_to_le64(0); - - dm_bm_unlock(sblock); + held_root = pmd->held_root; if (!held_root) { DMWARN("No pool metadata snapshot found: nothing to release."); @@ -1437,13 +1422,15 @@ static int __release_metadata_snap(struct dm_pool_metadata *pmd) if (r) return r; + pmd->held_root = 0; + disk_super = dm_block_data(copy); dm_btree_del(&pmd->info, le64_to_cpu(disk_super->data_mapping_root)); dm_btree_del(&pmd->details_info, le64_to_cpu(disk_super->device_details_root)); - dm_sm_dec_block(pmd->metadata_sm, held_root); - dm_tm_unlock(pmd->tm, copy); + dm_sm_dec_block(pmd->metadata_sm, held_root); + return 0; } @@ -1462,19 +1449,7 @@ int dm_pool_release_metadata_snap(struct dm_pool_metadata *pmd) static int __get_metadata_snap(struct dm_pool_metadata *pmd, dm_block_t *result) { - int r; - struct thin_disk_superblock *disk_super; - struct dm_block *sblock; - - r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION, - &sb_validator, &sblock); - if (r) - return r; - - disk_super = dm_block_data(sblock); - *result = le64_to_cpu(disk_super->held_root); - - dm_bm_unlock(sblock); + *result = pmd->held_root; return 0; } diff --git a/drivers/md/dm-vdo/indexer/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c index 5f4ce4ab1b1e..2d529250000e 100644 --- a/drivers/md/dm-vdo/indexer/index-layout.c +++ b/drivers/md/dm-vdo/indexer/index-layout.c @@ -282,7 +282,7 @@ static void create_unique_nonce_data(u8 *buffer) u32 rand; size_t offset = 0; - get_random_bytes(&rand, sizeof(u32)); + rand = get_random_u32(); memcpy(buffer + offset, &now, sizeof(now)); offset += sizeof(now); memcpy(buffer + offset, &rand, sizeof(rand)); diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index 85ad9dc210ff..c79f60df3a90 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -220,7 +220,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, PTR_ERR(bbuf)); /* assume the block is corrupted */ - if (neras && *neras <= v->fec->roots) + if (neras && *neras < v->fec->roots) fio->erasures[(*neras)++] = i; continue; @@ -238,7 +238,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, * skip if we have already found the theoretical * maximum number (i.e. fec->roots) of erasures */ - if (neras && *neras <= v->fec->roots && + if (neras && *neras < v->fec->roots && fec_is_erasure(v, io, want_digest, bbuf)) fio->erasures[(*neras)++] = i; } diff --git a/drivers/md/dm-verity-fec.h b/drivers/md/dm-verity-fec.h index 50b5e187d5cc..3885b514fc23 100644 --- a/drivers/md/dm-verity-fec.h +++ b/drivers/md/dm-verity-fec.h @@ -47,7 +47,7 @@ struct dm_verity_fec { /* per-bio data */ struct dm_verity_fec_io { struct rs_control *rs; /* Reed-Solomon state */ - int erasures[DM_VERITY_FEC_MAX_ROOTS + 1]; /* erasures for decode_rs8 */ + int erasures[DM_VERITY_FEC_MAX_ROOTS]; /* erasures for decode_rs8 */ u8 *output; /* buffer for corrected output */ unsigned int level; /* recursion level */ unsigned int nbufs; /* number of buffers allocated */ diff --git a/drivers/md/dm-verity-loadpin.c b/drivers/md/dm-verity-loadpin.c index 0666699b6858..9a64f575ae5f 100644 --- a/drivers/md/dm-verity-loadpin.c +++ b/drivers/md/dm-verity-loadpin.c @@ -70,7 +70,7 @@ bool dm_verity_loadpin_is_bdev_trusted(struct block_device *bdev) table = dm_get_live_table(md, &srcu_idx); - if (table->num_targets != 1) + if (!table || table->num_targets != 1) goto out; ti = dm_table_get_target(table, 0); diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 9a9847f94c46..1b0763091254 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -26,7 +26,7 @@ #define DM_MSG_PREFIX "verity" -#define DM_VERITY_ENV_LENGTH 42 +#define DM_VERITY_ENV_LENGTH 46 #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR" #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 @@ -180,14 +180,16 @@ static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, char *envp[] = { verity_env, NULL }; const char *type_str = ""; struct mapped_device *md = dm_table_get_md(v->ti->table); + int ce; /* Corruption should be visible in device status in all modes */ v->hash_failed = true; - if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS) - goto out; - - v->corrupted_errs++; + ce = atomic_read(&v->corrupted_errs); + do { + if (ce >= DM_VERITY_MAX_CORRUPTED_ERRS) + goto out; + } while (!atomic_try_cmpxchg(&v->corrupted_errs, &ce, ce + 1)); switch (type) { case DM_VERITY_BLOCK_TYPE_DATA: @@ -203,7 +205,7 @@ static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name, type_str, block); - if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) { + if (ce + 1 == DM_VERITY_MAX_CORRUPTED_ERRS) { DMERR("%s: reached maximum errors", v->data_dev->name); dm_audit_log_target(DM_MSG_PREFIX, "max-corrupted-errors", v->ti, 0); } @@ -1262,6 +1264,8 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, continue; } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { + if (v->use_bh_wq) + continue; v->use_bh_wq = true; static_branch_inc(&use_bh_wq_enabled); continue; diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index 2922263501f6..e104a651c657 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -68,7 +68,7 @@ struct dm_verity { unsigned int digest_size; /* digest size for the current hash algorithm */ enum verity_mode mode; /* mode for handling verification errors */ enum verity_mode error_mode;/* mode for handling I/O errors */ - unsigned int corrupted_errs;/* Number of errors for corrupted blocks */ + atomic_t corrupted_errs;/* Number of errors for corrupted blocks */ struct workqueue_struct *verify_wq; diff --git a/drivers/md/dm-zoned-metadata.c b/drivers/md/dm-zoned-metadata.c index ec605acddd88..f4f81c79a080 100644 --- a/drivers/md/dm-zoned-metadata.c +++ b/drivers/md/dm-zoned-metadata.c @@ -2871,7 +2871,7 @@ int dmz_ctr_metadata(struct dmz_dev *dev, int num_dev, if (!zmd) return -ENOMEM; - strcpy(zmd->devname, devname); + strscpy(zmd->devname, devname); zmd->dev = dev; zmd->nr_devs = num_dev; zmd->mblk_rbtree = RB_ROOT; diff --git a/drivers/md/dm.c b/drivers/md/dm.c index e178fe19973e..d413bfaf3527 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -735,7 +735,16 @@ static struct table_device *open_table_device(struct mapped_device *md, return ERR_PTR(-ENOMEM); refcount_set(&td->count, 1); - bdev_file = bdev_file_open_by_dev(dev, mode, _dm_claim_ptr, NULL); + /* + * Open the backing device with kernel rather than caller + * credentials. Otherwise the caller's credentials would be + * pinned in bdev_file->f_cred until the table device is closed. + * That would keep the caller's thread keyring alive long beyond the + * lifetime of the caller, breaking userspace expectation (e.g. + * cryptsetup(8) leaking the LUKS volume key). + */ + scoped_with_kernel_creds() + bdev_file = bdev_file_open_by_dev(dev, mode, _dm_claim_ptr, NULL); if (IS_ERR(bdev_file)) { r = PTR_ERR(bdev_file); goto out_free_td; @@ -2098,8 +2107,17 @@ static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob, WARN_ON_ONCE(!dm_tio_is_normal(&io->tio)); /* don't poll if the mapped io is done */ - if (atomic_read(&io->io_count) > 1) - bio_poll(&io->tio.clone, iob, flags); + if (atomic_read(&io->io_count) > 1) { + /* + * DM hides the target queues from the upper poller, which may + * decide it is safe to spin on a single stacked queue. Do not + * pass that spinning policy down to a target queue: one slow + * clone could keep the task inside dm_poll_bio() for a long + * time. Poll target bios once and let the caller decide + * whether to keep polling, reap completions or reschedule. + */ + bio_poll(&io->tio.clone, iob, flags | BLK_POLL_ONESHOT); + } /* bio_poll holds the last reference */ return atomic_read(&io->io_count) == 1; @@ -2546,7 +2564,7 @@ int dm_create(int minor, struct mapped_device **result) if (!md) return -ENXIO; - dm_ima_reset_data(md); + dm_ima_init(md); *result = md; return 0; diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 028b9ca8ce52..0f02e2956398 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -502,6 +502,18 @@ static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index, static void md_bitmap_file_kick(struct bitmap *bitmap); #ifdef CONFIG_MD_BITMAP_FILE +static void end_bitmap_write(struct bio *bio) +{ + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + struct bitmap *bitmap = bh->b_private; + + if (!uptodate) + set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); + if (atomic_dec_and_test(&bitmap->pending_writes)) + wake_up(&bitmap->write_wait); +} + static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) { struct buffer_head *bh = page_buffers(page); @@ -510,7 +522,7 @@ static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) atomic_inc(&bitmap->pending_writes); set_buffer_locked(bh); set_buffer_mapped(bh); - submit_bh(REQ_OP_WRITE | REQ_SYNC, bh); + bh_submit(bh, REQ_OP_WRITE | REQ_SYNC, end_bitmap_write); bh = bh->b_this_page; } @@ -519,16 +531,6 @@ static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) atomic_read(&bitmap->pending_writes) == 0); } -static void end_bitmap_write(struct buffer_head *bh, int uptodate) -{ - struct bitmap *bitmap = bh->b_private; - - if (!uptodate) - set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); - if (atomic_dec_and_test(&bitmap->pending_writes)) - wake_up(&bitmap->write_wait); -} - static void free_buffers(struct page *page) { struct buffer_head *bh; @@ -592,12 +594,11 @@ static int read_file_page(struct file *file, unsigned long index, else count -= blocksize; - bh->b_end_io = end_bitmap_write; bh->b_private = bitmap; atomic_inc(&bitmap->pending_writes); set_buffer_locked(bh); set_buffer_mapped(bh); - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, end_bitmap_write); } blk_cur++; bh = bh->b_this_page; @@ -2063,18 +2064,23 @@ static void bitmap_end_behind_write(struct mddev *mddev) bitmap->mddev->bitmap_info.max_write_behind); } -static void bitmap_wait_behind_writes(struct mddev *mddev) +static bool bitmap_wait_behind_writes(struct mddev *mddev, bool nowait) { struct bitmap *bitmap = mddev->bitmap; /* wait for behind writes to complete */ if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { + if (nowait) + return false; + pr_debug("md:%s: behind writes in progress - waiting to stop.\n", mdname(mddev)); /* need to kick something here to make sure I/O goes? */ wait_event(bitmap->behind_wait, atomic_read(&bitmap->behind_writes) == 0); } + + return true; } static void bitmap_destroy(struct mddev *mddev) @@ -2084,7 +2090,7 @@ static void bitmap_destroy(struct mddev *mddev) if (!bitmap) /* there was no bitmap */ return; - bitmap_wait_behind_writes(mddev); + bitmap_wait_behind_writes(mddev, false); if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags)) mddev_destroy_serial_pool(mddev, NULL); diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index 214f623c7e79..f46674bdfeb9 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -98,7 +98,7 @@ struct bitmap_operations { void (*start_behind_write)(struct mddev *mddev); void (*end_behind_write)(struct mddev *mddev); - void (*wait_behind_writes)(struct mddev *mddev); + bool (*wait_behind_writes)(struct mddev *mddev, bool nowait); md_bitmap_fn *start_write; md_bitmap_fn *end_write; diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 1adc5b117821..5a4e2abaa757 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1574,16 +1574,19 @@ static void llbitmap_end_behind_write(struct mddev *mddev) wake_up(&llbitmap->behind_wait); } -static void llbitmap_wait_behind_writes(struct mddev *mddev) +static bool llbitmap_wait_behind_writes(struct mddev *mddev, bool nowait) { struct llbitmap *llbitmap = mddev->bitmap; - if (!llbitmap) - return; + if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) { + if (nowait) + return false; - wait_event(llbitmap->behind_wait, - atomic_read(&llbitmap->behind_writes) == 0); + wait_event(llbitmap->behind_wait, + atomic_read(&llbitmap->behind_writes) == 0); + } + return true; } static ssize_t bits_show(struct mddev *mddev, char *page) diff --git a/drivers/md/md.c b/drivers/md/md.c index 8b568eee8743..d1465bcd86c8 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -395,17 +395,24 @@ static bool is_suspended(struct mddev *mddev, struct bio *bio) bool md_handle_request(struct mddev *mddev, struct bio *bio) { check_suspended: - if (is_suspended(mddev, bio)) { - /* Bail out if REQ_NOWAIT is set for the bio */ - if (bio->bi_opf & REQ_NOWAIT) { - bio_wouldblock_error(bio); - return true; + if (unlikely(md_cloned_bio(mddev, bio))) { + /* + * This bio is an MD cloned bio and already holds an + * active_io reference, so percpu_ref_get() is safe here. + */ + percpu_ref_get(&mddev->active_io); + } else { + if (is_suspended(mddev, bio)) { + /* Bail out if REQ_NOWAIT is set for the bio */ + if (bio->bi_opf & REQ_NOWAIT) { + bio_wouldblock_error(bio); + return true; + } + wait_event(mddev->sb_wait, !is_suspended(mddev, bio)); } - wait_event(mddev->sb_wait, !is_suspended(mddev, bio)); + if (!percpu_ref_tryget_live(&mddev->active_io)) + goto check_suspended; } - if (!percpu_ref_tryget_live(&mddev->active_io)) - goto check_suspended; - if (!mddev->pers->make_request(mddev, bio)) { percpu_ref_put(&mddev->active_io); if (mddev_is_dm(mddev) && mddev->pers->prepare_suspend) @@ -4414,9 +4421,10 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; - if (mddev->pers) - err = update_raid_disks(mddev, n); - else if (mddev->reshape_position != MaxSector) { + if (mddev->pers) { + if (n != mddev->raid_disks) + err = update_raid_disks(mddev, n); + } else if (mddev->reshape_position != MaxSector) { struct md_rdev *rdev; int olddisks = mddev->raid_disks - mddev->delta_disks; @@ -7042,7 +7050,7 @@ EXPORT_SYMBOL_GPL(md_stop_writes); static void mddev_detach(struct mddev *mddev) { if (md_bitmap_enabled(mddev, false)) - mddev->bitmap_ops->wait_behind_writes(mddev); + mddev->bitmap_ops->wait_behind_writes(mddev, false); if (mddev->pers && mddev->pers->quiesce && !is_md_suspended(mddev)) { mddev->pers->quiesce(mddev, 1); mddev->pers->quiesce(mddev, 0); diff --git a/drivers/md/md.h b/drivers/md/md.h index 52c378086046..d8daf0f75cbb 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -346,6 +346,7 @@ struct md_cluster_operations; * @MD_HAS_SUPERBLOCK: There is persistence sb in member disks. * @MD_FAILLAST_DEV: Allow last rdev to be removed. * @MD_SERIALIZE_POLICY: Enforce write IO is not reordered, just used by raid1. + * @MD_DM_SUSPENDING: This DM raid device is suspending. * * change UNSUPPORTED_MDDEV_FLAGS for each array type if new flag is added */ @@ -365,6 +366,7 @@ enum mddev_flags { MD_HAS_SUPERBLOCK, MD_FAILLAST_DEV, MD_SERIALIZE_POLICY, + MD_DM_SUSPENDING, }; enum mddev_sb_flags { @@ -1042,6 +1044,11 @@ void mddev_update_io_opt(struct mddev *mddev, unsigned int nr_stripes); extern const struct block_device_operations md_fops; +static inline bool md_cloned_bio(struct mddev *mddev, struct bio *bio) +{ + return bio->bi_pool == &mddev->io_clone_set; +} + /* * MD devices can be used undeneath by DM, in which case ->gendisk is NULL. */ diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index 5e38a51e349a..2c000b3a5f49 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -14,6 +14,7 @@ #include <linux/seq_file.h> #include <linux/module.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <trace/events/block.h> #include "md.h" #include "raid0.h" @@ -43,7 +44,7 @@ static void dump_zones(struct mddev *mddev) int raid_disks = conf->strip_zone[0].nb_dev; pr_debug("md: RAID0 configuration for %s - %d zone%s\n", mdname(mddev), - conf->nr_strip_zones, conf->nr_strip_zones==1?"":"s"); + conf->nr_strip_zones, str_plural(conf->nr_strip_zones)); for (j = 0; j < conf->nr_strip_zones; j++) { char line[200]; int len = 0; @@ -392,6 +393,7 @@ static int raid0_set_limits(struct mddev *mddev) lim.io_opt = lim.io_min * mddev->raid_disks; lim.chunk_sectors = mddev->chunk_sectors; lim.features |= BLK_FEAT_ATOMIC_WRITES; + lim.features |= BLK_FEAT_PCI_P2PDMA; err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY); if (err) return err; diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 64d970e2ef50..afe2ca96ad8c 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -359,8 +359,8 @@ static inline void update_head_pos(int disk, struct r1bio *r1_bio) { struct r1conf *conf = r1_bio->mddev->private; - conf->mirrors[disk].head_position = - r1_bio->sector + (r1_bio->sectors); + WRITE_ONCE(conf->mirrors[disk].head_position, + r1_bio->sector + r1_bio->sectors); } /* @@ -737,7 +737,7 @@ static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio) { /* TODO: address issues with this check and concurrency. */ return conf->mirrors[disk].next_seq_sect == r1_bio->sector || - conf->mirrors[disk].head_position == r1_bio->sector; + READ_ONCE(conf->mirrors[disk].head_position) == r1_bio->sector; } /* @@ -814,7 +814,8 @@ static int choose_best_rdev(struct r1conf *conf, struct r1bio *r1_bio) set_bit(R1BIO_FailFast, &r1_bio->state); pending = atomic_read(&rdev->nr_pending); - dist = abs(r1_bio->sector - conf->mirrors[disk].head_position); + dist = abs(r1_bio->sector - + READ_ONCE(conf->mirrors[disk].head_position)); /* Don't change to another disk for sequential reads */ if (is_sequential(conf, disk, r1_bio)) { @@ -1341,21 +1342,34 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, int max_sectors; int rdisk; bool r1bio_existed = !!r1_bio; + bool nowait = bio->bi_opf & REQ_NOWAIT; /* - * If r1_bio is set, we are blocking the raid1d thread - * so there is a tiny risk of deadlock. So ask for + * An md cloned bio indicates we are in the error path. + * This is more reliable than checking r1_bio, which might + * be NULL even in the error path if a failed bio was split. + */ + bool err_path = md_cloned_bio(mddev, bio); + + /* + * If we are in the error path, we are blocking the raid1d + * thread so there is a tiny risk of deadlock. So ask for * emergency memory if needed. */ - gfp_t gfp = r1_bio ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO; + gfp_t gfp = err_path ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO; /* * Still need barrier for READ in case that whole * array is frozen. */ - if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, - bio->bi_opf & REQ_NOWAIT)) { + if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, nowait)) { bio_wouldblock_error(bio); + + if (r1bio_existed) { + set_bit(R1BIO_Returned, &r1_bio->state); + raid_end_bio_io(r1_bio); + } + return; } @@ -1395,7 +1409,11 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, * over-take any writes that are 'behind' */ mddev_add_trace_msg(mddev, "raid1 wait behind writes"); - mddev->bitmap_ops->wait_behind_writes(mddev); + if (!mddev->bitmap_ops->wait_behind_writes(mddev, nowait)) { + bio_wouldblock_error(bio); + set_bit(R1BIO_Returned, &r1_bio->state); + goto err_handle; + } } if (max_sectors < bio_sectors(bio)) { @@ -1411,7 +1429,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, } r1_bio->read_disk = rdisk; - if (!r1bio_existed) { + if (likely(!md_cloned_bio(mddev, bio))) { md_account_bio(mddev, &bio); r1_bio->master_bio = bio; } @@ -1494,29 +1512,30 @@ static void raid1_start_write_behind(struct mddev *mddev, struct r1bio *r1_bio, } -static void raid1_write_request(struct mddev *mddev, struct bio *bio, - int max_write_sectors) +static bool raid1_write_request(struct mddev *mddev, struct bio *bio, + int max_sectors) { struct r1conf *conf = mddev->private; struct r1bio *r1_bio; int i, disks, k; unsigned long flags; int first_clone; - int max_sectors; bool write_behind = false; - bool is_discard = (bio_op(bio) == REQ_OP_DISCARD); + bool nowait = bio->bi_opf & REQ_NOWAIT; + bool is_discard = op_is_discard(bio->bi_opf); + sector_t sector = bio->bi_iter.bi_sector; if (mddev_is_clustered(mddev) && - mddev->cluster_ops->area_resyncing(mddev, WRITE, - bio->bi_iter.bi_sector, bio_end_sector(bio))) { + mddev->cluster_ops->area_resyncing(mddev, WRITE, sector, + bio_end_sector(bio))) { - if (bio->bi_opf & REQ_NOWAIT) { + if (nowait) { bio_wouldblock_error(bio); - return; + return false; } wait_event_idle(conf->wait_barrier, !mddev->cluster_ops->area_resyncing(mddev, WRITE, - bio->bi_iter.bi_sector, + sector, bio_end_sector(bio))); } @@ -1525,19 +1544,18 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, * thread has put up a bar for new requests. * Continue immediately if no resync is active currently. */ - if (!wait_barrier(conf, bio->bi_iter.bi_sector, - bio->bi_opf & REQ_NOWAIT)) { + if (!wait_barrier(conf, sector, nowait)) { bio_wouldblock_error(bio); - return; + return false; } if (!wait_blocked_rdev(mddev, bio)) { bio_wouldblock_error(bio); - return; + goto err_allow_barrier; } r1_bio = alloc_r1bio(mddev, bio); - r1_bio->sectors = max_write_sectors; + r1_bio->sectors = max_sectors; /* first select target devices under rcu_lock and * inc refcount on their rdev. Record them by setting @@ -1551,7 +1569,6 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, */ disks = conf->raid_disks * 2; - max_sectors = r1_bio->sectors; for (i = 0; i < disks; i++) { struct md_rdev *rdev = conf->mirrors[i].rdev; @@ -1567,23 +1584,21 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, if (!rdev || test_bit(Faulty, &rdev->flags)) continue; - atomic_inc(&rdev->nr_pending); if (test_bit(WriteErrorSeen, &rdev->flags)) { sector_t first_bad; sector_t bad_sectors; int is_bad; - is_bad = is_badblock(rdev, r1_bio->sector, max_sectors, + is_bad = is_badblock(rdev, sector, max_sectors, &first_bad, &bad_sectors); - if (is_bad && first_bad <= r1_bio->sector) { + if (is_bad && first_bad <= sector) { /* Cannot write here at all */ - bad_sectors -= (r1_bio->sector - first_bad); + bad_sectors -= (sector - first_bad); if (bad_sectors < max_sectors) /* mustn't write more than bad_sectors * to other devices yet */ max_sectors = bad_sectors; - rdev_dec_pending(rdev, mddev); continue; } if (is_bad) { @@ -1596,14 +1611,19 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, * complexity of supporting that is not worth * the benefit. */ - if (bio->bi_opf & REQ_ATOMIC) - goto err_handle; + if (bio->bi_opf & REQ_ATOMIC) { + bio->bi_status = BLK_STS_NOTSUPP; + bio_endio(bio); + goto err_dec_pending; + } - good_sectors = first_bad - r1_bio->sector; + good_sectors = first_bad - sector; if (good_sectors < max_sectors) max_sectors = good_sectors; } } + + atomic_inc(&rdev->nr_pending); r1_bio->bios[i] = bio; } @@ -1619,10 +1639,8 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, if (max_sectors < bio_sectors(bio)) { bio = bio_submit_split_bioset(bio, max_sectors, &conf->bio_split); - if (!bio) { - set_bit(R1BIO_Returned, &r1_bio->state); - goto err_handle; - } + if (!bio) + goto err_dec_pending; r1_bio->master_bio = bio; r1_bio->sectors = max_sectors; @@ -1666,7 +1684,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, mbio->bi_opf &= ~REQ_NOWAIT; r1_bio->bios[i] = mbio; - mbio->bi_iter.bi_sector = (r1_bio->sector + rdev->data_offset); + mbio->bi_iter.bi_sector = sector + rdev->data_offset; mbio->bi_end_io = raid1_end_write_request; if (test_bit(FailFast, &rdev->flags) && !test_bit(WriteMostly, &rdev->flags) && @@ -1675,7 +1693,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, mbio->bi_private = r1_bio; atomic_inc(&r1_bio->remaining); - mddev_trace_remap(mddev, mbio, r1_bio->sector); + mddev_trace_remap(mddev, mbio, sector); /* flush_pending_writes() needs access to the rdev so...*/ mbio->bi_bdev = (void *)rdev; if (!raid1_add_bio_to_plug(mddev, mbio, raid1_unplug, disks)) { @@ -1690,8 +1708,10 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, /* In case raid1d snuck in to freeze_array */ wake_up_barrier(conf); - return; -err_handle: + + return true; + +err_dec_pending: for (k = 0; k < i; k++) { if (r1_bio->bios[k]) { rdev_dec_pending(conf->mirrors[k].rdev, mddev); @@ -1699,7 +1719,12 @@ err_handle: } } - raid_end_bio_io(r1_bio); + free_r1bio(r1_bio); + +err_allow_barrier: + allow_barrier(conf, sector); + + return false; } static bool raid1_make_request(struct mddev *mddev, struct bio *bio) @@ -1723,8 +1748,9 @@ static bool raid1_make_request(struct mddev *mddev, struct bio *bio) if (bio_data_dir(bio) == READ) raid1_read_request(mddev, bio, sectors, NULL); else { - md_write_start(mddev,bio); - raid1_write_request(mddev, bio, sectors); + md_write_start(mddev, bio); + if (!raid1_write_request(mddev, bio, sectors)) + md_write_end(mddev); } return true; } @@ -2411,11 +2437,6 @@ static void fix_read_error(struct r1conf *conf, struct r1bio *r1_bio) struct mddev *mddev = conf->mddev; struct md_rdev *rdev = conf->mirrors[read_disk].rdev; - if (exceed_read_errors(mddev, rdev)) { - r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED; - return; - } - while(sectors) { int s = sectors; int d = read_disk; @@ -2627,35 +2648,36 @@ static void handle_write_finished(struct r1conf *conf, struct r1bio *r1_bio) static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio) { + struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev; + struct bio *bio = r1_bio->bios[r1_bio->read_disk]; struct mddev *mddev = conf->mddev; - struct bio *bio; - struct md_rdev *rdev; sector_t sector; clear_bit(R1BIO_ReadError, &r1_bio->state); - /* we got a read error. Maybe the drive is bad. Maybe just - * the block and we can fix it. - * We freeze all other IO, and try reading the block from - * other devices. When we find one, we re-write - * and check it that fixes the read error. - * This is all done synchronously while the array is - * frozen - */ - bio = r1_bio->bios[r1_bio->read_disk]; bio_put(bio); r1_bio->bios[r1_bio->read_disk] = NULL; - rdev = conf->mirrors[r1_bio->read_disk].rdev; - if (mddev->ro == 0 - && !test_bit(FailFast, &rdev->flags)) { - freeze_array(conf, 1); - fix_read_error(conf, r1_bio); - unfreeze_array(conf); - } else if (mddev->ro == 0 && test_bit(FailFast, &rdev->flags)) { + /* + * We got a read error. Maybe the drive is bad. Maybe just the block + * and we can fix it. + * + * If allowed, freeze all other IO, and try reading the block from other + * devices. If we find one, we re-write and check it that fixes the + * read error. This is all done synchronously while the array is + * frozen. + */ + if (mddev->ro) { + r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED; + } else if (test_bit(FailFast, &rdev->flags)) { md_error(mddev, rdev); } else { - r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED; + freeze_array(conf, 1); + if (exceed_read_errors(mddev, rdev)) + r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED; + else + fix_read_error(conf, r1_bio); + unfreeze_array(conf); } rdev_dec_pending(rdev, conf->mddev); @@ -3208,6 +3230,7 @@ static int raid1_set_limits(struct mddev *mddev) lim.max_hw_wzeroes_unmap_sectors = 0; lim.logical_block_size = mddev->logical_block_size; lim.features |= BLK_FEAT_ATOMIC_WRITES; + lim.features |= BLK_FEAT_PCI_P2PDMA; err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY); if (err) return err; diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 39085e7dd6d2..0a3cfdd3f5df 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1146,7 +1146,7 @@ static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf, } static void raid10_read_request(struct mddev *mddev, struct bio *bio, - struct r10bio *r10_bio, bool io_accounting) + struct r10bio *r10_bio) { struct r10conf *conf = mddev->private; struct bio *read_bio; @@ -1155,7 +1155,20 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, char b[BDEVNAME_SIZE]; int slot = r10_bio->read_slot; struct md_rdev *err_rdev = NULL; - gfp_t gfp = GFP_NOIO; + + /* + * An md cloned bio indicates we are in the error path. + * This is more reliable than checking slot, which might + * be -1 even in the error path if a failed bio was split. + */ + bool err_path = md_cloned_bio(mddev, bio); + + /* + * If we are in the error path, we are blocking the raid10d + * thread so there is a tiny risk of deadlock. So ask for + * emergency memory if needed. + */ + gfp_t gfp = err_path ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO; if (slot >= 0 && r10_bio->devs[slot].rdev) { /* @@ -1166,11 +1179,6 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, * we lose the device name in error messages. */ int disk; - /* - * As we are blocking raid10, it is a little safer to - * use __GFP_HIGH. - */ - gfp = GFP_NOIO | __GFP_HIGH; disk = r10_bio->devs[slot].devnum; err_rdev = conf->mirrors[disk].rdev; @@ -1218,7 +1226,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, } slot = r10_bio->read_slot; - if (io_accounting) { + if (likely(!md_cloned_bio(mddev, bio))) { md_account_bio(mddev, &bio); r10_bio->master_bio = bio; } @@ -1341,7 +1349,7 @@ retry_wait: } } -static void raid10_write_request(struct mddev *mddev, struct bio *bio, +static bool raid10_write_request(struct mddev *mddev, struct bio *bio, struct r10bio *r10_bio) { struct r10conf *conf = mddev->private; @@ -1357,7 +1365,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, /* Bail out if REQ_NOWAIT is set for the bio */ if (bio->bi_opf & REQ_NOWAIT) { bio_wouldblock_error(bio); - return; + return false; } for (;;) { prepare_to_wait(&conf->wait_barrier, @@ -1373,7 +1381,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, sectors = r10_bio->sectors; if (!regular_request_wait(mddev, conf, bio, sectors)) { free_r10bio(r10_bio); - return; + return false; } if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && @@ -1390,7 +1398,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, if (bio->bi_opf & REQ_NOWAIT) { allow_barrier(conf); bio_wouldblock_error(bio); - return; + return false; } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape metadata"); @@ -1506,7 +1514,8 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, raid10_write_one_disk(mddev, r10_bio, bio, true, i); } one_write_done(r10_bio); - return; + return true; + err_handle: for (k = 0; k < i; k++) { int d = r10_bio->devs[k].devnum; @@ -1524,10 +1533,12 @@ err_handle: } raid_end_bio_io(r10_bio); + return false; } -static void __make_request(struct mddev *mddev, struct bio *bio, int sectors) +static bool __make_request(struct mddev *mddev, struct bio *bio, int sectors) { + bool ret; struct r10conf *conf = mddev->private; struct r10bio *r10_bio; @@ -1543,10 +1554,13 @@ static void __make_request(struct mddev *mddev, struct bio *bio, int sectors) memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * conf->geo.raid_disks); + ret = true; if (bio_data_dir(bio) == READ) - raid10_read_request(mddev, bio, r10_bio, true); + raid10_read_request(mddev, bio, r10_bio); else - raid10_write_request(mddev, bio, r10_bio); + ret = raid10_write_request(mddev, bio, r10_bio); + + return ret; } static void raid_end_discard_bio(struct r10bio *r10bio) @@ -1625,6 +1639,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { bio_wouldblock_error(bio); + md_write_end(mddev); return 0; } @@ -1667,6 +1682,8 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (IS_ERR(split)) { bio->bi_status = errno_to_blk_status(PTR_ERR(split)); bio_endio(bio); + md_write_end(mddev); + allow_barrier(conf); return 0; } @@ -1684,6 +1701,8 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (IS_ERR(split)) { bio->bi_status = errno_to_blk_status(PTR_ERR(split)); bio_endio(bio); + md_write_end(mddev); + allow_barrier(conf); return 0; } @@ -1727,6 +1746,7 @@ retry_discard: r10_bio->mddev = mddev; r10_bio->state = 0; r10_bio->sectors = 0; + r10_bio->read_slot = -1; memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * geo->raid_disks); wait_blocked_dev(mddev, r10_bio); @@ -1891,7 +1911,8 @@ static bool raid10_make_request(struct mddev *mddev, struct bio *bio) sectors = chunk_sects - (bio->bi_iter.bi_sector & (chunk_sects - 1)); - __make_request(mddev, bio, sectors); + if (!__make_request(mddev, bio, sectors)) + md_write_end(mddev); /* In case raid10d snuck in to freeze_array */ wake_up_barrier(conf); @@ -2858,7 +2879,7 @@ static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio) rdev_dec_pending(rdev, mddev); r10_bio->state = 0; - raid10_read_request(mddev, r10_bio->master_bio, r10_bio, false); + raid10_read_request(mddev, r10_bio->master_bio, r10_bio); /* * allow_barrier after re-submit to ensure no sync io * can be issued while regular io pending. @@ -3941,6 +3962,7 @@ static int raid10_set_queue_limits(struct mddev *mddev) lim.chunk_sectors = mddev->chunk_sectors; lim.io_opt = lim.io_min * raid10_nr_stripes(conf); lim.features |= BLK_FEAT_ATOMIC_WRITES; + lim.features |= BLK_FEAT_PCI_P2PDMA; err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY); if (err) return err; diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 0d76e82f4506..ffb5fcde54a9 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -996,7 +996,7 @@ static void stripe_add_to_batch_list(struct r5conf *conf, if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) { int seq = sh->bm_seq; if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) && - sh->batch_head->bm_seq > seq) + sh->batch_head->bm_seq - seq > 0) seq = sh->batch_head->bm_seq; set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state); sh->batch_head->bm_seq = seq; @@ -1134,6 +1134,21 @@ static void defer_issue_bios(struct r5conf *conf, sector_t sector, dispatch_bio_list(&tmp); } +static bool raid5_discard_limits(struct mddev *mddev, struct bio *bi) +{ + struct r5conf *conf = mddev->private; + + if (mddev->bitmap_id == ID_LLBITMAP) + return true; + + if (!conf->raid5_discard_unsupported) + return true; + + bi->bi_status = BLK_STS_NOTSUPP; + bio_endio(bi); + return false; +} + static void raid5_end_read_request(struct bio *bi); static void @@ -4837,55 +4852,62 @@ static void break_stripe_batch_list(struct stripe_head *head_sh, { struct stripe_head *sh, *next; int i; + unsigned long state; list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) { list_del_init(&sh->batch_list); - WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) | - (1 << STRIPE_SYNCING) | - (1 << STRIPE_REPLACED) | - (1 << STRIPE_DELAYED) | - (1 << STRIPE_BIT_DELAY) | - (1 << STRIPE_FULL_WRITE) | - (1 << STRIPE_BIOFILL_RUN) | - (1 << STRIPE_COMPUTE_RUN) | - (1 << STRIPE_DISCARD) | - (1 << STRIPE_BATCH_READY) | - (1 << STRIPE_BATCH_ERR)), - "stripe state: %lx\n", sh->state); - WARN_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) | - (1 << STRIPE_REPLACED)), - "head stripe state: %lx\n", head_sh->state); + state = READ_ONCE(sh->state); + WARN_ONCE(state & ((1 << STRIPE_ACTIVE) | + (1 << STRIPE_SYNCING) | + (1 << STRIPE_REPLACED) | + (1 << STRIPE_DELAYED) | + (1 << STRIPE_BIT_DELAY) | + (1 << STRIPE_FULL_WRITE) | + (1 << STRIPE_BIOFILL_RUN) | + (1 << STRIPE_COMPUTE_RUN) | + (1 << STRIPE_DISCARD) | + (1 << STRIPE_BATCH_READY) | + (1 << STRIPE_BATCH_ERR)), + "stripe state: %lx\n", state); + + state = READ_ONCE(head_sh->state); + WARN_ONCE(state & ((1 << STRIPE_DISCARD) | + (1 << STRIPE_REPLACED)), + "head stripe state: %lx\n", state); set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS | (1 << STRIPE_PREREAD_ACTIVE) | (1 << STRIPE_ON_UNPLUG_LIST)), - head_sh->state & (1 << STRIPE_INSYNC)); + state & (1 << STRIPE_INSYNC)); sh->check_state = head_sh->check_state; sh->reconstruct_state = head_sh->reconstruct_state; spin_lock_irq(&sh->stripe_lock); - sh->batch_head = NULL; - spin_unlock_irq(&sh->stripe_lock); for (i = 0; i < sh->disks; i++) { if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) wake_up_bit(&sh->dev[i].flags, R5_Overlap); - sh->dev[i].flags = head_sh->dev[i].flags & + sh->dev[i].flags = READ_ONCE(head_sh->dev[i].flags) & (~((1 << R5_WriteError) | (1 << R5_Overlap))); } - if (handle_flags == 0 || - sh->state & handle_flags) + sh->batch_head = NULL; + spin_unlock_irq(&sh->stripe_lock); + + state = READ_ONCE(sh->state); + if (handle_flags == 0 || (state & handle_flags)) set_bit(STRIPE_HANDLE, &sh->state); raid5_release_stripe(sh); } spin_lock_irq(&head_sh->stripe_lock); - head_sh->batch_head = NULL; - spin_unlock_irq(&head_sh->stripe_lock); for (i = 0; i < head_sh->disks; i++) if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags)) wake_up_bit(&head_sh->dev[i].flags, R5_Overlap); - if (head_sh->state & handle_flags) + head_sh->batch_head = NULL; + spin_unlock_irq(&head_sh->stripe_lock); + + state = READ_ONCE(head_sh->state); + if (state & handle_flags) set_bit(STRIPE_HANDLE, &head_sh->state); } @@ -5690,7 +5712,10 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) { struct r5conf *conf = mddev->private; sector_t logical_sector, last_sector; + sector_t first_stripe, last_stripe; struct stripe_head *sh; + struct bvec_iter bi_iter; + struct bio *orig_bi = bi; int stripe_sectors; /* We need to handle this when io_uring supports discard/trim */ @@ -5701,19 +5726,38 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) /* Skip discard while reshape is happening */ return; - logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1); - last_sector = bio_end_sector(bi); - - bi->bi_next = NULL; + if (!raid5_discard_limits(mddev, bi)) + return; stripe_sectors = conf->chunk_sectors * (conf->raid_disks - conf->max_degraded); - logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, - stripe_sectors); - sector_div(last_sector, stripe_sectors); + first_stripe = DIV_ROUND_UP_SECTOR_T(bi->bi_iter.bi_sector, + stripe_sectors); + last_stripe = bio_end_sector(bi); + sector_div(last_stripe, stripe_sectors); - logical_sector *= conf->chunk_sectors; - last_sector *= conf->chunk_sectors; + if (first_stripe >= last_stripe) { + bio_endio(bi); + return; + } + + bi_iter = bi->bi_iter; + bi->bi_iter.bi_sector = first_stripe * stripe_sectors; + bi->bi_iter.bi_size = ((last_stripe - first_stripe) * + stripe_sectors) << 9; + md_account_bio(mddev, &bi); + orig_bi->bi_iter = bi_iter; + bi->bi_iter = bi_iter; + bi->bi_next = NULL; + + if (mddev->bitmap_id == ID_LLBITMAP && + conf->raid5_discard_unsupported) { + bio_endio(bi); + return; + } + + logical_sector = first_stripe * conf->chunk_sectors; + last_sector = last_stripe * conf->chunk_sectors; for (; logical_sector < last_sector; logical_sector += RAID5_STRIPE_SECTORS(conf)) { @@ -6042,8 +6086,11 @@ out_release: raid5_release_stripe(sh); out: if (ret == STRIPE_SCHEDULE_AND_RETRY && reshape_interrupted(mddev)) { - bi->bi_status = BLK_STS_RESOURCE; - ret = STRIPE_WAIT_RESHAPE; + if (!mddev_is_dm(mddev) || + test_bit(MD_DM_SUSPENDING, &mddev->flags)) { + bi->bi_status = BLK_STS_RESOURCE; + ret = STRIPE_WAIT_RESHAPE; + } pr_err_ratelimited("dm-raid456: io across reshape position while reshape can't make progress"); } return ret; @@ -6955,7 +7002,7 @@ raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len) if (kstrtoul(page, 10, &new)) return -EINVAL; - if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome) + if (new != PARITY_DISABLE_RMW && !raid6_can_xor_syndrome()) return -EINVAL; if (new != PARITY_DISABLE_RMW && @@ -7646,7 +7693,7 @@ static struct r5conf *setup_conf(struct mddev *mddev) conf->level = mddev->new_level; if (conf->level == 6) { conf->max_degraded = 2; - if (raid6_call.xor_syndrome) + if (raid6_can_xor_syndrome()) conf->rmw_level = PARITY_ENABLE_RMW; else conf->rmw_level = PARITY_DISABLE_RMW; @@ -7801,24 +7848,12 @@ static int raid5_set_limits(struct mddev *mddev) queue_limits_stack_bdev(&lim, rdev->bdev, rdev->new_data_offset, mddev->gendisk->disk_name); - /* - * Zeroing is required for discard, otherwise data could be lost. - * - * Consider a scenario: discard a stripe (the stripe could be - * inconsistent if discard_zeroes_data is 0); write one disk of the - * stripe (the stripe could be inconsistent again depending on which - * disks are used to calculate parity); the disk is broken; The stripe - * data of this disk is lost. - * - * We only allow DISCARD if the sysadmin has confirmed that only safe - * devices are in use by setting a module parameter. A better idea - * might be to turn DISCARD into WRITE_ZEROES requests, as that is - * required to be safe. - */ if (!devices_handle_discard_safely || lim.max_discard_sectors < (stripe >> 9) || lim.discard_granularity < stripe) - lim.max_hw_discard_sectors = 0; + conf->raid5_discard_unsupported = true; + else + conf->raid5_discard_unsupported = false; /* * Requests require having a bitmap for each stripe. @@ -7827,6 +7862,7 @@ static int raid5_set_limits(struct mddev *mddev) lim.max_hw_sectors = RAID5_MAX_REQ_STRIPES << RAID5_STRIPE_SHIFT(conf); if ((lim.max_hw_sectors << 9) < lim.io_opt) lim.max_hw_sectors = lim.io_opt >> 9; + lim.max_hw_discard_sectors = UINT_MAX; /* No restrictions on the number of segments in the request */ lim.max_segments = USHRT_MAX; diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h index 1c7b710fc9c1..cb5feae04db2 100644 --- a/drivers/md/raid5.h +++ b/drivers/md/raid5.h @@ -38,7 +38,7 @@ * Clean -> Dirty - on compute_parity to satisfy write/sync (RECONSTRUCT or RMW) * * The Want->Empty, Want->Clean, Dirty->Clean, transitions - * all happen in b_end_io at interrupt time. + * all happen in end_io at interrupt time. * Each sets the Uptodate bit before releasing the Lock bit. * This leaves one multi-stage transition: * Want->Dirty->Clean @@ -64,7 +64,7 @@ * together, but we are not guaranteed of that so we allow for more. * * If a buffer is on the read list when the associated cache buffer is - * Uptodate, the data is copied into the read buffer and it's b_end_io + * Uptodate, the data is copied into the read buffer and it's end_io * routine is called. This may happen in the end_request routine only * if the buffer has just successfully been read. end_request should * remove the buffers from the list and then set the Uptodate bit on @@ -76,7 +76,7 @@ * into the cache buffer, which is then marked dirty, and moved onto a * third list, the written list (bh_written). Once both the parity * block and the cached buffer are successfully written, any buffer on - * a written list can be returned with b_end_io. + * a written list can be returned with end_io. * * The write list and read list both act as fifos. The read list, * write list and written list are protected by the device_lock. @@ -689,6 +689,7 @@ struct r5conf { struct list_head pending_list; int pending_data_cnt; struct r5pending_data *next_pending_data; + bool raid5_discard_unsupported; mempool_t *ctx_pool; int ctx_size; |