dm-verity: Convert from tasklet to BH workqueue

The only generic interface to execute asynchronously in the BH context is
tasklet; however, it's marked deprecated and has some design flaws. To
replace tasklets, BH workqueue support was recently added. A BH workqueue
behaves similarly to regular workqueues except that the queued work items
are executed in the BH context.

This commit converts dm-verity from tasklet to BH workqueue. It
backfills tasklet code that was removed with commit 0a9bab391e33
("dm-crypt, dm-verity: disable tasklets") and tweaks to use BH
workqueue (and does some renaming).

This is a minimal conversion which doesn't rename the related names
including the "try_verify_in_tasklet" option. If this patch is applied, a
follow-up patch would be necessary. I couldn't decide whether the option
name would need to be updated too.

Signed-off-by: Tejun Heo <tj@kernel.org>
[snitzer: rename 'use_tasklet' to 'use_bh_wq' and 'in_tasklet' to 'in_bh']
Signed-off-by: Mike Snitzer <snitzer@kernel.org>

diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index 6b8f9698488e7864beda98c7b8f3cb4867993229..b6f62d6820ba7e11d55899255af92667d275dea4 100644 (file)
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -46,7 +46,7 @@ static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE
 
 module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644);
 
-static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled);
+static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled);
 
 struct dm_verity_prefetch_work {
        struct work_struct work;
@@ -299,7 +299,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
 
        verity_hash_at_level(v, block, level, &hash_block, &offset);
 
-       if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) {
+       if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
                data = dm_bufio_get(v->bufio, hash_block, &buf);
                if (data == NULL) {
                        /*
@@ -327,15 +327,14 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
 
                r = verity_hash(v, verity_io_hash_req(v, io),
                                data, 1 << v->hash_dev_block_bits,
-                               verity_io_real_digest(v, io), !io->in_tasklet);
+                               verity_io_real_digest(v, io), !io->in_bh);
                if (unlikely(r < 0))
                        goto release_ret_r;
 
                if (likely(memcmp(verity_io_real_digest(v, io), want_digest,
                                  v->digest_size) == 0))
                        aux->hash_verified = 1;
-               else if (static_branch_unlikely(&use_tasklet_enabled) &&
-                        io->in_tasklet) {
+               else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
                        /*
                         * Error handling code (FEC included) cannot be run in a
                         * tasklet since it may sleep, so fallback to work-queue.
@@ -576,7 +575,7 @@ static int verity_verify_io(struct dm_verity_io *io)
        struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
        unsigned int b;
 
-       if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) {
+       if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
                /*
                 * Copy the iterator in case we need to restart
                 * verification in a work-queue.
@@ -616,7 +615,7 @@ static int verity_verify_io(struct dm_verity_io *io)
                        continue;
                }
 
-               r = verity_hash_init(v, req, &wait, !io->in_tasklet);
+               r = verity_hash_init(v, req, &wait, !io->in_bh);
                if (unlikely(r < 0))
                        return r;
 
@@ -635,8 +634,7 @@ static int verity_verify_io(struct dm_verity_io *io)
                        if (v->validated_blocks)
                                set_bit(cur_block, v->validated_blocks);
                        continue;
-               } else if (static_branch_unlikely(&use_tasklet_enabled) &&
-                          io->in_tasklet) {
+               } else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
                        /*
                         * Error handling code (FEC included) cannot be run in a
                         * tasklet since it may sleep, so fallback to work-queue.
@@ -690,7 +688,7 @@ static void verity_finish_io(struct dm_verity_io *io, blk_status_t status)
        bio->bi_end_io = io->orig_bi_end_io;
        bio->bi_status = status;
 
-       if (!static_branch_unlikely(&use_tasklet_enabled) || !io->in_tasklet)
+       if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh)
                verity_fec_finish_io(io);
 
        bio_endio(bio);
@@ -700,11 +698,28 @@ static void verity_work(struct work_struct *w)
 {
        struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
 
-       io->in_tasklet = false;
+       io->in_bh = false;
 
        verity_finish_io(io, errno_to_blk_status(verity_verify_io(io)));
 }
 
+static void verity_bh_work(struct work_struct *w)
+{
+       struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work);
+       int err;
+
+       io->in_bh = true;
+       err = verity_verify_io(io);
+       if (err == -EAGAIN || err == -ENOMEM) {
+               /* fallback to retrying with work-queue */
+               INIT_WORK(&io->work, verity_work);
+               queue_work(io->v->verify_wq, &io->work);
+               return;
+       }
+
+       verity_finish_io(io, errno_to_blk_status(err));
+}
+
 static void verity_end_io(struct bio *bio)
 {
        struct dm_verity_io *io = bio->bi_private;
@@ -717,8 +732,13 @@ static void verity_end_io(struct bio *bio)
                return;
        }
 
-       INIT_WORK(&io->work, verity_work);
-       queue_work(io->v->verify_wq, &io->work);
+       if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq) {
+               INIT_WORK(&io->bh_work, verity_bh_work);
+               queue_work(system_bh_wq, &io->bh_work);
+       } else {
+               INIT_WORK(&io->work, verity_work);
+               queue_work(io->v->verify_wq, &io->work);
+       }
 }
 
 /*
@@ -885,7 +905,7 @@ static void verity_status(struct dm_target *ti, status_type_t type,
                        args++;
                if (v->validated_blocks)
                        args++;
-               if (v->use_tasklet)
+               if (v->use_bh_wq)
                        args++;
                if (v->signature_key_desc)
                        args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS;
@@ -912,7 +932,7 @@ static void verity_status(struct dm_target *ti, status_type_t type,
                        DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);
                if (v->validated_blocks)
                        DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);
-               if (v->use_tasklet)
+               if (v->use_bh_wq)
                        DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY);
                sz = verity_fec_status_table(v, sz, result, maxlen);
                if (v->signature_key_desc)
@@ -1031,8 +1051,8 @@ static void verity_dtr(struct dm_target *ti)
 
        kfree(v->signature_key_desc);
 
-       if (v->use_tasklet)
-               static_branch_dec(&use_tasklet_enabled);
+       if (v->use_bh_wq)
+               static_branch_dec(&use_bh_wq_enabled);
 
        kfree(v);
 
@@ -1166,8 +1186,8 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
                        continue;
 
                } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) {
-                       v->use_tasklet = true;
-                       static_branch_inc(&use_tasklet_enabled);
+                       v->use_bh_wq = true;
+                       static_branch_inc(&use_bh_wq_enabled);
                        continue;
 
                } else if (verity_is_fec_opt_arg(arg_name)) {
@@ -1338,7 +1358,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
        }
 
        v->tfm = crypto_alloc_ahash(v->alg_name, 0,
-                                   v->use_tasklet ? CRYPTO_ALG_ASYNC : 0);
+                                   v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0);
        if (IS_ERR(v->tfm)) {
                ti->error = "Cannot initialize hash function";
                r = PTR_ERR(v->tfm);
@@ -1463,7 +1483,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
        v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
                1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
                dm_bufio_alloc_callback, NULL,
-               v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
+               v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
        if (IS_ERR(v->bufio)) {
                ti->error = "Cannot initialize dm-bufio";
                r = PTR_ERR(v->bufio);
@@ -1482,7 +1502,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
         * reducing wait times when reading from a dm-verity device.
         *
         * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI
-        * allows verify_wq to preempt softirq since verification in tasklet
+        * allows verify_wq to preempt softirq since verification in BH workqueue
         * will fall-back to using it for error handling (or if the bufio cache
         * doesn't have required hashes).
         */

diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h
index db93a91169d5e6de31d344a6f37589bbc0bdb654..20b1bcf034742305976f1763da08e6f4964be75e 100644 (file)
--- a/drivers/md/dm-verity.h
+++ b/drivers/md/dm-verity.h
@@ -54,7 +54,7 @@ struct dm_verity {
        unsigned char levels;   /* the number of tree levels */
        unsigned char version;
        bool hash_failed:1;     /* set if hash of any block failed */
-       bool use_tasklet:1;     /* try to verify in tasklet before work-queue */
+       bool use_bh_wq:1;       /* try to verify in BH wq before normal work-queue */
        unsigned int digest_size;       /* digest size for the current hash algorithm */
        unsigned int ahash_reqsize;/* the size of temporary space for crypto */
        enum verity_mode mode;  /* mode for handling verification errors */
@@ -84,9 +84,10 @@ struct dm_verity_io {
 
        sector_t block;
        unsigned int n_blocks;
-       bool in_tasklet;
+       bool in_bh;
 
        struct work_struct work;
+       struct work_struct bh_work;
 
        char *recheck_buffer;