fuse: add kernel-enforced timeout option for requests

There are situations where fuse servers can become unresponsive or
stuck, for example if the server is deadlocked. Currently, there's no
good way to detect if a server is stuck and needs to be killed manually.

This commit adds an option for enforcing a timeout (in seconds) for
requests where if the timeout elapses without the server responding to
the request, the connection will be automatically aborted.

Please note that these timeouts are not 100% precise. For example, the
request may take roughly an extra FUSE_TIMEOUT_TIMER_FREQ seconds beyond
the requested timeout due to internal implementation, in order to
mitigate overhead.

[SzM: Bump the API version number]

Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
Reviewed-by: Jeff Layton <jlayton@kernel.org>
Signed-off-by: Miklos Szeredi <mszeredi@redhat.com>

diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index 2645cd8accfd081c518d3e22127e899ad5a09127..f48afffcb2a545e72e5975a2a6ce283359824ee9 100644 (file)
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -32,6 +32,103 @@ MODULE_ALIAS("devname:fuse");
 
 static struct kmem_cache *fuse_req_cachep;
 
+/* Frequency (in seconds) of request timeout checks, if opted into */
+#define FUSE_TIMEOUT_TIMER_FREQ 15
+
+const unsigned long fuse_timeout_timer_freq =
+       secs_to_jiffies(FUSE_TIMEOUT_TIMER_FREQ);
+
+bool fuse_request_expired(struct fuse_conn *fc, struct list_head *list)
+{
+       struct fuse_req *req;
+
+       req = list_first_entry_or_null(list, struct fuse_req, list);
+       if (!req)
+               return false;
+       return time_is_before_jiffies(req->create_time + fc->timeout.req_timeout);
+}
+
+bool fuse_fpq_processing_expired(struct fuse_conn *fc, struct list_head *processing)
+{
+       int i;
+
+       for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
+               if (fuse_request_expired(fc, &processing[i]))
+                       return true;
+
+       return false;
+}
+
+/*
+ * Check if any requests aren't being completed by the time the request timeout
+ * elapses. To do so, we:
+ * - check the fiq pending list
+ * - check the bg queue
+ * - check the fpq io and processing lists
+ *
+ * To make this fast, we only check against the head request on each list since
+ * these are generally queued in order of creation time (eg newer requests get
+ * queued to the tail). We might miss a few edge cases (eg requests transitioning
+ * between lists, re-sent requests at the head of the pending list having a
+ * later creation time than other requests on that list, etc.) but that is fine
+ * since if the request never gets fulfilled, it will eventually be caught.
+ */
+void fuse_check_timeout(struct work_struct *work)
+{
+       struct delayed_work *dwork = to_delayed_work(work);
+       struct fuse_conn *fc = container_of(dwork, struct fuse_conn,
+                                           timeout.work);
+       struct fuse_iqueue *fiq = &fc->iq;
+       struct fuse_dev *fud;
+       struct fuse_pqueue *fpq;
+       bool expired = false;
+
+       if (!atomic_read(&fc->num_waiting))
+           goto out;
+
+       spin_lock(&fiq->lock);
+       expired = fuse_request_expired(fc, &fiq->pending);
+       spin_unlock(&fiq->lock);
+       if (expired)
+               goto abort_conn;
+
+       spin_lock(&fc->bg_lock);
+       expired = fuse_request_expired(fc, &fc->bg_queue);
+       spin_unlock(&fc->bg_lock);
+       if (expired)
+               goto abort_conn;
+
+       spin_lock(&fc->lock);
+       if (!fc->connected) {
+               spin_unlock(&fc->lock);
+               return;
+       }
+       list_for_each_entry(fud, &fc->devices, entry) {
+               fpq = &fud->pq;
+               spin_lock(&fpq->lock);
+               if (fuse_request_expired(fc, &fpq->io) ||
+                   fuse_fpq_processing_expired(fc, fpq->processing)) {
+                       spin_unlock(&fpq->lock);
+                       spin_unlock(&fc->lock);
+                       goto abort_conn;
+               }
+
+               spin_unlock(&fpq->lock);
+       }
+       spin_unlock(&fc->lock);
+
+       if (fuse_uring_request_expired(fc))
+           goto abort_conn;
+
+out:
+       queue_delayed_work(system_wq, &fc->timeout.work,
+                          fuse_timeout_timer_freq);
+       return;
+
+abort_conn:
+       fuse_abort_conn(fc);
+}
+
 static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req)
 {
        INIT_LIST_HEAD(&req->list);
@@ -40,6 +137,7 @@ static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req)
        refcount_set(&req->count, 1);
        __set_bit(FR_PENDING, &req->flags);
        req->fm = fm;
+       req->create_time = jiffies;
 }
 
 static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags)
@@ -2291,6 +2389,9 @@ void fuse_abort_conn(struct fuse_conn *fc)
                LIST_HEAD(to_end);
                unsigned int i;
 
+               if (fc->timeout.req_timeout)
+                       cancel_delayed_work(&fc->timeout.work);
+
                /* Background queuing checks fc->connected under bg_lock */
                spin_lock(&fc->bg_lock);
                fc->connected = 0;

diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index add7273c8dc4a23a23e50b879db470fc06bd3d20..03062ee69b709fe205a6b8afab539523992bdc64 100644 (file)
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -140,6 +140,33 @@ void fuse_uring_abort_end_requests(struct fuse_ring *ring)
        }
 }
 
+bool fuse_uring_request_expired(struct fuse_conn *fc)
+{
+       struct fuse_ring *ring = fc->ring;
+       struct fuse_ring_queue *queue;
+       int qid;
+
+       if (!ring)
+               return false;
+
+       for (qid = 0; qid < ring->nr_queues; qid++) {
+               queue = READ_ONCE(ring->queues[qid]);
+               if (!queue)
+                       continue;
+
+               spin_lock(&queue->lock);
+               if (fuse_request_expired(fc, &queue->fuse_req_queue) ||
+                   fuse_request_expired(fc, &queue->fuse_req_bg_queue) ||
+                   fuse_fpq_processing_expired(fc, queue->fpq.processing)) {
+                       spin_unlock(&queue->lock);
+                       return true;
+               }
+               spin_unlock(&queue->lock);
+       }
+
+       return false;
+}
+
 void fuse_uring_destruct(struct fuse_conn *fc)
 {
        struct fuse_ring *ring = fc->ring;

diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h
index d87d0a55cb5f82633eeca67d58e7f5274e6e01b7..51a563922ce14158904a86c248c77767be4fe5ae 100644 (file)
--- a/fs/fuse/dev_uring_i.h
+++ b/fs/fuse/dev_uring_i.h
@@ -143,6 +143,7 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags);
 void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req);
 bool fuse_uring_queue_bq_req(struct fuse_req *req);
 bool fuse_uring_remove_pending_req(struct fuse_req *req);
+bool fuse_uring_request_expired(struct fuse_conn *fc);
 
 static inline void fuse_uring_abort(struct fuse_conn *fc)
 {
@@ -200,6 +201,11 @@ static inline bool fuse_uring_remove_pending_req(struct fuse_req *req)
        return false;
 }
 
+static inline bool fuse_uring_request_expired(struct fuse_conn *fc)
+{
+       return false;
+}
+
 #endif /* CONFIG_FUSE_IO_URING */
 
 #endif /* _FS_FUSE_DEV_URING_I_H */

diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h
index 2481da3388c5feec944143bfabb8d430a447d322..b3c2e32254bad639efa82b9e6537e4aaa4776851 100644 (file)
--- a/fs/fuse/fuse_dev_i.h
+++ b/fs/fuse/fuse_dev_i.h
@@ -63,5 +63,8 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq,
 void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req);
 bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock);
 
+bool fuse_request_expired(struct fuse_conn *fc, struct list_head *list);
+bool fuse_fpq_processing_expired(struct fuse_conn *fc, struct list_head *processing);
+
 #endif
 

diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
index 6f00b177a4343c9e839b31e631bae71dcc05ffea..519c93c3256edde29a2300add2f76e680fb2dbf6 100644 (file)
--- a/fs/fuse/fuse_i.h
+++ b/fs/fuse/fuse_i.h
@@ -44,6 +44,9 @@
 /** Number of dentries for each connection in the control filesystem */
 #define FUSE_CTL_NUM_DENTRIES 5
 
+/** Frequency (in jiffies) of request timeout checks, if opted into */
+extern const unsigned long fuse_timeout_timer_freq;
+
 /** Maximum of max_pages received in init_out */
 extern unsigned int fuse_max_pages_limit;
 
@@ -445,6 +448,8 @@ struct fuse_req {
        void *ring_entry;
        void *ring_queue;
 #endif
+       /** When (in jiffies) the request was created */
+       unsigned long create_time;
 };
 
 struct fuse_iqueue;
@@ -941,6 +946,15 @@ struct fuse_conn {
        /**  uring connection information*/
        struct fuse_ring *ring;
 #endif
+
+       /** Only used if the connection opts into request timeouts */
+       struct {
+               /* Worker for checking if any requests have timed out */
+               struct delayed_work work;
+
+               /* Request timeout (in jiffies). 0 = no timeout */
+               unsigned int req_timeout;
+       } timeout;
 };
 
 /*
@@ -1222,6 +1236,9 @@ void fuse_request_end(struct fuse_req *req);
 void fuse_abort_conn(struct fuse_conn *fc);
 void fuse_wait_aborted(struct fuse_conn *fc);
 
+/* Check if any requests timed out */
+void fuse_check_timeout(struct work_struct *work);
+
 /**
  * Invalidate inode attributes
  */

diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
index e9db2cb8c150878634728685af0fa15e7ade628f..79ebeb60015c8ff38ac7ab9c5eb4a2e155b243aa 100644 (file)
--- a/fs/fuse/inode.c
+++ b/fs/fuse/inode.c
@@ -979,6 +979,7 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm,
        fc->user_ns = get_user_ns(user_ns);
        fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ;
        fc->max_pages_limit = fuse_max_pages_limit;
+       fc->timeout.req_timeout = 0;
 
        if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
                fuse_backing_files_init(fc);
@@ -1007,6 +1008,8 @@ void fuse_conn_put(struct fuse_conn *fc)
 
                if (IS_ENABLED(CONFIG_FUSE_DAX))
                        fuse_dax_conn_free(fc);
+               if (fc->timeout.req_timeout)
+                       cancel_delayed_work_sync(&fc->timeout.work);
                if (fiq->ops->release)
                        fiq->ops->release(fiq);
                put_pid_ns(fc->pid_ns);
@@ -1257,6 +1260,14 @@ static void process_init_limits(struct fuse_conn *fc, struct fuse_init_out *arg)
        spin_unlock(&fc->bg_lock);
 }
 
+static void set_request_timeout(struct fuse_conn *fc, unsigned int timeout)
+{
+       fc->timeout.req_timeout = secs_to_jiffies(timeout);
+       INIT_DELAYED_WORK(&fc->timeout.work, fuse_check_timeout);
+       queue_delayed_work(system_wq, &fc->timeout.work,
+                          fuse_timeout_timer_freq);
+}
+
 struct fuse_init_args {
        struct fuse_args args;
        struct fuse_init_in in;
@@ -1392,6 +1403,9 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
                        }
                        if (flags & FUSE_OVER_IO_URING && fuse_uring_enabled())
                                fc->io_uring = 1;
+
+                       if ((flags & FUSE_REQUEST_TIMEOUT) && arg->request_timeout)
+                               set_request_timeout(fc, arg->request_timeout);
                } else {
                        ra_pages = fc->max_read / PAGE_SIZE;
                        fc->no_lock = 1;
@@ -1439,7 +1453,8 @@ void fuse_send_init(struct fuse_mount *fm)
                FUSE_HANDLE_KILLPRIV_V2 | FUSE_SETXATTR_EXT | FUSE_INIT_EXT |
                FUSE_SECURITY_CTX | FUSE_CREATE_SUPP_GROUP |
                FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_ALLOW_MMAP |
-               FUSE_NO_EXPORT_SUPPORT | FUSE_HAS_RESEND | FUSE_ALLOW_IDMAP;
+               FUSE_NO_EXPORT_SUPPORT | FUSE_HAS_RESEND | FUSE_ALLOW_IDMAP |
+               FUSE_REQUEST_TIMEOUT;
 #ifdef CONFIG_FUSE_DAX
        if (fm->fc->dax)
                flags |= FUSE_MAP_ALIGNMENT;

diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h
index 5e0eb41d967e9de5951673de4405a3ed22cdd8e2..5ec43ecbceb7831d51d537a04b6c574b33ad9f9d 100644 (file)
--- a/include/uapi/linux/fuse.h
+++ b/include/uapi/linux/fuse.h
@@ -229,6 +229,9 @@
  *    - FUSE_URING_IN_OUT_HEADER_SZ
  *    - FUSE_URING_OP_IN_OUT_SZ
  *    - enum fuse_uring_cmd
+ *
+ *  7.43
+ *  - add FUSE_REQUEST_TIMEOUT
  */
 
 #ifndef _LINUX_FUSE_H
@@ -264,7 +267,7 @@
 #define FUSE_KERNEL_VERSION 7
 
 /** Minor version number of this interface */
-#define FUSE_KERNEL_MINOR_VERSION 42
+#define FUSE_KERNEL_MINOR_VERSION 43
 
 /** The node ID of the root inode */
 #define FUSE_ROOT_ID 1
@@ -435,6 +438,8 @@ struct fuse_file_lock {
  *                 of the request ID indicates resend requests
  * FUSE_ALLOW_IDMAP: allow creation of idmapped mounts
  * FUSE_OVER_IO_URING: Indicate that client supports io-uring
+ * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests.
+ *                      init_out.request_timeout contains the timeout (in secs)
  */
 #define FUSE_ASYNC_READ                (1 << 0)
 #define FUSE_POSIX_LOCKS       (1 << 1)
@@ -477,11 +482,11 @@ struct fuse_file_lock {
 #define FUSE_PASSTHROUGH       (1ULL << 37)
 #define FUSE_NO_EXPORT_SUPPORT (1ULL << 38)
 #define FUSE_HAS_RESEND                (1ULL << 39)
-
 /* Obsolete alias for FUSE_DIRECT_IO_ALLOW_MMAP */
 #define FUSE_DIRECT_IO_RELAX   FUSE_DIRECT_IO_ALLOW_MMAP
 #define FUSE_ALLOW_IDMAP       (1ULL << 40)
 #define FUSE_OVER_IO_URING     (1ULL << 41)
+#define FUSE_REQUEST_TIMEOUT   (1ULL << 42)
 
 /**
  * CUSE INIT request/reply flags
@@ -909,7 +914,8 @@ struct fuse_init_out {
        uint16_t        map_alignment;
        uint32_t        flags2;
        uint32_t        max_stack_depth;
-       uint32_t        unused[6];
+       uint16_t        request_timeout;
+       uint16_t        unused[11];
 };
 
 #define CUSE_INIT_INFO_MAX 4096