io_uring: kbuf: inline io_kbuf_recycle_ring()

[linux-block.git] / io_uring / kbuf.h

diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h
index 721465c5d809fcdc26c1d32b8f0c14844fcc4b2d..b3e8c6c5fee1432d601a3ea64718e676d57599cf 100644 (file)
--- a/io_uring/kbuf.h
+++ b/io_uring/kbuf.h
@@ -49,7 +49,33 @@ int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
 unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags);
 
 void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
-void io_kbuf_recycle_ring(struct io_kiocb *req);
+
+static inline void io_kbuf_recycle_ring(struct io_kiocb *req)
+{
+       /*
+        * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
+        * the flag and hence ensure that bl->head doesn't get incremented.
+        * If the tail has already been incremented, hang on to it.
+        * The exception is partial io, that case we should increment bl->head
+        * to monopolize the buffer.
+        */
+       if (req->buf_list) {
+               if (req->flags & REQ_F_PARTIAL_IO) {
+                       /*
+                        * If we end up here, then the io_uring_lock has
+                        * been kept held since we retrieved the buffer.
+                        * For the io-wq case, we already cleared
+                        * req->buf_list when the buffer was retrieved,
+                        * hence it cannot be set here for that case.
+                        */
+                       req->buf_list->head++;
+                       req->buf_list = NULL;
+               } else {
+                       req->buf_index = req->buf_list->bgid;
+                       req->flags &= ~REQ_F_BUFFER_RING;
+               }
+       }
+}
 
 static inline bool io_do_buffer_select(struct io_kiocb *req)
 {