From: Chenliang Li <cliang01.li@samsung.com>
To: axboe@kernel.dk, asml.silence@gmail.com
Cc: io-uring@vger.kernel.org, peiwei.li@samsung.com,
joshi.k@samsung.com, kundan.kumar@samsung.com,
anuj20.g@samsung.com, gost.dev@samsung.com,
Chenliang Li <cliang01.li@samsung.com>
Subject: [PATCH v4 3/4] io_uring/rsrc: add init and account functions for coalesced imus
Date: Tue, 14 May 2024 15:54:43 +0800 [thread overview]
Message-ID: <20240514075444.590910-4-cliang01.li@samsung.com> (raw)
In-Reply-To: <20240514075444.590910-1-cliang01.li@samsung.com>
Introduce two functions to separate the coalesced imu alloc and
accounting path from the original one. This helps to keep the original
code path clean.
Signed-off-by: Chenliang Li <cliang01.li@samsung.com>
---
io_uring/rsrc.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 89 insertions(+)
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 578d382ca9bc..53fac5f27bbf 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -871,6 +871,45 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
return ret;
}
+static int io_coalesced_buffer_account_pin(struct io_ring_ctx *ctx,
+ struct page **pages,
+ struct io_mapped_ubuf *imu,
+ struct page **last_hpage,
+ struct io_imu_folio_data *data)
+{
+ int i, j, ret;
+
+ imu->acct_pages = 0;
+ j = 0;
+ for (i = 0; i < data->nr_folios; i++) {
+ struct page *hpage = pages[j];
+
+ if (hpage == *last_hpage)
+ continue;
+ *last_hpage = hpage;
+ /*
+ * Already checked the page array in try coalesce,
+ * so pass in nr_pages=0 here to waive that.
+ */
+ if (headpage_already_acct(ctx, pages, 0, hpage))
+ continue;
+ imu->acct_pages += data->nr_pages_mid;
+ if (i)
+ j += data->nr_pages_mid;
+ else
+ j = data->nr_pages_head;
+ }
+
+ if (!imu->acct_pages)
+ return 0;
+
+ ret = io_account_mem(ctx, imu->acct_pages);
+ if (!ret)
+ return 0;
+ imu->acct_pages = 0;
+ return ret;
+}
+
static bool __io_sqe_buffer_try_coalesce(struct page **pages, int nr_pages,
struct io_imu_folio_data *data)
{
@@ -949,6 +988,56 @@ static bool io_sqe_buffer_try_coalesce(struct page **pages, int nr_pages,
return true;
}
+static int io_coalesced_imu_alloc(struct io_ring_ctx *ctx, struct iovec *iov,
+ struct io_mapped_ubuf **pimu,
+ struct page **last_hpage, struct page **pages,
+ struct io_imu_folio_data *data)
+{
+ struct io_mapped_ubuf *imu = NULL;
+ unsigned long off;
+ size_t size, vec_len;
+ int ret, i, j;
+
+ ret = -ENOMEM;
+ imu = kvmalloc(struct_size(imu, bvec, data->nr_folios), GFP_KERNEL);
+ if (!imu)
+ return ret;
+
+ ret = io_coalesced_buffer_account_pin(ctx, pages, imu, last_hpage,
+ data);
+ if (ret) {
+ unpin_user_page(pages[0]);
+ j = data->nr_pages_head;
+ for (i = 1; i < data->nr_folios; i++) {
+ unpin_user_page(pages[j]);
+ j += data->nr_pages_mid;
+ }
+ return ret;
+ }
+ off = (unsigned long) iov->iov_base & ~PAGE_MASK;
+ size = iov->iov_len;
+ /* store original address for later verification */
+ imu->ubuf = (unsigned long) iov->iov_base;
+ imu->ubuf_end = imu->ubuf + iov->iov_len;
+ imu->nr_bvecs = data->nr_folios;
+ imu->folio_shift = data->folio_shift;
+ imu->folio_mask = ~((1UL << data->folio_shift) - 1);
+ *pimu = imu;
+ ret = 0;
+
+ vec_len = min_t(size_t, size, PAGE_SIZE * data->nr_pages_head - off);
+ bvec_set_page(&imu->bvec[0], pages[0], vec_len, off);
+ size -= vec_len;
+ j = data->nr_pages_head;
+ for (i = 1; i < data->nr_folios; i++) {
+ vec_len = min_t(size_t, size, data->folio_size);
+ bvec_set_page(&imu->bvec[i], pages[j], vec_len, 0);
+ size -= vec_len;
+ j += data->nr_pages_mid;
+ }
+ return ret;
+}
+
static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
struct io_mapped_ubuf **pimu,
struct page **last_hpage)
--
2.34.1
next prev parent reply other threads:[~2024-05-14 7:55 UTC|newest]
Thread overview: 10+ messages / expand[flat|nested] mbox.gz Atom feed top
[not found] <CGME20240514075453epcas5p17974fb62d65a88b1a1b55b97942ee2be@epcas5p1.samsung.com>
2024-05-14 7:54 ` [PATCH v4 0/4] io_uring/rsrc: coalescing multi-hugepage registered buffers Chenliang Li
[not found] ` <CGME20240514075457epcas5p10f02f1746f957df91353724ec859664f@epcas5p1.samsung.com>
2024-05-14 7:54 ` [PATCH v4 1/4] io_uring/rsrc: add hugepage buffer coalesce helpers Chenliang Li
2024-05-16 14:07 ` Anuj gupta
[not found] ` <CGME20240514075459epcas5p2275b4c26f16bcfcea200e97fc75c2a14@epcas5p2.samsung.com>
2024-05-14 7:54 ` [PATCH v4 2/4] io_uring/rsrc: store folio shift and mask into imu Chenliang Li
2024-05-16 14:08 ` Anuj gupta
[not found] ` <CGME20240514075500epcas5p1e638b1ae84727b3669ff6b780cd1cb23@epcas5p1.samsung.com>
2024-05-14 7:54 ` Chenliang Li [this message]
[not found] ` <CGME20240514075502epcas5p10be6bef71d284a110277575d6008563d@epcas5p1.samsung.com>
2024-05-14 7:54 ` [PATCH v4 4/4] io_uring/rsrc: enable multi-hugepage buffer coalescing Chenliang Li
2024-05-16 14:09 ` Anuj gupta
2024-05-16 14:01 ` [PATCH v4 0/4] io_uring/rsrc: coalescing multi-hugepage registered buffers Anuj gupta
2024-05-16 14:58 ` Jens Axboe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240514075444.590910-4-cliang01.li@samsung.com \
--to=cliang01.li@samsung.com \
--cc=anuj20.g@samsung.com \
--cc=asml.silence@gmail.com \
--cc=axboe@kernel.dk \
--cc=gost.dev@samsung.com \
--cc=io-uring@vger.kernel.org \
--cc=joshi.k@samsung.com \
--cc=kundan.kumar@samsung.com \
--cc=peiwei.li@samsung.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).