From: Chunhai Guo via Linux-erofs <linux-erofs@lists.ozlabs.org>
To: xiang@kernel.org
Cc: Chunhai Guo <guochunhai@vivo.com>,
linux-erofs@lists.ozlabs.org, huyue2@coolpad.com
Subject: [PATCH v2] erofs: add a global page pool for lz4 decompression
Date: Tue, 9 Jan 2024 00:41:43 -0700 [thread overview]
Message-ID: <20240109074143.4138783-1-guochunhai@vivo.com> (raw)
Using a global page pool for LZ4 decompression significantly reduces the
time spent on page allocation in low memory scenarios.
The table below shows the reduction in time spent on page allocation for
LZ4 decompression when using a global page pool. The results were
obtained from multi-app launch benchmarks on ARM64 Android devices
running the 5.15 kernel with an 8-core CPU and 8GB of memory. In the
benchmark, we launched 16 frequently-used apps, and the camera app was
the last one in each round. The data in the table is the average time of
camera app for each round.
After using the page pool, there was an average improvement of 150ms in
the launch time of the camera app, which was obtained from systrace log.
+--------------+---------------+--------------+---------+
| | w/o page pool | w/ page pool | diff |
+--------------+---------------+--------------+---------+
| Average (ms) | 3434 | 21 | -99.38% |
+--------------+---------------+--------------+---------+
Based on the benchmark logs, 64 pages are sufficient for 95% of
scenarios. This value can be adjusted from the module parameter. The
default value is 0.
This patch currently only supports the LZ4 decompressor, other
decompressors will be supported in the next step.
Signed-off-by: Chunhai Guo <guochunhai@vivo.com>
---
V1 -> V2: reuse shortlived interfaces for the global pool and support
configuration based on module parameters
---
fs/erofs/decompressor.c | 5 ++-
fs/erofs/internal.h | 15 +++++++-
fs/erofs/super.c | 1 +
fs/erofs/utils.c | 82 +++++++++++++++++++++++++++++++++++++++--
4 files changed, 97 insertions(+), 6 deletions(-)
diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
index d08a6ee23ac5..49aa40e5c260 100644
--- a/fs/erofs/decompressor.c
+++ b/fs/erofs/decompressor.c
@@ -54,6 +54,7 @@ static int z_erofs_load_lz4_config(struct super_block *sb,
sbi->lz4.max_distance_pages = distance ?
DIV_ROUND_UP(distance, PAGE_SIZE) + 1 :
LZ4_MAX_DISTANCE_PAGES;
+ erofs_global_page_pool_init();
return erofs_pcpubuf_growsize(sbi->lz4.max_pclusterblks);
}
@@ -111,8 +112,8 @@ static int z_erofs_lz4_prepare_dstpages(struct z_erofs_lz4_decompress_ctx *ctx,
victim = availables[--top];
get_page(victim);
} else {
- victim = erofs_allocpage(pagepool,
- GFP_KERNEL | __GFP_NOFAIL);
+ victim = erofs_allocpage_global(pagepool,
+ GFP_KERNEL | __GFP_NOFAIL);
set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
}
rq->out[i] = victim;
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index b0409badb017..d078ccae4f11 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -445,8 +445,21 @@ int erofs_register_sysfs(struct super_block *sb);
void erofs_unregister_sysfs(struct super_block *sb);
int __init erofs_init_sysfs(void);
void erofs_exit_sysfs(void);
+int erofs_global_page_pool_init(void);
+void erofs_global_page_pool_exit(void);
+struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp,
+ bool global_allowed);
+static inline struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)
+{
+ return __erofs_allocpage(pagepool, gfp, false);
+}
+
+static inline struct page *erofs_allocpage_global(struct page **pagepool,
+ gfp_t gfp)
+{
+ return __erofs_allocpage(pagepool, gfp, true);
+}
-struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp);
static inline void erofs_pagepool_add(struct page **pagepool, struct page *page)
{
set_page_private(page, (unsigned long)*pagepool);
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 5f60f163bd56..56ddacb1bd6e 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -946,6 +946,7 @@ static void __exit erofs_module_exit(void)
erofs_exit_shrinker();
kmem_cache_destroy(erofs_inode_cachep);
erofs_pcpubuf_exit();
+ erofs_global_page_pool_exit();
}
static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf)
diff --git a/fs/erofs/utils.c b/fs/erofs/utils.c
index 5dea308764b4..1b0cec47c00c 100644
--- a/fs/erofs/utils.c
+++ b/fs/erofs/utils.c
@@ -5,7 +5,19 @@
*/
#include "internal.h"
-struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)
+struct z_erofs_global_pool {
+ struct page *pagepool;
+ spinlock_t lock;
+ unsigned int total, unused;
+ bool inited;
+};
+
+static struct z_erofs_global_pool z_erofs_global_pool;
+
+module_param_named(global_pool_size, z_erofs_global_pool.total, uint, 0444);
+
+struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp,
+ bool from_global)
{
struct page *page = *pagepool;
@@ -13,7 +25,19 @@ struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)
DBG_BUGON(page_ref_count(page) != 1);
*pagepool = (struct page *)page_private(page);
} else {
- page = alloc_page(gfp);
+ if (from_global) {
+ spin_lock(&z_erofs_global_pool.lock);
+ page = z_erofs_global_pool.pagepool;
+ if (page) {
+ z_erofs_global_pool.pagepool =
+ (struct page *)page_private(page);
+ DBG_BUGON(page_ref_count(page) != 1);
+ z_erofs_global_pool.unused--;
+ }
+ spin_unlock(&z_erofs_global_pool.lock);
+ }
+ if (!page)
+ page = alloc_page(gfp);
}
return page;
}
@@ -24,7 +48,20 @@ void erofs_release_pages(struct page **pagepool)
struct page *page = *pagepool;
*pagepool = (struct page *)page_private(page);
- put_page(page);
+ if (z_erofs_global_pool.total) {
+ spin_lock(&z_erofs_global_pool.lock);
+ if (z_erofs_global_pool.unused
+ < z_erofs_global_pool.total) {
+ erofs_pagepool_add(&z_erofs_global_pool.pagepool,
+ page);
+ z_erofs_global_pool.unused++;
+ } else {
+ put_page(page);
+ }
+ spin_unlock(&z_erofs_global_pool.lock);
+ } else {
+ put_page(page);
+ }
}
}
@@ -284,4 +321,43 @@ void erofs_exit_shrinker(void)
{
shrinker_free(erofs_shrinker_info);
}
+
+int erofs_global_page_pool_init(void)
+{
+ int i;
+ struct page *page;
+
+ if (z_erofs_global_pool.inited)
+ return 0;
+
+ spin_lock_init(&z_erofs_global_pool.lock);
+ z_erofs_global_pool.unused = 0;
+ z_erofs_global_pool.pagepool = NULL;
+ for (i = 0; i < z_erofs_global_pool.total; i++) {
+ page = alloc_page(GFP_KERNEL);
+ if (!page) {
+ erofs_err(NULL, "failed to alloc page for erofs page pool\n");
+ return 0;
+ }
+ erofs_pagepool_add(&z_erofs_global_pool.pagepool, page);
+ z_erofs_global_pool.unused++;
+ }
+ z_erofs_global_pool.inited = true;
+ return 0;
+}
+
+void erofs_global_page_pool_exit(void)
+{
+ struct page *pagepool = z_erofs_global_pool.pagepool;
+
+ while (pagepool) {
+ struct page *page = pagepool;
+
+ pagepool = (struct page *)page_private(page);
+ put_page(page);
+ z_erofs_global_pool.unused--;
+ }
+ z_erofs_global_pool.total = 0;
+}
+
#endif /* !CONFIG_EROFS_FS_ZIP */
--
2.25.1
next reply other threads:[~2024-01-09 7:31 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-01-09 7:41 Chunhai Guo via Linux-erofs [this message]
2024-01-09 13:08 ` [PATCH v2] erofs: add a global page pool for lz4 decompression Gao Xiang
2024-01-10 6:45 ` Chunhai Guo via Linux-erofs
2024-01-12 1:58 ` Chunhai Guo via Linux-erofs
2024-01-22 4:41 ` Gao Xiang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240109074143.4138783-1-guochunhai@vivo.com \
--to=linux-erofs@lists.ozlabs.org \
--cc=guochunhai@vivo.com \
--cc=huyue2@coolpad.com \
--cc=xiang@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).