From: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
To: linux-erofs@lists.ozlabs.org
Cc: hsiangkao@linux.alibaba.com, Yifan Zhao <zhaoyifan@sjtu.edu.cn>,
xin_tong@sjtu.edu.cn
Subject: [PATCH 3/7] erofs-utils: mkfs: add --worker=# parameter
Date: Sun, 4 Feb 2024 18:33:53 +0800 [thread overview]
Message-ID: <20240204103353.141469-1-zhaoyifan@sjtu.edu.cn> (raw)
This patch introduces a --worker=# parameter for the incoming
multi-threaded compression support. It also introduces a segment size
used in multi-threaded compression, which has the default value 16MB
and cannot be modified.
Signed-off-by: Yifan Zhao <zhaoyifan@sjtu.edu.cn>
---
include/erofs/config.h | 4 ++++
lib/config.c | 4 ++++
mkfs/main.c | 38 ++++++++++++++++++++++++++++++++++++++
3 files changed, 46 insertions(+)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 73e3ac2..d19094e 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -75,6 +75,10 @@ struct erofs_configure {
char c_force_chunkformat;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
+#ifdef EROFS_MT_ENABLED
+ u64 c_mt_segment_size;
+ u32 c_mt_worker_num;
+#endif
u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;
u32 c_max_decompressed_extent_bytes;
diff --git a/lib/config.c b/lib/config.c
index 947a183..8add06d 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -38,6 +38,10 @@ void erofs_init_configure(void)
cfg.c_pclusterblks_max = 1;
cfg.c_pclusterblks_def = 1;
cfg.c_max_decompressed_extent_bytes = -1;
+#ifdef EROFS_MT_ENABLED
+ cfg.c_mt_segment_size = 16ULL * 1024 * 1024;
+ cfg.c_mt_worker_num = 1;
+#endif
erofs_stdout_tty = isatty(STDOUT_FILENO);
}
diff --git a/mkfs/main.c b/mkfs/main.c
index 7aea64a..3882533 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -73,6 +73,9 @@ static struct option long_options[] = {
{"gzip", no_argument, NULL, 517},
#endif
{"offset", required_argument, NULL, 518},
+#ifdef EROFS_MT_ENABLED
+ {"worker", required_argument, NULL, 519},
+#endif
{0, 0, 0, 0},
};
@@ -175,6 +178,9 @@ static void usage(int argc, char **argv)
" --product-out=X X=product_out directory\n"
" --fs-config-file=X X=fs_config file\n"
" --block-list-file=X X=block_list file\n"
+#endif
+#ifdef EROFS_MT_ENABLED
+ " --worker=# set the number of worker threads to # (default=1)\n"
#endif
);
}
@@ -404,6 +410,13 @@ static void erofs_rebuild_cleanup(void)
rebuild_src_count = 0;
}
+#ifdef EROFS_MT_ENABLED
+static u32 mkfs_max_worker_num() {
+ u32 ncpu = erofs_get_available_processors();
+ return ncpu ? ncpu : 16;
+}
+#endif
+
static int mkfs_parse_options_cfg(int argc, char *argv[])
{
char *endptr;
@@ -642,6 +655,21 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -EINVAL;
}
break;
+#ifdef EROFS_MT_ENABLED
+ case 519:
+ cfg.c_mt_worker_num = strtoul(optarg, &endptr, 0);
+ if (errno || *endptr != '\0') {
+ erofs_err("invalid worker number %s", optarg);
+ return -EINVAL;
+ }
+ if (cfg.c_mt_worker_num > mkfs_max_worker_num()) {
+ erofs_warn(
+ "worker number %s is too large, setting to %ud",
+ optarg, mkfs_max_worker_num());
+ cfg.c_mt_worker_num = mkfs_max_worker_num();
+ }
+ break;
+#endif
case 'V':
version();
exit(0);
@@ -784,6 +812,16 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
}
cfg.c_pclusterblks_packed = pclustersize_packed >> sbi.blkszbits;
}
+
+#ifdef EROFS_MT_ENABLED
+ if (cfg.c_mt_worker_num > 1 &&
+ (cfg.c_dedupe || cfg.c_fragments || cfg.c_ztailpacking)) {
+ cfg.c_mt_worker_num = 1;
+ erofs_warn("Please note that dedupe/fragments/ztailpacking"
+ "is NOT supported in multi-threaded mode now, using worker=1.");
+ }
+#endif
+
return 0;
}
--
2.43.0
reply other threads:[~2024-02-04 10:34 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240204103353.141469-1-zhaoyifan@sjtu.edu.cn \
--to=zhaoyifan@sjtu.edu.cn \
--cc=hsiangkao@linux.alibaba.com \
--cc=linux-erofs@lists.ozlabs.org \
--cc=xin_tong@sjtu.edu.cn \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).