From: Stephen Hemminger <stephen@networkplumber.org>
To: dev@dpdk.org
Cc: Stephen Hemminger <stephen@networkplumber.org>
Subject: [RFC] eal: make lcore_init aware of cgroup
Date: Mon, 29 Apr 2024 12:52:25 -0700 [thread overview]
Message-ID: <20240429195342.42711-1-stephen@networkplumber.org> (raw)
Some lcores maybe restricted from being used by DPDK by cgroups.
This should be detected at startup and only those cpu's allowed
to be used by the process should be marked as enabled.
This is a lightly tested patch, and parsing the cpuset info here
probably needs more checking. It is a response to the problem
reported with error handling.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
lib/eal/linux/eal_lcore.c | 95 ++++++++++++++++++++++++++++++++++++---
1 file changed, 88 insertions(+), 7 deletions(-)
diff --git a/lib/eal/linux/eal_lcore.c b/lib/eal/linux/eal_lcore.c
index 29b36dd610..098892fafa 100644
--- a/lib/eal/linux/eal_lcore.c
+++ b/lib/eal/linux/eal_lcore.c
@@ -4,6 +4,7 @@
#include <unistd.h>
#include <limits.h>
+#include <inttypes.h>
#include <rte_log.h>
@@ -11,23 +12,103 @@
#include "eal_filesystem.h"
#include "eal_thread.h"
+#define PROC_STATUS "/proc/%u/status"
#define SYS_CPU_DIR "/sys/devices/system/cpu/cpu%u"
#define CORE_ID_FILE "topology/core_id"
#define NUMA_NODE_PATH "/sys/devices/system/node"
-/* Check if a cpu is present by the presence of the cpu information for it */
+static cpu_set_t *cpus_allowed;
+
+/*
+ * Initialize allowed cpus from /proc/<pid>status
+ * The cpus allowed is a subset of available lcores on the system
+ * which maybe restricted by cgroups
+ */
+static void
+get_allowed_cpus(cpu_set_t *set)
+{
+ const char cpus_allowed[] = "Cpus_allowed:";
+ const size_t setsize = CPU_ALLOC_SIZE(RTE_MAX_LCORE);
+ char path[PATH_MAX];
+ char line[LINE_MAX];
+ FILE *f;
+
+ CPU_ZERO_S(setsize, set);
+
+ snprintf(path, sizeof(path), PROC_STATUS, getpid());
+ f = fopen(path, "r");
+ if (f == NULL) {
+ EAL_LOG(ERR, "%s(): cannot open %s: %s",
+ __func__, path, strerror(errno));
+ return;
+ }
+
+ while (fgets(line, sizeof(line), f)) {
+ char *cp;
+ unsigned int cpu;
+
+ if (strncmp(line, cpus_allowed, sizeof(cpus_allowed) - 1))
+ continue;
+
+ cp = line + sizeof(cpus_allowed);
+
+ while(*cp && isspace(*cp))
+ ++cp;
+
+ for (cpu = 0; cpu < RTE_MAX_LCORE; cpu += 32) {
+ uint32_t cpu_mask;
+ unsigned int i;
+
+ if (*cp == '\0')
+ break;
+
+ if (sscanf(cp, "%" SCNx32, &cpu_mask) != 1) {
+ EAL_LOG(NOTICE, "%s(): can not parse: %s",
+ __func__, line);
+ goto error;
+ }
+ for (i = 0; i < 32; i++) {
+ if (cpu_mask & (1u << i))
+ CPU_SET_S(cpu + i, setsize, set);
+ }
+
+ cp = strchr(cp, ',');
+ if (cp == NULL)
+ break;
+ cp += 1; /* skip the comma */
+ }
+ }
+
+error:
+ fclose(f);
+}
+
+/* Check if a cpu can be used by looking at /proc/<pid>/status */
int
-eal_cpu_detected(unsigned lcore_id)
+eal_cpu_detected(unsigned int lcore_id)
{
char path[PATH_MAX];
- int len = snprintf(path, sizeof(path), SYS_CPU_DIR
- "/"CORE_ID_FILE, lcore_id);
- if (len <= 0 || (unsigned)len >= sizeof(path))
+ int len;
+
+ if (cpus_allowed == NULL) {
+ cpus_allowed = CPU_ALLOC(RTE_MAX_LCORE);
+
+ if (cpus_allowed == NULL) {
+ EAL_LOG(ERR, "%s(): cannot allocate cpuset", __func__);
+ return 0;
+ }
+ get_allowed_cpus(cpus_allowed);
+ }
+
+ /* skip cpus blocked by cgroup */
+ if (!CPU_ISSET(lcore_id, cpus_allowed))
return 0;
- if (access(path, F_OK) != 0)
+
+ len = snprintf(path, sizeof(path), SYS_CPU_DIR "/"CORE_ID_FILE, lcore_id);
+ if (len <= 0 || (unsigned)len >= sizeof(path))
return 0;
- return 1;
+ return access(path, F_OK) == 0;
}
/*
--
2.43.0
next reply other threads:[~2024-04-29 19:53 UTC|newest]
Thread overview: 2+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-04-29 19:52 Stephen Hemminger [this message]
2024-04-30 0:49 ` [RFC] eal: make lcore_init aware of cgroup Stephen Hemminger
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240429195342.42711-1-stephen@networkplumber.org \
--to=stephen@networkplumber.org \
--cc=dev@dpdk.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).