From: Alexandru Elisei <alexandru.elisei@arm.com>
To: catalin.marinas@arm.com, will@kernel.org, oliver.upton@linux.dev,
maz@kernel.org, james.morse@arm.com, suzuki.poulose@arm.com,
yuzenghui@huawei.com, arnd@arndb.de, akpm@linux-foundation.org,
mingo@redhat.com, peterz@infradead.org, juri.lelli@redhat.com,
vincent.guittot@linaro.org, dietmar.eggemann@arm.com,
rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de,
bristot@redhat.com, vschneid@redhat.com, mhiramat@kernel.org,
rppt@kernel.org, hughd@google.com
Cc: pcc@google.com, steven.price@arm.com, anshuman.khandual@arm.com,
vincenzo.frascino@arm.com, david@redhat.com, eugenis@google.com,
kcc@google.com, hyesoo.yu@samsung.com,
linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, kvmarm@lists.linux.dev,
linux-fsdevel@vger.kernel.org, linux-arch@vger.kernel.org,
linux-mm@kvack.org, linux-trace-kernel@vger.kernel.org
Subject: [PATCH RFC v2 20/27] mm: hugepage: Handle huge page fault on access
Date: Sun, 19 Nov 2023 16:57:14 +0000 [thread overview]
Message-ID: <20231119165721.9849-21-alexandru.elisei@arm.com> (raw)
In-Reply-To: <20231119165721.9849-1-alexandru.elisei@arm.com>
Handle PAGE_FAULT_ON_ACCESS faults for huge pages in a similar way to
regular pages.
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
---
arch/arm64/include/asm/mte_tag_storage.h | 1 +
arch/arm64/include/asm/pgtable.h | 7 ++
arch/arm64/mm/fault.c | 81 ++++++++++++++++++++++++
include/linux/huge_mm.h | 2 +
include/linux/pgtable.h | 5 ++
mm/huge_memory.c | 4 +-
mm/memory.c | 3 +
7 files changed, 101 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/mte_tag_storage.h b/arch/arm64/include/asm/mte_tag_storage.h
index c70ced60a0cd..b97406d369ce 100644
--- a/arch/arm64/include/asm/mte_tag_storage.h
+++ b/arch/arm64/include/asm/mte_tag_storage.h
@@ -35,6 +35,7 @@ void free_tag_storage(struct page *page, int order);
bool page_tag_storage_reserved(struct page *page);
vm_fault_t handle_page_missing_tag_storage(struct vm_fault *vmf);
+vm_fault_t handle_huge_page_missing_tag_storage(struct vm_fault *vmf);
#else
static inline bool tag_storage_enabled(void)
{
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 8cc135f1c112..1704411c096d 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -477,6 +477,13 @@ static inline vm_fault_t arch_do_page_fault_on_access(struct vm_fault *vmf)
return handle_page_missing_tag_storage(vmf);
return VM_FAULT_SIGBUS;
}
+
+static inline vm_fault_t arch_do_huge_page_fault_on_access(struct vm_fault *vmf)
+{
+ if (tag_storage_enabled())
+ return handle_huge_page_missing_tag_storage(vmf);
+ return VM_FAULT_SIGBUS;
+}
#endif /* CONFIG_ARCH_HAS_FAULT_ON_ACCESS */
#define pmd_present_invalid(pmd) (!!(pmd_val(pmd) & PMD_PRESENT_INVALID))
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index f5fa583acf18..6730a0812a24 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -1041,6 +1041,87 @@ vm_fault_t handle_page_missing_tag_storage(struct vm_fault *vmf)
return 0;
+out_retry:
+ put_page(page);
+ if (vmf->flags & FAULT_FLAG_VMA_LOCK)
+ vma_end_read(vma);
+ if (fault_flag_allow_retry_first(vmf->flags)) {
+ err = VM_FAULT_RETRY;
+ } else {
+ /* Replay the fault. */
+ err = 0;
+ }
+ return err;
+}
+
+vm_fault_t handle_huge_page_missing_tag_storage(struct vm_fault *vmf)
+{
+ unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
+ struct vm_area_struct *vma = vmf->vma;
+ pmd_t old_pmd, new_pmd;
+ bool writable = false;
+ struct page *page;
+ vm_fault_t err;
+ int ret;
+
+ vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
+ if (unlikely(!pmd_same(vmf->orig_pmd, *vmf->pmd))) {
+ spin_unlock(vmf->ptl);
+ return 0;
+ }
+
+ old_pmd = vmf->orig_pmd;
+ new_pmd = pmd_modify(old_pmd, vma->vm_page_prot);
+
+ /*
+ * Detect now whether the PMD could be writable; this information
+ * is only valid while holding the PT lock.
+ */
+ writable = pmd_write(new_pmd);
+ if (!writable && vma_wants_manual_pte_write_upgrade(vma) &&
+ can_change_pmd_writable(vma, vmf->address, new_pmd))
+ writable = true;
+
+ page = vm_normal_page_pmd(vma, haddr, new_pmd);
+ if (!page)
+ goto out_map;
+
+ if (!(vma->vm_flags & VM_MTE))
+ goto out_map;
+
+ get_page(page);
+ vma_set_access_pid_bit(vma);
+
+ spin_unlock(vmf->ptl);
+ writable = false;
+
+ if (unlikely(is_migrate_isolate_page(page)))
+ goto out_retry;
+
+ ret = reserve_tag_storage(page, HPAGE_PMD_ORDER, GFP_HIGHUSER_MOVABLE);
+ if (ret)
+ goto out_retry;
+
+ put_page(page);
+
+ vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
+ if (unlikely(!pmd_same(old_pmd, *vmf->pmd))) {
+ spin_unlock(vmf->ptl);
+ return 0;
+ }
+
+out_map:
+ /* Restore the PMD */
+ new_pmd = pmd_modify(old_pmd, vma->vm_page_prot);
+ new_pmd = pmd_mkyoung(new_pmd);
+ if (writable)
+ new_pmd = pmd_mkwrite(new_pmd, vma);
+ set_pmd_at(vma->vm_mm, haddr, vmf->pmd, new_pmd);
+ update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
+ spin_unlock(vmf->ptl);
+
+ return 0;
+
out_retry:
put_page(page);
if (vmf->flags & FAULT_FLAG_VMA_LOCK)
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index fa0350b0812a..bb84291f9231 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -36,6 +36,8 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
pmd_t *pmd, unsigned long addr, pgprot_t newprot,
unsigned long cp_flags);
+bool can_change_pmd_writable(struct vm_area_struct *vma, unsigned long addr,
+ pmd_t pmd);
vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write);
vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write);
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index e2c761dd6c41..de45f475bf8d 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -1473,6 +1473,11 @@ static inline vm_fault_t arch_do_page_fault_on_access(struct vm_fault *vmf)
{
return VM_FAULT_SIGBUS;
}
+
+static inline vm_fault_t arch_do_huge_page_fault_on_access(struct vm_fault *vmf)
+{
+ return VM_FAULT_SIGBUS;
+}
#endif
#endif /* CONFIG_MMU */
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 9beead961a65..d1402b43ea39 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1406,8 +1406,8 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)
return VM_FAULT_FALLBACK;
}
-static inline bool can_change_pmd_writable(struct vm_area_struct *vma,
- unsigned long addr, pmd_t pmd)
+inline bool can_change_pmd_writable(struct vm_area_struct *vma,
+ unsigned long addr, pmd_t pmd)
{
struct page *page;
diff --git a/mm/memory.c b/mm/memory.c
index a04a971200b9..46b926625503 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -5168,6 +5168,9 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
return 0;
}
if (pmd_trans_huge(vmf.orig_pmd) || pmd_devmap(vmf.orig_pmd)) {
+ if (fault_on_access_pmd(vmf.orig_pmd) && vma_is_accessible(vma))
+ return arch_do_huge_page_fault_on_access(&vmf);
+
if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
return do_huge_pmd_numa_page(&vmf);
--
2.42.1
next prev parent reply other threads:[~2023-11-19 16:59 UTC|newest]
Thread overview: 98+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-11-19 16:56 [PATCH RFC v2 00/27] Add support for arm64 MTE dynamic tag storage reuse Alexandru Elisei
2023-11-19 16:56 ` [PATCH RFC v2 01/27] arm64: mte: Rework naming for tag manipulation functions Alexandru Elisei
2023-11-19 16:56 ` [PATCH RFC v2 02/27] arm64: mte: Rename __GFP_ZEROTAGS to __GFP_TAGGED Alexandru Elisei
2023-11-19 16:56 ` [PATCH RFC v2 03/27] mm: cma: Make CMA_ALLOC_SUCCESS/FAIL count the number of pages Alexandru Elisei
2023-11-19 16:56 ` [PATCH RFC v2 04/27] mm: migrate/mempolicy: Add hook to modify migration target gfp Alexandru Elisei
2023-11-25 10:03 ` Mike Rapoport
2023-11-27 11:52 ` Alexandru Elisei
2023-11-28 6:49 ` Mike Rapoport
2023-11-28 17:21 ` Alexandru Elisei
2023-11-19 16:56 ` [PATCH RFC v2 05/27] mm: page_alloc: Add an arch hook to allow prep_new_page() to fail Alexandru Elisei
2023-11-24 19:35 ` David Hildenbrand
2023-11-27 12:09 ` Alexandru Elisei
2023-11-28 16:57 ` David Hildenbrand
2023-11-28 17:17 ` Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 06/27] mm: page_alloc: Allow an arch to hook early into free_pages_prepare() Alexandru Elisei
2023-11-24 19:36 ` David Hildenbrand
2023-11-27 13:03 ` Alexandru Elisei
2023-11-28 16:58 ` David Hildenbrand
2023-11-28 17:17 ` Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 07/27] mm: page_alloc: Add an arch hook to filter MIGRATE_CMA allocations Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 08/27] mm: page_alloc: Partially revert "mm: page_alloc: remove stale CMA guard code" Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 09/27] mm: Allow an arch to hook into folio allocation when VMA is known Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 10/27] mm: Call arch_swap_prepare_to_restore() before arch_swap_restore() Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 11/27] arm64: mte: Reserve tag storage memory Alexandru Elisei
2023-11-29 8:44 ` Hyesoo Yu
2023-11-30 11:56 ` Alexandru Elisei
2023-12-03 12:14 ` Alexandru Elisei
2023-12-08 5:03 ` Hyesoo Yu
2023-12-11 14:45 ` Alexandru Elisei
2023-12-11 17:29 ` Rob Herring
2023-12-12 16:38 ` Alexandru Elisei
2023-12-12 18:44 ` Rob Herring
2023-12-13 13:04 ` Alexandru Elisei
2023-12-13 14:06 ` Rob Herring
2023-12-13 14:51 ` Alexandru Elisei
2023-12-13 17:22 ` Rob Herring
2023-12-13 17:44 ` Alexandru Elisei
2023-12-13 20:30 ` Rob Herring
2023-12-14 15:45 ` Alexandru Elisei
2023-12-14 18:55 ` Rob Herring
2023-12-18 10:59 ` Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 12/27] arm64: mte: Add tag storage pages to the MIGRATE_CMA migratetype Alexandru Elisei
2023-11-24 19:40 ` David Hildenbrand
2023-11-27 15:01 ` Alexandru Elisei
2023-11-28 17:03 ` David Hildenbrand
2023-11-29 10:44 ` Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 13/27] arm64: mte: Make tag storage depend on ARCH_KEEP_MEMBLOCK Alexandru Elisei
2023-11-24 19:51 ` David Hildenbrand
2023-11-27 15:04 ` Alexandru Elisei
2023-11-28 17:05 ` David Hildenbrand
2023-11-29 10:46 ` Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 14/27] arm64: mte: Disable dynamic tag storage management if HW KASAN is enabled Alexandru Elisei
2023-11-24 19:54 ` David Hildenbrand
2023-11-27 15:07 ` Alexandru Elisei
2023-11-28 17:05 ` David Hildenbrand
2023-11-19 16:57 ` [PATCH RFC v2 15/27] arm64: mte: Check that tag storage blocks are in the same zone Alexandru Elisei
2023-11-24 19:56 ` David Hildenbrand
2023-11-27 15:10 ` Alexandru Elisei
2023-11-29 8:57 ` Hyesoo Yu
2023-11-30 12:00 ` Alexandru Elisei
2023-12-08 5:27 ` Hyesoo Yu
2023-12-11 14:21 ` Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 16/27] arm64: mte: Manage tag storage on page allocation Alexandru Elisei
2023-11-29 9:10 ` Hyesoo Yu
2023-11-29 13:33 ` Alexandru Elisei
2023-12-08 5:29 ` Hyesoo Yu
2023-11-19 16:57 ` [PATCH RFC v2 17/27] arm64: mte: Perform CMOs for tag blocks on tagged page allocation/free Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 18/27] arm64: mte: Reserve tag block for the zero page Alexandru Elisei
2023-11-28 17:06 ` David Hildenbrand
2023-11-29 11:30 ` Alexandru Elisei
2023-11-29 13:13 ` David Hildenbrand
2023-11-29 13:41 ` Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 19/27] mm: mprotect: Introduce PAGE_FAULT_ON_ACCESS for mprotect(PROT_MTE) Alexandru Elisei
2023-11-28 17:55 ` David Hildenbrand
2023-11-28 18:00 ` David Hildenbrand
2023-11-29 11:55 ` Alexandru Elisei
2023-11-29 12:48 ` David Hildenbrand
2023-11-29 9:27 ` Hyesoo Yu
2023-11-30 12:06 ` Alexandru Elisei
2023-11-30 12:49 ` David Hildenbrand
2023-11-30 13:32 ` Alexandru Elisei
2023-11-30 13:43 ` David Hildenbrand
2023-11-30 14:33 ` Alexandru Elisei
2023-11-30 14:39 ` David Hildenbrand
2023-11-19 16:57 ` Alexandru Elisei [this message]
2023-11-22 1:28 ` [PATCH RFC v2 20/27] mm: hugepage: Handle huge page fault on access Peter Collingbourne
2023-11-22 9:22 ` Alexandru Elisei
2023-11-28 17:56 ` David Hildenbrand
2023-11-29 11:56 ` Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 21/27] mm: arm64: Handle tag storage pages mapped before mprotect(PROT_MTE) Alexandru Elisei
2023-11-28 5:39 ` Peter Collingbourne
2023-11-30 17:43 ` Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 22/27] arm64: mte: swap: Handle tag restoring when missing tag storage Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 23/27] arm64: mte: copypage: " Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 24/27] arm64: mte: Handle fatal signal in reserve_tag_storage() Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 25/27] KVM: arm64: Disable MTE if tag storage is enabled Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 26/27] arm64: mte: Fast track reserving tag storage when the block is free Alexandru Elisei
2023-11-19 16:57 ` [PATCH RFC v2 27/27] arm64: mte: Enable dynamic tag storage reuse Alexandru Elisei
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20231119165721.9849-21-alexandru.elisei@arm.com \
--to=alexandru.elisei@arm.com \
--cc=akpm@linux-foundation.org \
--cc=anshuman.khandual@arm.com \
--cc=arnd@arndb.de \
--cc=bristot@redhat.com \
--cc=bsegall@google.com \
--cc=catalin.marinas@arm.com \
--cc=david@redhat.com \
--cc=dietmar.eggemann@arm.com \
--cc=eugenis@google.com \
--cc=hughd@google.com \
--cc=hyesoo.yu@samsung.com \
--cc=james.morse@arm.com \
--cc=juri.lelli@redhat.com \
--cc=kcc@google.com \
--cc=kvmarm@lists.linux.dev \
--cc=linux-arch@vger.kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-trace-kernel@vger.kernel.org \
--cc=maz@kernel.org \
--cc=mgorman@suse.de \
--cc=mhiramat@kernel.org \
--cc=mingo@redhat.com \
--cc=oliver.upton@linux.dev \
--cc=pcc@google.com \
--cc=peterz@infradead.org \
--cc=rostedt@goodmis.org \
--cc=rppt@kernel.org \
--cc=steven.price@arm.com \
--cc=suzuki.poulose@arm.com \
--cc=vincent.guittot@linaro.org \
--cc=vincenzo.frascino@arm.com \
--cc=vschneid@redhat.com \
--cc=will@kernel.org \
--cc=yuzenghui@huawei.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).