diff options
Diffstat (limited to 'include/linux/mm.h')
| -rw-r--r-- | include/linux/mm.h | 83 |
1 files changed, 50 insertions, 33 deletions
diff --git a/include/linux/mm.h b/include/linux/mm.h index 113e1752548c..25e669632d2c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -354,6 +354,7 @@ enum { #endif DECLARE_VMA_BIT(UFFD_MINOR, 41), DECLARE_VMA_BIT(SEALED, 42), + DECLARE_VMA_BIT(UFFD_RWP, 43), /* Flags that reuse flags above. */ DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0), DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1), @@ -497,12 +498,17 @@ enum { #else #define VM_UFFD_MINOR VM_NONE #endif +#ifdef CONFIG_USERFAULTFD_RWP +#define VM_UFFD_RWP INIT_VM_FLAG(UFFD_RWP) +#else +#define VM_UFFD_RWP VM_NONE +#endif /* - * vma_flags_t masks for the userfaultfd VMA flags. VMA_UFFD_MINOR is gated on - * the same config as VM_UFFD_MINOR -- which implies 64BIT, where the bit fits - * -- so an out-of-range bit is never fed to mk_vma_flags() on a build whose - * bitmap cannot hold it. + * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are + * gated on the same configs as their VM_* flags above -- both of which imply + * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build + * whose bitmap cannot hold it. */ #define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT) #define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT) @@ -511,6 +517,11 @@ enum { #else #define VMA_UFFD_MINOR EMPTY_VMA_FLAGS #endif +#ifdef CONFIG_USERFAULTFD_RWP +#define VMA_UFFD_RWP mk_vma_flags(VMA_UFFD_RWP_BIT) +#else +#define VMA_UFFD_RWP EMPTY_VMA_FLAGS +#endif #ifdef CONFIG_64BIT #define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED) @@ -649,29 +660,32 @@ enum { * reconsistuted upon page fault, so necessitate page table copying upon fork. * * Note that these flags should be compared with the DESTINATION VMA not the - * source, as VM_UFFD_WP may not be propagated to destination, while all other - * flags will be. + * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination + * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did + * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate. * * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be * reasonably reconstructed on page fault. * * VM_UFFD_WP - Encodes metadata about an installed uffd - * write protect handler, which cannot be - * reconstructed on page fault. + * VM_UFFD_RWP write- or read-write-protect handler, which + * cannot be reconstructed on page fault. * - * We always copy pgtables when dst_vma has uffd-wp - * enabled even if it's file-backed - * (e.g. shmem). Because when uffd-wp is enabled, - * pgtable contains uffd-wp protection information, - * that's something we can't retrieve from page cache, - * and skip copying will lose those info. + * We always copy pgtables when dst_vma has the + * uffd PTE bit in use even if it's file-backed + * (e.g. shmem). Because when the uffd bit is + * in use, the pgtable contains the protection + * information, that's something we can't + * retrieve from page cache, and skip copying + * will lose those info. * * VM_MAYBE_GUARD - Could contain page guard region markers which * by design are a property of the page tables * only and thus cannot be reconstructed on page * fault. */ -#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD) +#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_UFFD_RWP | \ + VM_MAYBE_GUARD) /* * mapping from the currently active vm_flags protection bits (the @@ -2303,29 +2317,15 @@ static inline int memdesc_nid(const memdesc_flags_t *mdf) #endif #endif -#ifdef CONFIG_NUMA static inline int page_to_nid(const struct page *page) { - const struct page *p = PF_POISONED_CHECK(page); - - return memdesc_nid(&p->flags); + return memdesc_nid(&(PF_POISONED_CHECK(page)->flags)); } static inline int folio_nid(const struct folio *folio) { return memdesc_nid(&folio->flags); } -#else -static inline int page_to_nid(const struct page *page) -{ - return 0; -} - -static inline int folio_nid(const struct folio *folio) -{ - return 0; -} -#endif #ifdef CONFIG_NUMA_BALANCING /* page access time bits needs to hold at least 4 seconds */ @@ -2566,9 +2566,7 @@ static inline void set_page_section(struct page *page, unsigned long section) static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { -#if SECTIONS_WIDTH != 0 ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); -#endif return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ @@ -3343,6 +3341,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen); #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ MM_CP_UFFD_WP_RESOLVE) +/* Whether this change is for uffd RWP */ +#define MM_CP_UFFD_RWP (1UL << 4) /* do rwp */ +#define MM_CP_UFFD_RWP_RESOLVE (1UL << 5) /* resolve rwp */ +#define MM_CP_UFFD_RWP_ALL (MM_CP_UFFD_RWP | \ + MM_CP_UFFD_RWP_RESOLVE) bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte); @@ -4651,12 +4654,26 @@ static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) /* * Indicates whether GUP can follow a PROT_NONE mapped page, or whether - * a (NUMA hinting) fault is required. + * a (NUMA hinting or userfaultfd RWP) fault is required. */ static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma, unsigned int flags) { /* + * VM_UFFD_RWP uses protnone as an access-tracking marker, not for + * NUMA hinting. GUP must always take a fault so the access is + * delivered to userfaultfd, regardless of FOLL_HONOR_NUMA_FAULT. + * + * Only do so while the VMA is accessible. If it has been made + * inaccessible (e.g. mprotect(PROT_NONE)), fall through to the guard + * below: forcing a fault there would loop, as handle_mm_fault() makes + * no progress on protnone in an inaccessible VMA, and the access is + * denied regardless of RWP anyway. + */ + if (vma_test_single_mask(vma, VMA_UFFD_RWP) && vma_is_accessible(vma)) + return false; + + /* * If callers don't want to honor NUMA hinting faults, no need to * determine if we would actually have to trigger a NUMA hinting fault. */ |