diff options
Diffstat (limited to 'mm/vma.c')
| -rw-r--r-- | mm/vma.c | 113 |
1 files changed, 79 insertions, 34 deletions
@@ -18,6 +18,7 @@ struct mmap_state { unsigned long addr; unsigned long end; pgoff_t pgoff; + pgoff_t anon_pgoff; unsigned long pglen; union { vm_flags_t vm_flags; @@ -46,13 +47,14 @@ struct mmap_state { bool file_doesnt_need_get :1; }; -#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \ +#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, anon_pgoff_, vma_flags_, file_) \ struct mmap_state name = { \ .mm = mm_, \ .vmi = vmi_, \ .addr = addr_, \ .end = (addr_) + (len_), \ .pgoff = pgoff_, \ + .anon_pgoff = anon_pgoff_, \ .pglen = PHYS_PFN(len_), \ .vma_flags = vma_flags_, \ .file = file_, \ @@ -67,6 +69,7 @@ struct mmap_state { .end = (map_)->end, \ .vma_flags = (map_)->vma_flags, \ .pgoff = (map_)->pgoff, \ + .anon_pgoff = (map_)->anon_pgoff, \ .file = (map_)->file, \ .prev = (map_)->prev, \ .middle = vma_, \ @@ -82,10 +85,11 @@ static void __vma_set_range(struct vm_area_struct *vma, unsigned long start, } static void vma_set_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end, pgoff_t pgoff) + unsigned long end, pgoff_t pgoff, pgoff_t anon_pgoff) { __vma_set_range(vma, start, end); vma_set_pgoff(vma, pgoff); + vma_set_anon_pgoff(vma, anon_pgoff); } /* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */ @@ -201,6 +205,25 @@ static void init_multi_vma_prep(struct vma_prepare *vp, } /* + * Does this merge require that adjacent VMAs must have adjacent anonymous page + * offsets in addition to having adjacent vma->vm_pgoff? + * + * This is only required for MAP_PRIVATE-file backed mappings as the page offset + * for pure anonymous VMAs is equal to the anonymous page offset. + * + * Read-only shared mappings (with VMA_SHARED_BIT cleared) are always unfaulted + * so automatically have correct anonymous page offset (as it is always updated + * on remap). + * + * 'Special' mappings in the sense of VDSO, VVAR etc. have !file but would in + * any case not be candidates for merge nor be mergeable. + */ +static bool needs_adjacent_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->file && vma_flags_is_cow_mapping(&vmg->vma_flags); +} + +/* * Return true if we can merge this (vma_flags,anon_vma,file,vm_pgoff) * in front of (at a lower virtual address and file offset than) the vma. * @@ -221,6 +244,9 @@ static bool can_vma_merge_before(struct vma_merge_struct *vmg) return false; if (vmg_end_pgoff(vmg) != vma_start_pgoff(vmg->next)) return false; + if (needs_adjacent_anon_pgoff(vmg) && + vmg_end_anon_pgoff(vmg) != vma_start_anon_pgoff(vmg->next)) + return false; return true; } @@ -241,6 +267,9 @@ static bool can_vma_merge_after(struct vma_merge_struct *vmg) return false; if (vma_end_pgoff(vmg->prev) != vmg_start_pgoff(vmg)) return false; + if (needs_adjacent_anon_pgoff(vmg) && + vma_end_anon_pgoff(vmg->prev) != vmg_start_anon_pgoff(vmg)) + return false; return true; } @@ -812,7 +841,8 @@ static int commit_merge(struct vma_merge_struct *vmg) */ vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); - vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg)); + vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg), + vmg_start_anon_pgoff(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -982,6 +1012,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( vmg->start = prev->vm_start; vmg->end = next->vm_end; vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); /* * We already ensured anon_vma compatibility above, so now it's @@ -1000,6 +1031,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( */ vmg->start = prev->vm_start; vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -1022,12 +1054,14 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; vmg->pgoff = vma_start_pgoff(next) - pglen; + vmg->anon_pgoff = vma_start_anon_pgoff(next) - pglen; } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; vmg->pgoff = vma_start_pgoff(middle); + vmg->anon_pgoff = vma_start_anon_pgoff(middle); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1137,6 +1171,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) vmg->start = prev->vm_start; vmg->target = prev; vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); /* * If this merge would result in removal of the next VMA but we @@ -1908,23 +1943,27 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks) + pgoff_t anon_pgoff, bool *need_rmap_locks) { struct vm_area_struct *vma = *vmap; - unsigned long vma_start = vma->vm_start; + unsigned long old_vma_start = vma->vm_start; struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *new_vma; - bool faulted_in_anon_vma = true; + bool can_self_merge = false; VMA_ITERATOR(vmi, mm, addr); VMG_VMA_STATE(vmg, &vmi, NULL, vma, addr, addr + len); /* - * If anonymous vma has not yet been faulted, update new pgoff - * to match new location, to increase its chance of merging. + * If a vma has not yet been faulted, update its anonymous pgoff to + * match the new location to increase its chance of merging. */ - if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { - pgoff = addr >> PAGE_SHIFT; - faulted_in_anon_vma = false; + if (!vma->anon_vma) { + anon_pgoff = addr >> PAGE_SHIFT; + + if (vma_is_anonymous(vma)) { + pgoff = anon_pgoff; + can_self_merge = true; + } } /* @@ -1940,28 +1979,26 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, return NULL; /* should never get here */ vmg.pgoff = pgoff; + vmg.anon_pgoff = anon_pgoff; vmg.next = vma_iter_next_rewind(&vmi, NULL); new_vma = vma_merge_copied_range(&vmg); if (new_vma) { - /* - * Source vma may have been merged into new_vma - */ - if (unlikely(vma_start >= new_vma->vm_start && - vma_start < new_vma->vm_end)) { + /* Self-merged and VMA replaced. */ + if (unlikely(new_vma->vm_start < old_vma_start && + new_vma->vm_end > old_vma_start)) { /* - * The only way we can get a vma_merge with - * self during an mremap is if the vma hasn't - * been faulted in yet and we were allowed to - * reset the dst vma->vm_pgoff to the - * destination address of the mremap to allow - * the merge to happen. mremap must change the - * vm_pgoff linearity between src and dst vmas - * (in turn preventing a vma_merge) to be - * safe. It is only safe to keep the vm_pgoff - * linear if there are no pages mapped yet. + * The only way a VMA can both self-merge and be + * replaced is if the remap places the new VMA + * immediately prior to its old self ('next') and + * immediately after another VMA ('prev') causing the + * next to be removed and prev to be expanded to cover + * the entire range. + * + * This should only be possible if the anonymous page + * offset was updated, i.e. the VMA is unfaulted. */ - VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); + VM_WARN_ON_ONCE_VMA(!can_self_merge, new_vma); *vmap = vma = new_vma; } *need_rmap_locks = @@ -1970,7 +2007,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, new_vma = vm_area_dup(vma); if (!new_vma) goto out; - vma_set_range(new_vma, addr, addr + len, pgoff); + vma_set_range(new_vma, addr, addr + len, pgoff, anon_pgoff); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP)) @@ -2036,7 +2073,12 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * if (!vma_flags_empty(&diff)) return false; /* Page offset must align. */ - return vma_end_pgoff(a) == vma_start_pgoff(b); + if (vma_end_pgoff(a) != vma_start_pgoff(b)) + return false; + /* Only reached from anon path, so either MAP_PRIVATE file or anon. */ + if (vma_end_anon_pgoff(a) != vma_start_anon_pgoff(b)) + return false; + return true; } /* @@ -2612,7 +2654,7 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, if (is_anon) vma_set_anonymous(vma); - vma_set_range(vma, map->addr, map->end, map->pgoff); + vma_set_range(vma, map->addr, map->end, map->pgoff, map->anon_pgoff); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2801,7 +2843,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, struct vm_area_struct *vma = NULL; bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); - MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file); + const pgoff_t anon_pgoff = addr >> PAGE_SHIFT; + MMAP_STATE(map, mm, &vmi, addr, len, pgoff, anon_pgoff, vma_flags, file); struct vm_area_desc desc = { .mm = mm, .file = file, @@ -2946,6 +2989,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, unsigned long addr, unsigned long len, vma_flags_t vma_flags) { struct mm_struct *mm = current->mm; + const pgoff_t pgoff = addr >> PAGE_SHIFT; /* * Check against address space limits by the changed size @@ -2970,7 +3014,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, * occur after forking, so the expand will only happen on new VMAs. */ if (vma && vma->vm_end == addr) { - VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr)); + VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, pgoff, pgoff); vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ @@ -2990,7 +3034,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, goto unacct_fail; vma_set_anonymous(vma); - vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); + vma_set_range(vma, addr, addr + len, pgoff, pgoff); vma->flags = vma_flags; vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags)); vma_start_write(vma); @@ -3382,6 +3426,7 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) WARN_ON_ONCE(vma->anon_vma); vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT); } + vma_set_anon_pgoff(vma, vma->vm_start >> PAGE_SHIFT); if (vma_link(mm, vma)) { if (vma_test(vma, VMA_ACCOUNT_BIT)) @@ -3437,7 +3482,7 @@ struct vm_area_struct *__install_special_mapping( vma->vm_ops = ops; vma->vm_private_data = priv; - vma_set_range(vma, addr, addr + len, 0); + vma_set_range(vma, addr, addr + len, 0, addr >> PAGE_SHIFT); ret = insert_vm_struct(mm, vma); if (ret) |