diff options
Diffstat (limited to 'tools/testing/selftests/mm')
55 files changed, 2346 insertions, 1936 deletions
diff --git a/tools/testing/selftests/mm/.gitignore b/tools/testing/selftests/mm/.gitignore index b0c30c5ee9e3..9ccd9e1447e6 100644 --- a/tools/testing/selftests/mm/.gitignore +++ b/tools/testing/selftests/mm/.gitignore @@ -4,6 +4,10 @@ hugepage-mmap hugepage-mremap hugepage-shm hugepage-vmemmap +hugetlb-mmap +hugetlb-mremap +hugetlb-shm +hugetlb-vmemmap hugetlb-madvise hugetlb-read-hwpoison hugetlb-soft-offline diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index cd24596cdd27..e6df968f0971 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -61,16 +61,15 @@ TEST_GEN_FILES += gup_longterm TEST_GEN_FILES += gup_test TEST_GEN_FILES += hmm-tests TEST_GEN_FILES += hugetlb-madvise +TEST_GEN_FILES += hugetlb-mmap +TEST_GEN_FILES += hugetlb-mremap TEST_GEN_FILES += hugetlb-read-hwpoison +TEST_GEN_FILES += hugetlb-shm TEST_GEN_FILES += hugetlb-soft-offline -TEST_GEN_FILES += hugepage-mmap -TEST_GEN_FILES += hugepage-mremap -TEST_GEN_FILES += hugepage-shm -TEST_GEN_FILES += hugepage-vmemmap +TEST_GEN_FILES += hugetlb-vmemmap TEST_GEN_FILES += khugepaged TEST_GEN_FILES += madv_populate TEST_GEN_FILES += map_fixed_noreplace -TEST_GEN_FILES += map_hugetlb TEST_GEN_FILES += map_populate ifneq (,$(filter $(ARCH),arm64 riscv riscv64 x86 x86_64 loongarch32 loongarch64)) TEST_GEN_FILES += memfd_secret @@ -151,6 +150,7 @@ TEST_PROGS += ksft_gup_test.sh TEST_PROGS += ksft_hmm.sh TEST_PROGS += ksft_hugetlb.sh TEST_PROGS += ksft_hugevm.sh +TEST_PROGS += ksft_kmemleak_dedup.sh TEST_PROGS += ksft_ksm.sh TEST_PROGS += ksft_ksm_numa.sh TEST_PROGS += ksft_madv_guard.sh @@ -187,8 +187,8 @@ TEST_FILES += write_hugetlb_memory.sh include ../lib.mk -$(TEST_GEN_PROGS): vm_util.c thp_settings.c -$(TEST_GEN_FILES): vm_util.c thp_settings.c +$(TEST_GEN_PROGS): vm_util.c hugepage_settings.c +$(TEST_GEN_FILES): vm_util.c hugepage_settings.c $(OUTPUT)/uffd-stress: uffd-common.c $(OUTPUT)/uffd-unit-tests: uffd-common.c @@ -216,7 +216,8 @@ ifeq ($(CAN_BUILD_I386),1) $(BINARIES_32): CFLAGS += -m32 -mxsave $(BINARIES_32): LDLIBS += -lrt -ldl -lm $(BINARIES_32): $(OUTPUT)/%_32: %.c - $(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(notdir $^) $(LDLIBS) -o $@ + $(call msg,CC,,$@) + $(Q)$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(notdir $^) $(LDLIBS) -o $@ $(foreach t,$(VMTARGETS),$(eval $(call gen-target-rule-32,$(t)))) endif @@ -224,7 +225,8 @@ ifeq ($(CAN_BUILD_X86_64),1) $(BINARIES_64): CFLAGS += -m64 -mxsave $(BINARIES_64): LDLIBS += -lrt -ldl $(BINARIES_64): $(OUTPUT)/%_64: %.c - $(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(notdir $^) $(LDLIBS) -o $@ + $(call msg,CC,,$@) + $(Q)$(CC) $(CFLAGS) $(EXTRA_CFLAGS) $(notdir $^) $(LDLIBS) -o $@ $(foreach t,$(VMTARGETS),$(eval $(call gen-target-rule-64,$(t)))) endif @@ -261,7 +263,8 @@ $(OUTPUT)/migration: LDLIBS += -lnuma $(OUTPUT)/rmap: LDLIBS += -lnuma local_config.mk local_config.h: check_config.sh - CC="$(CC)" CFLAGS="$(CFLAGS)" ./check_config.sh + $(call msg,CHK,config,$@) + $(Q)CC="$(CC)" CFLAGS="$(CFLAGS)" ./check_config.sh EXTRA_CLEAN += local_config.mk local_config.h diff --git a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh index 44f4e703deb9..a1cfd3a349db 100755 --- a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh +++ b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh @@ -17,6 +17,7 @@ if ! command -v killall >/dev/null 2>&1; then fi nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) +trap 'echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages' EXIT INT TERM fault_limit_file=limit_in_bytes reservation_limit_file=rsvd.limit_in_bytes @@ -70,7 +71,6 @@ function cleanup() { if [[ -e $cgroup_path/hugetlb_cgroup_test2 ]]; then rmdir $cgroup_path/hugetlb_cgroup_test2 fi - echo 0 >/proc/sys/vm/nr_hugepages echo CLEANUP DONE } @@ -94,6 +94,15 @@ function get_machine_hugepage_size() { } MB=$(get_machine_hugepage_size) +if (( MB >= 1024 )); then + # For 1GB hugepages + UNIT="GB" + MB_DISPLAY=$((MB / 1024)) +else + # For 2MB hugepages + UNIT="MB" + MB_DISPLAY=$MB +fi function setup_cgroup() { local name="$1" @@ -103,11 +112,12 @@ function setup_cgroup() { mkdir $cgroup_path/$name echo writing cgroup limit: "$cgroup_limit" - echo "$cgroup_limit" >$cgroup_path/$name/hugetlb.${MB}MB.$fault_limit_file + echo "$cgroup_limit" > \ + $cgroup_path/$name/hugetlb.${MB_DISPLAY}${UNIT}.$fault_limit_file echo writing reservation limit: "$reservation_limit" echo "$reservation_limit" > \ - $cgroup_path/$name/hugetlb.${MB}MB.$reservation_limit_file + $cgroup_path/$name/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_limit_file if [ -e "$cgroup_path/$name/cpuset.cpus" ]; then echo 0 >$cgroup_path/$name/cpuset.cpus @@ -142,7 +152,7 @@ function wait_for_file_value() { function wait_for_hugetlb_memory_to_get_depleted() { local cgroup="$1" - local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file" + local path="$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file" wait_for_file_value "$path" "0" } @@ -150,7 +160,7 @@ function wait_for_hugetlb_memory_to_get_depleted() { function wait_for_hugetlb_memory_to_get_reserved() { local cgroup="$1" local size="$2" - local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file" + local path="$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file" wait_for_file_value "$path" "$size" } @@ -158,7 +168,7 @@ function wait_for_hugetlb_memory_to_get_reserved() { function wait_for_hugetlb_memory_to_get_written() { local cgroup="$1" local size="$2" - local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$fault_usage_file" + local path="$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file" wait_for_file_value "$path" "$size" } @@ -180,8 +190,8 @@ function write_hugetlbfs_and_get_usage() { hugetlb_difference=0 reserved_difference=0 - local hugetlb_usage=$cgroup_path/$cgroup/hugetlb.${MB}MB.$fault_usage_file - local reserved_usage=$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file + local hugetlb_usage=$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file + local reserved_usage=$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file local hugetlb_before=$(cat $hugetlb_usage) local reserved_before=$(cat $reserved_usage) @@ -312,8 +322,10 @@ function run_test() { cleanup_hugetlb_memory "hugetlb_cgroup_test" - local final_hugetlb=$(cat $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB}MB.$fault_usage_file) - local final_reservation=$(cat $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB}MB.$reservation_usage_file) + local final_hugetlb=$(cat \ + $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file) + local final_reservation=$(cat \ + $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file) echo $hugetlb_difference echo $reserved_difference @@ -369,10 +381,14 @@ function run_multiple_cgroup_test() { reservation_failed1=$reservation_failed oom_killed1=$oom_killed - local cgroup1_hugetlb_usage=$cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB}MB.$fault_usage_file - local cgroup1_reservation_usage=$cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB}MB.$reservation_usage_file - local cgroup2_hugetlb_usage=$cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB}MB.$fault_usage_file - local cgroup2_reservation_usage=$cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB}MB.$reservation_usage_file + local cgroup1_hugetlb_usage=\ + $cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file + local cgroup1_reservation_usage=\ + $cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file + local cgroup2_hugetlb_usage=\ + $cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file + local cgroup2_reservation_usage=\ + $cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file local usage_before_second_write=$(cat $cgroup1_hugetlb_usage) local reservation_usage_before_second_write=$(cat $cgroup1_reservation_usage) @@ -599,4 +615,3 @@ if [[ $do_umount ]]; then rmdir $cgroup_path fi -echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages diff --git a/tools/testing/selftests/mm/check_config.sh b/tools/testing/selftests/mm/check_config.sh index b84c82bbf875..32beaefe279e 100755 --- a/tools/testing/selftests/mm/check_config.sh +++ b/tools/testing/selftests/mm/check_config.sh @@ -16,7 +16,7 @@ echo "#include <sys/types.h>" > $tmpfile_c echo "#include <liburing.h>" >> $tmpfile_c echo "int func(void) { return 0; }" >> $tmpfile_c -$CC $CFLAGS -c $tmpfile_c -o $tmpfile_o +$CC $CFLAGS -c $tmpfile_c -o $tmpfile_o >/dev/null 2>&1 if [ -f $tmpfile_o ]; then echo "#define LOCAL_CONFIG_HAVE_LIBURING 1" > $OUTPUT_H_FILE diff --git a/tools/testing/selftests/mm/compaction_test.c b/tools/testing/selftests/mm/compaction_test.c index 30209c40b697..5b582588e015 100644 --- a/tools/testing/selftests/mm/compaction_test.c +++ b/tools/testing/selftests/mm/compaction_test.c @@ -17,6 +17,7 @@ #include <string.h> #include "kselftest.h" +#include "hugepage_settings.h" #define MAP_SIZE_MB 100 #define MAP_SIZE (MAP_SIZE_MB * 1024 * 1024) @@ -82,124 +83,44 @@ int prereq(void) return -1; } -int check_compaction(unsigned long mem_free, unsigned long hugepage_size, - unsigned long initial_nr_hugepages) +int check_compaction(unsigned long mem_free, unsigned long hugepage_size) { - unsigned long nr_hugepages_ul; - int fd, ret = -1; + unsigned long nr_hugepages; int compaction_index = 0; - char nr_hugepages[20] = {0}; - char init_nr_hugepages[24] = {0}; - char target_nr_hugepages[24] = {0}; - int slen; - - snprintf(init_nr_hugepages, sizeof(init_nr_hugepages), - "%lu", initial_nr_hugepages); + int ret = -1; /* We want to test with 80% of available memory. Else, OOM killer comes in to play */ mem_free = mem_free * 0.8; - fd = open("/proc/sys/vm/nr_hugepages", O_RDWR | O_NONBLOCK); - if (fd < 0) { - ksft_print_msg("Failed to open /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); - ret = -1; - goto out; - } - /* * Request huge pages for about half of the free memory. The Kernel * will allocate as much as it can, and we expect it will get at least 1/3 */ - nr_hugepages_ul = mem_free / hugepage_size / 2; - snprintf(target_nr_hugepages, sizeof(target_nr_hugepages), - "%lu", nr_hugepages_ul); - - slen = strlen(target_nr_hugepages); - if (write(fd, target_nr_hugepages, slen) != slen) { - ksft_print_msg("Failed to write %lu to /proc/sys/vm/nr_hugepages: %s\n", - nr_hugepages_ul, strerror(errno)); - goto close_fd; - } - - lseek(fd, 0, SEEK_SET); - - if (read(fd, nr_hugepages, sizeof(nr_hugepages)) <= 0) { - ksft_print_msg("Failed to re-read from /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); - goto close_fd; - } + nr_hugepages = mem_free / hugepage_size / 2; + hugetlb_set_nr_default_pages(nr_hugepages); /* We should have been able to request at least 1/3 rd of the memory in huge pages */ - nr_hugepages_ul = strtoul(nr_hugepages, NULL, 10); - if (!nr_hugepages_ul) { + nr_hugepages = hugetlb_nr_default_pages(); + if (!nr_hugepages) { ksft_print_msg("ERROR: No memory is available as huge pages\n"); - goto close_fd; - } - compaction_index = mem_free/(nr_hugepages_ul * hugepage_size); - - lseek(fd, 0, SEEK_SET); - - if (write(fd, init_nr_hugepages, strlen(init_nr_hugepages)) - != strlen(init_nr_hugepages)) { - ksft_print_msg("Failed to write value to /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); - goto close_fd; + goto out; } + compaction_index = mem_free/(nr_hugepages * hugepage_size); - ksft_print_msg("Number of huge pages allocated = %lu\n", - nr_hugepages_ul); + ksft_print_msg("Number of huge pages allocated = %lu\n", nr_hugepages); if (compaction_index > 3) { ksft_print_msg("ERROR: Less than 1/%d of memory is available\n" "as huge pages\n", compaction_index); - goto close_fd; - } - - ret = 0; - - close_fd: - close(fd); - out: - ksft_test_result(ret == 0, "check_compaction\n"); - return ret; -} - -int set_zero_hugepages(unsigned long *initial_nr_hugepages) -{ - int fd, ret = -1; - char nr_hugepages[20] = {0}; - - fd = open("/proc/sys/vm/nr_hugepages", O_RDWR | O_NONBLOCK); - if (fd < 0) { - ksft_print_msg("Failed to open /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); goto out; } - if (read(fd, nr_hugepages, sizeof(nr_hugepages)) <= 0) { - ksft_print_msg("Failed to read from /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); - goto close_fd; - } - - lseek(fd, 0, SEEK_SET); - - /* Start with the initial condition of 0 huge pages */ - if (write(fd, "0", sizeof(char)) != sizeof(char)) { - ksft_print_msg("Failed to write 0 to /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); - goto close_fd; - } - *initial_nr_hugepages = strtoul(nr_hugepages, NULL, 10); ret = 0; - close_fd: - close(fd); - out: + ksft_test_result(ret == 0, "check_compaction\n"); return ret; } @@ -212,18 +133,17 @@ int main(int argc, char **argv) unsigned long mem_free = 0; unsigned long hugepage_size = 0; long mem_fragmentable_MB = 0; - unsigned long initial_nr_hugepages; ksft_print_header(); if (prereq() || geteuid()) ksft_exit_skip("Prerequisites unsatisfied\n"); - ksft_set_plan(1); - /* Start the test without hugepages reducing mem_free */ - if (set_zero_hugepages(&initial_nr_hugepages)) - ksft_exit_fail(); + if (!hugetlb_setup_default_exact(0)) + ksft_exit_skip("Could not reset nr_hugepages\n"); + + ksft_set_plan(1); lim.rlim_cur = RLIM_INFINITY; lim.rlim_max = RLIM_INFINITY; @@ -261,6 +181,9 @@ int main(int argc, char **argv) mem_fragmentable_MB -= MAP_SIZE_MB; } + /* Unmap every other entry in the list to create fragmentation with + * locked pages before invoking check_compaction(). + */ for (entry = list; entry != NULL; entry = entry->next) { munmap(entry->map, MAP_SIZE); if (!entry->next) @@ -268,8 +191,7 @@ int main(int argc, char **argv) entry = entry->next; } - if (check_compaction(mem_free, hugepage_size, - initial_nr_hugepages) == 0) + if (check_compaction(mem_free, hugepage_size) == 0) ksft_exit_pass(); ksft_exit_fail(); diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index d9c69c04b67d..0c627ea89ff7 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -29,7 +29,7 @@ #include "../../../../mm/gup_test.h" #include "kselftest.h" #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" static size_t pagesize; static int pagemap_fd; @@ -37,7 +37,7 @@ static size_t pmdsize; static int nr_thpsizes; static size_t thpsizes[20]; static int nr_hugetlbsizes; -static size_t hugetlbsizes[10]; +static unsigned long hugetlbsizes[10]; static int gup_fd; static bool has_huge_zeropage; @@ -202,7 +202,7 @@ static void do_test_cow_in_parent(char *mem, size_t size, bool do_mprotect, log_test_result(KSFT_FAIL); goto close_comm_pipes; } else if (!ret) { - exit(fn(mem, size, &comm_pipes)); + _exit(fn(mem, size, &comm_pipes)); } while (read(comm_pipes.child_ready[0], &buf, 1) != 1) @@ -333,7 +333,7 @@ static void do_test_vmsplice_in_parent(char *mem, size_t size, ; /* Modify page content in the child. */ memset(mem, 0xff, size); - exit(0); + _exit(0); } if (!before_fork) { @@ -480,7 +480,7 @@ static void do_test_iouring(char *mem, size_t size, bool use_fork) write(comm_pipes.child_ready[1], "0", 1); while (read(comm_pipes.parent_ready[0], &buf, 1) != 1) ; - exit(0); + _exit(0); } while (read(comm_pipes.child_ready[0], &buf, 1) != 1) @@ -645,7 +645,7 @@ static void do_test_ro_pin(char *mem, size_t size, enum ro_pin_test test, write(comm_pipes.child_ready[1], "0", 1); while (read(comm_pipes.parent_ready[0], &buf, 1) != 1) ; - exit(0); + _exit(0); } /* Wait until our child is ready. */ @@ -956,7 +956,7 @@ static void do_run_with_thp(test_fn fn, enum thp_run thp_run, size_t thpsize) log_test_result(KSFT_FAIL); goto munmap; } else if (!ret) { - exit(0); + _exit(0); } wait(&ret); /* Allow for sharing all pages again. */ @@ -1347,13 +1347,13 @@ static void do_test_anon_thp_collapse(char *mem, size_t size, switch (test) { case ANON_THP_COLLAPSE_UNSHARED: case ANON_THP_COLLAPSE_FULLY_SHARED: - exit(child_memcmp_fn(mem, size, &comm_pipes)); + _exit(child_memcmp_fn(mem, size, &comm_pipes)); break; case ANON_THP_COLLAPSE_LOWER_SHARED: - exit(child_memcmp_fn(mem, size / 2, &comm_pipes)); + _exit(child_memcmp_fn(mem, size / 2, &comm_pipes)); break; case ANON_THP_COLLAPSE_UPPER_SHARED: - exit(child_memcmp_fn(mem + size / 2, size / 2, + _exit(child_memcmp_fn(mem + size / 2, size / 2, &comm_pipes)); break; default: @@ -1881,21 +1881,21 @@ int main(int argc, char **argv) ksft_print_header(); + thp_save_settings(); + pagesize = getpagesize(); pmdsize = read_pmd_pagesize(); if (pmdsize) { /* Only if THP is supported. */ thp_read_settings(&default_settings); default_settings.hugepages[sz2ord(pmdsize, pagesize)].enabled = THP_INHERIT; - thp_save_settings(); thp_push_settings(&default_settings); ksft_print_msg("[INFO] detected PMD size: %zu KiB\n", pmdsize / 1024); nr_thpsizes = detect_thp_sizes(thpsizes, ARRAY_SIZE(thpsizes)); } - nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes, - ARRAY_SIZE(hugetlbsizes)); + nr_hugetlbsizes = hugetlb_setup(2, hugetlbsizes, ARRAY_SIZE(hugetlbsizes)); has_huge_zeropage = detect_huge_zeropage(); ksft_set_plan(ARRAY_SIZE(anon_test_cases) * tests_per_anon_test_case() + @@ -1911,10 +1911,5 @@ int main(int argc, char **argv) run_anon_thp_test_cases(); run_non_anon_test_cases(); - if (pmdsize) { - /* Only if THP is supported. */ - thp_restore_settings(); - } - ksft_finished(); } diff --git a/tools/testing/selftests/mm/droppable.c b/tools/testing/selftests/mm/droppable.c index 44940f75c461..57e1b6fc5569 100644 --- a/tools/testing/selftests/mm/droppable.c +++ b/tools/testing/selftests/mm/droppable.c @@ -17,37 +17,50 @@ int main(int argc, char *argv[]) { - size_t alloc_size = 134217728; - size_t page_size = getpagesize(); + const size_t alloc_size = 2 * 1024 * 1024; + int retry_count = 10; + bool dropped; void *alloc; - pid_t child; ksft_print_header(); ksft_set_plan(1); alloc = mmap(0, alloc_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_DROPPABLE, -1, 0); - assert(alloc != MAP_FAILED); - memset(alloc, 'A', alloc_size); - for (size_t i = 0; i < alloc_size; i += page_size) - assert(*(uint8_t *)(alloc + i)); - - child = fork(); - assert(child >= 0); - if (!child) { - for (;;) - *(char *)malloc(page_size) = 'B'; + if (alloc == MAP_FAILED) { + if ((errno == EOPNOTSUPP) || (errno == EINVAL)) { + ksft_test_result_skip("MAP_DROPPABLE not supported\n"); + exit(KSFT_SKIP); + } + ksft_test_result_fail("mmap error: %s\n", strerror(errno)); + exit(KSFT_FAIL); } + memset(alloc, 'A', alloc_size); - for (bool done = false; !done;) { - for (size_t i = 0; i < alloc_size; i += page_size) { - if (!*(uint8_t *)(alloc + i)) { - done = true; - break; + while (retry_count--) { + if (madvise(alloc, alloc_size, MADV_PAGEOUT)) { + if (errno == EINVAL) { + ksft_test_result_skip("madvise(MADV_PAGEOUT) not supported\n"); + exit(KSFT_SKIP); } + ksft_test_result_fail("madvise(MADV_PAGEOUT) error: %s\n", strerror(errno)); + exit(KSFT_FAIL); } + + dropped = memchr(alloc, 'A', alloc_size) == NULL; + + /* + * Speculative reference can temporarily prevent some + * pages from getting dropped. So sleep and retry. + * + * If a page is not droppable for 10s, something + * is seriously messed up and we want to fail. + */ + if (dropped) + break; + sleep(1); } - kill(child, SIGTERM); - ksft_test_result_pass("MAP_DROPPABLE: PASS\n"); - exit(KSFT_PASS); + ksft_test_result(dropped, "madvise(MADV_PAGEOUT) behavior\n"); + + ksft_finished(); } diff --git a/tools/testing/selftests/mm/folio_split_race_test.c b/tools/testing/selftests/mm/folio_split_race_test.c index ff026f183ac7..6329e37fff4c 100644 --- a/tools/testing/selftests/mm/folio_split_race_test.c +++ b/tools/testing/selftests/mm/folio_split_race_test.c @@ -25,7 +25,7 @@ #include <unistd.h> #include "vm_util.h" #include "kselftest.h" -#include "thp_settings.h" +#include "hugepage_settings.h" uint64_t page_size; uint64_t pmd_pagesize; @@ -226,23 +226,6 @@ static uint64_t run_iteration(void) return reader_failures; } -static void thp_cleanup_handler(int signum) -{ - thp_restore_settings(); - /* - * Restore default handler and re-raise the signal to exit. - * This is to ensure the test process exits with the correct - * status code corresponding to the signal. - */ - signal(signum, SIG_DFL); - raise(signum); -} - -static void thp_settings_cleanup(void) -{ - thp_restore_settings(); -} - int main(void) { struct thp_settings current_settings; @@ -261,12 +244,6 @@ int main(void) ksft_exit_skip("Please run the test as root\n"); thp_save_settings(); - /* make sure thp settings are restored */ - if (atexit(thp_settings_cleanup) != 0) - ksft_exit_fail_msg("atexit failed\n"); - - signal(SIGINT, thp_cleanup_handler); - signal(SIGTERM, thp_cleanup_handler); thp_read_settings(¤t_settings); current_settings.shmem_enabled = SHMEM_ADVISE; diff --git a/tools/testing/selftests/mm/guard-regions.c b/tools/testing/selftests/mm/guard-regions.c index 48e8b1539be3..b21df3040b1c 100644 --- a/tools/testing/selftests/mm/guard-regions.c +++ b/tools/testing/selftests/mm/guard-regions.c @@ -21,7 +21,7 @@ #include <sys/uio.h> #include <unistd.h> #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #include "../pidfd/pidfd.h" @@ -2203,17 +2203,6 @@ TEST_F(guard_regions, collapse) if (variant->backing != ANON_BACKED) ASSERT_EQ(ftruncate(self->fd, size), 0); - /* - * We must close and re-open local-file backed as read-only for - * CONFIG_READ_ONLY_THP_FOR_FS to work. - */ - if (variant->backing == LOCAL_FILE_BACKED) { - ASSERT_EQ(close(self->fd), 0); - - self->fd = open(self->path, O_RDONLY); - ASSERT_GE(self->fd, 0); - } - ptr = mmap_(self, variant, NULL, size, PROT_READ, 0, 0); ASSERT_NE(ptr, MAP_FAILED); @@ -2237,9 +2226,10 @@ TEST_F(guard_regions, collapse) /* * Now collapse the entire region. This should fail in all cases. * - * The madvise() call will also fail if CONFIG_READ_ONLY_THP_FOR_FS is - * not set for the local file case, but we can't differentiate whether - * this occurred or if the collapse was rightly rejected. + * The madvise() call will also fail if the file system does not support + * large folio or the supported orders do not include PMD_ORDER for the + * local file case, but we can't differentiate whether this occurred or + * if the collapse was rightly rejected. */ EXPECT_NE(madvise(ptr, size, MADV_COLLAPSE), 0); diff --git a/tools/testing/selftests/mm/gup_longterm.c b/tools/testing/selftests/mm/gup_longterm.c index f61150d28eb2..eb8963e9d98f 100644 --- a/tools/testing/selftests/mm/gup_longterm.c +++ b/tools/testing/selftests/mm/gup_longterm.c @@ -29,10 +29,11 @@ #include "../../../../mm/gup_test.h" #include "kselftest.h" #include "vm_util.h" +#include "hugepage_settings.h" static size_t pagesize; static int nr_hugetlbsizes; -static size_t hugetlbsizes[10]; +static unsigned long hugetlbsizes[10]; static int gup_fd; static __fsword_t get_fs_type(int fd) @@ -509,7 +510,7 @@ int main(int argc, char **argv) int i; pagesize = getpagesize(); - nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes, + nr_hugetlbsizes = hugetlb_setup(2, hugetlbsizes, ARRAY_SIZE(hugetlbsizes)); ksft_print_header(); diff --git a/tools/testing/selftests/mm/gup_test.c b/tools/testing/selftests/mm/gup_test.c index fb8f9ae49efa..3f841a96f870 100644 --- a/tools/testing/selftests/mm/gup_test.c +++ b/tools/testing/selftests/mm/gup_test.c @@ -14,6 +14,7 @@ #include <mm/gup_test.h> #include "kselftest.h" #include "vm_util.h" +#include "hugepage_settings.h" #define MB (1UL << 20) @@ -94,6 +95,7 @@ int main(int argc, char **argv) int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret; int flags = MAP_PRIVATE; char *file = "/dev/zero"; + bool hugetlb = false; pthread_t *tid; char *p; @@ -168,6 +170,7 @@ int main(int argc, char **argv) break; case 'H': flags |= (MAP_HUGETLB | MAP_ANONYMOUS); + hugetlb = true; break; default: ksft_exit_fail_msg("Wrong argument\n"); @@ -199,6 +202,18 @@ int main(int argc, char **argv) } ksft_print_header(); + + if (hugetlb) { + unsigned long hp_size = default_huge_page_size(); + + if (!hp_size) + ksft_exit_skip("HugeTLB is unavailable\n"); + + size = (size + hp_size - 1) & ~(hp_size - 1); + if (!hugetlb_setup_default(size / hp_size)) + ksft_exit_skip("Not enough huge pages\n"); + } + ksft_set_plan(nthreads); filed = open(file, O_RDWR|O_CREAT, 0664); diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 77fb4c5d871b..2f2b9879d100 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -11,6 +11,7 @@ */ #include "kselftest_harness.h" +#include "hugepage_settings.h" #include <errno.h> #include <fcntl.h> @@ -21,13 +22,14 @@ #include <strings.h> #include <time.h> #include <pthread.h> +#include <limits.h> +#include <linux/mman.h> #include <sys/types.h> #include <sys/stat.h> #include <sys/mman.h> #include <sys/ioctl.h> #include <sys/time.h> - /* * This is a private UAPI to the kernel test module so it isn't exported * in the usual include/uapi/... directory. @@ -69,6 +71,9 @@ enum { #ifndef FOLL_LONGTERM #define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */ #endif + +HUGETLB_SETUP_DEFAULT_PAGES(1) + FIXTURE(hmm) { int fd; @@ -632,7 +637,7 @@ TEST_F(hmm, anon_write_child) } close(child_fd); - exit(0); + _exit(0); } } } @@ -712,7 +717,7 @@ TEST_F(hmm, anon_write_child_shared) ASSERT_EQ(ptr[i], -i); close(child_fd); - exit(0); + _exit(0); } /* @@ -784,8 +789,8 @@ TEST_F(hmm, anon_write_hugetlbfs) int *ptr; int ret; - if (!default_hsize) - SKIP(return, "Huge page size could not be determined"); + if (!hugetlb_free_default_pages()) + SKIP(return, "Not enough huge pages"); size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; @@ -1599,8 +1604,8 @@ TEST_F(hmm2, snapshot) } /* - * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that - * should be mapped by a large page table entry. + * Test the hmm_range_fault() handling of large pages (PMD or PUD) + * that should be mapped by a large page table entry. */ TEST_F(hmm, compound) { @@ -1610,13 +1615,13 @@ TEST_F(hmm, compound) unsigned long default_hsize = default_huge_page_size(); int *ptr; unsigned char *m; + unsigned char prot; int ret; unsigned long i; /* Skip test if we can't allocate a hugetlbfs page. */ - - if (!default_hsize) - SKIP(return, "Huge page size could not be determined"); + if (!hugetlb_free_default_pages()) + SKIP(return, "Not enough huge pages"); size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; @@ -1646,11 +1651,20 @@ TEST_F(hmm, compound) ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); - /* Check what the device saw. */ + /* + * Check what the device saw. The region is backed by a single huge + * page that the device reports either at PMD or at PUD level depending + * on the configured default hugepage size. Determine that level from + * the first page and require every page in the range to match it + * exactly, so that a fragmented mapping mixing levels (or a missing + * large-page bit) is still caught and reported with its actual value. + */ m = buffer->mirror; + prot = HMM_DMIRROR_PROT_WRITE | + ((m[0] & HMM_DMIRROR_PROT_PUD) ? HMM_DMIRROR_PROT_PUD : + HMM_DMIRROR_PROT_PMD); for (i = 0; i < npages; ++i) - ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE | - HMM_DMIRROR_PROT_PMD); + ASSERT_EQ(m[i], prot); /* Make the region read-only. */ ret = mprotect(buffer->ptr, size, PROT_READ); @@ -1661,11 +1675,17 @@ TEST_F(hmm, compound) ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); - /* Check what the device saw. */ + /* + * Check what the device saw after mprotect(PROT_READ). Same + * approach as above: determine the mapping level from the first + * page and require every page to match it exactly. + */ m = buffer->mirror; + prot = HMM_DMIRROR_PROT_READ | + ((m[0] & HMM_DMIRROR_PROT_PUD) ? HMM_DMIRROR_PROT_PUD : + HMM_DMIRROR_PROT_PMD); for (i = 0; i < npages; ++i) - ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ | - HMM_DMIRROR_PROT_PMD); + ASSERT_EQ(m[i], prot); munmap(buffer->ptr, buffer->size); buffer->ptr = NULL; @@ -1865,6 +1885,8 @@ TEST_F(hmm, exclusive_cow) unsigned long i; int *ptr; int ret; + pid_t pid; + int status; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); @@ -1893,14 +1915,37 @@ TEST_F(hmm, exclusive_cow) ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); - fork(); + pid = fork(); + if (pid == -1) + ASSERT_EQ(pid, 0); - /* Fault pages back to system memory and check them. */ + if (pid == 0) { + /* + * Child verifies COW independently, then _exit(0)s so it does + * not run the test teardown. A failed ASSERT_* here makes the + * harness abort() the child, so the parent sees + * !WIFEXITED(status) below and fails in turn. + */ + for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) + ASSERT_EQ(ptr[i]++, i); + + for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) + ASSERT_EQ(ptr[i], i + 1); + + _exit(0); + } + + /* Parent: also increment to verify COW works for both processes. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i]++, i); for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) - ASSERT_EQ(ptr[i], i+1); + ASSERT_EQ(ptr[i], i + 1); + + /* Parent: wait for child and then free the buffer. */ + ASSERT_EQ(waitpid(pid, &status, 0), pid); + ASSERT_TRUE(WIFEXITED(status)); + ASSERT_EQ(WEXITSTATUS(status), 0); hmm_buffer_free(buffer); } @@ -2062,7 +2107,7 @@ TEST_F(hmm, hmm_cow_in_device) if (pid == -1) ASSERT_EQ(pid, 0); if (!pid) { - /* Child process waits for SIGTERM from the parent. */ + /* Child process waits for SIGKILL from the parent. */ while (1) { } /* Should not reach this */ @@ -2075,10 +2120,10 @@ TEST_F(hmm, hmm_cow_in_device) ptr[i] = i; /* Terminate child and wait */ - EXPECT_EQ(0, kill(pid, SIGTERM)); + EXPECT_EQ(0, kill(pid, SIGKILL)); EXPECT_EQ(pid, waitpid(pid, &status, 0)); EXPECT_NE(0, WIFSIGNALED(status)); - EXPECT_EQ(SIGTERM, WTERMSIG(status)); + EXPECT_EQ(SIGKILL, WTERMSIG(status)); /* Take snapshot to CPU pagetables */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); @@ -2274,8 +2319,11 @@ TEST_F(hmm, migrate_anon_huge_fault) unsigned long npages; unsigned long size; unsigned long i; + unsigned char *m; + uint64_t entry; void *old_ptr; void *map; + int pagemap_fd; int *ptr; int ret; @@ -2298,8 +2346,6 @@ TEST_F(hmm, migrate_anon_huge_fault) npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); - ret = madvise(map, size, MADV_HUGEPAGE); - ASSERT_EQ(ret, 0); old_ptr = buffer->ptr; buffer->ptr = map; @@ -2307,6 +2353,9 @@ TEST_F(hmm, migrate_anon_huge_fault) for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; + ret = madvise(map, size, MADV_COLLAPSE); + ASSERT_EQ(ret, 0); + /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, 0); @@ -2316,6 +2365,32 @@ TEST_F(hmm, migrate_anon_huge_fault) for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); + if (!hmm_is_coherent_type(variant->device_number)) { + ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, + buffer, npages); + ASSERT_EQ(ret, 0); + ASSERT_EQ(buffer->cpages, npages); + + m = buffer->mirror; + for (i = 0; i < npages; ++i) + ASSERT_EQ(m[i], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL | + HMM_DMIRROR_PROT_WRITE | + HMM_DMIRROR_PROT_PMD); + + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + ASSERT_GE(pagemap_fd, 0); + + for (i = 0; i < npages; ++i) { + entry = pagemap_get_entry(pagemap_fd, + (char *)buffer->ptr + i * self->page_size); + + ASSERT_NE(entry & PM_SWAP, 0); + ASSERT_FALSE(PAGEMAP_PRESENT(entry)); + } + + close(pagemap_fd); + } + /* Fault pages back to system memory and check them. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); @@ -2332,12 +2407,21 @@ TEST_F(hmm, migrate_partial_unmap_fault) struct hmm_buffer *buffer; unsigned long npages; unsigned long size = read_pmd_pagesize(); + unsigned long unmap_size; + unsigned long offsets[3]; unsigned long i; void *old_ptr; void *map; int *ptr; int ret, j, use_thp; - int offsets[] = { 0, 512 * ONEKB, ONEMEG }; + + if (!size) + size = TWOMEG; + + unmap_size = size / 2; + offsets[0] = 0; + offsets[1] = size / 4; + offsets[2] = size / 2; for (use_thp = 0; use_thp < 2; ++use_thp) { for (j = 0; j < ARRAY_SIZE(offsets); ++j) { @@ -2379,12 +2463,12 @@ TEST_F(hmm, migrate_partial_unmap_fault) for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); - munmap(buffer->ptr + offsets[j], ONEMEG); + munmap(buffer->ptr + offsets[j], unmap_size); /* Fault pages back to system memory and check them. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) if (i * sizeof(int) < offsets[j] || - i * sizeof(int) >= offsets[j] + ONEMEG) + i * sizeof(int) >= offsets[j] + unmap_size) ASSERT_EQ(ptr[i], i); buffer->ptr = old_ptr; @@ -2398,12 +2482,19 @@ TEST_F(hmm, migrate_remap_fault) struct hmm_buffer *buffer; unsigned long npages; unsigned long size = read_pmd_pagesize(); + unsigned long offsets[3]; unsigned long i; void *old_ptr, *new_ptr = NULL; void *map; int *ptr; int ret, j, use_thp, dont_unmap, before; - int offsets[] = { 0, 512 * ONEKB, ONEMEG }; + + if (!size) + size = TWOMEG; + + offsets[0] = 0; + offsets[1] = size / 4; + offsets[2] = size / 2; for (before = 0; before < 2; ++before) { for (dont_unmap = 0; dont_unmap < 2; ++dont_unmap) { @@ -2738,7 +2829,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (!buffer->ptr) + if (buffer->ptr == MAP_FAILED) return -1; /* Apply THP hint if requested */ @@ -2806,38 +2897,45 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) { struct benchmark_results thp_results, regular_results; - size_t thp_size = 2 * 1024 * 1024; /* 2MB - typical THP size */ + size_t thp_size = read_pmd_pagesize(); int iterations = 5; + if (!thp_size) + thp_size = TWOMEG; + printf("\nHMM THP Migration Benchmark\n"); printf("---------------------------\n"); printf("System page size: %ld bytes\n", sysconf(_SC_PAGESIZE)); /* Test different buffer sizes */ size_t test_sizes[] = { - thp_size / 4, /* 512KB - smaller than THP */ - thp_size / 2, /* 1MB - half THP */ - thp_size, /* 2MB - single THP */ - thp_size * 2, /* 4MB - two THPs */ - thp_size * 4, /* 8MB - four THPs */ - thp_size * 8, /* 16MB - eight THPs */ - thp_size * 128, /* 256MB - one twenty eight THPs */ + thp_size / 4, /* quarter THP */ + thp_size / 2, /* half THP */ + thp_size, /* single THP */ + thp_size * 2, /* two THPs */ + thp_size * 4, /* four THPs */ + thp_size * 8, /* eight THPs */ + thp_size * 128, /* one twenty eight THPs */ }; static const char *const test_names[] = { - "Small Buffer (512KB)", - "Half THP Size (1MB)", - "Single THP Size (2MB)", - "Two THP Size (4MB)", - "Four THP Size (8MB)", - "Eight THP Size (16MB)", - "One twenty eight THP Size (256MB)" + "Small Buffer", + "Half THP Size", + "Single THP Size", + "Two THP Size", + "Four THP Size", + "Eight THP Size", + "One twenty eight THP Size" }; int num_tests = ARRAY_SIZE(test_sizes); /* Run all tests */ for (int i = 0; i < num_tests; i++) { + /* Skip test sizes exceeding INT_MAX to avoid overflow */ + if (test_sizes[i] > INT_MAX) + break; + /* Test with THP */ ASSERT_EQ(run_migration_benchmark(self->fd, 1, test_sizes[i], iterations, &thp_results), 0); diff --git a/tools/testing/selftests/mm/hugepage-mmap.c b/tools/testing/selftests/mm/hugepage-mmap.c deleted file mode 100644 index d543419de040..000000000000 --- a/tools/testing/selftests/mm/hugepage-mmap.c +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * hugepage-mmap: - * - * Example of using huge page memory in a user application using the mmap - * system call. Before running this application, make sure that the - * administrator has mounted the hugetlbfs filesystem (on some directory - * like /mnt) using the command mount -t hugetlbfs nodev /mnt. In this - * example, the app is requesting memory of size 256MB that is backed by - * huge pages. - */ -#define _GNU_SOURCE -#include <stdlib.h> -#include <stdio.h> -#include <unistd.h> -#include <sys/mman.h> -#include <fcntl.h> -#include "kselftest.h" - -#define LENGTH (256UL*1024*1024) -#define PROTECTION (PROT_READ | PROT_WRITE) - -static void check_bytes(char *addr) -{ - ksft_print_msg("First hex is %x\n", *((unsigned int *)addr)); -} - -static void write_bytes(char *addr) -{ - unsigned long i; - - for (i = 0; i < LENGTH; i++) - *(addr + i) = (char)i; -} - -static int read_bytes(char *addr) -{ - unsigned long i; - - check_bytes(addr); - for (i = 0; i < LENGTH; i++) - if (*(addr + i) != (char)i) { - ksft_print_msg("Error: Mismatch at %lu\n", i); - return 1; - } - return 0; -} - -int main(void) -{ - void *addr; - int fd, ret; - - ksft_print_header(); - ksft_set_plan(1); - - fd = memfd_create("hugepage-mmap", MFD_HUGETLB); - if (fd < 0) - ksft_exit_fail_msg("memfd_create() failed: %s\n", strerror(errno)); - - addr = mmap(NULL, LENGTH, PROTECTION, MAP_SHARED, fd, 0); - if (addr == MAP_FAILED) { - close(fd); - ksft_exit_fail_msg("mmap(): %s\n", strerror(errno)); - } - - ksft_print_msg("Returned address is %p\n", addr); - check_bytes(addr); - write_bytes(addr); - ret = read_bytes(addr); - - munmap(addr, LENGTH); - close(fd); - - ksft_test_result(!ret, "Read same data\n"); - - ksft_exit(!ret); -} diff --git a/tools/testing/selftests/mm/thp_settings.c b/tools/testing/selftests/mm/hugepage_settings.c index e748ebfb3d4e..2eab2110ac6a 100644 --- a/tools/testing/selftests/mm/thp_settings.c +++ b/tools/testing/selftests/mm/hugepage_settings.c @@ -1,13 +1,16 @@ // SPDX-License-Identifier: GPL-2.0 +#include <dirent.h> #include <fcntl.h> #include <limits.h> +#include <signal.h> +#include <stddef.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <unistd.h> #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" #define MAX_SETTINGS_DEPTH 4 @@ -15,6 +18,7 @@ static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH]; static int settings_index; static struct thp_settings saved_settings; static char dev_queue_read_ahead_path[PATH_MAX]; +static bool thp_settings_saved; static const char * const thp_enabled_strings[] = { "never", @@ -44,47 +48,6 @@ static const char * const shmem_enabled_strings[] = { NULL }; -int read_file(const char *path, char *buf, size_t buflen) -{ - int fd; - ssize_t numread; - - fd = open(path, O_RDONLY); - if (fd == -1) - return 0; - - numread = read(fd, buf, buflen - 1); - if (numread < 1) { - close(fd); - return 0; - } - - buf[numread] = '\0'; - close(fd); - - return (unsigned int) numread; -} - -unsigned long read_num(const char *path) -{ - char buf[21]; - - if (read_file(path, buf, sizeof(buf)) < 0) { - perror("read_file()"); - exit(EXIT_FAILURE); - } - - return strtoul(buf, NULL, 10); -} - -void write_num(const char *path, unsigned long num) -{ - char buf[21]; - - sprintf(buf, "%ld", num); - write_file(path, buf, strlen(buf) + 1); -} - int thp_read_string(const char *name, const char * const strings[]) { char path[PATH_MAX]; @@ -298,12 +261,20 @@ void thp_pop_settings(void) void thp_restore_settings(void) { - thp_write_settings(&saved_settings); + if (thp_settings_saved) + thp_write_settings(&saved_settings); } -void thp_save_settings(void) +static void __thp_save_settings(void) { + if (!thp_available()) + return; + + if (thp_settings_saved) + return; + thp_read_settings(&saved_settings); + thp_settings_saved = true; } void thp_set_read_ahead_path(char *path) @@ -370,3 +341,260 @@ bool thp_is_enabled(void) /* THP is considered enabled if it's either "always" or "madvise" */ return mode == 1 || mode == 3; } + +#define HUGETLB_MAX_NR_PAGESIZES 10 +struct hugetlb_settings { + unsigned long nr_hugepages[HUGETLB_MAX_NR_PAGESIZES]; + unsigned long sizes[HUGETLB_MAX_NR_PAGESIZES]; + unsigned long default_size; + int nr_sizes; +}; + +static struct hugetlb_settings hugetlb_saved_settings; +static bool hugetlb_settings_saved; + +int detect_hugetlb_page_sizes(unsigned long sizes[], int max) +{ + static struct hugetlb_settings *settings = &hugetlb_saved_settings; + DIR *dir; + int count = 0; + + if (settings->nr_sizes) { + if (settings->nr_sizes < max) + max = settings->nr_sizes; + for (count = 0; count < max; count++) + sizes[count] = settings->sizes[count]; + return count; + } + + dir = opendir("/sys/kernel/mm/hugepages/"); + if (!dir) + return 0; + + while (count < max) { + struct dirent *entry = readdir(dir); + size_t kb; + + if (!entry) + break; + if (entry->d_type != DT_DIR) + continue; + if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) + continue; + sizes[count++] = kb * 1024; + ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", + kb); + } + closedir(dir); + return count; +} + +unsigned long default_huge_page_size(void) +{ + static struct hugetlb_settings *settings = &hugetlb_saved_settings; + unsigned long hps = 0; + char *line = NULL; + size_t linelen = 0; + FILE *f; + + if (settings->default_size) + return settings->default_size; + + f = fopen("/proc/meminfo", "r"); + if (!f) + return 0; + while (getline(&line, &linelen, f) > 0) { + if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { + hps <<= 10; + break; + } + } + + free(line); + fclose(f); + return hps; +} + +static void hugetlb_sysfs_path(char *buf, size_t buflen, + unsigned long size, const char *attr) +{ + snprintf(buf, buflen, "/sys/kernel/mm/hugepages/hugepages-%lukB/%s", + size / 1024, attr); +} + +unsigned long hugetlb_nr_pages(unsigned long size) +{ + char path[PATH_MAX]; + + hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); + + return read_num(path); +} + +void hugetlb_set_nr_pages(unsigned long size, unsigned long nr) +{ + char path[PATH_MAX]; + + hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); + + write_num(path, nr); +} + +unsigned long hugetlb_free_pages(unsigned long size) +{ + char path[PATH_MAX]; + + hugetlb_sysfs_path(path, sizeof(path), size, "free_hugepages"); + + return read_num(path); +} + +static bool __hugetlb_setup(unsigned long size, unsigned long nr) +{ + unsigned long free = hugetlb_free_pages(size); + unsigned long total = hugetlb_nr_pages(size); + + if (free >= nr) + return true; + + hugetlb_set_nr_pages(size, total + (nr - free)); + + return hugetlb_free_pages(size) >= nr; +} + +bool hugetlb_setup_default(unsigned long nr) +{ + unsigned long size; + + hugetlb_save_settings(); + size = default_huge_page_size(); + if (!size) + return false; + + return __hugetlb_setup(size, nr); +} + +bool hugetlb_setup_default_exact(unsigned long nr) +{ + unsigned long size; + + hugetlb_save_settings(); + size = default_huge_page_size(); + if (!size) + return false; + + hugetlb_set_nr_pages(size, nr); + + return hugetlb_free_pages(size) == nr; +} + +unsigned long hugetlb_setup(unsigned long nr, unsigned long sizes[], + int max) +{ + unsigned long enabled[10]; + int nr_sizes = 0; + int nr_enabled; + + hugetlb_save_settings(); + + nr_enabled = detect_hugetlb_page_sizes(enabled, ARRAY_SIZE(enabled)); + if (!nr_enabled) + return 0; + + if (nr_enabled > max) { + ksft_print_msg("detected %d huge page sizes, will only test %d\n", nr_enabled, max); + nr_enabled = max; + } + + /* request nr HugeTLB pages of every size. */ + for (int i = 0; i < nr_enabled; i++) { + if (!__hugetlb_setup(enabled[i], nr)) + continue; + sizes[nr_sizes++] = enabled[i]; + } + + return nr_sizes; +} + +static void __hugetlb_save_settings(void) +{ + struct hugetlb_settings *settings = &hugetlb_saved_settings; + int nr_sizes; + + if (hugetlb_settings_saved) + return; + + settings->default_size = default_huge_page_size(); + if (!settings->default_size) + return; + + nr_sizes = detect_hugetlb_page_sizes(settings->sizes, + HUGETLB_MAX_NR_PAGESIZES); + if (!nr_sizes) { + settings->default_size = 0; + return; + } + + for (int i = 0; i < nr_sizes; i++) { + unsigned long sz = settings->sizes[i]; + + if (!sz) + continue; + settings->nr_hugepages[i] = hugetlb_nr_pages(sz); + } + + settings->nr_sizes = nr_sizes; + hugetlb_settings_saved = true; +} + +void hugetlb_restore_settings(void) +{ + struct hugetlb_settings *settings = &hugetlb_saved_settings; + + if (!hugetlb_settings_saved || !settings->default_size) + return; + + for (int i = 0; i < HUGETLB_MAX_NR_PAGESIZES; i++) { + unsigned long sz = settings->sizes[i]; + + if (!sz) + continue; + + hugetlb_set_nr_pages(sz, settings->nr_hugepages[i]); + } +} + +static void hugepage_restore_settings_atexit(void) +{ + if (thp_settings_saved) + thp_restore_settings(); + if (hugetlb_settings_saved) + hugetlb_restore_settings(); +} + +static void hugepage_restore_settings_sighandler(int sig) +{ + /* exit() will invoke the hugepage_restore_settings_atexit handler. */ + exit(KSFT_FAIL); +} + +void hugepage_save_settings(bool thp, bool hugetlb) +{ + if (!thp && !hugetlb) + return; + + if (thp) + __thp_save_settings(); + if (hugetlb) + __hugetlb_save_settings(); + + /* + * setup exit hooks to make sure THP and HugeTLB settings are + * restored on graceful and error exits and signals + */ + atexit(hugepage_restore_settings_atexit); + signal(SIGTERM, hugepage_restore_settings_sighandler); + signal(SIGINT, hugepage_restore_settings_sighandler); + signal(SIGHUP, hugepage_restore_settings_sighandler); + signal(SIGQUIT, hugepage_restore_settings_sighandler); +} diff --git a/tools/testing/selftests/mm/thp_settings.h b/tools/testing/selftests/mm/hugepage_settings.h index 7748a9009191..726c73c43c05 100644 --- a/tools/testing/selftests/mm/thp_settings.h +++ b/tools/testing/selftests/mm/hugepage_settings.h @@ -1,11 +1,15 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __THP_SETTINGS_H__ -#define __THP_SETTINGS_H__ +#ifndef __HUGEPAGE_SETTINGS_H__ +#define __HUGEPAGE_SETTINGS_H__ #include <stdbool.h> #include <stddef.h> #include <stdint.h> +void hugepage_save_settings(bool thp, bool hugetlb); + +/* Transparent Huge Pages (THP) */ + enum thp_enabled { THP_NEVER, THP_ALWAYS, @@ -62,10 +66,6 @@ struct thp_settings { struct shmem_hugepages_settings shmem_hugepages[NR_ORDERS]; }; -int read_file(const char *path, char *buf, size_t buflen); -unsigned long read_num(const char *path); -void write_num(const char *path, unsigned long num); - int thp_read_string(const char *name, const char * const strings[]); void thp_write_string(const char *name, const char *val); unsigned long thp_read_num(const char *name); @@ -77,7 +77,11 @@ struct thp_settings *thp_current_settings(void); void thp_push_settings(struct thp_settings *settings); void thp_pop_settings(void); void thp_restore_settings(void); -void thp_save_settings(void); + +static inline void thp_save_settings(void) +{ + hugepage_save_settings(/* thp = */ true, /* hugetlb = */ false); +} void thp_set_read_ahead_path(char *path); unsigned long thp_supported_orders(void); @@ -86,4 +90,66 @@ unsigned long thp_shmem_supported_orders(void); bool thp_available(void); bool thp_is_enabled(void); -#endif /* __THP_SETTINGS_H__ */ +/* HugeTLB */ + +int detect_hugetlb_page_sizes(unsigned long sizes[], int max); +unsigned long default_huge_page_size(void); + +unsigned long hugetlb_nr_pages(unsigned long size); +void hugetlb_set_nr_pages(unsigned long size, unsigned long nr); +unsigned long hugetlb_free_pages(unsigned long size); + +static inline void hugetlb_save_settings(void) +{ + hugepage_save_settings(/* thp = */ false, /* hugetlb = */ true); +} + +void hugetlb_restore_settings(void); + +static inline unsigned long hugetlb_nr_default_pages(void) +{ + unsigned long size = default_huge_page_size(); + + if (!size) + return 0; + + return hugetlb_nr_pages(size); +} + +static inline void hugetlb_set_nr_default_pages(unsigned long nr) +{ + unsigned long size = default_huge_page_size(); + + if (!size) + return; + + hugetlb_set_nr_pages(size, nr); +} + +static inline unsigned long hugetlb_free_default_pages(void) +{ + unsigned long size = default_huge_page_size(); + + if (!size) + return 0; + + return hugetlb_free_pages(size); +} + +static inline bool hugetlb_available(void) +{ + return default_huge_page_size() != 0; +} + +bool hugetlb_setup_default(unsigned long nr); +bool hugetlb_setup_default_exact(unsigned long nr); +unsigned long hugetlb_setup(unsigned long nr, unsigned long sizes[], + int max); + +#define HUGETLB_SETUP_DEFAULT_PAGES(nr_pages) \ +static void __attribute__((constructor)) __hugetlb_setup_default(void) \ +{ \ + hugetlb_setup_default((nr_pages)); \ +} + +#endif /* __HUGEPAGE_SETTINGS_H__ */ diff --git a/tools/testing/selftests/mm/hugetlb-madvise.c b/tools/testing/selftests/mm/hugetlb-madvise.c index 5b12041fa310..555b4b3d1430 100644 --- a/tools/testing/selftests/mm/hugetlb-madvise.c +++ b/tools/testing/selftests/mm/hugetlb-madvise.c @@ -1,15 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 /* - * hugepage-madvise: + * hugetlb-madvise: * * Basic functional testing of madvise MADV_DONTNEED and MADV_REMOVE * on hugetlb mappings. - * - * Before running this test, make sure the administrator has pre-allocated - * at least MIN_FREE_PAGES hugetlb pages and they are free. In addition, - * the test takes an argument that is the path to a file in a hugetlbfs - * filesystem. Therefore, a hugetlbfs filesystem must be mounted on some - * directory. */ #define _GNU_SOURCE @@ -20,18 +14,18 @@ #include <fcntl.h> #include "vm_util.h" #include "kselftest.h" +#include "hugepage_settings.h" #define MIN_FREE_PAGES 20 #define NR_HUGE_PAGES 10 /* common number of pages to map/allocate */ #define validate_free_pages(exp_free) \ do { \ - int fhp = get_free_hugepages(); \ - if (fhp != (exp_free)) { \ - printf("Unexpected number of free huge " \ - "pages line %d\n", __LINE__); \ - exit(1); \ - } \ + unsigned long fhp = hugetlb_free_default_pages(); \ + if (fhp != (exp_free)) \ + ksft_exit_fail_msg("Unexpected number of free " \ + "huge pages %lu, expected %lu line %d\n", \ + fhp, (exp_free), __LINE__); \ } while (0) unsigned long huge_page_size; @@ -57,28 +51,24 @@ int main(int argc, char **argv) int fd; int ret; + ksft_print_header(); + ksft_set_plan(1); + huge_page_size = default_huge_page_size(); - if (!huge_page_size) { - printf("Unable to determine huge page size, exiting!\n"); - exit(1); - } + if (!huge_page_size) + ksft_exit_skip("Unable to determine huge page size\n"); + base_page_size = sysconf(_SC_PAGE_SIZE); - if (!huge_page_size) { - printf("Unable to determine base page size, exiting!\n"); - exit(1); - } + if (!base_page_size) + ksft_exit_fail_msg("Unable to determine base page size\n"); - free_hugepages = get_free_hugepages(); - if (free_hugepages < MIN_FREE_PAGES) { - printf("Not enough free huge pages to test, exiting!\n"); - exit(KSFT_SKIP); - } + if (!hugetlb_setup_default(MIN_FREE_PAGES)) + ksft_exit_skip("Not enough free huge pages (have %lu, need %d)\n", hugetlb_free_default_pages(), MIN_FREE_PAGES); + free_hugepages = hugetlb_free_default_pages(); fd = memfd_create(argv[0], MFD_HUGETLB); - if (fd < 0) { - perror("memfd_create() failed"); - exit(1); - } + if (fd < 0) + ksft_exit_fail_perror("memfd_create"); /* * Test validity of MADV_DONTNEED addr and length arguments. mmap @@ -90,16 +80,13 @@ int main(int argc, char **argv) PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); + if (munmap(addr, huge_page_size) || munmap(addr + (NR_HUGE_PAGES + 1) * huge_page_size, - huge_page_size)) { - perror("munmap"); - exit(1); - } + huge_page_size)) + ksft_exit_fail_perror("munmap"); addr = addr + huge_page_size; write_fault_pages(addr, NR_HUGE_PAGES); @@ -108,20 +95,14 @@ int main(int argc, char **argv) /* addr before mapping should fail */ ret = madvise(addr - base_page_size, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED); - if (!ret) { - printf("Unexpected success of madvise call with invalid addr line %d\n", - __LINE__); - exit(1); - } + if (!ret) + ksft_exit_fail_msg("madvise with invalid addr unexpectedly succeeded line %d\n", __LINE__); /* addr + length after mapping should fail */ ret = madvise(addr, (NR_HUGE_PAGES * huge_page_size) + base_page_size, MADV_DONTNEED); - if (!ret) { - printf("Unexpected success of madvise call with invalid length line %d\n", - __LINE__); - exit(1); - } + if (!ret) + ksft_exit_fail_msg("madvise with invalid length unexpectedly succeeded line %d\n", __LINE__); (void)munmap(addr, NR_HUGE_PAGES * huge_page_size); @@ -132,10 +113,9 @@ int main(int argc, char **argv) PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); + write_fault_pages(addr, NR_HUGE_PAGES); validate_free_pages(free_hugepages - NR_HUGE_PAGES); @@ -143,19 +123,14 @@ int main(int argc, char **argv) ret = madvise(addr + base_page_size, NR_HUGE_PAGES * huge_page_size - base_page_size, MADV_DONTNEED); - if (!ret) { - printf("Unexpected success of madvise call with unaligned start address %d\n", - __LINE__); - exit(1); - } + if (!ret) + ksft_exit_fail_msg("madvise with unaligned start unexpectedly succeeded line %d\n", __LINE__); /* addr + length should be aligned down to huge page size */ if (madvise(addr, ((NR_HUGE_PAGES - 1) * huge_page_size) + base_page_size, - MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); /* should free all but last page in mapping */ validate_free_pages(free_hugepages - 1); @@ -170,17 +145,14 @@ int main(int argc, char **argv) PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); + write_fault_pages(addr, NR_HUGE_PAGES); validate_free_pages(free_hugepages - NR_HUGE_PAGES); - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); /* should free all pages in mapping */ validate_free_pages(free_hugepages); @@ -190,29 +162,25 @@ int main(int argc, char **argv) /* * Test MADV_DONTNEED on private mapping of hugetlb file */ - if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) { - perror("fallocate"); - exit(1); - } + if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) + ksft_exit_fail_perror("fallocate"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); /* read should not consume any pages */ read_fault_pages(addr, NR_HUGE_PAGES); validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* madvise should not free any pages */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* writes should allocate private pages */ @@ -220,10 +188,9 @@ int main(int argc, char **argv) validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES)); /* madvise should free private pages */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* writes should allocate private pages */ @@ -238,10 +205,9 @@ int main(int argc, char **argv) * implementation. */ if (fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, - 0, NR_HUGE_PAGES * huge_page_size)) { - perror("fallocate"); - exit(1); - } + 0, NR_HUGE_PAGES * huge_page_size)) + ksft_exit_fail_perror("fallocate"); + validate_free_pages(free_hugepages); (void)munmap(addr, NR_HUGE_PAGES * huge_page_size); @@ -249,29 +215,25 @@ int main(int argc, char **argv) /* * Test MADV_DONTNEED on shared mapping of hugetlb file */ - if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) { - perror("fallocate"); - exit(1); - } + if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) + ksft_exit_fail_perror("fallocate"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); /* write should not consume any pages */ write_fault_pages(addr, NR_HUGE_PAGES); validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* madvise should not free any pages */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* @@ -279,29 +241,25 @@ int main(int argc, char **argv) * * madvise is same as hole punch and should free all pages. */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages); (void)munmap(addr, NR_HUGE_PAGES * huge_page_size); /* * Test MADV_REMOVE on shared and private mapping of hugetlb file */ - if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) { - perror("fallocate"); - exit(1); - } + if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) + ksft_exit_fail_perror("fallocate"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); /* shared write should not consume any additional pages */ write_fault_pages(addr, NR_HUGE_PAGES); @@ -310,10 +268,8 @@ int main(int argc, char **argv) addr2 = mmap(NULL, NR_HUGE_PAGES * huge_page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - if (addr2 == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr2 == MAP_FAILED) + ksft_exit_fail_perror("mmap"); /* private read should not consume any pages */ read_fault_pages(addr2, NR_HUGE_PAGES); @@ -324,17 +280,15 @@ int main(int argc, char **argv) validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES)); /* madvise of shared mapping should not free any pages */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES)); /* madvise of private mapping should free private pages */ - if (madvise(addr2, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr2, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* private write should consume additional pages again */ @@ -346,15 +300,16 @@ int main(int argc, char **argv) * not correct. private pages should not be freed, but this is * expected. See comment associated with FALLOC_FL_PUNCH_HOLE call. */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages); (void)munmap(addr, NR_HUGE_PAGES * huge_page_size); (void)munmap(addr2, NR_HUGE_PAGES * huge_page_size); close(fd); - return 0; + + ksft_test_result_pass("MADV_DONTNEED and MADV_REMOVE on hugetlb\n"); + ksft_finished(); } diff --git a/tools/testing/selftests/mm/hugetlb-mmap.c b/tools/testing/selftests/mm/hugetlb-mmap.c new file mode 100644 index 000000000000..0f2aad1b7dbd --- /dev/null +++ b/tools/testing/selftests/mm/hugetlb-mmap.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * hugetlb-mmap: + * + * Example of using huge page memory in a user application using the mmap + * system call. Before running this application, make sure that the + * administrator has mounted the hugetlbfs filesystem (on some directory + * like /mnt) using the command mount -t hugetlbfs nodev /mnt. In this + * example, the app is requesting memory of size 256MB that is backed by + * huge pages. + */ +#define _GNU_SOURCE +#include <stdlib.h> +#include <stdio.h> +#include <unistd.h> +#include <sys/mman.h> +#include <fcntl.h> +#include <linux/memfd.h> +#include "vm_util.h" +#include "kselftest.h" +#include "hugepage_settings.h" + +#define LENGTH (256UL*1024*1024) +#define PROTECTION (PROT_READ | PROT_WRITE) + +static void check_bytes(char *addr) +{ + ksft_print_msg("First hex is %x\n", *((unsigned int *)addr)); +} + +static void write_bytes(char *addr, size_t length) +{ + unsigned long i; + + for (i = 0; i < length; i++) + *(addr + i) = (char)i; +} + +static bool verify_bytes(char *addr, size_t length) +{ + unsigned long i; + + check_bytes(addr); + for (i = 0; i < length; i++) + if (*(addr + i) != (char)i) { + ksft_print_msg("Error: Mismatch at %lu(%p)\n", i, addr); + return false; + } + + return true; +} + +static void test_mmap(size_t length, int mmap_flags, int fd, + const char *test_name) +{ + bool passed = true; + void *addr; + + addr = mmap(NULL, length, PROTECTION, mmap_flags, fd, 0); + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); + + ksft_print_msg("Returned address is %p\n", addr); + check_bytes(addr); + write_bytes(addr, length); + if (!verify_bytes(addr, length)) + passed = false; + + /* munmap() length of MAP_HUGETLB memory must be hugepage aligned */ + if (munmap(addr, length)) + ksft_exit_fail_perror("munmap"); + + ksft_test_result(passed, "%s\n", test_name); +} + +static void test_anon_mmap(size_t length, int shift) +{ + const char *test_name = "hugetlb anonymous mmap"; + int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB; + + if (shift) + mmap_flags |= (shift & MAP_HUGE_MASK) << MAP_HUGE_SHIFT; + + test_mmap(length, mmap_flags, -1, test_name); +} + +static void test_file_mmap(size_t length, int shift) +{ + const char *test_name = "hugetlb file mmap"; + int mfd_flags = MFD_HUGETLB; + int fd; + + if (shift) + mfd_flags |= (shift & MFD_HUGE_MASK) << MFD_HUGE_SHIFT; + + fd = memfd_create("hugetlb-mmap", mfd_flags); + if (fd < 0) + ksft_exit_fail_perror("memfd_create"); + + test_mmap(length, MAP_SHARED, fd, test_name); + close(fd); +} + +int main(int argc, char **argv) +{ + size_t hugepage_size; + size_t length = LENGTH; + int shift = 0, nr; + + ksft_print_header(); + + if (argc > 1) + length = atol(argv[1]) << 20; + if (argc > 2) + shift = atoi(argv[2]); + + hugetlb_save_settings(); + if (shift) { + hugepage_size = (1UL << shift); + ksft_print_msg("%lu kB hugepages\n", 1UL << (shift - 10)); + } else { + hugepage_size = default_huge_page_size(); + if (!hugepage_size) + ksft_exit_skip("Could not detect default hugetlb page size."); + ksft_print_msg("Default size hugepages (%lu kB)\n", hugepage_size >> 10); + } + + /* munmap will fail if the length is not page aligned */ + length = (length + hugepage_size - 1) & ~(hugepage_size - 1); + nr = length / hugepage_size; + + hugetlb_set_nr_pages(hugepage_size, nr); + if (hugetlb_free_pages(hugepage_size) < nr) + ksft_exit_skip("Not enough %lu Kb pages\n", hugepage_size >> 10); + + ksft_set_plan(2); + ksft_print_msg("Mapping %lu Mbytes\n", (unsigned long)length >> 20); + + test_anon_mmap(length, shift); + test_file_mmap(length, shift); + + ksft_finished(); +} diff --git a/tools/testing/selftests/mm/hugepage-mremap.c b/tools/testing/selftests/mm/hugetlb-mremap.c index b8f7d92e5a35..ed3d92e862d8 100644 --- a/tools/testing/selftests/mm/hugepage-mremap.c +++ b/tools/testing/selftests/mm/hugetlb-mremap.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * hugepage-mremap: + * hugetlb-mremap: * * Example of remapping huge page memory in a user application using the * mremap system call. The path to a file in a hugetlbfs filesystem must @@ -26,12 +26,13 @@ #include <stdbool.h> #include "kselftest.h" #include "vm_util.h" +#include "hugepage_settings.h" #define DEFAULT_LENGTH_MB 10UL #define MB_TO_BYTES(x) (x * 1024 * 1024) #define PROTECTION (PROT_READ | PROT_WRITE | PROT_EXEC) -#define FLAGS (MAP_SHARED | MAP_ANONYMOUS) +#define FLAGS (MAP_HUGETLB | MAP_SHARED) static void check_bytes(char *addr) { @@ -85,31 +86,21 @@ static void register_region_with_uffd(char *addr, size_t len) if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1) ksft_exit_fail_msg("ioctl-UFFDIO_API: %s\n", strerror(errno)); - /* Create a private anonymous mapping. The memory will be - * demand-zero paged--that is, not yet allocated. When we - * actually touch the memory, it will be allocated via - * the userfaultfd. - */ - - addr = mmap(NULL, len, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (addr == MAP_FAILED) - ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); - - ksft_print_msg("Address returned by mmap() = %p\n", addr); - - /* Register the memory range of the mapping we just created for - * handling by the userfaultfd object. In mode, we request to track - * missing pages (i.e., pages that have not yet been faulted in). + /* Register the passed memory range for handling by the userfaultfd object. + * In mode, we request to track missing pages + * (i.e., pages that have not yet been faulted in). */ if (uffd_register(uffd, addr, len, true, false, false)) ksft_exit_fail_msg("ioctl-UFFDIO_REGISTER: %s\n", strerror(errno)); + + ksft_print_msg("Registered memory at address %p with userfaultfd\n", addr); } int main(int argc, char *argv[]) { + unsigned long hugepage_size; + int ret = 0, fd, nr; size_t length = 0; - int ret = 0, fd; ksft_print_header(); ksft_set_plan(1); @@ -125,30 +116,36 @@ int main(int argc, char *argv[]) else length = DEFAULT_LENGTH_MB; + hugepage_size = default_huge_page_size(); + if (!hugepage_size) + ksft_exit_skip("Could not detect default hugetlb page size\n"); length = MB_TO_BYTES(length); + length = (length + hugepage_size - 1) & ~(hugepage_size - 1); + nr = length / hugepage_size; + + if (!hugetlb_setup_default(nr)) + ksft_exit_skip("Not enough huge pages\n"); + fd = memfd_create(argv[0], MFD_HUGETLB); if (fd < 0) ksft_exit_fail_msg("Open failed: %s\n", strerror(errno)); /* mmap to a PUD aligned address to hopefully trigger pmd sharing. */ unsigned long suggested_addr = 0x7eaa40000000; - void *haddr = mmap((void *)suggested_addr, length, PROTECTION, - MAP_HUGETLB | MAP_SHARED | MAP_POPULATE, fd, 0); + void *haddr = mmap((void *)suggested_addr, length, PROTECTION, FLAGS, fd, 0); ksft_print_msg("Map haddr: Returned address is %p\n", haddr); if (haddr == MAP_FAILED) ksft_exit_fail_msg("mmap1: %s\n", strerror(errno)); /* mmap again to a dummy address to hopefully trigger pmd sharing. */ suggested_addr = 0x7daa40000000; - void *daddr = mmap((void *)suggested_addr, length, PROTECTION, - MAP_HUGETLB | MAP_SHARED | MAP_POPULATE, fd, 0); + void *daddr = mmap((void *)suggested_addr, length, PROTECTION, FLAGS, fd, 0); ksft_print_msg("Map daddr: Returned address is %p\n", daddr); if (daddr == MAP_FAILED) ksft_exit_fail_msg("mmap3: %s\n", strerror(errno)); suggested_addr = 0x7faa40000000; - void *vaddr = - mmap((void *)suggested_addr, length, PROTECTION, FLAGS, -1, 0); + void *vaddr = mmap((void *)suggested_addr, length, PROTECTION, FLAGS, fd, 0); ksft_print_msg("Map vaddr: Returned address is %p\n", vaddr); if (vaddr == MAP_FAILED) ksft_exit_fail_msg("mmap2: %s\n", strerror(errno)); diff --git a/tools/testing/selftests/mm/hugetlb-read-hwpoison.c b/tools/testing/selftests/mm/hugetlb-read-hwpoison.c index 46230462ad48..70b24e3660c4 100644 --- a/tools/testing/selftests/mm/hugetlb-read-hwpoison.c +++ b/tools/testing/selftests/mm/hugetlb-read-hwpoison.c @@ -10,12 +10,10 @@ #include <sys/statfs.h> #include <errno.h> #include <stdbool.h> +#include <signal.h> #include "kselftest.h" -#define PREFIX " ... " -#define ERROR_PREFIX " !!! " - #define MAX_WRITE_READ_CHUNK_SIZE (getpagesize() * 16) #define MAX(a, b) (((a) > (b)) ? (a) : (b)) @@ -25,17 +23,22 @@ enum test_status { TEST_SKIPPED = 2, }; -static char *status_to_str(enum test_status status) +static void report_status(enum test_status status, const char *test_name, + size_t chunk_size) { switch (status) { case TEST_PASSED: - return "TEST_PASSED"; + ksft_test_result_pass("%s chunk_size=0x%lx\n", + test_name, chunk_size); + break; case TEST_FAILED: - return "TEST_FAILED"; + ksft_test_result_fail("%s chunk_size=0x%lx\n", + test_name, chunk_size); + break; case TEST_SKIPPED: - return "TEST_SKIPPED"; - default: - return "TEST_???"; + ksft_test_result_skip("%s chunk_size=0x%lx\n", + test_name, chunk_size); + break; } } @@ -58,8 +61,8 @@ static bool verify_chunk(char *buf, size_t len, char val) for (i = 0; i < len; ++i) { if (buf[i] != val) { - printf(PREFIX ERROR_PREFIX "check fail: buf[%lu] = %u != %u\n", - i, buf[i], val); + ksft_print_msg("check fail: buf[%lu] = %u != %u\n", + i, buf[i], val); return false; } } @@ -75,21 +78,21 @@ static bool seek_read_hugepage_filemap(int fd, size_t len, size_t wr_chunk_size, ssize_t total_ret_count = 0; char val = offset / wr_chunk_size + offset % wr_chunk_size; - printf(PREFIX PREFIX "init val=%u with offset=0x%lx\n", val, offset); - printf(PREFIX PREFIX "expect to read 0x%lx bytes of data in total\n", - expected); + ksft_print_msg("init val=%u with offset=0x%lx\n", val, offset); + ksft_print_msg("expect to read 0x%lx bytes of data in total\n", + expected); if (lseek(fd, offset, SEEK_SET) < 0) { - perror(PREFIX ERROR_PREFIX "seek failed"); + ksft_perror("seek failed"); return false; } while (offset + total_ret_count < len) { ret_count = read(fd, buf, wr_chunk_size); if (ret_count == 0) { - printf(PREFIX PREFIX "read reach end of the file\n"); + ksft_print_msg("read reach end of the file\n"); break; } else if (ret_count < 0) { - perror(PREFIX ERROR_PREFIX "read failed"); + ksft_perror("read failed"); break; } ++val; @@ -98,8 +101,8 @@ static bool seek_read_hugepage_filemap(int fd, size_t len, size_t wr_chunk_size, total_ret_count += ret_count; } - printf(PREFIX PREFIX "actually read 0x%lx bytes of data in total\n", - total_ret_count); + ksft_print_msg("actually read 0x%lx bytes of data in total\n", + total_ret_count); return total_ret_count == expected; } @@ -112,15 +115,15 @@ static bool read_hugepage_filemap(int fd, size_t len, ssize_t total_ret_count = 0; char val = 0; - printf(PREFIX PREFIX "expect to read 0x%lx bytes of data in total\n", - expected); + ksft_print_msg("expect to read 0x%lx bytes of data in total\n", + expected); while (total_ret_count < len) { ret_count = read(fd, buf, wr_chunk_size); if (ret_count == 0) { - printf(PREFIX PREFIX "read reach end of the file\n"); + ksft_print_msg("read reach end of the file\n"); break; } else if (ret_count < 0) { - perror(PREFIX ERROR_PREFIX "read failed"); + ksft_perror("read failed"); break; } ++val; @@ -129,8 +132,8 @@ static bool read_hugepage_filemap(int fd, size_t len, total_ret_count += ret_count; } - printf(PREFIX PREFIX "actually read 0x%lx bytes of data in total\n", - total_ret_count); + ksft_print_msg("actually read 0x%lx bytes of data in total\n", + total_ret_count); return total_ret_count == expected; } @@ -142,14 +145,14 @@ test_hugetlb_read(int fd, size_t len, size_t wr_chunk_size) char *filemap = NULL; if (ftruncate(fd, len) < 0) { - perror(PREFIX ERROR_PREFIX "ftruncate failed"); + ksft_perror("ftruncate failed"); return status; } filemap = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, 0); if (filemap == MAP_FAILED) { - perror(PREFIX ERROR_PREFIX "mmap for primary mapping failed"); + ksft_perror("mmap for primary mapping failed"); goto done; } @@ -162,7 +165,7 @@ test_hugetlb_read(int fd, size_t len, size_t wr_chunk_size) munmap(filemap, len); done: if (ftruncate(fd, 0) < 0) { - perror(PREFIX ERROR_PREFIX "ftruncate back to 0 failed"); + ksft_perror("ftruncate back to 0 failed"); status = TEST_FAILED; } @@ -179,14 +182,14 @@ test_hugetlb_read_hwpoison(int fd, size_t len, size_t wr_chunk_size, const unsigned long pagesize = getpagesize(); if (ftruncate(fd, len) < 0) { - perror(PREFIX ERROR_PREFIX "ftruncate failed"); + ksft_perror("ftruncate failed"); return status; } filemap = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, 0); if (filemap == MAP_FAILED) { - perror(PREFIX ERROR_PREFIX "mmap for primary mapping failed"); + ksft_perror("mmap for primary mapping failed"); goto done; } @@ -201,7 +204,7 @@ test_hugetlb_read_hwpoison(int fd, size_t len, size_t wr_chunk_size, */ hwp_addr = filemap + len / 2 + pagesize; if (madvise(hwp_addr, pagesize, MADV_HWPOISON) < 0) { - perror(PREFIX ERROR_PREFIX "MADV_HWPOISON failed"); + ksft_perror("MADV_HWPOISON failed"); goto unmap; } @@ -228,7 +231,7 @@ unmap: munmap(filemap, len); done: if (ftruncate(fd, 0) < 0) { - perror(PREFIX ERROR_PREFIX "ftruncate back to 0 failed"); + ksft_perror("ftruncate back to 0 failed"); status = TEST_FAILED; } @@ -241,17 +244,17 @@ static int create_hugetlbfs_file(struct statfs *file_stat) fd = memfd_create("hugetlb_tmp", MFD_HUGETLB); if (fd < 0) { - perror(PREFIX ERROR_PREFIX "could not open hugetlbfs file"); + ksft_perror("could not open hugetlbfs file"); return -1; } memset(file_stat, 0, sizeof(*file_stat)); if (fstatfs(fd, file_stat)) { - perror(PREFIX ERROR_PREFIX "fstatfs failed"); + ksft_perror("fstatfs failed"); goto close; } if (file_stat->f_type != HUGETLBFS_MAGIC) { - printf(PREFIX ERROR_PREFIX "not hugetlbfs file\n"); + ksft_print_msg("not hugetlbfs file\n"); goto close; } @@ -261,6 +264,10 @@ close: return -1; } +static void sigbus_handler(int sig) +{ +} + int main(void) { int fd; @@ -273,50 +280,44 @@ int main(void) }; size_t i; + ksft_print_header(); + ksft_set_plan(ARRAY_SIZE(wr_chunk_sizes) * 3); + + signal(SIGBUS, sigbus_handler); for (i = 0; i < ARRAY_SIZE(wr_chunk_sizes); ++i) { - printf("Write/read chunk size=0x%lx\n", - wr_chunk_sizes[i]); + ksft_print_msg("Write/read chunk size=0x%lx\n", + wr_chunk_sizes[i]); fd = create_hugetlbfs_file(&file_stat); if (fd < 0) - goto create_failure; - printf(PREFIX "HugeTLB read regression test...\n"); + ksft_exit_fail_msg("Failed to create hugetlbfs file\n"); + status = test_hugetlb_read(fd, file_stat.f_bsize, wr_chunk_sizes[i]); - printf(PREFIX "HugeTLB read regression test...%s\n", - status_to_str(status)); close(fd); - if (status == TEST_FAILED) - return -1; + report_status(status, "HugeTLB read regression", + wr_chunk_sizes[i]); fd = create_hugetlbfs_file(&file_stat); if (fd < 0) - goto create_failure; - printf(PREFIX "HugeTLB read HWPOISON test...\n"); + ksft_exit_fail_msg("Failed to create hugetlbfs file\n"); + status = test_hugetlb_read_hwpoison(fd, file_stat.f_bsize, wr_chunk_sizes[i], false); - printf(PREFIX "HugeTLB read HWPOISON test...%s\n", - status_to_str(status)); close(fd); - if (status == TEST_FAILED) - return -1; + report_status(status, "HugeTLB read HWPOISON", + wr_chunk_sizes[i]); fd = create_hugetlbfs_file(&file_stat); if (fd < 0) - goto create_failure; - printf(PREFIX "HugeTLB seek then read HWPOISON test...\n"); + ksft_exit_fail_msg("Failed to create hugetlbfs file\n"); + status = test_hugetlb_read_hwpoison(fd, file_stat.f_bsize, wr_chunk_sizes[i], true); - printf(PREFIX "HugeTLB seek then read HWPOISON test...%s\n", - status_to_str(status)); close(fd); - if (status == TEST_FAILED) - return -1; + report_status(status, "HugeTLB seek then read HWPOISON", + wr_chunk_sizes[i]); } - return 0; - -create_failure: - printf(ERROR_PREFIX "Abort test: failed to create hugetlbfs file\n"); - return -1; + ksft_finished(); } diff --git a/tools/testing/selftests/mm/hugepage-shm.c b/tools/testing/selftests/mm/hugetlb-shm.c index ef06260802b5..3ff7f062b7eb 100644 --- a/tools/testing/selftests/mm/hugepage-shm.c +++ b/tools/testing/selftests/mm/hugetlb-shm.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * hugepage-shm: + * hugetlb-shm: * * Example of using huge page memory in a user application using Sys V shared * memory system calls. In this example the app is requesting 256MB of @@ -28,9 +28,27 @@ #include <sys/shm.h> #include <sys/mman.h> +#include "vm_util.h" +#include "hugepage_settings.h" + #define LENGTH (256UL*1024*1024) -#define dprintf(x) printf(x) +static void prepare(void) +{ + unsigned long length, hugepage_size, nr; + + hugepage_size = default_huge_page_size(); + if (!hugepage_size) + ksft_exit_skip("Unable to determine huge page size\n"); + + length = (LENGTH + hugepage_size - 1) & ~(hugepage_size - 1); + nr = length / hugepage_size; + + if (!hugetlb_setup_default(nr)) + ksft_exit_skip("Not enough free huge pages\n"); + + shm_limits_prepare(length); +} int main(void) { @@ -38,44 +56,45 @@ int main(void) unsigned long i; char *shmaddr; + ksft_print_header(); + ksft_set_plan(1); + + prepare(); + shmid = shmget(2, LENGTH, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); - if (shmid < 0) { - perror("shmget"); - exit(1); - } - printf("shmid: 0x%x\n", shmid); + if (shmid < 0) + ksft_exit_fail_perror("shmget"); + + ksft_print_msg("shmid: 0x%x\n", shmid); shmaddr = shmat(shmid, NULL, 0); if (shmaddr == (char *)-1) { - perror("Shared memory attach failure"); + ksft_perror("Shared memory attach failure"); shmctl(shmid, IPC_RMID, NULL); - exit(2); + ksft_exit_fail(); } - printf("shmaddr: %p\n", shmaddr); + ksft_print_msg("shmaddr: %p\n", shmaddr); - dprintf("Starting the writes:\n"); - for (i = 0; i < LENGTH; i++) { + ksft_print_msg("Starting the writes:\n"); + for (i = 0; i < LENGTH; i++) shmaddr[i] = (char)(i); - if (!(i % (1024 * 1024))) - dprintf("."); - } - dprintf("\n"); - dprintf("Starting the Check..."); + ksft_print_msg("Starting the Check..."); for (i = 0; i < LENGTH; i++) - if (shmaddr[i] != (char)i) { - printf("\nIndex %lu mismatched\n", i); - exit(3); - } - dprintf("Done.\n"); + if (shmaddr[i] != (char)i) + ksft_exit_fail_msg("Data mismatch at index %lu\n", i); + ksft_print_msg("Done.\n"); if (shmdt((const void *)shmaddr) != 0) { - perror("Detach failure"); + ksft_perror("Detach failure"); shmctl(shmid, IPC_RMID, NULL); - exit(4); + ksft_exit_fail(); } shmctl(shmid, IPC_RMID, NULL); - return 0; + ksft_test_result_pass("hugepage using SysV shmget/shmat\n"); + ksft_finished(); } + +SHM_LIMITS_RESTORE() diff --git a/tools/testing/selftests/mm/hugetlb-soft-offline.c b/tools/testing/selftests/mm/hugetlb-soft-offline.c index a8bc02688085..bc202e4ed2bd 100644 --- a/tools/testing/selftests/mm/hugetlb-soft-offline.c +++ b/tools/testing/selftests/mm/hugetlb-soft-offline.c @@ -6,9 +6,7 @@ * - if enable_soft_offline = 1, a hugepage should be dissolved and * nr_hugepages/free_hugepages should be reduced by 1. * - * Before running, make sure more than 2 hugepages of default_hugepagesz - * are allocated. For example, if /proc/meminfo/Hugepagesize is 2048kB: - * echo 8 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + * The test allocates 8 default hugepages */ #define _GNU_SOURCE @@ -25,6 +23,7 @@ #include <sys/types.h> #include "kselftest.h" +#include "hugepage_settings.h" #ifndef MADV_SOFT_OFFLINE #define MADV_SOFT_OFFLINE 101 @@ -100,32 +99,6 @@ static int set_enable_soft_offline(int value) return 0; } -static int read_nr_hugepages(unsigned long hugepage_size, - unsigned long *nr_hugepages) -{ - char buffer[256] = {0}; - char cmd[256] = {0}; - - sprintf(cmd, "cat /sys/kernel/mm/hugepages/hugepages-%ldkB/nr_hugepages", - hugepage_size); - FILE *cmdfile = popen(cmd, "r"); - - if (cmdfile == NULL) { - ksft_perror(EPREFIX "failed to popen nr_hugepages"); - return -1; - } - - if (!fgets(buffer, sizeof(buffer), cmdfile)) { - ksft_perror(EPREFIX "failed to read nr_hugepages"); - pclose(cmdfile); - return -1; - } - - *nr_hugepages = atoll(buffer); - pclose(cmdfile); - return 0; -} - static int create_hugetlbfs_file(struct statfs *file_stat) { int fd; @@ -177,20 +150,14 @@ static void test_soft_offline_common(int enable_soft_offline) ksft_exit_fail_msg("Failed to set enable_soft_offline\n"); } - if (read_nr_hugepages(hugepagesize_kb, &nr_hugepages_before) != 0) { - close(fd); - ksft_exit_fail_msg("Failed to read nr_hugepages\n"); - } + nr_hugepages_before = hugetlb_nr_default_pages(); ksft_print_msg("Before MADV_SOFT_OFFLINE nr_hugepages=%ld\n", nr_hugepages_before); ret = do_soft_offline(fd, 2 * file_stat.f_bsize, expect_errno); - if (read_nr_hugepages(hugepagesize_kb, &nr_hugepages_after) != 0) { - close(fd); - ksft_exit_fail_msg("Failed to read nr_hugepages\n"); - } + nr_hugepages_after = hugetlb_nr_default_pages(); ksft_print_msg("After MADV_SOFT_OFFLINE nr_hugepages=%ld\n", nr_hugepages_after); @@ -219,6 +186,10 @@ static void test_soft_offline_common(int enable_soft_offline) int main(int argc, char **argv) { ksft_print_header(); + + if (!hugetlb_setup_default(8)) + ksft_exit_skip("not enough hugetlb pages\n"); + ksft_set_plan(2); test_soft_offline_common(1); diff --git a/tools/testing/selftests/mm/hugepage-vmemmap.c b/tools/testing/selftests/mm/hugetlb-vmemmap.c index df366a4d1b92..507df78a158d 100644 --- a/tools/testing/selftests/mm/hugepage-vmemmap.c +++ b/tools/testing/selftests/mm/hugetlb-vmemmap.c @@ -11,6 +11,7 @@ #include <sys/mman.h> #include <fcntl.h> #include "vm_util.h" +#include "hugepage_settings.h" #define PAGE_COMPOUND_HEAD (1UL << 15) #define PAGE_COMPOUND_TAIL (1UL << 16) @@ -63,7 +64,7 @@ static int check_page_flags(unsigned long pfn) read(fd, &pageflags, sizeof(pageflags)); if ((pageflags & HEAD_PAGE_FLAGS) != HEAD_PAGE_FLAGS) { close(fd); - printf("Head page flags (%lx) is invalid\n", pageflags); + ksft_print_msg("Head page flags (%lx) is invalid\n", pageflags); return -1; } @@ -77,7 +78,7 @@ static int check_page_flags(unsigned long pfn) if ((pageflags & TAIL_PAGE_FLAGS) != TAIL_PAGE_FLAGS || (pageflags & HEAD_PAGE_FLAGS) == HEAD_PAGE_FLAGS) { close(fd); - printf("Tail page flags (%lx) is invalid\n", pageflags); + ksft_print_msg("Tail page flags (%lx) is invalid\n", pageflags); return -1; } } @@ -91,44 +92,41 @@ int main(int argc, char **argv) { void *addr; unsigned long pfn; + int ret; + + ksft_print_header(); + ksft_set_plan(1); + + if (!hugetlb_setup_default(1)) + ksft_exit_skip("Not enough free huge pages\n"); pagesize = psize(); maplength = default_huge_page_size(); - if (!maplength) { - printf("Unable to determine huge page size\n"); - exit(1); - } + if (!maplength) + ksft_exit_skip("Unable to determine huge page size\n"); addr = mmap(NULL, maplength, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); /* Trigger allocation of HugeTLB page. */ write_bytes(addr, maplength); pfn = virt_to_pfn(addr); if (pfn == -1UL) { + ksft_perror("virt_to_pfn"); munmap(addr, maplength); - perror("virt_to_pfn"); - exit(1); + ksft_exit_fail(); } - printf("Returned address is %p whose pfn is %lx\n", addr, pfn); + ksft_print_msg("Returned address is %p whose pfn is %lx\n", addr, pfn); - if (check_page_flags(pfn) < 0) { - munmap(addr, maplength); - perror("check_page_flags"); - exit(1); - } + ret = check_page_flags(pfn); - /* munmap() length of MAP_HUGETLB memory must be hugepage aligned */ - if (munmap(addr, maplength)) { - perror("munmap"); - exit(1); - } + if (munmap(addr, maplength)) + ksft_exit_fail_perror("munmap"); - return 0; + ksft_test_result(!ret, "HugeTLB vmemmap page flags\n"); + ksft_finished(); } diff --git a/tools/testing/selftests/mm/hugetlb_dio.c b/tools/testing/selftests/mm/hugetlb_dio.c index 31a054fa8134..fb4600570e13 100644 --- a/tools/testing/selftests/mm/hugetlb_dio.c +++ b/tools/testing/selftests/mm/hugetlb_dio.c @@ -20,6 +20,7 @@ #include <sys/syscall.h> #include "vm_util.h" #include "kselftest.h" +#include "hugepage_settings.h" #ifndef STATX_DIOALIGN #define STATX_DIOALIGN 0x00002000U @@ -84,19 +85,13 @@ static void run_dio_using_hugetlb(int fd, unsigned int start_off, /* Get the default huge page size */ h_pagesize = default_huge_page_size(); - if (!h_pagesize) - ksft_exit_fail_msg("Unable to determine huge page size\n"); /* Reset file position since fd is shared across tests */ if (lseek(fd, 0, SEEK_SET) < 0) ksft_exit_fail_perror("lseek failed\n"); /* Get the free huge pages before allocation */ - free_hpage_b = get_free_hugepages(); - if (free_hpage_b == 0) { - close(fd); - ksft_exit_skip("No free hugepage, exiting!\n"); - } + free_hpage_b = hugetlb_free_default_pages(); /* Allocate a hugetlb page */ orig_buffer = mmap(NULL, h_pagesize, mmap_prot, mmap_flags, -1, 0); @@ -120,7 +115,7 @@ static void run_dio_using_hugetlb(int fd, unsigned int start_off, munmap(orig_buffer, h_pagesize); /* Get the free huge pages after unmap*/ - free_hpage_a = get_free_hugepages(); + free_hpage_a = hugetlb_free_default_pages(); ksft_print_msg("No. Free pages before allocation : %d\n", free_hpage_b); ksft_print_msg("No. Free pages after munmap : %d\n", free_hpage_a); @@ -140,8 +135,8 @@ int main(void) ksft_print_header(); - /* Check if huge pages are free */ - if (!get_free_hugepages()) + /* request a huge page */ + if (!hugetlb_setup_default(1)) ksft_exit_skip("No free hugepage, exiting\n"); fd = open("/tmp", O_TMPFILE | O_RDWR | O_DIRECT, 0664); diff --git a/tools/testing/selftests/mm/hugetlb_fault_after_madv.c b/tools/testing/selftests/mm/hugetlb_fault_after_madv.c index b4b257775b74..2dc158054f66 100644 --- a/tools/testing/selftests/mm/hugetlb_fault_after_madv.c +++ b/tools/testing/selftests/mm/hugetlb_fault_after_madv.c @@ -10,6 +10,7 @@ #include "vm_util.h" #include "kselftest.h" +#include "hugepage_settings.h" #define INLOOP_ITER 100 @@ -53,7 +54,6 @@ void *madv(void *unused) int main(void) { - unsigned long free_hugepages; pthread_t thread1, thread2; /* * On kernel 6.4, we are able to reproduce the problem with ~1000 @@ -77,11 +77,8 @@ int main(void) ksft_print_msg("[INFO] detected default hugetlb page size: %zu KiB\n", huge_page_size / 1024); - free_hugepages = get_free_hugepages(); - if (free_hugepages != 1) { - ksft_exit_skip("This test needs one and only one page to execute. Got %lu\n", - free_hugepages); - } + if (!hugetlb_setup_default(1)) + ksft_exit_skip("Not enough HugeTLB pages\n"); while (max--) { huge_ptr = mmap(NULL, huge_page_size, PROT_READ | PROT_WRITE, diff --git a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c index efd774b41389..f94549efcc6f 100644 --- a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c +++ b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c @@ -25,7 +25,7 @@ #include <unistd.h> #include "vm_util.h" -#include "kselftest.h" +#include "hugepage_settings.h" #define INLOOP_ITER 100 @@ -77,7 +77,6 @@ void *map_extra(void *unused) int main(void) { pthread_t thread1, thread2, thread3; - unsigned long free_hugepages; void *ret; /* @@ -86,12 +85,12 @@ int main(void) */ int max = 10; - free_hugepages = get_free_hugepages(); + ksft_print_header(); + ksft_set_plan(1); - if (free_hugepages != 1) { + if (!hugetlb_setup_default_exact(1)) ksft_exit_skip("This test needs one and only one page to execute. Got %lu\n", - free_hugepages); - } + hugetlb_free_default_pages()); mmap_size = default_huge_page_size(); @@ -100,10 +99,8 @@ int main(void) MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if ((unsigned long)huge_ptr == -1) { - ksft_test_result_fail("Failed to allocate huge page\n"); - return KSFT_FAIL; - } + if ((unsigned long)huge_ptr == -1) + ksft_exit_fail_msg("Failed to allocate huge page\n"); pthread_create(&thread1, NULL, madv, NULL); pthread_create(&thread2, NULL, touch, NULL); @@ -115,12 +112,13 @@ int main(void) if (ret) { ksft_test_result_fail("Unexpected huge page allocation\n"); - return KSFT_FAIL; + ksft_finished(); } /* Unmap and restart */ munmap(huge_ptr, mmap_size); } - return KSFT_PASS; + ksft_test_result_pass("No unexpected huge page allocations\n"); + ksft_finished(); } diff --git a/tools/testing/selftests/mm/hugetlb_reparenting_test.sh b/tools/testing/selftests/mm/hugetlb_reparenting_test.sh index 0dd31892ff67..95f517c3bd16 100755 --- a/tools/testing/selftests/mm/hugetlb_reparenting_test.sh +++ b/tools/testing/selftests/mm/hugetlb_reparenting_test.sh @@ -12,6 +12,8 @@ if [[ $(id -u) -ne 0 ]]; then fi nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) +trap 'echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages' EXIT INT TERM + usage_file=usage_in_bytes if [[ "$1" == "-cgroup-v2" ]]; then @@ -46,6 +48,13 @@ function get_machine_hugepage_size() { } MB=$(get_machine_hugepage_size) +if (( MB >= 1024 )); then + UNIT="GB" + MB_DISPLAY=$((MB / 1024)) +else + UNIT="MB" + MB_DISPLAY=$MB +fi function cleanup() { echo cleanup @@ -56,7 +65,6 @@ function cleanup() { rmdir "$CGROUP_ROOT"/a/b 2>/dev/null rmdir "$CGROUP_ROOT"/a 2>/dev/null rmdir "$CGROUP_ROOT"/test1 2>/dev/null - echo $nr_hugepgs >/proc/sys/vm/nr_hugepages set -e } @@ -87,6 +95,7 @@ function assert_with_retry() { if [[ $elapsed -ge $timeout ]]; then echo "actual = $((${actual%% *} / 1024 / 1024)) MB" echo "expected = $((${expected%% *} / 1024 / 1024)) MB" + echo FAIL cleanup exit 1 fi @@ -96,22 +105,19 @@ function assert_with_retry() { } function assert_state() { - local expected_a="$1" - local expected_a_hugetlb="$2" - local expected_b="" + local expected_a_hugetlb="$1" local expected_b_hugetlb="" - if [ ! -z ${3:-} ] && [ ! -z ${4:-} ]; then - expected_b="$3" - expected_b_hugetlb="$4" + if [ ! -z ${2:-} ]; then + expected_b_hugetlb="$2" fi - assert_with_retry "$CGROUP_ROOT/a/memory.$usage_file" "$expected_a" - assert_with_retry "$CGROUP_ROOT/a/hugetlb.${MB}MB.$usage_file" "$expected_a_hugetlb" + assert_with_retry \ + "$CGROUP_ROOT/a/hugetlb.${MB_DISPLAY}${UNIT}.$usage_file" "$expected_a_hugetlb" - if [[ -n "$expected_b" && -n "$expected_b_hugetlb" ]]; then - assert_with_retry "$CGROUP_ROOT/a/b/memory.$usage_file" "$expected_b" - assert_with_retry "$CGROUP_ROOT/a/b/hugetlb.${MB}MB.$usage_file" "$expected_b_hugetlb" + if [[ -n "$expected_b_hugetlb" ]]; then + assert_with_retry \ + "$CGROUP_ROOT/a/b/hugetlb.${MB_DISPLAY}${UNIT}.$usage_file" "$expected_b_hugetlb" fi } @@ -143,18 +149,17 @@ write_hugetlbfs() { local size="$3" if [[ $cgroup2 ]]; then - echo $$ >$CGROUP_ROOT/$cgroup/cgroup.procs + cg_file="$CGROUP_ROOT/$cgroup/cgroup.procs" else echo 0 >$CGROUP_ROOT/$cgroup/cpuset.mems echo 0 >$CGROUP_ROOT/$cgroup/cpuset.cpus - echo $$ >"$CGROUP_ROOT/$cgroup/tasks" - fi - ./write_to_hugetlbfs -p "$path" -s "$size" -m 0 -o - if [[ $cgroup2 ]]; then - echo $$ >$CGROUP_ROOT/cgroup.procs - else - echo $$ >"$CGROUP_ROOT/tasks" + cg_file="$CGROUP_ROOT/$cgroup/tasks" fi + + # Spawn helper to join cgroup before exec to ensure correct cgroup accounting + bash -c 'echo $$ > "$1"; exec ./write_to_hugetlbfs -p "$2" -s "$3" -m 0 -o' _ \ + "$cg_file" "$path" "$size" & pid=$! + wait "$pid" echo } @@ -192,21 +197,21 @@ if [[ ! $cgroup2 ]]; then write_hugetlbfs a "$MNT"/test $size echo Assert memory charged correctly for parent use. - assert_state 0 $size 0 0 + assert_state $size 0 write_hugetlbfs a/b "$MNT"/test2 $size echo Assert memory charged correctly for child use. - assert_state 0 $(($size * 2)) 0 $size + assert_state $(($size * 2)) $size rmdir "$CGROUP_ROOT"/a/b echo Assert memory reparent correctly. - assert_state 0 $(($size * 2)) + assert_state $(($size * 2)) rm -rf "$MNT"/* umount "$MNT" echo Assert memory uncharged correctly. - assert_state 0 0 + assert_state 0 cleanup fi @@ -220,16 +225,16 @@ echo write write_hugetlbfs a/b "$MNT"/test2 $size echo Assert memory charged correctly for child only use. -assert_state 0 $(($size)) 0 $size +assert_state $(($size)) $size rmdir "$CGROUP_ROOT"/a/b echo Assert memory reparent correctly. -assert_state 0 $size +assert_state $size rm -rf "$MNT"/* umount "$MNT" echo Assert memory uncharged correctly. -assert_state 0 0 +assert_state 0 cleanup @@ -240,4 +245,3 @@ if [[ $do_umount ]]; then rm -rf $CGROUP_ROOT fi -echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 3fe7ef04ac62..10e8dedcb087 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -22,7 +22,7 @@ #include "linux/magic.h" #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #define BASE_ADDR ((void *)(1UL << 30)) static unsigned long hpage_pmd_size; @@ -41,6 +41,12 @@ enum vma_type { VMA_SHMEM, }; +enum file_setup_ops { + FILE_SETUP_READ_ONLY_FS, + FILE_SETUP_READ_WRITE_FS_READ_DATA, + FILE_SETUP_READ_WRITE_FS_WRITE_DATA, +}; + struct mem_ops { void *(*setup_area)(int nr_hpages); void (*cleanup_area)(void *p, unsigned long size); @@ -49,7 +55,9 @@ struct mem_ops { const char *name; }; -static struct mem_ops *file_ops; +static struct mem_ops *read_only_file_ops; +static struct mem_ops *read_write_file_read_ops; +static struct mem_ops *read_write_file_write_ops; static struct mem_ops *anon_ops; static struct mem_ops *shmem_ops; @@ -72,57 +80,36 @@ struct file_info { }; static struct file_info finfo; -static bool skip_settings_restore; static int exit_status; static void success(const char *msg) { printf(" \e[32m%s\e[0m\n", msg); + exit_status = KSFT_PASS; } static void fail(const char *msg) { printf(" \e[31m%s\e[0m\n", msg); - exit_status++; + exit_status = KSFT_FAIL; } static void skip(const char *msg) { printf(" \e[33m%s\e[0m\n", msg); -} - -static void restore_settings_atexit(void) -{ - if (skip_settings_restore) - return; - - printf("Restore THP and khugepaged settings..."); - thp_restore_settings(); - success("OK"); - - skip_settings_restore = true; -} - -static void restore_settings(int sig) -{ - /* exit() will invoke the restore_settings_atexit handler. */ - exit(sig ? EXIT_FAILURE : exit_status); + exit_status = KSFT_SKIP; } static void save_settings(void) { - printf("Save THP and khugepaged settings..."); - if (file_ops && finfo.type == VMA_FILE) + ksft_print_msg("Save THP and khugepaged settings..."); + if ((read_only_file_ops || read_write_file_read_ops || + read_write_file_write_ops) && + finfo.type == VMA_FILE) thp_set_read_ahead_path(finfo.dev_queue_read_ahead_path); thp_save_settings(); success("OK"); - - atexit(restore_settings_atexit); - signal(SIGTERM, restore_settings); - signal(SIGINT, restore_settings); - signal(SIGHUP, restore_settings); - signal(SIGQUIT, restore_settings); } static void get_finfo(const char *dir) @@ -135,19 +122,13 @@ static void get_finfo(const char *dir) finfo.dir = dir; stat(finfo.dir, &path_stat); - if (!S_ISDIR(path_stat.st_mode)) { - printf("%s: Not a directory (%s)\n", __func__, finfo.dir); - exit(EXIT_FAILURE); - } + if (!S_ISDIR(path_stat.st_mode)) + ksft_exit_fail_msg("%s: Not a directory (%s)\n", __func__, finfo.dir); if (snprintf(finfo.path, sizeof(finfo.path), "%s/" TEST_FILE, - finfo.dir) >= sizeof(finfo.path)) { - printf("%s: Pathname is too long\n", __func__); - exit(EXIT_FAILURE); - } - if (statfs(finfo.dir, &fs)) { - perror("statfs()"); - exit(EXIT_FAILURE); - } + finfo.dir) >= sizeof(finfo.path)) + ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); + if (statfs(finfo.dir, &fs)) + ksft_exit_fail_perror("statfs()"); finfo.type = fs.f_type == TMPFS_MAGIC ? VMA_SHMEM : VMA_FILE; if (finfo.type == VMA_SHMEM) return; @@ -155,40 +136,30 @@ static void get_finfo(const char *dir) /* Find owning device's queue/read_ahead_kb control */ if (snprintf(path, sizeof(path), "/sys/dev/block/%d:%d/uevent", major(path_stat.st_dev), minor(path_stat.st_dev)) - >= sizeof(path)) { - printf("%s: Pathname is too long\n", __func__); - exit(EXIT_FAILURE); - } - if (read_file(path, buf, sizeof(buf)) < 0) { - perror("read_file(read_num)"); - exit(EXIT_FAILURE); - } + >= sizeof(path)) + ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); + if (read_file(path, buf, sizeof(buf)) < 0) + ksft_exit_fail_perror("read_file(read_num)"); if (strstr(buf, "DEVTYPE=disk")) { /* Found it */ if (snprintf(finfo.dev_queue_read_ahead_path, sizeof(finfo.dev_queue_read_ahead_path), "/sys/dev/block/%d:%d/queue/read_ahead_kb", major(path_stat.st_dev), minor(path_stat.st_dev)) - >= sizeof(finfo.dev_queue_read_ahead_path)) { - printf("%s: Pathname is too long\n", __func__); - exit(EXIT_FAILURE); - } + >= sizeof(finfo.dev_queue_read_ahead_path)) + ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); return; } - if (!strstr(buf, "DEVTYPE=partition")) { - printf("%s: Unknown device type: %s\n", __func__, path); - exit(EXIT_FAILURE); - } + if (!strstr(buf, "DEVTYPE=partition")) + ksft_exit_fail_msg("%s: Unknown device type: %s\n", __func__, path); /* * Partition of block device - need to find actual device. * Using naming convention that devnameN is partition of * device devname. */ str = strstr(buf, "DEVNAME="); - if (!str) { - printf("%s: Could not read: %s", __func__, path); - exit(EXIT_FAILURE); - } + if (!str) + ksft_exit_fail_msg("%s: Could not read: %s", __func__, path); str += 8; end = str; while (*end) { @@ -197,16 +168,13 @@ static void get_finfo(const char *dir) if (snprintf(finfo.dev_queue_read_ahead_path, sizeof(finfo.dev_queue_read_ahead_path), "/sys/block/%s/queue/read_ahead_kb", - str) >= sizeof(finfo.dev_queue_read_ahead_path)) { - printf("%s: Pathname is too long\n", __func__); - exit(EXIT_FAILURE); - } + str) >= sizeof(finfo.dev_queue_read_ahead_path)) + ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); return; } ++end; } - printf("%s: Could not read: %s\n", __func__, path); - exit(EXIT_FAILURE); + ksft_exit_fail_msg("%s: Could not read: %s\n", __func__, path); } static bool check_swap(void *addr, unsigned long size) @@ -219,26 +187,19 @@ static bool check_swap(void *addr, unsigned long size) ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", (unsigned long) addr); - if (ret >= MAX_LINE_LENGTH) { - printf("%s: Pattern is too long\n", __func__); - exit(EXIT_FAILURE); - } - + if (ret >= MAX_LINE_LENGTH) + ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); fp = fopen(PID_SMAPS, "r"); - if (!fp) { - printf("%s: Failed to open file %s\n", __func__, PID_SMAPS); - exit(EXIT_FAILURE); - } + if (!fp) + ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, PID_SMAPS); if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer))) goto err_out; ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "Swap:%19ld kB", size >> 10); - if (ret >= MAX_LINE_LENGTH) { - printf("%s: Pattern is too long\n", __func__); - exit(EXIT_FAILURE); - } + if (ret >= MAX_LINE_LENGTH) + ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); /* * Fetch the Swap: in the same block and check whether it got * the expected number of hugeepages next. @@ -261,10 +222,8 @@ static void *alloc_mapping(int nr) p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (p != BASE_ADDR) { - printf("Failed to allocate VMA at %p\n", BASE_ADDR); - exit(EXIT_FAILURE); - } + if (p != BASE_ADDR) + ksft_exit_fail_msg("Failed to allocate VMA at %p\n", BASE_ADDR); return p; } @@ -314,19 +273,13 @@ static void *alloc_hpage(struct mem_ops *ops) * khugepaged on low-load system (like a test machine), which * would cause MADV_COLLAPSE to fail with EAGAIN. */ - printf("Allocate huge page..."); - if (madvise_collapse_retry(p, hpage_pmd_size)) { - perror("madvise(MADV_COLLAPSE)"); - exit(EXIT_FAILURE); - } - if (!ops->check_huge(p, 1)) { - perror("madvise(MADV_COLLAPSE)"); - exit(EXIT_FAILURE); - } - if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) { - perror("madvise(MADV_HUGEPAGE)"); - exit(EXIT_FAILURE); - } + ksft_print_msg("Allocate huge page..."); + if (madvise_collapse_retry(p, hpage_pmd_size)) + ksft_exit_fail_perror("madvise(MADV_COLLAPSE)"); + if (!ops->check_huge(p, 1)) + ksft_exit_fail_perror("madvise(MADV_COLLAPSE)"); + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); success("OK"); return p; } @@ -336,11 +289,9 @@ static void validate_memory(int *p, unsigned long start, unsigned long end) int i; for (i = start / page_size; i < end / page_size; i++) { - if (p[i * page_size / sizeof(*p)] != i + 0xdead0000) { - printf("Page %d is corrupted: %#x\n", - i, p[i * page_size / sizeof(*p)]); - exit(EXIT_FAILURE); - } + if (p[i * page_size / sizeof(*p)] != i + 0xdead0000) + ksft_exit_fail_msg("Page %d is corrupted: %#x\n", + i, p[i * page_size / sizeof(*p)]); } } @@ -364,42 +315,52 @@ static bool anon_check_huge(void *addr, int nr_hpages) return check_huge_anon(addr, nr_hpages, hpage_pmd_size); } -static void *file_setup_area(int nr_hpages) +static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup) { + const int open_opt = setup == FILE_SETUP_READ_ONLY_FS ? O_RDONLY : O_RDWR; + const int mmap_prot = setup == FILE_SETUP_READ_ONLY_FS ? PROT_READ : (PROT_READ | PROT_WRITE); int fd; void *p; unsigned long size; unlink(finfo.path); /* Cleanup from previous failed tests */ - printf("Creating %s for collapse%s...", finfo.path, - finfo.type == VMA_SHMEM ? " (tmpfs)" : ""); - fd = open(finfo.path, O_DSYNC | O_CREAT | O_RDWR | O_TRUNC | O_EXCL, + ksft_print_msg("Creating %s for collapse%s...", finfo.path, + finfo.type == VMA_SHMEM ? " (tmpfs)" : ""); + fd = open(finfo.path, O_CREAT | O_RDWR | O_TRUNC | O_EXCL, 777); - if (fd < 0) { - perror("open()"); - exit(EXIT_FAILURE); - } + if (fd < 0) + ksft_exit_fail_perror("open()"); size = nr_hpages * hpage_pmd_size; - p = alloc_mapping(nr_hpages); - fill_memory(p, 0, size); - write(fd, p, size); - close(fd); - munmap(p, size); - success("OK"); - - printf("Opening %s read only for collapse...", finfo.path); - finfo.fd = open(finfo.path, O_RDONLY, 777); - if (finfo.fd < 0) { - perror("open()"); + if (ftruncate(fd, size)) { + perror("ftruncate()"); exit(EXIT_FAILURE); } - p = mmap(BASE_ADDR, size, PROT_READ, - MAP_PRIVATE, finfo.fd, 0); - if (p == MAP_FAILED || p != BASE_ADDR) { + p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p != BASE_ADDR) { perror("mmap()"); exit(EXIT_FAILURE); } + fill_memory(p, 0, size); + if (msync(p, size, MS_SYNC)) { + perror("msync()"); + exit(EXIT_FAILURE); + } + close(fd); + munmap(p, size); + success("OK"); + ksft_print_msg("Opening %s %s for collapse...", finfo.path, + setup == FILE_SETUP_READ_ONLY_FS ? "read-only" : + setup == FILE_SETUP_READ_WRITE_FS_READ_DATA ? + "read-write (read)" : + "read-write (write)"); + finfo.fd = open(finfo.path, open_opt, 777); + if (finfo.fd < 0) + ksft_exit_fail_perror("open()"); + p = mmap(BASE_ADDR, size, mmap_prot, MAP_SHARED, finfo.fd, 0); + if (p == MAP_FAILED || p != BASE_ADDR) + ksft_exit_fail_perror("mmap()"); /* Drop page cache */ write_file("/proc/sys/vm/drop_caches", "3", 2); @@ -407,6 +368,21 @@ static void *file_setup_area(int nr_hpages) return p; } +static void *file_setup_read_only_area(int nr_hpages) +{ + return file_setup_area_common(nr_hpages, FILE_SETUP_READ_ONLY_FS); +} + +static void *file_setup_read_write_fs_read_area(int nr_hpages) +{ + return file_setup_area_common(nr_hpages, FILE_SETUP_READ_WRITE_FS_READ_DATA); +} + +static void *file_setup_read_write_fs_write_area(int nr_hpages) +{ + return file_setup_area_common(nr_hpages, FILE_SETUP_READ_WRITE_FS_WRITE_DATA); +} + static void file_cleanup_area(void *p, unsigned long size) { munmap(p, size); @@ -414,12 +390,26 @@ static void file_cleanup_area(void *p, unsigned long size) unlink(finfo.path); } -static void file_fault(void *p, unsigned long start, unsigned long end) +static void file_fault_read(void *p, unsigned long start, unsigned long end) { - if (madvise(((char *)p) + start, end - start, MADV_POPULATE_READ)) { - perror("madvise(MADV_POPULATE_READ"); - exit(EXIT_FAILURE); - } + if (madvise(((char *)p) + start, end - start, MADV_POPULATE_READ)) + ksft_exit_fail_perror("madvise(MADV_POPULATE_READ)"); +} + +static void file_fault_read_and_flush(void *p, unsigned long start, unsigned long end) +{ + file_fault_read(p, start, end); + /* + * make folio clean, since dirty folios from read&write file are + * rejected and not flushed + */ + msync((char *)p + start, end - start, MS_SYNC); +} + +static void file_fault_write(void *p, unsigned long start, unsigned long end) +{ + if (madvise(((char *)p) + start, end - start, MADV_POPULATE_WRITE)) + ksft_exit_fail_perror("madvise(MADV_POPULATE_WRITE)"); } static bool file_check_huge(void *addr, int nr_hpages) @@ -441,20 +431,14 @@ static void *shmem_setup_area(int nr_hpages) unsigned long size = nr_hpages * hpage_pmd_size; finfo.fd = memfd_create("khugepaged-selftest-collapse-shmem", 0); - if (finfo.fd < 0) { - perror("memfd_create()"); - exit(EXIT_FAILURE); - } - if (ftruncate(finfo.fd, size)) { - perror("ftruncate()"); - exit(EXIT_FAILURE); - } + if (finfo.fd < 0) + ksft_exit_fail_perror("memfd_create()"); + if (ftruncate(finfo.fd, size)) + ksft_exit_fail_perror("ftruncate()"); p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, MAP_SHARED, finfo.fd, 0); - if (p != BASE_ADDR) { - perror("mmap()"); - exit(EXIT_FAILURE); - } + if (p != BASE_ADDR) + ksft_exit_fail_perror("mmap()"); return p; } @@ -477,10 +461,26 @@ static struct mem_ops __anon_ops = { .name = "anon", }; -static struct mem_ops __file_ops = { - .setup_area = &file_setup_area, +static struct mem_ops __read_only_file_ops = { + .setup_area = &file_setup_read_only_area, + .cleanup_area = &file_cleanup_area, + .fault = &file_fault_read, + .check_huge = &file_check_huge, + .name = "file", +}; + +static struct mem_ops __read_write_file_read_ops = { + .setup_area = &file_setup_read_write_fs_read_area, + .cleanup_area = &file_cleanup_area, + .fault = &file_fault_read_and_flush, + .check_huge = &file_check_huge, + .name = "file", +}; + +static struct mem_ops __read_write_file_write_ops = { + .setup_area = &file_setup_read_write_fs_write_area, .cleanup_area = &file_cleanup_area, - .fault = &file_fault, + .fault = &file_fault_write, .check_huge = &file_check_huge, .name = "file", }; @@ -493,13 +493,32 @@ static struct mem_ops __shmem_ops = { .name = "shmem", }; +static bool is_tmpfs(struct mem_ops *ops) +{ + return (ops == &__read_only_file_ops || + ops == &__read_write_file_read_ops || + ops == &__read_write_file_write_ops) && + finfo.type == VMA_SHMEM; +} + +static bool is_anon(struct mem_ops *ops) +{ + return ops == &__anon_ops; +} + static void __madvise_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { int ret; struct thp_settings settings = *thp_current_settings(); - printf("%s...", msg); + ksft_print_msg("%s...", msg); + + /* + * read&write file collapse succeeds for MADV_COLLAPSE because dirty + * folios are written back after collapse fails for dirty folios and + * another collapse is attempted. + */ /* * Prevent khugepaged interference and tests that MADV_COLLAPSE @@ -526,10 +545,8 @@ static void madvise_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { /* Sanity check */ - if (!ops->check_huge(p, 0)) { - printf("Unexpected huge page\n"); - exit(EXIT_FAILURE); - } + if (!ops->check_huge(p, 0)) + ksft_exit_fail_msg("Unexpected huge page\n"); __madvise_collapse(msg, p, nr_hpages, ops, expect); } @@ -541,17 +558,15 @@ static bool wait_for_scan(const char *msg, char *p, int nr_hpages, int timeout = 6; /* 3 seconds */ /* Sanity check */ - if (!ops->check_huge(p, 0)) { - printf("Unexpected huge page\n"); - exit(EXIT_FAILURE); - } + if (!ops->check_huge(p, 0)) + ksft_exit_fail_msg("Unexpected huge page\n"); madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); /* Wait until the second full_scan completed */ full_scans = thp_read_num("khugepaged/full_scans") + 2; - printf("%s...", msg); + ksft_print_msg("%s...", msg); while (timeout--) { if (ops->check_huge(p, nr_hpages)) break; @@ -567,6 +582,13 @@ static bool wait_for_scan(const char *msg, char *p, int nr_hpages, static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { + /* + * read&write file collapse fails since khugepaged does not flush + * the target dirty folios + */ + if (!is_tmpfs(ops) && ops == &__read_write_file_write_ops) + expect = false; + if (wait_for_scan(msg, p, nr_hpages, ops)) { if (expect) fail("Timeout"); @@ -601,16 +623,6 @@ static struct collapse_context __madvise_context = { .name = "madvise", }; -static bool is_tmpfs(struct mem_ops *ops) -{ - return ops == &__file_ops && finfo.type == VMA_SHMEM; -} - -static bool is_anon(struct mem_ops *ops) -{ - return ops == &__anon_ops; -} - static void alloc_at_fault(void) { struct thp_settings settings = *thp_current_settings(); @@ -621,7 +633,7 @@ static void alloc_at_fault(void) p = alloc_mapping(1); *p = 1; - printf("Allocate huge page on fault..."); + ksft_print_msg("Allocate huge page on fault..."); if (check_huge_anon(p, 1, hpage_pmd_size)) success("OK"); else @@ -630,12 +642,14 @@ static void alloc_at_fault(void) thp_pop_settings(); madvise(p, page_size, MADV_DONTNEED); - printf("Split huge PMD on MADV_DONTNEED..."); + ksft_print_msg("Split huge PMD on MADV_DONTNEED..."); if (check_huge_anon(p, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); munmap(p, hpage_pmd_size); + + ksft_test_result_report(exit_status, "allocate on fault and split\n"); } static void collapse_full(struct collapse_context *c, struct mem_ops *ops) @@ -650,6 +664,8 @@ static void collapse_full(struct collapse_context *c, struct mem_ops *ops) ops, true); validate_memory(p, 0, size); ops->cleanup_area(p, size); + + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_empty(struct collapse_context *c, struct mem_ops *ops) @@ -659,6 +675,7 @@ static void collapse_empty(struct collapse_context *c, struct mem_ops *ops) p = ops->setup_area(1); c->collapse("Do not collapse empty PTE table", p, 1, ops, false); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_single_pte_entry(struct collapse_context *c, struct mem_ops *ops) @@ -670,6 +687,7 @@ static void collapse_single_pte_entry(struct collapse_context *c, struct mem_ops c->collapse("Collapse PTE table with single PTE entry present", p, 1, ops, true); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *ops) @@ -697,6 +715,9 @@ static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *o validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - fault_nr_pages) * page_size); if (c->enforce_pte_scan_limits) { + ops->cleanup_area(p, hpage_pmd_size); + p = ops->setup_area(1); + ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ops, true); @@ -706,6 +727,7 @@ static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *o skip: ops->cleanup_area(p, hpage_pmd_size); thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_ops *ops) @@ -715,11 +737,9 @@ static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_op p = ops->setup_area(1); ops->fault(p, 0, hpage_pmd_size); - printf("Swapout one page..."); - if (madvise(p, page_size, MADV_PAGEOUT)) { - perror("madvise(MADV_PAGEOUT)"); - exit(EXIT_FAILURE); - } + ksft_print_msg("Swapout one page..."); + if (madvise(p, page_size, MADV_PAGEOUT)) + ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); if (check_swap(p, page_size)) { success("OK"); } else { @@ -732,6 +752,7 @@ static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_op validate_memory(p, 0, hpage_pmd_size); out: ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *ops) @@ -742,11 +763,9 @@ static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *o p = ops->setup_area(1); ops->fault(p, 0, hpage_pmd_size); - printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); - if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) { - perror("madvise(MADV_PAGEOUT)"); - exit(EXIT_FAILURE); - } + ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); + if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) + ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); if (check_swap(p, (max_ptes_swap + 1) * page_size)) { success("OK"); } else { @@ -760,12 +779,10 @@ static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *o if (c->enforce_pte_scan_limits) { ops->fault(p, 0, hpage_pmd_size); - printf("Swapout %d of %d pages...", max_ptes_swap, + ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr); - if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { - perror("madvise(MADV_PAGEOUT)"); - exit(EXIT_FAILURE); - } + if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) + ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); if (check_swap(p, max_ptes_swap * page_size)) { success("OK"); } else { @@ -779,6 +796,7 @@ static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *o } out: ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_single_pte_entry_compound(struct collapse_context *c, struct mem_ops *ops) @@ -795,7 +813,7 @@ static void collapse_single_pte_entry_compound(struct collapse_context *c, struc } madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); - printf("Split huge page leaving single PTE mapping compound page..."); + ksft_print_msg("Split huge page leaving single PTE mapping compound page..."); madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); if (ops->check_huge(p, 0)) success("OK"); @@ -807,6 +825,7 @@ static void collapse_single_pte_entry_compound(struct collapse_context *c, struc validate_memory(p, 0, page_size); skip: ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops *ops) @@ -814,7 +833,7 @@ static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops void *p; p = alloc_hpage(ops); - printf("Split huge page leaving single PTE page table full of compound pages..."); + ksft_print_msg("Split huge page leaving single PTE page table full of compound pages..."); madvise(p, page_size, MADV_NOHUGEPAGE); madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); if (ops->check_huge(p, 0)) @@ -826,6 +845,7 @@ static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops true); validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops *ops) @@ -834,16 +854,12 @@ static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops int i; p = ops->setup_area(1); + ksft_print_msg("Construct PTE page table full of different PTE-mapped compound pages\n"); for (i = 0; i < hpage_pmd_nr; i++) { - printf("\rConstruct PTE page table full of different PTE-mapped compound pages %3d/%d...", - i + 1, hpage_pmd_nr); - madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE); ops->fault(BASE_ADDR, 0, hpage_pmd_size); - if (!ops->check_huge(BASE_ADDR, 1)) { - printf("Failed to allocate huge page\n"); - exit(EXIT_FAILURE); - } + if (!ops->check_huge(BASE_ADDR, 1)) + ksft_exit_fail_msg("Failed to allocate huge page\n"); madvise(BASE_ADDR, hpage_pmd_size, MADV_NOHUGEPAGE); p = mremap(BASE_ADDR - i * page_size, @@ -851,20 +867,16 @@ static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops (i + 1) * page_size, MREMAP_MAYMOVE | MREMAP_FIXED, BASE_ADDR + 2 * hpage_pmd_size); - if (p == MAP_FAILED) { - perror("mremap+unmap"); - exit(EXIT_FAILURE); - } + if (p == MAP_FAILED) + ksft_exit_fail_perror("mremap+unmap"); p = mremap(BASE_ADDR + 2 * hpage_pmd_size, (i + 1) * page_size, (i + 1) * page_size + hpage_pmd_size, MREMAP_MAYMOVE | MREMAP_FIXED, BASE_ADDR - (i + 1) * page_size); - if (p == MAP_FAILED) { - perror("mremap+alloc"); - exit(EXIT_FAILURE); - } + if (p == MAP_FAILED) + ksft_exit_fail_perror("mremap+alloc"); } ops->cleanup_area(BASE_ADDR, hpage_pmd_size); @@ -879,6 +891,7 @@ static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) @@ -888,19 +901,16 @@ static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) p = ops->setup_area(1); - printf("Allocate small page..."); + ksft_print_msg("Allocate small page..."); ops->fault(p, 0, page_size); if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); - printf("Share small page over fork()..."); + ksft_print_msg("Share small page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ - skip_settings_restore = true; - exit_status = 0; - if (ops->check_huge(p, 0)) success("OK"); else @@ -912,19 +922,20 @@ static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) validate_memory(p, 0, page_size); ops->cleanup_area(p, hpage_pmd_size); - exit(exit_status); + _exit(exit_status); } wait(&wstatus); - exit_status += WEXITSTATUS(wstatus); + exit_status = WEXITSTATUS(wstatus); - printf("Check if parent still has small page..."); + ksft_print_msg("Check if parent still has small page..."); if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); validate_memory(p, 0, page_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *ops) @@ -933,18 +944,15 @@ static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *o void *p; p = alloc_hpage(ops); - printf("Share huge page over fork()..."); + ksft_print_msg("Share huge page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ - skip_settings_restore = true; - exit_status = 0; - if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); - printf("Split huge page PMD in child process..."); + ksft_print_msg("Split huge page PMD in child process..."); madvise(p, page_size, MADV_NOHUGEPAGE); madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); if (ops->check_huge(p, 0)) @@ -961,19 +969,20 @@ static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *o validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); - exit(exit_status); + _exit(exit_status); } wait(&wstatus); - exit_status += WEXITSTATUS(wstatus); + exit_status = WEXITSTATUS(wstatus); - printf("Check if parent still has huge page..."); + ksft_print_msg("Check if parent still has huge page..."); if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops *ops) @@ -983,18 +992,15 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops void *p; p = alloc_hpage(ops); - printf("Share huge page over fork()..."); + ksft_print_msg("Share huge page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ - skip_settings_restore = true; - exit_status = 0; - if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); - printf("Trigger CoW on page %d of %d...", + ksft_print_msg("Trigger CoW on page %d of %d...", hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); if (ops->check_huge(p, 0)) @@ -1006,7 +1012,7 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops 1, ops, !c->enforce_pte_scan_limits); if (c->enforce_pte_scan_limits) { - printf("Trigger CoW on page %d of %d...", + ksft_print_msg("Trigger CoW on page %d of %d...", hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size); @@ -1021,19 +1027,20 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); - exit(exit_status); + _exit(exit_status); } wait(&wstatus); - exit_status += WEXITSTATUS(wstatus); + exit_status = WEXITSTATUS(wstatus); - printf("Check if parent still has huge page..."); + ksft_print_msg("Check if parent still has huge page..."); if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void madvise_collapse_existing_thps(struct collapse_context *c, @@ -1050,6 +1057,7 @@ static void madvise_collapse_existing_thps(struct collapse_context *c, __madvise_collapse("Re-collapse PMD-mapped hugepage", p, 1, ops, true); validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } /* @@ -1077,6 +1085,7 @@ static void madvise_retracted_page_tables(struct collapse_context *c, true); validate_memory(p, 0, size); ops->cleanup_area(p, size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void usage(void) @@ -1086,8 +1095,8 @@ static void usage(void) fprintf(stderr, "\t<context>\t: [all|khugepaged|madvise]\n"); fprintf(stderr, "\t<mem_type>\t: [all|anon|file|shmem]\n"); fprintf(stderr, "\n\t\"file,all\" mem_type requires [dir] argument\n"); - fprintf(stderr, "\n\t\"file,all\" mem_type requires kernel built with\n"); - fprintf(stderr, "\tCONFIG_READ_ONLY_THP_FOR_FS=y\n"); + fprintf(stderr, "\n\t\"file,all\" mem_type requires a file system\n"); + fprintf(stderr, "\twith PMD-sized large folio support\n"); fprintf(stderr, "\n\tif [dir] is a (sub)directory of a tmpfs mount, tmpfs must be\n"); fprintf(stderr, "\tmounted with huge=advise option for khugepaged tests to work\n"); fprintf(stderr, "\n\tSupported Options:\n"); @@ -1143,20 +1152,25 @@ static void parse_test_type(int argc, char **argv) usage(); if (!strcmp(buf, "all")) { - file_ops = &__file_ops; + read_only_file_ops = &__read_only_file_ops; + read_write_file_read_ops = &__read_write_file_read_ops; + read_write_file_write_ops = &__read_write_file_write_ops; anon_ops = &__anon_ops; shmem_ops = &__shmem_ops; } else if (!strcmp(buf, "anon")) { anon_ops = &__anon_ops; } else if (!strcmp(buf, "file")) { - file_ops = &__file_ops; + read_only_file_ops = &__read_only_file_ops; + read_write_file_read_ops = &__read_write_file_read_ops; + read_write_file_write_ops = &__read_write_file_write_ops; } else if (!strcmp(buf, "shmem")) { shmem_ops = &__shmem_ops; } else { usage(); } - if (!file_ops) + if (!read_only_file_ops && !read_write_file_read_ops && + !read_write_file_write_ops) return; if (argc != 2) @@ -1165,6 +1179,32 @@ static void parse_test_type(int argc, char **argv) get_finfo(argv[1]); } +typedef void (*test_fn)(struct collapse_context *c, struct mem_ops *ops); + +struct test_case { + struct collapse_context *ctx; + struct mem_ops *ops; + const char *desc; + test_fn fn; +}; + +#define MAX_TEST_CASES 64 +static struct test_case test_cases[MAX_TEST_CASES]; +static int nr_test_cases; + +#define TEST(t, c, o) do { \ + if (c && o) { \ + if (nr_test_cases >= MAX_TEST_CASES) \ + ksft_exit_fail_msg("MAX_TEST_CASES is too small\n"); \ + test_cases[nr_test_cases++] = (struct test_case){ \ + .ctx = c, \ + .ops = o, \ + .desc = #t, \ + .fn = t, \ + }; \ + } \ + } while (0) + int main(int argc, char **argv) { int hpage_pmd_order; @@ -1188,10 +1228,10 @@ int main(int argc, char **argv) .read_ahead_kb = 0, }; - if (!thp_is_enabled()) { - printf("Transparent Hugepages not available\n"); - return KSFT_SKIP; - } + ksft_print_header(); + + if (!thp_is_enabled()) + ksft_exit_skip("Transparent Hugepages not available\n"); parse_test_type(argc, argv); @@ -1199,10 +1239,8 @@ int main(int argc, char **argv) page_size = getpagesize(); hpage_pmd_size = read_pmd_pagesize(); - if (!hpage_pmd_size) { - printf("Reading PMD pagesize failed"); - exit(EXIT_FAILURE); - } + if (!hpage_pmd_size) + ksft_exit_fail_msg("Reading PMD pagesize failed\n"); hpage_pmd_nr = hpage_pmd_size / page_size; hpage_pmd_order = __builtin_ctz(hpage_pmd_nr); @@ -1218,47 +1256,54 @@ int main(int argc, char **argv) save_settings(); thp_push_settings(&default_settings); - alloc_at_fault(); - -#define TEST(t, c, o) do { \ - if (c && o) { \ - printf("\nRun test: " #t " (%s:%s)\n", c->name, o->name); \ - t(c, o); \ - } \ - } while (0) - TEST(collapse_full, khugepaged_context, anon_ops); - TEST(collapse_full, khugepaged_context, file_ops); + TEST(collapse_full, khugepaged_context, read_only_file_ops); + TEST(collapse_full, khugepaged_context, read_write_file_read_ops); + TEST(collapse_full, khugepaged_context, read_write_file_write_ops); TEST(collapse_full, khugepaged_context, shmem_ops); TEST(collapse_full, madvise_context, anon_ops); - TEST(collapse_full, madvise_context, file_ops); + TEST(collapse_full, madvise_context, read_only_file_ops); + TEST(collapse_full, madvise_context, read_write_file_read_ops); + TEST(collapse_full, madvise_context, read_write_file_write_ops); TEST(collapse_full, madvise_context, shmem_ops); TEST(collapse_empty, khugepaged_context, anon_ops); TEST(collapse_empty, madvise_context, anon_ops); TEST(collapse_single_pte_entry, khugepaged_context, anon_ops); - TEST(collapse_single_pte_entry, khugepaged_context, file_ops); + TEST(collapse_single_pte_entry, khugepaged_context, read_only_file_ops); + TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_read_ops); + TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_write_ops); TEST(collapse_single_pte_entry, khugepaged_context, shmem_ops); TEST(collapse_single_pte_entry, madvise_context, anon_ops); - TEST(collapse_single_pte_entry, madvise_context, file_ops); + TEST(collapse_single_pte_entry, madvise_context, read_only_file_ops); + TEST(collapse_single_pte_entry, madvise_context, read_write_file_read_ops); + TEST(collapse_single_pte_entry, madvise_context, read_write_file_write_ops); TEST(collapse_single_pte_entry, madvise_context, shmem_ops); TEST(collapse_max_ptes_none, khugepaged_context, anon_ops); - TEST(collapse_max_ptes_none, khugepaged_context, file_ops); + TEST(collapse_max_ptes_none, khugepaged_context, read_only_file_ops); + TEST(collapse_max_ptes_none, khugepaged_context, read_write_file_read_ops); + TEST(collapse_max_ptes_none, khugepaged_context, read_write_file_write_ops); TEST(collapse_max_ptes_none, madvise_context, anon_ops); - TEST(collapse_max_ptes_none, madvise_context, file_ops); + TEST(collapse_max_ptes_none, madvise_context, read_only_file_ops); + TEST(collapse_max_ptes_none, madvise_context, read_write_file_read_ops); + TEST(collapse_max_ptes_none, madvise_context, read_write_file_write_ops); TEST(collapse_single_pte_entry_compound, khugepaged_context, anon_ops); - TEST(collapse_single_pte_entry_compound, khugepaged_context, file_ops); + TEST(collapse_single_pte_entry_compound, khugepaged_context, read_only_file_ops); + TEST(collapse_single_pte_entry_compound, khugepaged_context, read_write_file_read_ops); TEST(collapse_single_pte_entry_compound, madvise_context, anon_ops); - TEST(collapse_single_pte_entry_compound, madvise_context, file_ops); + TEST(collapse_single_pte_entry_compound, madvise_context, read_only_file_ops); + TEST(collapse_single_pte_entry_compound, madvise_context, read_write_file_read_ops); TEST(collapse_full_of_compound, khugepaged_context, anon_ops); - TEST(collapse_full_of_compound, khugepaged_context, file_ops); + TEST(collapse_full_of_compound, khugepaged_context, read_only_file_ops); + TEST(collapse_full_of_compound, khugepaged_context, read_write_file_read_ops); TEST(collapse_full_of_compound, khugepaged_context, shmem_ops); TEST(collapse_full_of_compound, madvise_context, anon_ops); - TEST(collapse_full_of_compound, madvise_context, file_ops); + TEST(collapse_full_of_compound, madvise_context, read_only_file_ops); + TEST(collapse_full_of_compound, madvise_context, read_write_file_read_ops); TEST(collapse_full_of_compound, madvise_context, shmem_ops); TEST(collapse_compound_extreme, khugepaged_context, anon_ops); @@ -1280,11 +1325,23 @@ int main(int argc, char **argv) TEST(collapse_max_ptes_shared, madvise_context, anon_ops); TEST(madvise_collapse_existing_thps, madvise_context, anon_ops); - TEST(madvise_collapse_existing_thps, madvise_context, file_ops); + TEST(madvise_collapse_existing_thps, madvise_context, read_only_file_ops); + TEST(madvise_collapse_existing_thps, madvise_context, read_write_file_read_ops); TEST(madvise_collapse_existing_thps, madvise_context, shmem_ops); - TEST(madvise_retracted_page_tables, madvise_context, file_ops); + TEST(madvise_retracted_page_tables, madvise_context, read_only_file_ops); + TEST(madvise_retracted_page_tables, madvise_context, read_write_file_read_ops); TEST(madvise_retracted_page_tables, madvise_context, shmem_ops); - restore_settings(0); + ksft_set_plan(nr_test_cases + 1); + + alloc_at_fault(); + for (int i = 0; i < nr_test_cases; i++) { + struct test_case *t = &test_cases[i]; + + ksft_print_msg("\n# Run test: %s (%s:%s)\n", t->desc, t->ctx->name, t->ops->name); + t->fn(t->ctx, t->ops); + } + + ksft_finished(); } diff --git a/tools/testing/selftests/mm/ksft_kmemleak_dedup.sh b/tools/testing/selftests/mm/ksft_kmemleak_dedup.sh new file mode 100755 index 000000000000..d01950244490 --- /dev/null +++ b/tools/testing/selftests/mm/ksft_kmemleak_dedup.sh @@ -0,0 +1,222 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Regression test for kmemleak's per-scan verbose dedup. +# +# Loads samples/kmemleak's helper module to generate orphan allocations +# (some of which share an allocation backtrace), runs a few kmemleak +# scans with verbose printing enabled, and verifies that no two +# "unreferenced object" reports within a single scan share the same +# backtrace - which would mean dedup failed to collapse them. +# +# This test is intentionally permissive: the kmemleak-test module's +# leaks frequently get reported across many separate scans (per-CPU +# chunk reuse, slab freelist pointers, kernel stack residue), so dedup +# may never have anything to fold within one scan. That is not a +# regression. The test only fails when it actually catches dedup not +# happening on input that should have triggered it - i.e. two reports +# with identical backtraces in the same scan. +# +# Author: Breno Leitao <leitao@debian.org> + +ksft_skip=4 +KMEMLEAK=/sys/kernel/debug/kmemleak +VERBOSE_PARAM=/sys/module/kmemleak/parameters/verbose +MODULE=kmemleak-test + +skip() { + echo "SKIP: $*" + exit $ksft_skip +} + +fail() { + echo "FAIL: $*" + exit 1 +} + +pass() { + echo "PASS: $*" + exit 0 +} + +[ "$(id -u)" -eq 0 ] || skip "must run as root" +[ -r "$KMEMLEAK" ] || skip "no kmemleak debugfs (CONFIG_DEBUG_KMEMLEAK)" +[ -w "$VERBOSE_PARAM" ] || skip "kmemleak verbose param missing" +modinfo "$MODULE" >/dev/null 2>&1 || + skip "$MODULE not built (CONFIG_SAMPLE_KMEMLEAK)" + +# The verdict depends entirely on dmesg contents, so a silently-empty +# dmesg (dmesg_restrict=1 with CAP_SYSLOG dropped, restricted container, +# etc.) would let the script report PASS without parsing anything. Probe +# both read and clear up front and skip cleanly if either is denied. +dmesg >/dev/null 2>&1 || + skip "cannot read dmesg (need CAP_SYSLOG or dmesg_restrict=0)" +dmesg -C >/dev/null 2>&1 || + skip "cannot clear dmesg (need CAP_SYSLOG or dmesg_restrict=0)" + +# kmemleak can be present but disabled at runtime (boot arg kmemleak=off, +# or it self-disabled after an internal error). In that state writes other +# than "clear" return EPERM, so probe once and skip if so. +if ! echo scan > "$KMEMLEAK" 2>/dev/null; then + skip "kmemleak is disabled (check dmesg or kmemleak= boot arg)" +fi + +prev_verbose=$(cat "$VERBOSE_PARAM") +# shellcheck disable=SC2317 # invoked indirectly via trap +cleanup() { + echo "$prev_verbose" > "$VERBOSE_PARAM" 2>/dev/null + rmmod "$MODULE" 2>/dev/null + # Drain the leak set we generated. Subsequent selftests (e.g. + # tools/testing/selftests/net/netfilter/nft_interface_stress.sh) + # fail on any non-empty kmemleak report, so leaving the helper + # module's intentional leaks behind would poison the rest of a + # kselftest run. + # + # Caveat: kmemleak_clear() only greys objects that have already + # been reported (OBJECT_REPORTED && unreferenced_object()). Helper + # allocations that stayed "still referenced" throughout the test + # (stale pointers in per-CPU chunks, slab freelists, kernel stacks) + # were never reported and are therefore not greyed by this clear - + # they remain tracked and a later scan can still surface them. Such + # leftovers are inherent to the kmemleak-test sample module and are + # not specific to this test; consumers that fail on any kmemleak + # output (rather than on the test-specific backtraces) need to be + # robust to that, or this test should be excluded from the run. + echo clear > "$KMEMLEAK" 2>/dev/null +} +trap cleanup EXIT + +echo 1 > "$VERBOSE_PARAM" + +# Drain the existing leak set so the next scan only reports our objects. +echo clear > "$KMEMLEAK" + +# Re-clear dmesg now (the up-front probe also cleared it, but anything +# logged between then and here - module unload chatter, the probe scan, +# the verbose-param write - would otherwise pollute the parse window). +dmesg -C >/dev/null + +# If the module was left loaded by a previous aborted run, modprobe would +# be a no-op and the init function would not run, so no new leaks would be +# generated. Force a clean state first. +rmmod "$MODULE" 2>/dev/null +modprobe "$MODULE" || skip "failed to load $MODULE" +# Removing the module orphans the list elements without freeing them. +rmmod "$MODULE" || skip "failed to unload $MODULE" + +# Run a handful of scans so kmemleak has the chance to age and report +# the orphans. We do not require any particular number to be reported: +# the regression check below operates on whatever lands in dmesg. +# +# Note: with CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN=y the kernel's own scan +# thread can report and mark these orphans (OBJECT_REPORTED) before our +# manual scans run, after which our scans will see nothing. The +# lower-bound check below catches the case where that happens and the +# manual scans also produce nothing. +SCAN_COUNT=4 +SCAN_SLEEP=6 +for _ in $(seq 1 "$SCAN_COUNT"); do + echo scan > "$KMEMLEAK" + sleep "$SCAN_SLEEP" +done + +# Strip the leading "[ nnn.nnnnnn] " dmesg timestamp prefix. Without +# this, two identical stack frames printed from two reports in the same +# scan would produce different per-frame strings (different timestamps) +# and the duplicate-backtrace check below would not match them, silently +# passing a real dedup regression. Doing the strip here makes the rest +# of the parser timestamp-agnostic regardless of what dmesg defaults to. +log=$(dmesg | sed 's/^\[[^]]*\] //') + +# After running the workload (modprobe + scans), dmesg should contain at +# least the helper module's pr_info lines and our manual-scan output. An +# empty capture here means dmesg succeeded earlier but is now denying us +# the buffer (race with dmesg_restrict toggling, etc.); refuse to give a +# verdict on no evidence. +[ -n "$log" ] || skip "dmesg returned empty after running workload" + +# Lower bound: if kmemleak's own per-scan tally counted leaks but the +# verbose path emitted no "unreferenced object" line, the verbose printer +# itself is regressed - fail rather than silently passing on no input. +new_leaks=$(echo "$log" | + sed -n 's/.*kmemleak: \([0-9]\+\) new suspected.*/\1/p' | + awk '{s+=$1} END{print s+0}') +printed=$(echo "$log" | grep -c 'kmemleak: unreferenced object') +if [ "$new_leaks" -gt 0 ] && [ "$printed" -eq 0 ]; then + fail "verbose path broken: $new_leaks leaks counted, 0 printed in $SCAN_COUNT scans" +fi + +# Walk the log: split into per-scan chunks at "N new suspected memory +# leaks" boundaries; within each chunk, capture each "unreferenced +# object" report's backtrace and check that no backtrace is reported +# more than once. A duplicate within a single scan means dedup failed +# to collapse two leaks that share an allocation site. +violations=$(echo "$log" | awk ' + function flush_block() { + if (in_block) { + # Skip empty backtraces: leaks with trace_handle == 0 + # (early-boot allocations or stack_depot_save() failures + # under memory pressure) are intentionally not deduped, + # so multiple such reports in one scan are expected and + # must not be flagged as a regression. + if (bt != "") + seen[bt]++ + in_block = 0 + collecting = 0 + bt = "" + } + } + function check_and_reset( b) { + for (b in seen) + if (seen[b] > 1) + printf("backtrace seen %d times in one scan:\n%s\n", + seen[b], b) + delete seen + } + # Scan boundary: the per-scan summary line. + /kmemleak: [0-9]+ new suspected memory leaks/ { + flush_block() + check_and_reset() + next + } + # Start of a new "unreferenced object" report. + /kmemleak: unreferenced object/ { + flush_block() + in_block = 1 + next + } + # Inside a report, the "backtrace (crc ...):" line switches us to + # backtrace-collecting mode. + in_block && /kmemleak:[[:space:]]+backtrace \(crc/ { + collecting = 1 + next + } + # Once collecting, capture only deeply-indented "kmemleak: " lines + # (stack frames have 4+ spaces of indentation under "kmemleak: "; + # headers and the "... and N more" tail line have less). This stops + # unrelated kmemleak warns landing between reports from being lumped + # into the backtrace key, which would mask a genuine duplicate. + in_block && collecting && /kmemleak:[[:space:]]{4,}/ { + bt = bt $0 "\n" + next + } + END { + flush_block() + check_and_reset() + } +') + +if [ -n "$violations" ]; then + echo "$violations" + fail "kmemleak dedup regression: same backtrace reported more than once in a single scan" +fi + +# Count the dedup summary lines so the report distinguishes "dedup +# actually fired" from "no same-backtrace leaks turned up to dedup". +dedup_lines=$(echo "$log" | grep -c 'more object(s) with the same backtrace') + +if [ "$dedup_lines" -gt 0 ]; then + pass "no dedup violations across $SCAN_COUNT scans; dedup fired ($dedup_lines summary line(s) observed)" +else + pass "no dedup violations across $SCAN_COUNT scans; dedup had nothing to collapse" +fi diff --git a/tools/testing/selftests/mm/ksft_process_madv.sh b/tools/testing/selftests/mm/ksft_process_madv.sh index 2c3137ae8bc8..edad2d2d888f 100755 --- a/tools/testing/selftests/mm/ksft_process_madv.sh +++ b/tools/testing/selftests/mm/ksft_process_madv.sh @@ -1,4 +1,4 @@ #!/bin/sh -e # SPDX-License-Identifier: GPL-2.0 -./run_vmtests.sh -t mmap +./run_vmtests.sh -t process_madv diff --git a/tools/testing/selftests/mm/ksm_functional_tests.c b/tools/testing/selftests/mm/ksm_functional_tests.c index 8d874c4754f3..31c06c72203f 100644 --- a/tools/testing/selftests/mm/ksm_functional_tests.c +++ b/tools/testing/selftests/mm/ksm_functional_tests.c @@ -498,6 +498,7 @@ static void test_prctl_fork(void) static int start_ksmd_and_set_frequency(char *pages_to_scan, char *sleep_ms) { int ksm_fd; + size_t len; ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR); if (ksm_fd < 0) @@ -506,11 +507,13 @@ static int start_ksmd_and_set_frequency(char *pages_to_scan, char *sleep_ms) if (write(ksm_fd, "1", 1) != 1) return -errno; - if (write(pages_to_scan_fd, pages_to_scan, strlen(pages_to_scan)) <= 0) - return -errno; + len = strlen(pages_to_scan); + if (write(pages_to_scan_fd, pages_to_scan, len) != len) + return -1; - if (write(sleep_millisecs_fd, sleep_ms, strlen(sleep_ms)) <= 0) - return -errno; + len = strlen(sleep_ms); + if (write(sleep_millisecs_fd, sleep_ms, len) != len) + return -1; return 0; } @@ -526,11 +529,11 @@ static int stop_ksmd_and_restore_frequency(void) if (write(ksm_fd, "2", 1) != 1) return -errno; - if (write(pages_to_scan_fd, "100", 3) <= 0) - return -errno; + if (write(pages_to_scan_fd, "100", 3) != 3) + return -1; - if (write(sleep_millisecs_fd, "20", 2) <= 0) - return -errno; + if (write(sleep_millisecs_fd, "20", 2) != 2) + return -1; return 0; } diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index a0b48b839d54..a050f4840cfa 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -15,7 +15,7 @@ #include "kselftest.h" #include <include/vdso/time64.h> #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" #define KSM_FP(s) (KSM_SYSFS_PATH s) @@ -174,13 +174,13 @@ static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_ { void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); - if (!map_ptr) { - perror("mmap"); + if (map_ptr == MAP_FAILED) { + ksft_perror("mmap"); return NULL; } memset(map_ptr, data, map_size); if (mprotect(map_ptr, map_size, prot)) { - perror("mprotect"); + ksft_perror("mprotect"); munmap(map_ptr, map_size); return NULL; } @@ -201,11 +201,11 @@ static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout) if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan)) return 1; if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return 1; } if ((cur_time.tv_sec - start_time.tv_sec) > timeout) { - printf("Scan time limit exceeded\n"); + ksft_print_msg("Scan time limit exceeded\n"); return 1; } } @@ -218,12 +218,12 @@ static int ksm_merge_pages(int merge_type, void *addr, size_t size, { if (merge_type == KSM_MERGE_MADVISE) { if (madvise(addr, size, MADV_MERGEABLE)) { - perror("madvise"); + ksft_perror("madvise"); return 1; } } else if (merge_type == KSM_MERGE_PRCTL) { if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) { - perror("prctl"); + ksft_perror("prctl"); return 1; } } @@ -242,7 +242,7 @@ static int ksm_unmerge_pages(void *addr, size_t size, struct timespec start_time, int timeout) { if (madvise(addr, size, MADV_UNMERGEABLE)) { - perror("madvise"); + ksft_perror("madvise"); return 1; } return 0; @@ -324,7 +324,7 @@ static int check_ksm_merge(int merge_type, int mapping, int prot, struct timespec start_time; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } @@ -338,7 +338,6 @@ static int check_ksm_merge(int merge_type, int mapping, int prot, /* verify that the right number of pages are merged */ if (assert_ksm_pages_count(page_count)) { - printf("OK\n"); munmap(map_ptr, page_size * page_count); if (merge_type == KSM_MERGE_PRCTL) prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); @@ -346,7 +345,6 @@ static int check_ksm_merge(int merge_type, int mapping, int prot, } err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -358,7 +356,7 @@ static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, int page_count = 2; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } @@ -380,13 +378,11 @@ static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, /* check that unmerging was successful and 0 pages are currently merged */ if (assert_ksm_pages_count(0)) { - printf("OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_PASS; } err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -398,7 +394,7 @@ static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long struct timespec start_time; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } @@ -425,12 +421,10 @@ static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long else if (!use_zero_pages && !assert_ksm_pages_count(page_count)) goto err_out; - printf("OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -465,16 +459,16 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo int first_node; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } if (numa_available() < 0) { - perror("NUMA support not enabled"); + ksft_print_msg("NUMA support not enabled\n"); return KSFT_SKIP; } if (numa_num_configured_nodes() <= 1) { - printf("At least 2 NUMA nodes must be available\n"); + ksft_print_msg("At least 2 NUMA nodes must be available\n"); return KSFT_SKIP; } if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) @@ -485,7 +479,7 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo numa1_map_ptr = numa_alloc_onnode(page_size, first_node); numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); if (!numa1_map_ptr || !numa2_map_ptr) { - perror("numa_alloc_onnode"); + ksft_perror("numa_alloc_onnode"); return KSFT_FAIL; } @@ -510,13 +504,11 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo numa_free(numa1_map_ptr, page_size); numa_free(numa2_map_ptr, page_size); - printf("OK\n"); return KSFT_PASS; err_out: numa_free(numa1_map_ptr, page_size); numa_free(numa2_map_ptr, page_size); - printf("Not OK\n"); return KSFT_FAIL; } @@ -529,7 +521,7 @@ static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, int pagemap_fd, n_normal_pages, n_huge_pages; if (!thp_is_enabled()) { - printf("Transparent Hugepages not available\n"); + ksft_print_msg("Transparent Hugepages not available\n"); return KSFT_SKIP; } @@ -559,36 +551,35 @@ static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, else n_huge_pages++; } - printf("Number of normal pages: %d\n", n_normal_pages); - printf("Number of huge pages: %d\n", n_huge_pages); + ksft_print_msg("Number of normal pages: %d\n", n_normal_pages); + ksft_print_msg("Number of huge pages: %d\n", n_huge_pages); memset(map_ptr, '*', len); if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n", map_size / MB); - printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n", map_size / MB); + ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, scan_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", (map_size / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / ((double)scan_time_ns / NSEC_PER_SEC)); munmap(map_ptr_orig, len + HPAGE_SIZE); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr_orig, len + HPAGE_SIZE); return KSFT_FAIL; } @@ -606,30 +597,29 @@ static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, si return KSFT_FAIL; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n", map_size / MB); - printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n", map_size / MB); + ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, scan_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", (map_size / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / ((double)scan_time_ns / NSEC_PER_SEC)); munmap(map_ptr, map_size); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, map_size); return KSFT_FAIL; } @@ -646,37 +636,36 @@ static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, if (!map_ptr) return KSFT_FAIL; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n", map_size / MB); - printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n", map_size / MB); + ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, scan_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", (map_size / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / ((double)scan_time_ns / NSEC_PER_SEC)); munmap(map_ptr, map_size); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, map_size); return KSFT_FAIL; } @@ -695,24 +684,24 @@ static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size return KSFT_FAIL; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } for (size_t i = 0; i < page_count - 1; i = i + 2) memset(map_ptr + page_size * i, '-', 1); if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n\n", (page_size * page_count) / MB); - printf("Not merged pages:\n"); - printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n\n", (page_size * page_count) / MB); + ksft_print_msg("Not merged pages:\n"); + ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, cow_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / ((double)cow_time_ns / NSEC_PER_SEC)); /* Create 2000 pairs of duplicate pages */ @@ -724,30 +713,29 @@ static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } for (size_t i = 0; i < page_count - 1; i = i + 2) memset(map_ptr + page_size * i, '-', 1); if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Merged pages:\n"); - printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, + ksft_print_msg("Merged pages:\n"); + ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, cow_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / ((double)cow_time_ns / NSEC_PER_SEC)); munmap(map_ptr, page_size * page_count); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -765,6 +753,10 @@ int main(int argc, char *argv[]) bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT; bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT; long size_MB = 0; + const char *test_descr = "KSM merging"; + + ksft_print_header(); + ksft_set_plan(1); while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) { switch (opt) { @@ -773,17 +765,13 @@ int main(int argc, char *argv[]) break; case 'p': page_count = atol(optarg); - if (page_count <= 0) { - printf("The number of pages must be greater than 0\n"); - return KSFT_FAIL; - } + if (page_count <= 0) + ksft_exit_fail_msg("The number of pages must be greater than 0\n"); break; case 'l': ksm_scan_limit_sec = atoi(optarg); - if (ksm_scan_limit_sec <= 0) { - printf("Timeout value must be greater than 0\n"); - return KSFT_FAIL; - } + if (ksm_scan_limit_sec <= 0) + ksft_exit_fail_msg("Timeout value must be greater than 0\n"); break; case 'h': print_help(); @@ -805,19 +793,15 @@ int main(int argc, char *argv[]) break; case 's': size_MB = atoi(optarg); - if (size_MB <= 0) { - printf("Size must be greater than 0\n"); - return KSFT_FAIL; - } + if (size_MB <= 0) + ksft_exit_fail_msg("Size must be greater than 0\n"); break; case 't': { int tmp = atoi(optarg); - if (tmp < 0 || tmp > KSM_MERGE_LAST) { - printf("Invalid merge type\n"); - return KSFT_FAIL; - } + if (tmp < 0 || tmp > KSM_MERGE_LAST) + ksft_exit_fail_msg("Invalid merge type\n"); merge_type = tmp; } break; @@ -845,82 +829,80 @@ int main(int argc, char *argv[]) test_name = KSM_COW_TIME; break; default: - return KSFT_FAIL; + ksft_exit_fail_msg("Unknown option\n"); } } if (prot == 0) prot = str_to_prot(KSM_PROT_STR_DEFAULT); - if (access(KSM_SYSFS_PATH, F_OK)) { - printf("Config KSM not enabled\n"); - return KSFT_SKIP; - } + if (access(KSM_SYSFS_PATH, F_OK)) + ksft_exit_skip("Config KSM not enabled\n"); - if (ksm_save_def(&ksm_sysfs_old)) { - printf("Cannot save default tunables\n"); - return KSFT_FAIL; - } + if (ksm_save_def(&ksm_sysfs_old)) + ksft_exit_fail_msg("Cannot save default tunables\n"); if (ksm_write_sysfs(KSM_FP("run"), 2) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || numa_available() ? 0 : ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) - return KSFT_FAIL; + ksft_exit_fail_msg("Cannot set up KSM tunables\n"); switch (test_name) { case CHECK_KSM_MERGE: + test_descr = "KSM merging"; ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, ksm_scan_limit_sec, page_size); break; case CHECK_KSM_UNMERGE: + test_descr = "KSM unmerging"; ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, page_size); break; case CHECK_KSM_ZERO_PAGE_MERGE: + test_descr = "KSM zero page merging"; ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, ksm_scan_limit_sec, use_zero_pages, page_size); break; case CHECK_KSM_NUMA_MERGE: + test_descr = "KSM NUMA merging"; ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, merge_across_nodes, page_size); break; case KSM_MERGE_TIME: - if (size_MB == 0) { - printf("Option '-s' is required.\n"); - return KSFT_FAIL; - } + if (size_MB == 0) + ksft_exit_fail_msg("Option '-s' is required\n"); + test_descr = "KSM merge time"; ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, size_MB); break; case KSM_MERGE_TIME_HUGE_PAGES: - if (size_MB == 0) { - printf("Option '-s' is required.\n"); - return KSFT_FAIL; - } + if (size_MB == 0) + ksft_exit_fail_msg("Option '-s' is required\n"); + test_descr = "KSM merge time with huge pages"; ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, size_MB); break; case KSM_UNMERGE_TIME: - if (size_MB == 0) { - printf("Option '-s' is required.\n"); - return KSFT_FAIL; - } + if (size_MB == 0) + ksft_exit_fail_msg("Option '-s' is required\n"); + test_descr = "KSM unmerge time"; ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, size_MB); break; case KSM_COW_TIME: + test_descr = "KSM COW time"; ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, page_size); break; } - if (ksm_restore(&ksm_sysfs_old)) { - printf("Cannot restore default tunables\n"); - return KSFT_FAIL; - } + if (ksm_restore(&ksm_sysfs_old)) + ksft_print_msg("Cannot restore default tunables\n"); + + ksft_test_result_report(ret, "%s\n", test_descr); - return ret; + ksft_finished(); } diff --git a/tools/testing/selftests/mm/madv_populate.c b/tools/testing/selftests/mm/madv_populate.c index 88050e0f829a..7fce5d0b622b 100644 --- a/tools/testing/selftests/mm/madv_populate.c +++ b/tools/testing/selftests/mm/madv_populate.c @@ -34,7 +34,7 @@ static void sense_support(void) addr = mmap(0, pagesize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - if (!addr) + if (addr == MAP_FAILED) ksft_exit_fail_msg("mmap failed\n"); ret = madvise(addr, pagesize, MADV_POPULATE_READ); diff --git a/tools/testing/selftests/mm/map_hugetlb.c b/tools/testing/selftests/mm/map_hugetlb.c deleted file mode 100644 index aa409107611b..000000000000 --- a/tools/testing/selftests/mm/map_hugetlb.c +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Example of using hugepage memory in a user application using the mmap - * system call with MAP_HUGETLB flag. Before running this program make - * sure the administrator has allocated enough default sized huge pages - * to cover the 256 MB allocation. - */ -#include <stdlib.h> -#include <stdio.h> -#include <unistd.h> -#include <sys/mman.h> -#include <fcntl.h> -#include "vm_util.h" -#include "kselftest.h" - -#define LENGTH (256UL*1024*1024) -#define PROTECTION (PROT_READ | PROT_WRITE) - -static void check_bytes(char *addr) -{ - ksft_print_msg("First hex is %x\n", *((unsigned int *)addr)); -} - -static void write_bytes(char *addr, size_t length) -{ - unsigned long i; - - for (i = 0; i < length; i++) - *(addr + i) = (char)i; -} - -static void read_bytes(char *addr, size_t length) -{ - unsigned long i; - - check_bytes(addr); - for (i = 0; i < length; i++) - if (*(addr + i) != (char)i) - ksft_exit_fail_msg("Mismatch at %lu\n", i); - - ksft_test_result_pass("Read correct data\n"); -} - -int main(int argc, char **argv) -{ - void *addr; - size_t hugepage_size; - size_t length = LENGTH; - int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB; - int shift = 0; - - hugepage_size = default_huge_page_size(); - /* munmap with fail if the length is not page aligned */ - if (hugepage_size > length) - length = hugepage_size; - - ksft_print_header(); - ksft_set_plan(1); - - if (argc > 1) - length = atol(argv[1]) << 20; - if (argc > 2) { - shift = atoi(argv[2]); - if (shift) - flags |= (shift & MAP_HUGE_MASK) << MAP_HUGE_SHIFT; - } - - if (shift) - ksft_print_msg("%u kB hugepages\n", 1 << (shift - 10)); - else - ksft_print_msg("Default size hugepages\n"); - ksft_print_msg("Mapping %lu Mbytes\n", (unsigned long)length >> 20); - - addr = mmap(NULL, length, PROTECTION, flags, -1, 0); - if (addr == MAP_FAILED) - ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); - - ksft_print_msg("Returned address is %p\n", addr); - check_bytes(addr); - write_bytes(addr, length); - read_bytes(addr, length); - - /* munmap() length of MAP_HUGETLB memory must be hugepage aligned */ - if (munmap(addr, length)) - ksft_exit_fail_msg("munmap: %s\n", strerror(errno)); - - ksft_finished(); -} diff --git a/tools/testing/selftests/mm/migration.c b/tools/testing/selftests/mm/migration.c index 60e78bbfc0e3..29f7492453d4 100644 --- a/tools/testing/selftests/mm/migration.c +++ b/tools/testing/selftests/mm/migration.c @@ -5,7 +5,7 @@ */ #include "kselftest_harness.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #include <strings.h> #include <pthread.h> @@ -23,6 +23,8 @@ #define MAX_RETRIES 100 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) +HUGETLB_SETUP_DEFAULT_PAGES(1) + FIXTURE(migration) { pthread_t *threads; @@ -32,13 +34,26 @@ FIXTURE(migration) int n2; }; +static void reset_signals(void) +{ + struct sigaction sa = { .sa_handler = SIG_DFL }; + + sigemptyset(&sa.sa_mask); + sigaction(SIGTERM, &sa, NULL); + sigaction(SIGHUP, &sa, NULL); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGQUIT, &sa, NULL); +} + FIXTURE_SETUP(migration) { int n; + reset_signals(); + if (numa_available() < 0) SKIP(return, "NUMA not available"); - self->nthreads = numa_num_task_cpus() - 1; + self->nthreads = numa_num_task_cpus() - 2; self->n1 = -1; self->n2 = -1; @@ -52,6 +67,9 @@ FIXTURE_SETUP(migration) } } + if (self->nthreads < 1 || self->n1 < 0 || self->n2 < 0) + SKIP(return, "Not enough threads or NUMA nodes available"); + self->threads = malloc(self->nthreads * sizeof(*self->threads)); ASSERT_NE(self->threads, NULL); self->pids = malloc(self->nthreads * sizeof(*self->pids)); @@ -64,6 +82,29 @@ FIXTURE_TEARDOWN(migration) free(self->pids); } +static bool kill_children(FIXTURE_DATA(migration) * self) +{ + bool err = false; + pid_t pid; + int i; + + for (i = 0; i < self->nthreads; i++) { + int status = 0; + + pid = self->pids[i]; + if (pid < 0) + continue; + if (kill(pid, SIGTERM)) + err = true; + if (pid != waitpid(pid, &status, 0)) + err = true; + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGTERM) + err = true; + } + + return !err; +} + int migrate(uint64_t *ptr, int n1, int n2) { int ret, tmp; @@ -127,20 +168,17 @@ TEST_F_TIMEOUT(migration, private_anon, 2*RUNTIME) uint64_t *ptr; int i; - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); - ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) + for (i = 0; i < self->nthreads; i++) if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) perror("Couldn't create thread"); ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) + for (i = 0; i < self->nthreads; i++) ASSERT_EQ(pthread_cancel(self->threads[i]), 0); } @@ -151,17 +189,14 @@ TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME) { pid_t pid; uint64_t *ptr; - int i; - - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); + int i, err; ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) { + for (i = 0; i < self->nthreads; i++) { pid = fork(); if (!pid) { prctl(PR_SET_PDEATHSIG, SIGHUP); @@ -174,9 +209,9 @@ TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME) } } - ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) - ASSERT_EQ(kill(self->pids[i], SIGTERM), 0); + err = migrate(ptr, self->n1, self->n2); + ASSERT_EQ(kill_children(self), true); + ASSERT_EQ(err, 0); } /* @@ -184,28 +219,30 @@ TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME) */ TEST_F_TIMEOUT(migration, private_anon_thp, 2*RUNTIME) { + uint64_t pmdsize; uint64_t *ptr; int i; if (!thp_is_enabled()) SKIP(return, "Transparent Hugepages not available"); - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); + pmdsize = read_pmd_pagesize(); + if (!pmdsize) + SKIP(return, "Reading PMD pagesize failed"); - ptr = mmap(NULL, 2*TWOMEG, PROT_READ | PROT_WRITE, + ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); - ptr = (uint64_t *) ALIGN((uintptr_t) ptr, TWOMEG); - ASSERT_EQ(madvise(ptr, TWOMEG, MADV_HUGEPAGE), 0); - memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) + ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize); + ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0); + memset(ptr, 0xde, pmdsize); + for (i = 0; i < self->nthreads; i++) if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) perror("Couldn't create thread"); ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) + for (i = 0; i < self->nthreads; i++) ASSERT_EQ(pthread_cancel(self->threads[i]), 0); } @@ -215,25 +252,27 @@ TEST_F_TIMEOUT(migration, private_anon_thp, 2*RUNTIME) TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME) { + uint64_t pmdsize; pid_t pid; uint64_t *ptr; - int i; + int i, err; if (!thp_is_enabled()) SKIP(return, "Transparent Hugepages not available"); - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); + pmdsize = read_pmd_pagesize(); + if (!pmdsize) + SKIP(return, "Reading PMD pagesize failed"); - ptr = mmap(NULL, 2 * TWOMEG, PROT_READ | PROT_WRITE, + ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); - ptr = (uint64_t *) ALIGN((uintptr_t) ptr, TWOMEG); - ASSERT_EQ(madvise(ptr, TWOMEG, MADV_HUGEPAGE), 0); + ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize); + ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0); - memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) { + memset(ptr, 0xde, pmdsize); + for (i = 0; i < self->nthreads; i++) { pid = fork(); if (!pid) { prctl(PR_SET_PDEATHSIG, SIGHUP); @@ -246,9 +285,9 @@ TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME) } } - ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) - ASSERT_EQ(kill(self->pids[i], SIGTERM), 0); + err = migrate(ptr, self->n1, self->n2); + ASSERT_EQ(kill_children(self), true); + ASSERT_EQ(err, 0); } /* @@ -256,23 +295,28 @@ TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME) */ TEST_F_TIMEOUT(migration, private_anon_htlb, 2*RUNTIME) { + unsigned long hugepage_size; uint64_t *ptr; int i; - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); + hugepage_size = default_huge_page_size(); + if (!hugepage_size) + SKIP(return, "Reading HugeTLB pagesize failed"); - ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, + if (hugetlb_free_default_pages() < 1) + SKIP(return, "Not enough huge pages"); + + ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); ASSERT_NE(ptr, MAP_FAILED); - memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) + memset(ptr, 0xde, hugepage_size); + for (i = 0; i < self->nthreads; i++) if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) perror("Couldn't create thread"); ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) + for (i = 0; i < self->nthreads; i++) ASSERT_EQ(pthread_cancel(self->threads[i]), 0); } @@ -281,19 +325,24 @@ TEST_F_TIMEOUT(migration, private_anon_htlb, 2*RUNTIME) */ TEST_F_TIMEOUT(migration, shared_anon_htlb, 2*RUNTIME) { + unsigned long hugepage_size; pid_t pid; uint64_t *ptr; - int i; + int i, err; - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); + hugepage_size = default_huge_page_size(); + if (!hugepage_size) + SKIP(return, "Reading HugeTLB pagesize failed"); - ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, + if (hugetlb_free_default_pages() < 1) + SKIP(return, "Not enough huge pages"); + + ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); ASSERT_NE(ptr, MAP_FAILED); - memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) { + memset(ptr, 0xde, hugepage_size); + for (i = 0; i < self->nthreads; i++) { pid = fork(); if (!pid) { prctl(PR_SET_PDEATHSIG, SIGHUP); @@ -306,9 +355,9 @@ TEST_F_TIMEOUT(migration, shared_anon_htlb, 2*RUNTIME) } } - ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) - ASSERT_EQ(kill(self->pids[i], SIGTERM), 0); + err = migrate(ptr, self->n1, self->n2); + ASSERT_EQ(kill_children(self), true); + ASSERT_EQ(err, 0); } TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/mm/mlock2-tests.c b/tools/testing/selftests/mm/mlock2-tests.c index b474f2b20def..e16e288cc7c1 100644 --- a/tools/testing/selftests/mm/mlock2-tests.c +++ b/tools/testing/selftests/mm/mlock2-tests.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #define _GNU_SOURCE #include <sys/mman.h> +#include <linux/mman.h> #include <stdint.h> #include <unistd.h> #include <string.h> @@ -163,14 +164,17 @@ static int lock_check(unsigned long addr) return (vma_rss == vma_size); } -static int unlock_lock_check(char *map) +static int unlock_lock_check(char *map, bool mlock_supported) { - if (is_vmflag_set((unsigned long)map, LOCKED)) { + if (!is_vmflag_set((unsigned long)map, LOCKED)) + return 0; + + if (mlock_supported) ksft_print_msg("VMA flag %s is present on page 1 after unlock\n", LOCKED); - return 1; - } + else + ksft_print_msg("VMA flag %s is present on an unsupported VMA\n", LOCKED); - return 0; + return 1; } static void test_mlock_lock(void) @@ -196,7 +200,7 @@ static void test_mlock_lock(void) ksft_exit_fail_msg("munlock(): %s\n", strerror(errno)); } - ksft_test_result(!unlock_lock_check(map), "%s: Unlocked\n", __func__); + ksft_test_result(!unlock_lock_check(map, true), "%s: Unlocked\n", __func__); munmap(map, 2 * page_size); } @@ -296,7 +300,7 @@ static void test_munlockall0(void) ksft_exit_fail_msg("munlockall(): %s\n", strerror(errno)); } - ksft_test_result(!unlock_lock_check(map), "%s: No locked memory\n", __func__); + ksft_test_result(!unlock_lock_check(map, true), "%s: No locked memory\n", __func__); munmap(map, 2 * page_size); } @@ -336,7 +340,67 @@ static void test_munlockall1(void) ksft_exit_fail_msg("munlockall() %s\n", strerror(errno)); } - ksft_test_result(!unlock_lock_check(map), "%s: No locked memory\n", __func__); + ksft_test_result(!unlock_lock_check(map, true), "%s: No locked memory\n", __func__); + munmap(map, 2 * page_size); +} + +/* Droppable memory should not be lockable. */ +static void test_mlock_droppable(void) +{ + char *map; + unsigned long page_size = getpagesize(); + + /* Ensure MCL_FUTURE is not set. */ + if (munlockall()) { + ksft_test_result_fail("munlockall() %s\n", strerror(errno)); + return; + } + + map = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_DROPPABLE, -1, 0); + if (map == MAP_FAILED) { + if ((errno == EOPNOTSUPP) || (errno == EINVAL)) + ksft_test_result_skip("%s: MAP_DROPPABLE not supported\n", __func__); + else + ksft_test_result_fail("mmap error: %s\n", strerror(errno)); + return; + } + + if (mlock2_(map, 2 * page_size, 0)) + ksft_test_result_fail("mlock2(0): %s\n", strerror(errno)); + else + ksft_test_result(!unlock_lock_check(map, false), + "%s: droppable memory not locked\n", __func__); + + munmap(map, 2 * page_size); +} + +static void test_mlockall_future_droppable(void) +{ + char *map; + unsigned long page_size = getpagesize(); + + if (mlockall(MCL_CURRENT | MCL_FUTURE)) { + ksft_test_result_fail("mlockall(MCL_CURRENT | MCL_FUTURE): %s\n", strerror(errno)); + return; + } + + map = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_DROPPABLE, -1, 0); + + if (map == MAP_FAILED) { + if ((errno == EOPNOTSUPP) || (errno == EINVAL)) + ksft_test_result_skip("%s: MAP_DROPPABLE not supported\n", __func__); + else + ksft_test_result_fail("mmap error: %s\n", strerror(errno)); + munlockall(); + return; + } + + ksft_test_result(!unlock_lock_check(map, false), "%s: droppable memory not locked\n", + __func__); + + munlockall(); munmap(map, 2 * page_size); } @@ -442,7 +506,7 @@ int main(int argc, char **argv) munmap(map, size); - ksft_set_plan(13); + ksft_set_plan(15); test_mlock_lock(); test_mlock_onfault(); @@ -451,6 +515,8 @@ int main(int argc, char **argv) test_lock_onfault_of_present(); test_vma_management(true); test_mlockall(); + test_mlock_droppable(); + test_mlockall_future_droppable(); ksft_finished(); } diff --git a/tools/testing/selftests/mm/mremap_test.c b/tools/testing/selftests/mm/mremap_test.c index 308576437228..131d9d6db867 100644 --- a/tools/testing/selftests/mm/mremap_test.c +++ b/tools/testing/selftests/mm/mremap_test.c @@ -76,27 +76,6 @@ enum { .expect_failure = should_fail \ } -/* compute square root using binary search */ -static unsigned long get_sqrt(unsigned long val) -{ - unsigned long low = 1; - - /* assuming rand_size is less than 1TB */ - unsigned long high = (1UL << 20); - - while (low <= high) { - unsigned long mid = low + (high - low) / 2; - unsigned long temp = mid * mid; - - if (temp == val) - return mid; - if (temp < val) - low = mid + 1; - high = mid - 1; - } - return low; -} - /* * Returns false if the requested remap region overlaps with an * existing mapping (e.g text, stack) else returns true. @@ -995,11 +974,9 @@ static long long remap_region(struct config c, unsigned int threshold_mb, char *rand_addr) { void *addr, *tmp_addr, *src_addr, *dest_addr, *dest_preamble_addr = NULL; - unsigned long long t, d; struct timespec t_start = {0, 0}, t_end = {0, 0}; long long start_ns, end_ns, align_mask, ret, offset; unsigned long long threshold; - unsigned long num_chunks; if (threshold_mb == VALIDATION_NO_THRESHOLD) threshold = c.region_size; @@ -1068,87 +1045,21 @@ static long long remap_region(struct config c, unsigned int threshold_mb, goto clean_up_dest_preamble; } - /* - * Verify byte pattern after remapping. Employ an algorithm with a - * square root time complexity in threshold: divide the range into - * chunks, if memcmp() returns non-zero, only then perform an - * iteration in that chunk to find the mismatch index. - */ - num_chunks = get_sqrt(threshold); - for (unsigned long i = 0; i < num_chunks; ++i) { - size_t chunk_size = threshold / num_chunks; - unsigned long shift = i * chunk_size; - - if (!memcmp(dest_addr + shift, rand_addr + shift, chunk_size)) - continue; - - /* brute force iteration only over mismatch segment */ - for (t = shift; t < shift + chunk_size; ++t) { - if (((char *) dest_addr)[t] != rand_addr[t]) { - ksft_print_msg("Data after remap doesn't match at offset %llu\n", - t); - ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff, - ((char *) dest_addr)[t] & 0xff); - ret = -1; - goto clean_up_dest; - } - } - } - - /* - * if threshold is not divisible by num_chunks, then check the - * last chunk - */ - for (t = num_chunks * (threshold / num_chunks); t < threshold; ++t) { - if (((char *) dest_addr)[t] != rand_addr[t]) { - ksft_print_msg("Data after remap doesn't match at offset %llu\n", - t); - ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff, - ((char *) dest_addr)[t] & 0xff); - ret = -1; - goto clean_up_dest; - } + /* Verify byte pattern after remapping */ + if (memcmp(dest_addr, rand_addr, threshold)) { + ksft_print_msg("Data after remap doesn't match\n"); + ret = -1; + goto clean_up_dest; } /* Verify the dest preamble byte pattern after remapping */ - if (!c.dest_preamble_size) - goto no_preamble; - - num_chunks = get_sqrt(c.dest_preamble_size); - - for (unsigned long i = 0; i < num_chunks; ++i) { - size_t chunk_size = c.dest_preamble_size / num_chunks; - unsigned long shift = i * chunk_size; - - if (!memcmp(dest_preamble_addr + shift, rand_addr + shift, - chunk_size)) - continue; - - /* brute force iteration only over mismatched segment */ - for (d = shift; d < shift + chunk_size; ++d) { - if (((char *) dest_preamble_addr)[d] != rand_addr[d]) { - ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n", - d); - ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff, - ((char *) dest_preamble_addr)[d] & 0xff); - ret = -1; - goto clean_up_dest; - } - } - } - - for (d = num_chunks * (c.dest_preamble_size / num_chunks); d < c.dest_preamble_size; ++d) { - if (((char *) dest_preamble_addr)[d] != rand_addr[d]) { - ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n", - d); - ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff, - ((char *) dest_preamble_addr)[d] & 0xff); - ret = -1; - goto clean_up_dest; - } + if (c.dest_preamble_size && + memcmp(dest_preamble_addr, rand_addr, c.dest_preamble_size)) { + ksft_print_msg("Preamble data after remap doesn't match\n"); + ret = -1; + goto clean_up_dest; } -no_preamble: start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec; end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec; ret = end_ns - start_ns; diff --git a/tools/testing/selftests/mm/page_frag/page_frag_test.c b/tools/testing/selftests/mm/page_frag/page_frag_test.c index e806c1866e36..c8584d0fdeab 100644 --- a/tools/testing/selftests/mm/page_frag/page_frag_test.c +++ b/tools/testing/selftests/mm/page_frag/page_frag_test.c @@ -131,6 +131,8 @@ static int __init page_frag_test_init(void) init_completion(&wait); if (test_alloc_len > PAGE_SIZE || test_alloc_len <= 0 || + test_push_cpu < 0 || test_push_cpu >= nr_cpu_ids || + test_pop_cpu < 0 || test_pop_cpu >= nr_cpu_ids || !cpu_active(test_push_cpu) || !cpu_active(test_pop_cpu)) return -EINVAL; diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index 7f9428d6062c..6f8971d5b3ce 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -7,8 +7,6 @@ #include <sys/mman.h> #include <errno.h> #include <malloc.h> -#include "vm_util.h" -#include "kselftest.h" #include <linux/types.h> #include <linux/memfd.h> #include <linux/userfaultfd.h> @@ -23,6 +21,10 @@ #include <sys/ipc.h> #include <sys/shm.h> +#include "vm_util.h" +#include "kselftest.h" +#include "hugepage_settings.h" + #define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ PAGE_IS_FILE | PAGE_IS_PRESENT | \ PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ @@ -1366,7 +1368,7 @@ void *thread_proc(void *mem) ksft_exit_fail_msg("pthread_barrier_wait\n"); for (i = 0; i < access_per_thread; ++i) - __atomic_add_fetch(m + i * (0x1000 / sizeof(*m)), 1, __ATOMIC_SEQ_CST); + __atomic_add_fetch(m + i * (page_size / sizeof(*m)), 1, __ATOMIC_SEQ_CST); ret = pthread_barrier_wait(&end_barrier); if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) @@ -1401,15 +1403,15 @@ static void transact_test(int page_size) if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1)) ksft_exit_fail_msg("pthread_barrier_init\n"); - mem = mmap(NULL, 0x1000 * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, + mem = mmap(NULL, page_size * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (mem == MAP_FAILED) ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno)); - wp_init(mem, 0x1000 * nthreads * pages_per_thread); - wp_addr_range(mem, 0x1000 * nthreads * pages_per_thread); + wp_init(mem, page_size * nthreads * pages_per_thread); + wp_addr_range(mem, page_size * nthreads * pages_per_thread); - memset(mem, 0, 0x1000 * nthreads * pages_per_thread); + memset(mem, 0, page_size * nthreads * pages_per_thread); count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); ksft_test_result(count > 0, "%s count %u\n", __func__, count); @@ -1418,7 +1420,7 @@ static void transact_test(int page_size) finish = 0; for (i = 0; i < nthreads; ++i) - pthread_create(&th, NULL, thread_proc, mem + 0x1000 * i * pages_per_thread); + pthread_create(&th, NULL, thread_proc, mem + page_size * i * pages_per_thread); extra_pages = 0; for (i = 0; i < iter_count; ++i) { @@ -1554,6 +1556,9 @@ int main(int __attribute__((unused)) argc, char *argv[]) if (init_uffd()) ksft_exit_skip("Failed to initialize userfaultfd\n"); + if (!hugetlb_setup_default(4)) + ksft_print_msg("HugeTLB test will be skipped\n"); + ksft_set_plan(117); page_size = getpagesize(); @@ -1605,7 +1610,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) } /* 5. SHM Hugetlb page testing */ - mem_size = 2*1024*1024; + mem_size = default_huge_page_size(); mem = gethugetlb_mem(mem_size, &shmid); if (mem) { wp_init(mem, mem_size); @@ -1633,7 +1638,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) } /* 7. File Hugetlb testing */ - mem_size = 2*1024*1024; + mem_size = default_huge_page_size(); fd = memfd_create("uffd-test", MFD_HUGETLB | MFD_NOEXEC_SEAL); if (fd < 0) ksft_exit_fail_msg("uffd-test creation failed %d %s\n", errno, strerror(errno)); diff --git a/tools/testing/selftests/mm/pkey-helpers.h b/tools/testing/selftests/mm/pkey-helpers.h index 7c29f075e40b..2c377f4e9df1 100644 --- a/tools/testing/selftests/mm/pkey-helpers.h +++ b/tools/testing/selftests/mm/pkey-helpers.h @@ -71,13 +71,14 @@ static inline void sigsafe_printf(const char *format, ...) extern void abort_hooks(void); #define pkey_assert(condition) do { \ if (!(condition)) { \ - dprintf0("assert() at %s::%d test_nr: %d iteration: %d\n", \ - __FILE__, __LINE__, \ - test_nr, iteration_nr); \ - dprintf0("errno at assert: %d", errno); \ - abort_hooks(); \ - exit(__LINE__); \ - } \ + dprintf0("# assert() at %s::%d test_nr: %d iteration: %d\n", \ + __FILE__, __LINE__, \ + test_nr, iteration_nr); \ + dprintf0("# errno at assert: %d\n", errno); \ + abort_hooks(); \ + ksft_exit_fail_msg("test %d (iteration %d)\n", \ + test_nr, iteration_nr); \ + } \ } while (0) #define barrier() __asm__ __volatile__("": : :"memory") diff --git a/tools/testing/selftests/mm/prctl_thp_disable.c b/tools/testing/selftests/mm/prctl_thp_disable.c index ca27200596a4..d8d9d1de57b8 100644 --- a/tools/testing/selftests/mm/prctl_thp_disable.c +++ b/tools/testing/selftests/mm/prctl_thp_disable.c @@ -14,7 +14,7 @@ #include <sys/wait.h> #include "kselftest_harness.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #include "vm_util.h" #ifndef PR_THP_DISABLE_EXCEPT_ADVISED diff --git a/tools/testing/selftests/mm/process_madv.c b/tools/testing/selftests/mm/process_madv.c index cd4610baf5d7..3fffd5f7e6fb 100644 --- a/tools/testing/selftests/mm/process_madv.c +++ b/tools/testing/selftests/mm/process_madv.c @@ -310,6 +310,34 @@ TEST_F(process_madvise, invalid_vlen) } /* + * Test that invalid advice is rejected even when the iovec has zero total + * length. A request with valid advice and zero length is a noop, but + * invalid advice should still fail with EINVAL. + */ +TEST_F(process_madvise, invalid_advice_zero_length) +{ + struct iovec vec = { + .iov_base = NULL, + .iov_len = 0, + }; + int pidfd = self->pidfd; + ssize_t ret; + + errno = 0; + ret = sys_process_madvise(pidfd, &vec, 1, -1, 0); + ASSERT_EQ(ret, -1); + ASSERT_EQ(errno, EINVAL); + + errno = 0; + ret = sys_process_madvise(pidfd, &vec, 1, MADV_DONTNEED, 0); + ASSERT_EQ(ret, 0); + + ret = sys_process_madvise(pidfd, NULL, 0, -1, 0); + ASSERT_EQ(ret, -1); + ASSERT_EQ(errno, EINVAL); +} + +/* * Test process_madvise() with an invalid flag value. Currently, only a flag * value of 0 is supported. This test is reserved for the future, e.g., if * synchronous flags are added. diff --git a/tools/testing/selftests/mm/protection_keys.c b/tools/testing/selftests/mm/protection_keys.c index 2085982dba69..9a6d954ee371 100644 --- a/tools/testing/selftests/mm/protection_keys.c +++ b/tools/testing/selftests/mm/protection_keys.c @@ -46,6 +46,7 @@ #include <sys/ptrace.h> #include <setjmp.h> +#include "hugepage_settings.h" #include "pkey-helpers.h" int iteration_nr = 1; @@ -61,6 +62,7 @@ noinline int read_ptr(int *ptr) return *ptr; } +#if CONTROL_TRACING > 0 static void cat_into_file(char *str, char *file) { int fd = open(file, O_RDWR); @@ -86,7 +88,6 @@ static void cat_into_file(char *str, char *file) close(fd); } -#if CONTROL_TRACING > 0 static int warned_tracing; static int tracing_root_ok(void) { @@ -136,6 +137,7 @@ static void tracing_off(void) void abort_hooks(void) { + fflush(stdout); fprintf(stderr, "running %s()...\n", __func__); tracing_off(); #ifdef SLEEP_ON_ABORT @@ -370,8 +372,8 @@ static void signal_handler(int signum, siginfo_t *si, void *vucontext) if ((si->si_code == SEGV_MAPERR) || (si->si_code == SEGV_ACCERR) || (si->si_code == SEGV_BNDERR)) { - printf("non-PK si_code, exiting...\n"); - exit(4); + dprintf0("# non-PK si_code: %d, exiting...\n", si->si_code); + exit(1); } si_pkey_ptr = siginfo_get_pkey_ptr(si); @@ -708,50 +710,28 @@ static void *malloc_pkey_anon_huge(long size, int prot, u16 pkey) } static int hugetlb_setup_ok; -#define SYSFS_FMT_NR_HUGE_PAGES "/sys/kernel/mm/hugepages/hugepages-%ldkB/nr_hugepages" #define GET_NR_HUGE_PAGES 10 static void setup_hugetlbfs(void) { - int err; - int fd; - char buf[256]; - long hpagesz_kb; - long hpagesz_mb; + long hpagesz_mb = HPAGE_SIZE / 1024 / 1024; + unsigned long free_pages; if (geteuid() != 0) { - fprintf(stderr, "WARNING: not run as root, can not do hugetlb test\n"); + ksft_print_msg("WARNING: not run as root, can not do hugetlb test\n"); return; } - cat_into_file(__stringify(GET_NR_HUGE_PAGES), "/proc/sys/vm/nr_hugepages"); - /* - * Now go make sure that we got the pages and that they + * Make sure that we got the pages and that they * are PMD-level pages. Someone might have made PUD-level * pages the default. */ - hpagesz_kb = HPAGE_SIZE / 1024; - hpagesz_mb = hpagesz_kb / 1024; - sprintf(buf, SYSFS_FMT_NR_HUGE_PAGES, hpagesz_kb); - fd = open(buf, O_RDONLY); - if (fd < 0) { - fprintf(stderr, "opening sysfs %ldM hugetlb config: %s\n", - hpagesz_mb, strerror(errno)); - return; - } - - /* -1 to guarantee leaving the trailing \0 */ - err = read(fd, buf, sizeof(buf)-1); - close(fd); - if (err <= 0) { - fprintf(stderr, "reading sysfs %ldM hugetlb config: %s\n", - hpagesz_mb, strerror(errno)); - return; - } - - if (atoi(buf) != GET_NR_HUGE_PAGES) { - fprintf(stderr, "could not confirm %ldM pages, got: '%s' expected %d\n", - hpagesz_mb, buf, GET_NR_HUGE_PAGES); + hugetlb_save_settings(); + hugetlb_set_nr_pages(HPAGE_SIZE, GET_NR_HUGE_PAGES); + free_pages = hugetlb_free_pages(HPAGE_SIZE); + if (free_pages < GET_NR_HUGE_PAGES) { + ksft_print_msg("could not confirm %ldM pages, got: '%lu' expected %d\n", + hpagesz_mb, free_pages, GET_NR_HUGE_PAGES); return; } @@ -855,7 +835,7 @@ void expected_pkey_fault(int pkey) #define do_not_expect_pkey_fault(msg) do { \ if (last_pkey_faults != pkey_faults) \ - dprintf0("unexpected PKey fault: %s\n", msg); \ + dprintf0("# unexpected PKey fault: %s\n", msg); \ pkey_assert(last_pkey_faults == pkey_faults); \ } while (0) @@ -1128,7 +1108,7 @@ static void become_child(void) /* in the child */ return; } - exit(0); + _exit(0); } /* Assumes that all pkeys other than 'pkey' are unallocated */ @@ -1507,18 +1487,18 @@ static void test_ptrace_modifies_pkru(int *ptr, u16 pkey) * checking */ if (__read_pkey_reg() != new_pkru) - exit(1); + _exit(1); /* Stop and allow the tracer to clear XSTATE_BV for PKRU */ raise(SIGSTOP); if (__read_pkey_reg() != 0) - exit(1); + _exit(1); /* Stop and allow the tracer to examine PKRU */ raise(SIGSTOP); - exit(0); + _exit(0); } pkey_assert(child == waitpid(child, &status, 0)); @@ -1692,29 +1672,36 @@ static void test_mprotect_pkey_on_unsupported_cpu(int *ptr, u16 pkey) pkey_assert(sret < 0); } -static void (*pkey_tests[])(int *ptr, u16 pkey) = { - test_read_of_write_disabled_region, - test_read_of_access_disabled_region, - test_read_of_access_disabled_region_with_page_already_mapped, - test_write_of_write_disabled_region, - test_write_of_write_disabled_region_with_page_already_mapped, - test_write_of_access_disabled_region, - test_write_of_access_disabled_region_with_page_already_mapped, - test_kernel_write_of_access_disabled_region, - test_kernel_write_of_write_disabled_region, - test_kernel_gup_of_access_disabled_region, - test_kernel_gup_write_to_write_disabled_region, - test_executing_on_unreadable_memory, - test_implicit_mprotect_exec_only_memory, - test_mprotect_with_pkey_0, - test_ptrace_of_child, - test_pkey_init_state, - test_pkey_syscalls_on_non_allocated_pkey, - test_pkey_syscalls_bad_args, - test_pkey_alloc_exhaust, - test_pkey_alloc_free_attach_pkey0, +struct pkey_test { + void (*func)(int *ptr, u16 pkey); + const char *name; +}; + +#define PKEY_TEST(fn) { fn, #fn } + +static struct pkey_test pkey_tests[] = { + PKEY_TEST(test_read_of_write_disabled_region), + PKEY_TEST(test_read_of_access_disabled_region), + PKEY_TEST(test_read_of_access_disabled_region_with_page_already_mapped), + PKEY_TEST(test_write_of_write_disabled_region), + PKEY_TEST(test_write_of_write_disabled_region_with_page_already_mapped), + PKEY_TEST(test_write_of_access_disabled_region), + PKEY_TEST(test_write_of_access_disabled_region_with_page_already_mapped), + PKEY_TEST(test_kernel_write_of_access_disabled_region), + PKEY_TEST(test_kernel_write_of_write_disabled_region), + PKEY_TEST(test_kernel_gup_of_access_disabled_region), + PKEY_TEST(test_kernel_gup_write_to_write_disabled_region), + PKEY_TEST(test_executing_on_unreadable_memory), + PKEY_TEST(test_implicit_mprotect_exec_only_memory), + PKEY_TEST(test_mprotect_with_pkey_0), + PKEY_TEST(test_ptrace_of_child), + PKEY_TEST(test_pkey_init_state), + PKEY_TEST(test_pkey_syscalls_on_non_allocated_pkey), + PKEY_TEST(test_pkey_syscalls_bad_args), + PKEY_TEST(test_pkey_alloc_exhaust), + PKEY_TEST(test_pkey_alloc_free_attach_pkey0), #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) - test_ptrace_modifies_pkru, + PKEY_TEST(test_ptrace_modifies_pkru), #endif }; @@ -1735,7 +1722,7 @@ static void run_tests_once(void) dprintf1("test %d starting with pkey: %d\n", test_nr, pkey); ptr = malloc_pkey(PAGE_SIZE, prot, pkey); dprintf1("test %d starting...\n", test_nr); - pkey_tests[test_nr](ptr, pkey); + pkey_tests[test_nr].func(ptr, pkey); dprintf1("freeing test memory: %p\n", ptr); free_pkey_malloc(ptr); sys_pkey_free(pkey); @@ -1746,7 +1733,7 @@ static void run_tests_once(void) tracing_off(); close_test_fds(); - printf("test %2d PASSED (iteration %d)\n", test_nr, iteration_nr); + ksft_test_result_pass("test %s (iteration %d)\n", pkey_tests[test_nr].name, iteration_nr); dprintf1("======================\n\n"); } iteration_nr++; @@ -1766,27 +1753,30 @@ int main(void) setup_handlers(); - printf("has pkeys: %d\n", pkeys_supported); + ksft_print_header(); if (!pkeys_supported) { int size = PAGE_SIZE; int *ptr; - printf("running PKEY tests for unsupported CPU/OS\n"); + ksft_set_plan(1); + ksft_print_msg("running PKEY tests for unsupported CPU/OS\n"); ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); assert(ptr != (void *)-1); test_mprotect_pkey_on_unsupported_cpu(ptr, 1); - exit(0); + ksft_test_result_pass("pkey on unsupported CPU/OS\n"); + ksft_finished(); } + ksft_set_plan(ARRAY_SIZE(pkey_tests) * nr_iterations); + pkey_setup_shadow(); - printf("startup pkey_reg: %016llx\n", read_pkey_reg()); + ksft_print_msg("startup pkey_reg: %016llx\n", read_pkey_reg()); setup_hugetlbfs(); while (nr_iterations-- > 0) run_tests_once(); - printf("done (all tests OK)\n"); - return 0; + ksft_finished(); } diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index c17b133a81d2..8c296dedf047 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -132,7 +132,7 @@ test_selected() { run_gup_matrix() { # -t: thp=on, -T: thp=off, -H: hugetlb=on - local hugetlb_mb=$(( needmem_KB / 1024 )) + local hugetlb_mb=256 for huge in -t -T "-H -m $hugetlb_mb"; do # -u: gup-fast, -U: gup-basic, -a: pin-fast, -b: pin-basic, -L: pin-longterm @@ -154,60 +154,6 @@ run_gup_matrix() { done } -# get huge pagesize and freepages from /proc/meminfo -while read -r name size unit; do - if [ "$name" = "HugePages_Free:" ]; then - freepgs="$size" - fi - if [ "$name" = "Hugepagesize:" ]; then - hpgsize_KB="$size" - fi -done < /proc/meminfo - -# Simple hugetlbfs tests have a hardcoded minimum requirement of -# huge pages totaling 256MB (262144KB) in size. The userfaultfd -# hugetlb test requires a minimum of 2 * nr_cpus huge pages. Take -# both of these requirements into account and attempt to increase -# number of huge pages available. -nr_cpus=$(nproc) -uffd_min_KB=$((hpgsize_KB * nr_cpus * 2)) -hugetlb_min_KB=$((256 * 1024)) -if [[ $uffd_min_KB -gt $hugetlb_min_KB ]]; then - needmem_KB=$uffd_min_KB -else - needmem_KB=$hugetlb_min_KB -fi - -# set proper nr_hugepages -if [ -n "$freepgs" ] && [ -n "$hpgsize_KB" ]; then - orig_nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) - needpgs=$((needmem_KB / hpgsize_KB)) - tries=2 - while [ "$tries" -gt 0 ] && [ "$freepgs" -lt "$needpgs" ]; do - lackpgs=$((needpgs - freepgs)) - echo 3 > /proc/sys/vm/drop_caches - if ! echo $((lackpgs + orig_nr_hugepgs)) > /proc/sys/vm/nr_hugepages; then - echo "Please run this test as root" - exit $ksft_skip - fi - while read -r name size unit; do - if [ "$name" = "HugePages_Free:" ]; then - freepgs=$size - fi - done < /proc/meminfo - tries=$((tries - 1)) - done - nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) - if [ "$freepgs" -lt "$needpgs" ]; then - printf "Not enough huge pages available (%d < %d)\n" \ - "$freepgs" "$needpgs" - fi - HAVE_HUGEPAGES=1 -else - echo "no hugetlbfs support in kernel?" - HAVE_HUGEPAGES=0 -fi - # filter 64bit architectures ARCH64STR="arm64 mips64 parisc64 ppc64 ppc64le riscv64 s390x sparc64 x86_64" if [ -z "$ARCH" ]; then @@ -235,32 +181,61 @@ pretty_name() { run_test() { if test_selected ${CATEGORY}; then local skip=0 + local LOADED_HWPOISON_INJECT_MOD=0 # On memory constrainted systems some tests can fail to allocate hugepages. # perform some cleanup before the test for a higher success rate. if [ ${CATEGORY} == "thp" -o ${CATEGORY} == "hugetlb" ]; then - if [ "${HAVE_HUGEPAGES}" = "1" ]; then + mem_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) + mem_Mb=$((mem_kb / 1024)) + + if (( $mem_Mb < 256 )); then echo 3 > /proc/sys/vm/drop_caches sleep 2 echo 1 > /proc/sys/vm/compact_memory sleep 2 - else - echo "hugepages not supported" | tap_prefix - skip=1 fi fi + # Ensure hwpoison_inject is available for memory-failure tests + if [ "${CATEGORY}" = "memory-failure" ]; then + # Try to load hwpoison_inject if not present. + HWPOISON_DIR=/sys/kernel/debug/hwpoison/ + if [ ! -d "$HWPOISON_DIR" ]; then + if ! modprobe -n hwpoison_inject > /dev/null 2>&1; then + echo "Module hwpoison_inject not found, skipping..." \ + | tap_prefix + skip=1 + else + modprobe hwpoison_inject > /dev/null 2>&1 + LOADED_HWPOISON_INJECT_MOD=1 + if [ ! -d "$HWPOISON_DIR" ]; then + echo "hwpoison debugfs interface not present" \ + | tap_prefix + skip=1 + fi + fi + fi + + fi + local test=$(pretty_name "$*") local title="running $*" local sep=$(echo -n "$title" | tr "[:graph:][:space:]" -) printf "%s\n%s\n%s\n" "$sep" "$title" "$sep" | tap_prefix - if [ "${skip}" != "1" ]; then + if [ $skip -eq 1 ]; then + local ret=$ksft_skip + else ("$@" 2>&1) | tap_prefix local ret=${PIPESTATUS[0]} - else - local ret=$ksft_skip fi + + # Unload hwpoison_inject if we loaded it + if [ "${LOADED_HWPOISON_INJECT_MOD}" = "1" ]; then + modprobe -r hwpoison_inject > /dev/null 2>&1 + fi + count_total=$(( count_total + 1 )) if [ $ret -eq 0 ]; then count_pass=$(( count_pass + 1 )) @@ -270,7 +245,9 @@ run_test() { count_skip=$(( count_skip + 1 )) echo "[SKIP]" | tap_prefix echo "ok ${count_total} ${test} # SKIP" | tap_output - exitcode=$ksft_skip + if [ $exitcode -eq 0 ]; then + exitcode=$ksft_skip + fi else count_fail=$(( count_fail + 1 )) echo "[FAIL]" | tap_prefix @@ -282,31 +259,14 @@ run_test() { echo "TAP version 13" | tap_output -CATEGORY="hugetlb" run_test ./hugepage-mmap - -shmmax=$(cat /proc/sys/kernel/shmmax) -shmall=$(cat /proc/sys/kernel/shmall) -echo 268435456 > /proc/sys/kernel/shmmax -echo 4194304 > /proc/sys/kernel/shmall -CATEGORY="hugetlb" run_test ./hugepage-shm -echo "$shmmax" > /proc/sys/kernel/shmmax -echo "$shmall" > /proc/sys/kernel/shmall - -CATEGORY="hugetlb" run_test ./map_hugetlb -CATEGORY="hugetlb" run_test ./hugepage-mremap -CATEGORY="hugetlb" run_test ./hugepage-vmemmap +CATEGORY="hugetlb" run_test ./hugetlb-mmap +CATEGORY="hugetlb" run_test ./hugetlb-shm +CATEGORY="hugetlb" run_test ./hugetlb-mremap +CATEGORY="hugetlb" run_test ./hugetlb-vmemmap CATEGORY="hugetlb" run_test ./hugetlb-madvise CATEGORY="hugetlb" run_test ./hugetlb_dio - -if [ "${HAVE_HUGEPAGES}" = "1" ]; then - nr_hugepages_tmp=$(cat /proc/sys/vm/nr_hugepages) - # For this test, we need one and just one huge page - echo 1 > /proc/sys/vm/nr_hugepages - CATEGORY="hugetlb" run_test ./hugetlb_fault_after_madv - CATEGORY="hugetlb" run_test ./hugetlb_madv_vs_map - # Restore the previous number of huge pages, since further tests rely on it - echo "$nr_hugepages_tmp" > /proc/sys/vm/nr_hugepages -fi +CATEGORY="hugetlb" run_test ./hugetlb_fault_after_madv +CATEGORY="hugetlb" run_test ./hugetlb_madv_vs_map if test_selected "hugetlb"; then echo "NOTE: These hugetlb tests provide minimal coverage. Use" | tap_prefix @@ -333,44 +293,11 @@ CATEGORY="gup_test" run_test ./gup_longterm CATEGORY="userfaultfd" run_test ./uffd-unit-tests uffd_stress_bin=./uffd-stress CATEGORY="userfaultfd" run_test ${uffd_stress_bin} anon 20 16 -# Hugetlb tests require source and destination huge pages. Pass in almost half -# the size of the free pages we have, which is used for *each*. An adjustment -# of (nr_parallel - 1) is done (see nr_parallel in uffd-stress.c) to have some -# extra hugepages - this is done to prevent the test from failing by racily -# reserving more hugepages than strictly required. -# uffd-stress expects a region expressed in MiB, so we adjust -# half_ufd_size_MB accordingly. -adjustment=$(( (31 < (nr_cpus - 1)) ? 31 : (nr_cpus - 1) )) -half_ufd_size_MB=$((((freepgs - adjustment) * hpgsize_KB) / 1024 / 2)) -CATEGORY="userfaultfd" run_test ${uffd_stress_bin} hugetlb "$half_ufd_size_MB" 32 -CATEGORY="userfaultfd" run_test ${uffd_stress_bin} hugetlb-private "$half_ufd_size_MB" 32 +CATEGORY="userfaultfd" run_test ${uffd_stress_bin} hugetlb 128 32 +CATEGORY="userfaultfd" run_test ${uffd_stress_bin} hugetlb-private 128 32 CATEGORY="userfaultfd" run_test ${uffd_stress_bin} shmem 20 16 CATEGORY="userfaultfd" run_test ${uffd_stress_bin} shmem-private 20 16 -# uffd-wp-mremap requires at least one page of each size. -have_all_size_hugepgs=true -declare -A nr_size_hugepgs -for f in /sys/kernel/mm/hugepages/**/nr_hugepages; do - old=$(cat $f) - nr_size_hugepgs["$f"]="$old" - if [ "$old" == 0 ]; then - echo 1 > "$f" - fi - if [ $(cat "$f") == 0 ]; then - have_all_size_hugepgs=false - break - fi -done -if $have_all_size_hugepgs; then - CATEGORY="userfaultfd" run_test ./uffd-wp-mremap -else - echo "# SKIP ./uffd-wp-mremap" -fi - -#cleanup -for f in "${!nr_size_hugepgs[@]}"; do - echo "${nr_size_hugepgs["$f"]}" > "$f" -done -echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages +CATEGORY="userfaultfd" run_test ./uffd-wp-mremap CATEGORY="compaction" run_test ./compaction_test @@ -382,6 +309,7 @@ else fi CATEGORY="mmap" run_test ./map_populate +CATEGORY="mmap" run_test ./droppable CATEGORY="mlock" run_test ./mlock-random-test @@ -394,12 +322,10 @@ CATEGORY="mremap" run_test ./mremap_test CATEGORY="hugetlb" run_test ./thuge-gen CATEGORY="hugetlb" run_test ./charge_reserved_hugetlb.sh -cgroup-v2 CATEGORY="hugetlb" run_test ./hugetlb_reparenting_test.sh -cgroup-v2 + if $RUN_DESTRUCTIVE; then -nr_hugepages_tmp=$(cat /proc/sys/vm/nr_hugepages) enable_soft_offline=$(cat /proc/sys/vm/enable_soft_offline) -echo 8 > /proc/sys/vm/nr_hugepages CATEGORY="hugetlb" run_test ./hugetlb-soft-offline -echo "$nr_hugepages_tmp" > /proc/sys/vm/nr_hugepages echo "$enable_soft_offline" > /proc/sys/vm/enable_soft_offline CATEGORY="hugetlb" run_test ./hugetlb-read-hwpoison fi @@ -436,9 +362,7 @@ CATEGORY="memfd_secret" run_test ./memfd_secret fi # KSM KSM_MERGE_TIME_HUGE_PAGES test with size of 100 -if [ "${HAVE_HUGEPAGES}" = "1" ]; then - CATEGORY="ksm" run_test ./ksm_tests -H -s 100 -fi +CATEGORY="ksm" run_test ./ksm_tests -H -s 100 # KSM KSM_MERGE_TIME test with size of 100 CATEGORY="ksm" run_test ./ksm_tests -P -s 100 # KSM MADV_MERGEABLE test with 10 identical pages @@ -457,7 +381,6 @@ CATEGORY="ksm_numa" run_test ./ksm_tests -N -m 0 CATEGORY="ksm" run_test ./ksm_functional_tests # protection_keys tests -nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) if [ -x ./protection_keys_32 ] then CATEGORY="pkey" run_test ./protection_keys_32 @@ -467,7 +390,6 @@ if [ -x ./protection_keys_64 ] then CATEGORY="pkey" run_test ./protection_keys_64 fi -echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages if [ -x ./soft-dirty ] then @@ -489,24 +411,28 @@ CATEGORY="thp" run_test ./khugepaged all:shmem CATEGORY="thp" run_test ./khugepaged -s 4 all:shmem -CATEGORY="thp" run_test ./transhuge-stress -d 20 - # Try to create XFS if not provided if [ -z "${SPLIT_HUGE_PAGE_TEST_XFS_PATH}" ]; then - if [ "${HAVE_HUGEPAGES}" = "1" ]; then - if test_selected "thp"; then - if grep xfs /proc/filesystems &>/dev/null; then - XFS_IMG=$(mktemp /tmp/xfs_img_XXXXXX) - SPLIT_HUGE_PAGE_TEST_XFS_PATH=$(mktemp -d /tmp/xfs_dir_XXXXXX) - truncate -s 314572800 ${XFS_IMG} - mkfs.xfs -q ${XFS_IMG} - mount -o loop ${XFS_IMG} ${SPLIT_HUGE_PAGE_TEST_XFS_PATH} - MOUNTED_XFS=1 - fi + if test_selected "thp"; then + if grep xfs /proc/filesystems &>/dev/null; then + XFS_IMG=$(mktemp /tmp/xfs_img_XXXXXX) + SPLIT_HUGE_PAGE_TEST_XFS_PATH=$(mktemp -d /tmp/xfs_dir_XXXXXX) + truncate -s 314572800 ${XFS_IMG} + mkfs.xfs -q ${XFS_IMG} + mount -o loop ${XFS_IMG} ${SPLIT_HUGE_PAGE_TEST_XFS_PATH} + MOUNTED_XFS=1 fi fi fi +if [ -n "${SPLIT_HUGE_PAGE_TEST_XFS_PATH}" ]; then +CATEGORY="thp" run_test ./khugepaged all:file ${SPLIT_HUGE_PAGE_TEST_XFS_PATH} +elif test_selected thp; then + count_total=$(( count_total + 1 )) + count_skip=$(( count_skip + 1 )) + echo "[SKIP] ./khugepaged all:file" | tap_prefix +fi + CATEGORY="thp" run_test ./split_huge_page_test ${SPLIT_HUGE_PAGE_TEST_XFS_PATH} if [ -n "${MOUNTED_XFS}" ]; then @@ -515,6 +441,8 @@ if [ -n "${MOUNTED_XFS}" ]; then rm -f ${XFS_IMG} fi +CATEGORY="thp" run_test ./transhuge-stress -d 20 + CATEGORY="thp" run_test ./folio_split_race_test CATEGORY="migration" run_test ./migration @@ -531,28 +459,7 @@ CATEGORY="page_frag" run_test ./test_page_frag.sh nonaligned CATEGORY="rmap" run_test ./rmap -# Try to load hwpoison_inject if not present. -HWPOISON_DIR=/sys/kernel/debug/hwpoison/ -if [ ! -d "$HWPOISON_DIR" ]; then - if ! modprobe -q -R hwpoison_inject; then - echo "Module hwpoison_inject not found, skipping..." - else - modprobe hwpoison_inject > /dev/null 2>&1 - LOADED_MOD=1 - fi -fi - -if [ -d "$HWPOISON_DIR" ]; then - CATEGORY="memory-failure" run_test ./memory-failure -fi - -if [ -n "${LOADED_MOD}" ]; then - modprobe -r hwpoison_inject > /dev/null 2>&1 -fi - -if [ "${HAVE_HUGEPAGES}" = 1 ]; then - echo "$orig_nr_hugepgs" > /proc/sys/vm/nr_hugepages -fi +CATEGORY="memory-failure" run_test ./memory-failure echo "SUMMARY: PASS=${count_pass} SKIP=${count_skip} FAIL=${count_fail}" | tap_prefix echo "1..${count_total}" | tap_output diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c index bcfcac99b436..fb1864a68e1c 100644 --- a/tools/testing/selftests/mm/soft-dirty.c +++ b/tools/testing/selftests/mm/soft-dirty.c @@ -9,7 +9,7 @@ #include "kselftest.h" #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #define PAGEMAP_FILE_PATH "/proc/self/pagemap" #define TEST_ITERATIONS 10000 @@ -143,7 +143,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon) if (anon) { map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - if (!map) + if (map == MAP_FAILED) ksft_exit_fail_msg("anon mmap failed\n"); } else { test_fd = open(fname, O_RDWR | O_CREAT, 0664); @@ -155,7 +155,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon) ftruncate(test_fd, pagesize); map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE, MAP_SHARED, test_fd, 0); - if (!map) + if (map == MAP_FAILED) ksft_exit_fail_msg("file mmap failed\n"); } diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 500d07c4938b..32b991472f74 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -21,7 +21,7 @@ #include <time.h> #include "vm_util.h" #include "kselftest.h" -#include "thp_settings.h" +#include "hugepage_settings.h" uint64_t pagesize; unsigned int pageshift; @@ -470,13 +470,18 @@ static void split_file_backed_thp(int order) char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; const char *tmpfs_loc = mkdtemp(tmpfs_template); char testfile[INPUT_MAX]; + unsigned long size = 2 * pmd_pagesize; + char opts[64]; ssize_t num_written, num_read; - char *file_buf1, *file_buf2; + char *file_buf1 = NULL, *file_buf2 = NULL; uint64_t pgoff_start = 0, pgoff_end = 1024; int i; ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n"); + if (!tmpfs_loc) + ksft_exit_fail_msg("mkdtemp failed\n"); + file_buf1 = (char *)malloc(pmd_pagesize); file_buf2 = (char *)malloc(pmd_pagesize); @@ -489,10 +494,13 @@ static void split_file_backed_thp(int order) file_buf1[i] = (char)i; memset(file_buf2, 0, pmd_pagesize); - status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, "huge=always,size=4m"); + snprintf(opts, sizeof(opts), "huge=always,size=%lu", size); + status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, opts); - if (status) - ksft_exit_fail_msg("Unable to create a tmpfs for testing\n"); + if (status) { + ksft_print_msg("Unable to create a tmpfs for testing\n"); + goto out; + } status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); if (status >= INPUT_MAX) { @@ -544,10 +552,13 @@ static void split_file_backed_thp(int order) status = umount(tmpfs_loc); if (status) { - rmdir(tmpfs_loc); - ksft_exit_fail_msg("Unable to umount %s\n", tmpfs_loc); + ksft_print_msg("Unable to umount %s\n", tmpfs_loc); + goto out; } + free(file_buf1); + free(file_buf2); + status = rmdir(tmpfs_loc); if (status) ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno)); @@ -560,8 +571,10 @@ close_file: close(fd); cleanup: umount(tmpfs_loc); - rmdir(tmpfs_loc); out: + free(file_buf1); + free(file_buf2); + rmdir(tmpfs_loc); ksft_exit_fail_msg("Error occurred\n"); } @@ -609,9 +622,13 @@ static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, assert(fd_size % sizeof(buf) == 0); for (i = 0; i < sizeof(buf); i++) buf[i] = (unsigned char)i; - for (i = 0; i < fd_size; i += sizeof(buf)) - write(*fd, buf, sizeof(buf)); - + for (i = 0; i < fd_size; i += sizeof(buf)) { + if (write(*fd, buf, sizeof(buf)) != sizeof(buf)) { + ksft_perror("write testfile"); + close(*fd); + goto err_out_unlink; + } + } close(*fd); sync(); *fd = open("/proc/sys/vm/drop_caches", O_WRONLY); @@ -621,7 +638,7 @@ static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, } if (write(*fd, "3", 1) != 1) { ksft_perror("write to drop_caches"); - goto err_out_unlink; + goto err_out_close; } close(*fd); diff --git a/tools/testing/selftests/mm/thuge-gen.c b/tools/testing/selftests/mm/thuge-gen.c index 77813d34dcc2..22b9c2f1c35d 100644 --- a/tools/testing/selftests/mm/thuge-gen.c +++ b/tools/testing/selftests/mm/thuge-gen.c @@ -1,17 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Test selecting other page sizes for mmap/shmget. - - Before running this huge pages for each huge page size must have been - reserved. - For large pages beyond MAX_PAGE_ORDER (like 1GB on x86) boot options must - be used. 1GB wouldn't be tested if it isn't available. - Also shmmax must be increased. - And you need to run as root to work around some weird permissions in shm. - And nothing using huge pages should run in parallel. - When the program aborts you may need to clean up the shm segments with - ipcrm -m by hand, like this - sudo ipcs | awk '$1 == "0x00000000" {print $2}' | xargs -n1 sudo ipcrm -m - (warning this will remove all if someone else uses them) */ +/* Test selecting other page sizes for mmap/shmget. */ #define _GNU_SOURCE #include <sys/mman.h> @@ -21,13 +9,12 @@ #include <sys/ipc.h> #include <sys/shm.h> #include <sys/stat.h> -#include <glob.h> -#include <assert.h> #include <unistd.h> #include <stdarg.h> #include <string.h> #include "vm_util.h" #include "kselftest.h" +#include "hugepage_settings.h" #if !defined(MAP_HUGETLB) #define MAP_HUGETLB 0x40000 @@ -37,15 +24,6 @@ #ifndef SHM_HUGE_SHIFT #define SHM_HUGE_SHIFT 26 #endif -#ifndef SHM_HUGE_MASK -#define SHM_HUGE_MASK 0x3f -#endif -#ifndef SHM_HUGE_2MB -#define SHM_HUGE_2MB (21 << SHM_HUGE_SHIFT) -#endif -#ifndef SHM_HUGE_1GB -#define SHM_HUGE_1GB (30 << SHM_HUGE_SHIFT) -#endif #define NUM_PAGESIZES 5 #define NUM_PAGES 4 @@ -63,32 +41,10 @@ int ilog2(unsigned long v) void show(unsigned long ps) { - char buf[100]; - if (ps == getpagesize()) return; - ksft_print_msg("%luMB: ", ps >> 20); - - fflush(stdout); - snprintf(buf, sizeof buf, - "cat /sys/kernel/mm/hugepages/hugepages-%lukB/free_hugepages", - ps >> 10); - system(buf); -} - -unsigned long read_free(unsigned long ps) -{ - unsigned long val = 0; - char buf[100]; - - snprintf(buf, sizeof(buf), - "/sys/kernel/mm/hugepages/hugepages-%lukB/free_hugepages", - ps >> 10); - if (read_sysfs(buf, &val) && ps != getpagesize()) - ksft_print_msg("missing %s\n", buf); - - return val; + ksft_print_msg("%luMB: %lu\n", ps >> 20, hugetlb_free_pages(ps)); } void test_mmap(unsigned long size, unsigned flags) @@ -96,14 +52,14 @@ void test_mmap(unsigned long size, unsigned flags) char *map; unsigned long before, after; - before = read_free(size); + before = hugetlb_free_pages(size); map = mmap(NULL, size*NUM_PAGES, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB|flags, -1, 0); if (map == MAP_FAILED) ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); memset(map, 0xff, size*NUM_PAGES); - after = read_free(size); + after = hugetlb_free_pages(size); show(size); ksft_test_result(size == getpagesize() || (before - after) == NUM_PAGES, @@ -120,7 +76,7 @@ void test_shmget(unsigned long size, unsigned flags) struct shm_info i; char *map; - before = read_free(size); + before = hugetlb_free_pages(size); id = shmget(IPC_PRIVATE, size * NUM_PAGES, IPC_CREAT|0600|flags); if (id < 0) { if (errno == EPERM) { @@ -141,7 +97,7 @@ void test_shmget(unsigned long size, unsigned flags) shmctl(id, IPC_RMID, NULL); memset(map, 0xff, size*NUM_PAGES); - after = read_free(size); + after = hugetlb_free_pages(size); show(size); ksft_test_result(size == getpagesize() || (before - after) == NUM_PAGES, @@ -153,43 +109,15 @@ void test_shmget(unsigned long size, unsigned flags) void find_pagesizes(void) { unsigned long largest = getpagesize(); - unsigned long shmmax_val = 0; int i; - glob_t g; - glob("/sys/kernel/mm/hugepages/hugepages-*kB", 0, NULL, &g); - assert(g.gl_pathc <= NUM_PAGESIZES); - for (i = 0; (i < g.gl_pathc) && (num_page_sizes < NUM_PAGESIZES); i++) { - sscanf(g.gl_pathv[i], "/sys/kernel/mm/hugepages/hugepages-%lukB", - &page_sizes[num_page_sizes]); - page_sizes[num_page_sizes] <<= 10; - ksft_print_msg("Found %luMB\n", page_sizes[i] >> 20); + num_page_sizes = hugetlb_setup(NUM_PAGES, page_sizes, ARRAY_SIZE(page_sizes)); - if (page_sizes[num_page_sizes] > largest) + for (i = 0; i < num_page_sizes; i++) + if (page_sizes[i] > largest) largest = page_sizes[i]; - if (read_free(page_sizes[num_page_sizes]) >= NUM_PAGES) - num_page_sizes++; - else - ksft_print_msg("SKIP for size %lu MB as not enough huge pages, need %u\n", - page_sizes[num_page_sizes] >> 20, NUM_PAGES); - } - globfree(&g); - - read_sysfs("/proc/sys/kernel/shmmax", &shmmax_val); - if (shmmax_val < NUM_PAGES * largest) { - ksft_print_msg("WARNING: shmmax is too small to run this test.\n"); - ksft_print_msg("Please run the following command to increase shmmax:\n"); - ksft_print_msg("echo %lu > /proc/sys/kernel/shmmax\n", largest * NUM_PAGES); - ksft_exit_skip("Test skipped due to insufficient shmmax value.\n"); - } - -#if defined(__x86_64__) - if (largest != 1U<<30) { - ksft_exit_skip("No GB pages available on x86-64\n" - "Please boot with hugepagesz=1G hugepages=%d\n", NUM_PAGES); - } -#endif + shm_limits_prepare(NUM_PAGES * largest); } int main(void) @@ -232,3 +160,5 @@ int main(void) ksft_finished(); } + +SHM_LIMITS_RESTORE() diff --git a/tools/testing/selftests/mm/transhuge-stress.c b/tools/testing/selftests/mm/transhuge-stress.c index 7a9f1035099b..8eb0c5630e7e 100644 --- a/tools/testing/selftests/mm/transhuge-stress.c +++ b/tools/testing/selftests/mm/transhuge-stress.c @@ -17,7 +17,7 @@ #include <sys/mman.h> #include "vm_util.h" #include "kselftest.h" -#include "thp_settings.h" +#include "hugepage_settings.h" int backing_fd = -1; int mmap_flags = MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE; diff --git a/tools/testing/selftests/mm/uffd-common.h b/tools/testing/selftests/mm/uffd-common.h index 844a85ab31eb..92a21b97f745 100644 --- a/tools/testing/selftests/mm/uffd-common.h +++ b/tools/testing/selftests/mm/uffd-common.h @@ -37,24 +37,24 @@ #include "kselftest.h" #include "vm_util.h" +#include "hugepage_settings.h" #define UFFD_FLAGS (O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY) -#define _err(fmt, ...) \ - do { \ - int ret = errno; \ - fprintf(stderr, "ERROR: " fmt, ##__VA_ARGS__); \ - fprintf(stderr, " (errno=%d, @%s:%d)\n", \ - ret, __FILE__, __LINE__); \ +#define _err(fmt, ...) \ + do { \ + int ret = errno; \ + ksft_print_msg("ERROR: " fmt " (errno=%d, @%s:%d)\n", \ + ##__VA_ARGS__, ret, __FILE__, __LINE__); \ } while (0) -#define errexit(exitcode, fmt, ...) \ +#define errexit(fmt, ...) \ do { \ _err(fmt, ##__VA_ARGS__); \ - exit(exitcode); \ + ksft_exit_fail(); \ } while (0) -#define err(fmt, ...) errexit(1, fmt, ##__VA_ARGS__) +#define err(fmt, ...) errexit(fmt, ##__VA_ARGS__) struct uffd_global_test_opts { unsigned long nr_parallel, nr_pages, nr_pages_per_cpu, page_size; diff --git a/tools/testing/selftests/mm/uffd-stress.c b/tools/testing/selftests/mm/uffd-stress.c index 700fbaa18d44..3401dd6028f0 100644 --- a/tools/testing/selftests/mm/uffd-stress.c +++ b/tools/testing/selftests/mm/uffd-stress.c @@ -286,18 +286,12 @@ static int userfaultfd_stress(uffd_global_test_opts_t *gopts) pthread_attr_setstacksize(&attr, 16*1024*1024); while (bounces--) { - printf("bounces: %d, mode:", bounces); - if (bounces & BOUNCE_RANDOM) - printf(" rnd"); - if (bounces & BOUNCE_RACINGFAULTS) - printf(" racing"); - if (bounces & BOUNCE_VERIFY) - printf(" ver"); - if (bounces & BOUNCE_POLL) - printf(" poll"); - else - printf(" read"); - printf(", "); + ksft_print_msg("bounces: %d, mode:%s%s%s%s, ", + bounces, + bounces & BOUNCE_RANDOM ? " rnd" : "", + bounces & BOUNCE_RACINGFAULTS ? " racing" : "", + bounces & BOUNCE_VERIFY ? " ver" : "", + bounces & BOUNCE_POLL ? " poll" : " read"); fflush(stdout); if (bounces & BOUNCE_POLL) @@ -461,6 +455,9 @@ int main(int argc, char **argv) if (argc < 4) usage(); + ksft_print_header(); + ksft_set_plan(1); + if (signal(SIGALRM, sigalrm) == SIG_ERR) err("failed to arm SIGALRM"); alarm(ALARM_INTERVAL_SECS); @@ -483,17 +480,17 @@ int main(int argc, char **argv) * Ensure nr_parallel - 1 hugepages on top of that to account * for racy extra reservation of hugepages. */ - if (gopts->test_type == TEST_HUGETLB && - get_free_hugepages() < 2 * (bytes / gopts->page_size) + gopts->nr_parallel - 1) { - printf("skip: Skipping userfaultfd... not enough hugepages\n"); - return KSFT_SKIP; + if (gopts->test_type == TEST_HUGETLB) { + unsigned long nr = 2 * (bytes / gopts->page_size) + gopts->nr_parallel - 1; + + if (!hugetlb_setup_default(nr)) + ksft_exit_skip("Skipping userfaultfd... not enough hugepages\n"); } gopts->nr_pages_per_cpu = bytes / gopts->page_size / gopts->nr_parallel; if (!gopts->nr_pages_per_cpu) { - _err("pages_per_cpu = 0, cannot test (%lu / %lu / %lu)", - bytes, gopts->page_size, gopts->nr_parallel); - usage(); + ksft_exit_skip("pages_per_cpu = 0, cannot test (%zu / %lu / %lu)\n", + bytes, gopts->page_size, gopts->nr_parallel); } bounces = atoi(argv[3]); @@ -503,9 +500,12 @@ int main(int argc, char **argv) } gopts->nr_pages = gopts->nr_pages_per_cpu * gopts->nr_parallel; - printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n", - gopts->nr_pages, gopts->nr_pages_per_cpu); - return userfaultfd_stress(gopts); + ksft_print_msg("nr_pages: %lu, nr_pages_per_cpu: %lu\n", + gopts->nr_pages, gopts->nr_pages_per_cpu); + + ksft_test_result(!userfaultfd_stress(gopts), + "uffd-stress %s\n", argv[1]); + ksft_finished(); } #else /* __NR_userfaultfd */ @@ -514,8 +514,8 @@ int main(int argc, char **argv) int main(void) { - printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n"); - return KSFT_SKIP; + ksft_print_header(); + ksft_exit_skip("missing __NR_userfaultfd definition\n"); } #endif /* __NR_userfaultfd */ diff --git a/tools/testing/selftests/mm/uffd-unit-tests.c b/tools/testing/selftests/mm/uffd-unit-tests.c index 6f5e404a446c..a6c14109e818 100644 --- a/tools/testing/selftests/mm/uffd-unit-tests.c +++ b/tools/testing/selftests/mm/uffd-unit-tests.c @@ -86,47 +86,28 @@ typedef struct { uffd_test_case_ops_t *test_case_ops; } uffd_test_case_t; -static void uffd_test_report(void) -{ - printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n", - ksft_get_pass_cnt(), - ksft_get_xskip_cnt(), - ksft_get_fail_cnt(), - ksft_test_num()); -} +static char current_test[256]; static void uffd_test_pass(void) { - printf("done\n"); - ksft_inc_pass_cnt(); + ksft_test_result_pass("%s\n", current_test); } #define uffd_test_start(...) do { \ - printf("Testing "); \ - printf(__VA_ARGS__); \ - printf("... "); \ - fflush(stdout); \ + snprintf(current_test, sizeof(current_test), __VA_ARGS__); \ } while (0) -#define uffd_test_fail(...) do { \ - printf("failed [reason: "); \ - printf(__VA_ARGS__); \ - printf("]\n"); \ - ksft_inc_fail_cnt(); \ +#define uffd_test_fail(fmt, ...) do { \ + ksft_print_msg("failed reason: [" fmt "]\n", ##__VA_ARGS__); \ + ksft_test_result_fail("%s\n", current_test); \ } while (0) static void uffd_test_skip(const char *message) { - printf("skipped [reason: %s]\n", message); - ksft_inc_xskip_cnt(); + ksft_test_result_skip("%s (%s)\n", current_test, message); } -/* - * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll - * return 1 even if some test failed as long as uffd supported, because in - * that case we still want to proceed with the rest uffd unit tests. - */ -static int test_uffd_api(bool use_dev) +static void test_uffd_api(bool use_dev) { struct uffdio_api uffdio_api; int uffd; @@ -140,7 +121,7 @@ static int test_uffd_api(bool use_dev) uffd = uffd_open_sys(UFFD_FLAGS); if (uffd < 0) { uffd_test_skip("cannot open userfaultfd handle"); - return 0; + return; } /* Test wrong UFFD_API */ @@ -177,8 +158,6 @@ static int test_uffd_api(bool use_dev) uffd_test_pass(); out: close(uffd); - /* We have a valid uffd handle */ - return 1; } @@ -320,7 +299,7 @@ static int pagemap_test_fork(uffd_global_test_opts_t *gopts, bool with_event, bo if (test_pin) unpin_pages(&args); /* Succeed */ - exit(0); + _exit(0); } waitpid(child, &result, 0); @@ -788,7 +767,7 @@ static void uffd_sigbus_test_common(uffd_global_test_opts_t *gopts, bool wp) err("fork"); if (!pid) - exit(faulting_process(gopts, 2, wp)); + _exit(faulting_process(gopts, 2, wp)); waitpid(pid, &err, 0); if (err) @@ -842,7 +821,7 @@ static void uffd_events_test_common(uffd_global_test_opts_t *gopts, bool wp) err("fork"); if (!pid) - exit(faulting_process(gopts, 0, wp)); + _exit(faulting_process(gopts, 0, wp)); waitpid(pid, &err, 0); if (err) @@ -1701,18 +1680,58 @@ static void usage(const char *prog) exit(KSFT_FAIL); } +static int uffd_count_tests(int n_tests, int n_mems, const char *test_filter) +{ + uffd_test_case_t *test; + int i, j, count = 0; + + if (!test_filter) + count += 2; /* test_uffd_api(false) + test_uffd_api(true) */ + + for (i = 0; i < n_tests; i++) { + test = &uffd_tests[i]; + if (test_filter && !strstr(test->name, test_filter)) + continue; + for (j = 0; j < n_mems; j++) + if (test->mem_targets & mem_types[j].mem_flag) + count++; + } + + return count; +} + +static unsigned long uffd_setup_hugetlb(void) +{ + unsigned long nr_hugepages, hp_size; + + hugetlb_save_settings(); + hp_size = default_huge_page_size(); + + if (!hp_size) + return 0; + + /* need twice UFFD_TEST_MEM_SIZE, one for src area and one for dst */ + nr_hugepages = 2 * MAX(UFFD_TEST_MEM_SIZE, hp_size * 2) / hp_size; + hugetlb_set_nr_default_pages(nr_hugepages); + + if (hugetlb_free_default_pages() < nr_hugepages) + return 0; + + return hp_size; +} + int main(int argc, char *argv[]) { int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t); int n_mems = sizeof(mem_types) / sizeof(mem_type_t); const char *test_filter = NULL; + unsigned long hugepage_size; bool list_only = false; uffd_test_case_t *test; mem_type_t *mem_type; uffd_test_args_t args; const char *errmsg; - int has_uffd, opt; - int i, j; + int i, j, opt; while ((opt = getopt(argc, argv, "f:hl")) != -1) { switch (opt) { @@ -1730,24 +1749,30 @@ int main(int argc, char *argv[]) } } - if (!test_filter && !list_only) { - has_uffd = test_uffd_api(false); - has_uffd |= test_uffd_api(true); - - if (!has_uffd) { - printf("Userfaultfd not supported or unprivileged, skip all tests\n"); - exit(KSFT_SKIP); + if (list_only) { + for (i = 0; i < n_tests; i++) { + test = &uffd_tests[i]; + if (test_filter && !strstr(test->name, test_filter)) + continue; + printf("%s\n", test->name); } + return KSFT_PASS; + } + + hugepage_size = uffd_setup_hugetlb(); + + ksft_print_header(); + ksft_set_plan(uffd_count_tests(n_tests, n_mems, test_filter)); + + if (!test_filter) { + test_uffd_api(false); + test_uffd_api(true); } for (i = 0; i < n_tests; i++) { test = &uffd_tests[i]; if (test_filter && !strstr(test->name, test_filter)) continue; - if (list_only) { - printf("%s\n", test->name); - continue; - } for (j = 0; j < n_mems; j++) { mem_type = &mem_types[j]; @@ -1758,10 +1783,14 @@ int main(int argc, char *argv[]) uffd_test_ops = mem_type->mem_ops; uffd_test_case_ops = test->test_case_ops; + if (!(test->mem_targets & mem_type->mem_flag)) + continue; + + uffd_test_start("%s on %s", test->name, mem_type->name); if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB)) { - gopts.page_size = default_huge_page_size(); + gopts.page_size = hugepage_size; if (gopts.page_size == 0) { - uffd_test_skip("huge page size is 0, feature missing?"); + uffd_test_skip("not enough HugeTLB pages"); continue; } } else { @@ -1777,10 +1806,6 @@ int main(int argc, char *argv[]) /* Initialize test arguments */ args.mem_type = mem_type; - if (!(test->mem_targets & mem_type->mem_flag)) - continue; - - uffd_test_start("%s on %s", test->name, mem_type->name); if (!uffd_feature_supported(test)) { uffd_test_skip("feature missing"); continue; @@ -1794,10 +1819,7 @@ int main(int argc, char *argv[]) } } - if (!list_only) - uffd_test_report(); - - return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; + ksft_finished(); } #else /* __NR_userfaultfd */ @@ -1806,8 +1828,8 @@ int main(int argc, char *argv[]) int main(void) { - printf("Skipping %s (missing __NR_userfaultfd)\n", __file__); - return KSFT_SKIP; + ksft_print_header(); + ksft_exit_skip("missing __NR_userfaultfd definition\n"); } #endif /* __NR_userfaultfd */ diff --git a/tools/testing/selftests/mm/uffd-wp-mremap.c b/tools/testing/selftests/mm/uffd-wp-mremap.c index 17186d4a4147..c973d6722720 100644 --- a/tools/testing/selftests/mm/uffd-wp-mremap.c +++ b/tools/testing/selftests/mm/uffd-wp-mremap.c @@ -8,16 +8,27 @@ #include <linux/mman.h> #include <sys/mman.h> #include "kselftest.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #include "uffd-common.h" static int pagemap_fd; -static size_t pagesize; static int nr_pagesizes = 1; +static unsigned long pagesize; static int nr_thpsizes; static size_t thpsizes[20]; static int nr_hugetlbsizes; -static size_t hugetlbsizes[10]; +static unsigned long hugetlbsizes[10]; + +static void check_uffd_wp_feature_supported(void) +{ + uint64_t features = 0; + + if (uffd_get_features(&features)) + ksft_exit_skip("failed to get available features (%d)\n", errno); + + if (!(features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) + ksft_exit_skip("uffd-wp feature not supported\n"); +} static int detect_thp_sizes(size_t sizes[], int max) { @@ -245,7 +256,7 @@ out: } struct testcase { - size_t *sizes; + unsigned long *sizes; int *nr_sizes; bool private; bool swapout; @@ -336,14 +347,16 @@ int main(int argc, char **argv) struct thp_settings settings; int i, j, plan = 0; + hugepage_save_settings(true, true); + + check_uffd_wp_feature_supported(); + pagesize = getpagesize(); nr_thpsizes = detect_thp_sizes(thpsizes, ARRAY_SIZE(thpsizes)); - nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes, - ARRAY_SIZE(hugetlbsizes)); + nr_hugetlbsizes = hugetlb_setup(1, hugetlbsizes, ARRAY_SIZE(hugetlbsizes)); - /* If THP is supported, save THP settings and initially disable THP. */ + /* If THP is supported, initially disable THP. */ if (nr_thpsizes) { - thp_save_settings(); thp_read_settings(&settings); for (i = 0; i < NR_ORDERS; i++) { settings.hugepages[i].enabled = THP_NEVER; @@ -368,10 +381,6 @@ int main(int argc, char **argv) tc->swapout, tc->hugetlb); } - /* If THP is supported, restore original THP settings. */ - if (nr_thpsizes) - thp_restore_settings(); - i = ksft_get_fail_cnt(); if (i) ksft_exit_fail_msg("%d out of %d tests failed\n", diff --git a/tools/testing/selftests/mm/va_high_addr_switch.c b/tools/testing/selftests/mm/va_high_addr_switch.c index 51401e081b20..e24d7ba00b44 100644 --- a/tools/testing/selftests/mm/va_high_addr_switch.c +++ b/tools/testing/selftests/mm/va_high_addr_switch.c @@ -11,6 +11,7 @@ #include "vm_util.h" #include "kselftest.h" +#include "hugepage_settings.h" /* * The hint addr value is used to allocate addresses @@ -257,40 +258,35 @@ void testcases_init(void) switch_hint = addr_switch_hint; } -static int run_test(struct testcase *test, int count) +static void run_test(struct testcase *test, int count) { void *p; - int i, ret = KSFT_PASS; + int i; for (i = 0; i < count; i++) { struct testcase *t = test + i; p = mmap(t->addr, t->size, PROT_READ | PROT_WRITE, t->flags, -1, 0); - - printf("%s: %p - ", t->msg, p); - if (p == MAP_FAILED) { - printf("FAILED\n"); - ret = KSFT_FAIL; + ksft_perror("MAP_FAILED"); + ksft_test_result_fail("%s\n", t->msg); continue; } if (t->low_addr_required && p >= (void *)(switch_hint)) { - printf("FAILED\n"); - ret = KSFT_FAIL; + ksft_print_msg("%p not below switch hint\n", p); + ksft_test_result_fail("%s\n", t->msg); } else { /* * Do a dereference of the address returned so that we catch * bugs in page fault handling */ memset(p, 0, t->size); - printf("OK\n"); + ksft_test_result_pass("%s\n", t->msg); } if (!t->keep_mapped) munmap(p, t->size); } - - return ret; } #ifdef __aarch64__ @@ -322,19 +318,23 @@ static int supported_arch(void) int main(int argc, char **argv) { - int ret, hugetlb_ret = KSFT_PASS; + bool run_hugetlb = false; + + ksft_print_header(); if (!supported_arch()) - return KSFT_SKIP; + ksft_exit_skip("Architecture not supported\n"); + + if (hugetlb_setup_default(6)) + run_hugetlb = true; testcases_init(); - ret = run_test(testcases, sz_testcases); - if (argc == 2 && !strcmp(argv[1], "--run-hugetlb")) - hugetlb_ret = run_test(hugetlb_testcases, sz_hugetlb_testcases); + ksft_set_plan(sz_testcases + (run_hugetlb ? sz_hugetlb_testcases : 0)); + + run_test(testcases, sz_testcases); + if (run_hugetlb) + run_test(hugetlb_testcases, sz_hugetlb_testcases); - if (ret == KSFT_PASS && hugetlb_ret == KSFT_PASS) - return KSFT_PASS; - else - return KSFT_FAIL; + ksft_finished(); } diff --git a/tools/testing/selftests/mm/va_high_addr_switch.sh b/tools/testing/selftests/mm/va_high_addr_switch.sh index 9492c2d72634..01c15fe3c799 100755 --- a/tools/testing/selftests/mm/va_high_addr_switch.sh +++ b/tools/testing/selftests/mm/va_high_addr_switch.sh @@ -9,7 +9,6 @@ # Kselftest framework requirement - SKIP code is 4. ksft_skip=4 -orig_nr_hugepages=0 skip() { @@ -77,43 +76,5 @@ check_test_requirements() esac } -save_nr_hugepages() -{ - orig_nr_hugepages=$(cat /proc/sys/vm/nr_hugepages) -} - -restore_nr_hugepages() -{ - echo "$orig_nr_hugepages" > /proc/sys/vm/nr_hugepages -} - -setup_nr_hugepages() -{ - local needpgs=$1 - while read -r name size unit; do - if [ "$name" = "HugePages_Free:" ]; then - freepgs="$size" - break - fi - done < /proc/meminfo - if [ "$freepgs" -ge "$needpgs" ]; then - return - fi - local hpgs=$((orig_nr_hugepages + needpgs)) - echo $hpgs > /proc/sys/vm/nr_hugepages - - local nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) - if [ "$nr_hugepgs" != "$hpgs" ]; then - restore_nr_hugepages - skip "$0: no enough hugepages for testing" - fi -} - check_test_requirements -save_nr_hugepages -# The HugeTLB tests require 6 pages -setup_nr_hugepages 6 -./va_high_addr_switch --run-hugetlb -retcode=$? -restore_nr_hugepages -exit $retcode +./va_high_addr_switch diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index db94564f4431..311fc5b4513e 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -2,7 +2,6 @@ #include <string.h> #include <errno.h> #include <fcntl.h> -#include <dirent.h> #include <inttypes.h> #include <sys/ioctl.h> #include <linux/userfaultfd.h> @@ -291,53 +290,6 @@ int64_t allocate_transhuge(void *ptr, int pagemap_fd) return -1; } -unsigned long default_huge_page_size(void) -{ - unsigned long hps = 0; - char *line = NULL; - size_t linelen = 0; - FILE *f = fopen("/proc/meminfo", "r"); - - if (!f) - return 0; - while (getline(&line, &linelen, f) > 0) { - if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { - hps <<= 10; - break; - } - } - - free(line); - fclose(f); - return hps; -} - -int detect_hugetlb_page_sizes(size_t sizes[], int max) -{ - DIR *dir = opendir("/sys/kernel/mm/hugepages/"); - int count = 0; - - if (!dir) - return 0; - - while (count < max) { - struct dirent *entry = readdir(dir); - size_t kb; - - if (!entry) - break; - if (entry->d_type != DT_DIR) - continue; - if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) - continue; - sizes[count++] = kb * 1024; - ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", - kb); - } - closedir(dir); - return count; -} - int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags) { size_t count; @@ -396,25 +348,6 @@ int uffd_unregister(int uffd, void *addr, uint64_t len) return ret; } -unsigned long get_free_hugepages(void) -{ - unsigned long fhp = 0; - char *line = NULL; - size_t linelen = 0; - FILE *f = fopen("/proc/meminfo", "r"); - - if (!f) - return fhp; - while (getline(&line, &linelen, f) > 0) { - if (sscanf(line, "HugePages_Free: %lu", &fhp) == 1) - break; - } - - free(line); - fclose(f); - return fhp; -} - static bool check_vmflag(void *addr, const char *flag) { char buffer[MAX_LINE_LENGTH]; @@ -463,7 +396,7 @@ bool softdirty_supported(void) /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */ addr = mmap(0, pagesize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - if (!addr) + if (addr == MAP_FAILED) ksft_exit_fail_msg("mmap failed\n"); supported = check_vmflag(addr, "sd"); @@ -765,6 +698,27 @@ int unpoison_memory(unsigned long pfn) return ret > 0 ? 0 : -errno; } +int read_file(const char *path, char *buf, size_t buflen) +{ + int fd; + ssize_t numread; + + fd = open(path, O_RDONLY); + if (fd == -1) + return 0; + + numread = read(fd, buf, buflen - 1); + if (numread < 1) { + close(fd); + return 0; + } + + buf[numread] = '\0'; + close(fd); + + return (unsigned int) numread; +} + void write_file(const char *path, const char *buf, size_t buflen) { int fd, saved_errno; @@ -788,3 +742,49 @@ void write_file(const char *path, const char *buf, size_t buflen) ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n", path, (int)(buflen - 1), buf, buflen - 1, numwritten); } + +unsigned long read_num(const char *path) +{ + char buf[21]; + + if (read_file(path, buf, sizeof(buf)) < 0) + ksft_exit_fail_perror("read_file()"); + + return strtoul(buf, NULL, 10); +} + +void write_num(const char *path, unsigned long num) +{ + char buf[21]; + + sprintf(buf, "%lu", num); + write_file(path, buf, strlen(buf) + 1); +} + +static unsigned long shmall, shmmax; + +void __shm_limits_restore(void) +{ + if (shmmax) + write_num("/proc/sys/kernel/shmmax", shmmax); + if (shmall) + write_num("/proc/sys/kernel/shmall", shmall); +} + +void shm_limits_prepare(unsigned long length) +{ + unsigned long nr = length / psize(); + unsigned long val; + + val = read_num("/proc/sys/kernel/shmmax"); + if (val < length) { + write_num("/proc/sys/kernel/shmmax", length); + shmmax = val; + } + + val = read_num("/proc/sys/kernel/shmall"); + if (val < nr) { + write_num("/proc/sys/kernel/shmall", nr); + shmall = val; + } +} diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 1a07305ceff4..ea8fc8fdf0eb 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -94,8 +94,6 @@ bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size); bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size); bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size); int64_t allocate_transhuge(void *ptr, int pagemap_fd); -unsigned long default_huge_page_size(void); -int detect_hugetlb_page_sizes(size_t sizes[], int max); int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags); int uffd_register(int uffd, void *addr, uint64_t len, @@ -103,7 +101,6 @@ int uffd_register(int uffd, void *addr, uint64_t len, int uffd_unregister(int uffd, void *addr, uint64_t len); int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor, uint64_t *ioctls); -unsigned long get_free_hugepages(void); bool check_vmflag_io(void *addr); bool check_vmflag_pfnmap(void *addr); bool check_vmflag_guard(void *addr); @@ -168,3 +165,15 @@ int unpoison_memory(unsigned long pfn); #define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1)) void write_file(const char *path, const char *buf, size_t buflen); +int read_file(const char *path, char *buf, size_t buflen); +unsigned long read_num(const char *path); +void write_num(const char *path, unsigned long num); + +void shm_limits_prepare(unsigned long length); +void __shm_limits_restore(void); + +#define SHM_LIMITS_RESTORE() \ +static void __attribute__((destructor)) shm_limits_restore(void) \ +{ \ + __shm_limits_restore(); \ +} |