diff options
Diffstat (limited to 'tools')
60 files changed, 4027 insertions, 947 deletions
diff --git a/tools/include/linux/gfp_types.h b/tools/include/linux/gfp_types.h index 54ca0c88bab6..da47aa089f01 100644 --- a/tools/include/linux/gfp_types.h +++ b/tools/include/linux/gfp_types.h @@ -55,7 +55,6 @@ enum { #ifdef CONFIG_LOCKDEP ___GFP_NOLOCKDEP_BIT, #endif - ___GFP_NO_OBJ_EXT_BIT, ___GFP_LAST_BIT }; @@ -96,7 +95,6 @@ enum { #else #define ___GFP_NOLOCKDEP 0 #endif -#define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT) /* * Physical address zone modifiers (see linux/mmzone.h - low four bits) @@ -137,17 +135,12 @@ enum { * node with no fallbacks or placement policy enforcements. * * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. - * - * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. - * mark_obj_codetag_empty() should be called upon freeing for objects allocated - * with this flag to indicate that their NULL tags are expected and normal. */ #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) #define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) #define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) -#define __GFP_NO_OBJ_EXT ((__force gfp_t)___GFP_NO_OBJ_EXT) /** * DOC: Watermark modifiers diff --git a/tools/include/uapi/linux/fs.h b/tools/include/uapi/linux/fs.h index 24ddf7bc4f25..f0a26309b6d5 100644 --- a/tools/include/uapi/linux/fs.h +++ b/tools/include/uapi/linux/fs.h @@ -364,6 +364,7 @@ typedef int __bitwise __kernel_rwf_t; #define PAGE_IS_HUGE (1 << 6) #define PAGE_IS_SOFT_DIRTY (1 << 7) #define PAGE_IS_GUARD (1 << 8) +#define PAGE_IS_ACCESSED (1 << 9) /* * struct page_region - Page region with flags diff --git a/tools/include/uapi/linux/userfaultfd.h b/tools/include/uapi/linux/userfaultfd.h index 4283de22d5b6..c8e99df95219 100644 --- a/tools/include/uapi/linux/userfaultfd.h +++ b/tools/include/uapi/linux/userfaultfd.h @@ -25,7 +25,8 @@ #define UFFD_API ((__u64)0xAA) #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \ UFFDIO_REGISTER_MODE_WP | \ - UFFDIO_REGISTER_MODE_MINOR) + UFFDIO_REGISTER_MODE_MINOR | \ + UFFDIO_REGISTER_MODE_RWP) #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \ UFFD_FEATURE_EVENT_FORK | \ UFFD_FEATURE_EVENT_REMAP | \ @@ -42,10 +43,13 @@ UFFD_FEATURE_WP_UNPOPULATED | \ UFFD_FEATURE_POISON | \ UFFD_FEATURE_WP_ASYNC | \ - UFFD_FEATURE_MOVE) + UFFD_FEATURE_MOVE | \ + UFFD_FEATURE_RWP | \ + UFFD_FEATURE_RWP_ASYNC) #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ + (__u64)1 << _UFFDIO_SET_MODE | \ (__u64)1 << _UFFDIO_API) #define UFFD_API_RANGE_IOCTLS \ ((__u64)1 << _UFFDIO_WAKE | \ @@ -54,13 +58,15 @@ (__u64)1 << _UFFDIO_MOVE | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) #define UFFD_API_RANGE_IOCTLS_BASIC \ ((__u64)1 << _UFFDIO_WAKE | \ (__u64)1 << _UFFDIO_COPY | \ (__u64)1 << _UFFDIO_WRITEPROTECT | \ (__u64)1 << _UFFDIO_CONTINUE | \ - (__u64)1 << _UFFDIO_POISON) + (__u64)1 << _UFFDIO_POISON | \ + (__u64)1 << _UFFDIO_RWPROTECT) /* * Valid ioctl command number range with this API is from 0x00 to @@ -79,6 +85,8 @@ #define _UFFDIO_WRITEPROTECT (0x06) #define _UFFDIO_CONTINUE (0x07) #define _UFFDIO_POISON (0x08) +#define _UFFDIO_RWPROTECT (0x09) +#define _UFFDIO_SET_MODE (0x0A) #define _UFFDIO_API (0x3F) /* userfaultfd ioctl ids */ @@ -103,6 +111,10 @@ struct uffdio_continue) #define UFFDIO_POISON _IOWR(UFFDIO, _UFFDIO_POISON, \ struct uffdio_poison) +#define UFFDIO_RWPROTECT _IOWR(UFFDIO, _UFFDIO_RWPROTECT, \ + struct uffdio_rwprotect) +#define UFFDIO_SET_MODE _IOW(UFFDIO, _UFFDIO_SET_MODE, \ + struct uffdio_set_mode) /* read() structure */ struct uffd_msg { @@ -158,6 +170,7 @@ struct uffd_msg { #define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */ #define UFFD_PAGEFAULT_FLAG_WP (1<<1) /* If reason is VM_UFFD_WP */ #define UFFD_PAGEFAULT_FLAG_MINOR (1<<2) /* If reason is VM_UFFD_MINOR */ +#define UFFD_PAGEFAULT_FLAG_RWP (1<<3) /* If reason is VM_UFFD_RWP */ struct uffdio_api { /* userland asks for an API number and the features to enable */ @@ -230,6 +243,18 @@ struct uffdio_api { * * UFFD_FEATURE_MOVE indicates that the kernel supports moving an * existing page contents from userspace. + * + * UFFD_FEATURE_RWP indicates that the kernel supports + * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking. + * Pages are made inaccessible via UFFDIO_RWPROTECT and faults + * are delivered when the pages are re-accessed. + * + * UFFD_FEATURE_RWP_ASYNC indicates asynchronous mode for + * UFFDIO_REGISTER_MODE_RWP. When set, faults on read-write + * protected pages are auto-resolved by the kernel (PTE + * permissions restored immediately) without delivering a message + * to the userfaultfd handler. Use PAGEMAP_SCAN with inverted + * PAGE_IS_ACCESSED to find pages that were not re-accessed. */ #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) #define UFFD_FEATURE_EVENT_FORK (1<<1) @@ -248,6 +273,8 @@ struct uffdio_api { #define UFFD_FEATURE_POISON (1<<14) #define UFFD_FEATURE_WP_ASYNC (1<<15) #define UFFD_FEATURE_MOVE (1<<16) +#define UFFD_FEATURE_RWP (1<<17) +#define UFFD_FEATURE_RWP_ASYNC (1<<18) __u64 features; __u64 ioctls; @@ -263,6 +290,7 @@ struct uffdio_register { #define UFFDIO_REGISTER_MODE_MISSING ((__u64)1<<0) #define UFFDIO_REGISTER_MODE_WP ((__u64)1<<1) #define UFFDIO_REGISTER_MODE_MINOR ((__u64)1<<2) +#define UFFDIO_REGISTER_MODE_RWP ((__u64)1<<3) __u64 mode; /* @@ -356,6 +384,14 @@ struct uffdio_poison { __s64 updated; }; +struct uffdio_rwprotect { + struct uffdio_range range; + /* !RWP means undo RWP-protection */ +#define UFFDIO_RWPROTECT_MODE_RWP ((__u64)1<<0) +#define UFFDIO_RWPROTECT_MODE_DONTWAKE ((__u64)1<<1) + __u64 mode; +}; + struct uffdio_move { __u64 dst; __u64 src; @@ -374,6 +410,16 @@ struct uffdio_move { __s64 move; }; +struct uffdio_set_mode { + /* + * Toggle async mode for features at runtime. + * Supported: UFFD_FEATURE_RWP_ASYNC. + * Setting a bit in both enable and disable is invalid. + */ + __u64 enable; + __u64 disable; +}; + /* * Flags for the userfaultfd(2) system call itself. */ diff --git a/tools/mm/.gitignore b/tools/mm/.gitignore index 922879f93fc8..1446a659e540 100644 --- a/tools/mm/.gitignore +++ b/tools/mm/.gitignore @@ -2,3 +2,4 @@ slabinfo page-types page_owner_sort +thp_swap_allocator_test diff --git a/tools/mm/Makefile b/tools/mm/Makefile index f5725b5c23aa..858186a6eefd 100644 --- a/tools/mm/Makefile +++ b/tools/mm/Makefile @@ -3,7 +3,7 @@ # include ../scripts/Makefile.include -BUILD_TARGETS=page-types slabinfo page_owner_sort thp_swap_allocator_test +BUILD_TARGETS=page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test INSTALL_TARGETS = $(BUILD_TARGETS) thpmaps LIB_DIR = ../lib/api @@ -23,7 +23,7 @@ $(LIBS): $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) clean: - $(RM) page-types slabinfo page_owner_sort thp_swap_allocator_test + $(RM) page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test make -C $(LIB_DIR) clean sbindir ?= /usr/sbin diff --git a/tools/mm/page_owner_filter.c b/tools/mm/page_owner_filter.c new file mode 100644 index 000000000000..1d1f0a38678a --- /dev/null +++ b/tools/mm/page_owner_filter.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * User-space helper to filter page_owner output per-fd + * + * Example use: + * ./page_owner_filter -m handle + * ./page_owner_filter -m stack_handle + * ./page_owner_filter -n 0,1,2 + * + * See Documentation/mm/page_owner.rst + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> +#include <ctype.h> +#include <getopt.h> +#include <signal.h> + +#define MAX_CMD_LEN 512 + +static void usage(const char *prog) +{ + fprintf(stderr, "Usage: %s [OPTIONS]\n", prog); + fprintf(stderr, "\nOptions:\n"); + fprintf(stderr, " -m, --mode MODE : print_mode (stack, handle, or stack_handle)\n"); + fprintf(stderr, " -n, --nid NID_LIST : NUMA node IDs (comma-separated or ranges)\n"); + fprintf(stderr, " -o, --output FILE : output file (default: stdout)\n"); + fprintf(stderr, " -h, --help : show this help message\n"); + fprintf(stderr, "\nExamples:\n"); + fprintf(stderr, " %s -m stack\n", prog); + fprintf(stderr, " %s -m handle\n", prog); + fprintf(stderr, " %s -m stack_handle\n", prog); + fprintf(stderr, " %s -m stack -o output.txt\n", prog); + fprintf(stderr, " %s -n 0,1,2\n", prog); + fprintf(stderr, " %s -m stack -n 0\n", prog); +} + +static int validate_mode(const char *mode) +{ + if (strcmp(mode, "stack") == 0 || + strcmp(mode, "handle") == 0 || + strcmp(mode, "stack_handle") == 0) + return 0; + + fprintf(stderr, "Error: Invalid mode '%s'\n", mode); + fprintf(stderr, "Valid modes: stack, handle, stack_handle\n"); + return -1; +} + +static int validate_nid_list(const char *nid_list) +{ + const char *p; + int i = 0; + int has_digit = 0; + int in_range = 0; + int prev_num = 0; + int curr_num = 0; + + if (!nid_list || strlen(nid_list) == 0) + return 0; + + for (p = nid_list; *p; p++) { + if (*p == ',') { + if (!has_digit) { + fprintf(stderr, "Error: Invalid nid_list format\n"); + return -1; + } + if (in_range && prev_num > curr_num) { + fprintf(stderr, + "Error: Invalid range %d-%d (start must be <= end)\n", + prev_num, curr_num); + return -1; + } + i = 0; + has_digit = 0; + in_range = 0; + prev_num = 0; + curr_num = 0; + continue; + } + + if (*p == '-') { + if (!has_digit) { + fprintf(stderr, + "Error: Invalid nid_list format "); + fprintf(stderr, + "(dash without preceding number)\n"); + return -1; + } + if (in_range) { + fprintf(stderr, "Error: Multiple dashes in nid_list\n"); + return -1; + } + prev_num = curr_num; + curr_num = 0; + i = 0; + has_digit = 0; + in_range = 1; + continue; + } + + if (!isdigit((unsigned char)*p)) { + fprintf(stderr, "Error: Invalid character '%c' in nid_list\n", *p); + return -1; + } + + if (i > 5) { + fprintf(stderr, "Error: NID too long (max 65536)\n"); + return -1; + } + curr_num = curr_num * 10 + (*p - '0'); + i++; + has_digit = 1; + } + + if (!has_digit) { + fprintf(stderr, "Error: Invalid nid_list format\n"); + return -1; + } + + if (in_range && prev_num > curr_num) { + fprintf(stderr, + "Error: Invalid range %d-%d (start must be <= end)\n", + prev_num, curr_num); + return -1; + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + const char *output_file = NULL; + char filter_cmd[MAX_CMD_LEN]; + FILE *output = NULL; + int fd = -1; + ssize_t ret; + char buf[4096]; + int opt; + size_t cmd_len = 0; + + signal(SIGPIPE, SIG_IGN); + + static struct option long_options[] = { + {"mode", required_argument, 0, 'm'}, + {"nid", required_argument, 0, 'n'}, + {"output", required_argument, 0, 'o'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + filter_cmd[0] = '\0'; + + if (argc > 1) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + usage(argv[0]); + return 0; + } + } + } + + /* Check if page_owner exists and is readable */ + if (access("/sys/kernel/debug/page_owner", F_OK) != 0) { + if (errno == ENOENT) + fprintf(stderr, "Error: /sys/kernel/debug/page_owner does not exist\n"); + else + perror("Error accessing /sys/kernel/debug/page_owner"); + fprintf(stderr, "Make sure page_owner is enabled in kernel\n"); + return 1; + } + + while ((opt = getopt_long(argc, argv, "m:n:o:h", long_options, NULL)) != -1) { + int len; + + switch (opt) { + case 'm': { + const char *mode = optarg; + + if (validate_mode(mode) < 0) + return 1; + len = snprintf(filter_cmd + cmd_len, MAX_CMD_LEN - cmd_len, + "%smode=%s", cmd_len > 0 ? " " : "", mode); + if (len < 0 || cmd_len + len >= MAX_CMD_LEN) { + fprintf(stderr, "Error: Command too long\n"); + return 1; + } + cmd_len += len; + break; + } + case 'n': { + const char *nid_list = optarg; + + if (validate_nid_list(nid_list) < 0) + return 1; + len = snprintf(filter_cmd + cmd_len, MAX_CMD_LEN - cmd_len, + "%snid=%s", cmd_len > 0 ? " " : "", nid_list); + if (len < 0 || cmd_len + len >= MAX_CMD_LEN) { + fprintf(stderr, "Error: Command too long\n"); + return 1; + } + cmd_len += len; + break; + } + case 'o': + output_file = optarg; + break; + case 'h': + /* Already handled above */ + break; + default: + usage(argv[0]); + return 1; + } + } + + /* At least one filter must be specified */ + if (cmd_len == 0) { + fprintf(stderr, "Error: At least one filter (-m or -n) must be specified\n\n"); + usage(argv[0]); + return 1; + } + + /* Open page_owner for read-write - this will fail if kernel doesn't support write */ + fd = open("/sys/kernel/debug/page_owner", O_RDWR); + if (fd < 0) { + if (errno == EACCES || errno == EPERM) { + fprintf(stderr, "Error: /sys/kernel/debug/page_owner "); + fprintf(stderr, "does not support write access\n"); + fprintf(stderr, "This kernel does not support "); + fprintf(stderr, "per-fd filtering.\n"); + fprintf(stderr, "Please ensure you have a kernel with "); + fprintf(stderr, "per-fd filtering support.\n"); + } else { + perror("Error opening /sys/kernel/debug/page_owner"); + } + return 1; + } + + if (output_file) { + output = fopen(output_file, "w"); + if (!output) { + perror("open output file"); + close(fd); + return 1; + } + } else { + output = stdout; + } + + ret = write(fd, filter_cmd, strlen(filter_cmd)); + + if (ret < 0) { + if (errno == EINVAL) { + fprintf(stderr, "Error: Kernel rejected the filter command.\n"); + fprintf(stderr, "Possible causes:\n"); + fprintf(stderr, " - Kernel does not support per-fd filtering\n"); + fprintf(stderr, " - NUMA node has no memory\n"); + fprintf(stderr, " - Unknown reason\n"); + } else { + perror("write filter command"); + } + goto out; + } + + if ((size_t)ret != strlen(filter_cmd)) + fprintf(stderr, "Warning: Partial write (%zd/%zu)\n", ret, strlen(filter_cmd)); + + /* Read and display filtered output */ + ret = 0; + while ((ret = read(fd, buf, sizeof(buf))) > 0) { + size_t written = fwrite(buf, 1, ret, output); + + if (written != (size_t)ret) { + if (errno == EPIPE) { + /* Pipe closed, treat as success */ + ret = 0; + goto out; + } + perror("write output"); + ret = -1; + goto out; + } + } + + if (ret < 0) { + perror("read page_owner"); + goto out; + } + + if (fflush(output)) { + if (errno == EPIPE) { + /* Pipe closed, treat as success */ + ret = 0; + } else { + perror("flush output"); + ret = -1; + } + } + +out: + close(fd); + if (output != stdout) + fclose(output); + return ret < 0 ? 1 : 0; +} diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index e6954909401c..35d3d254941c 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -43,6 +43,13 @@ enum FILTER_BIT { FILTER_TGID = 1<<2, FILTER_COMM = 1<<3 }; + +enum FILTER_RESULT { + FILTER_ERROR, + FILTER_SKIP, + FILTER_MATCH +}; + enum CULL_BIT { CULL_PID = 1<<1, CULL_TGID = 1<<2, @@ -230,7 +237,8 @@ static int remove_pattern(regex_t *pattern, char *buf, int len) return len - (pmatch[1].rm_eo - pmatch[1].rm_so); } -static int search_pattern(regex_t *pattern, char *pattern_str, char *buf) +static int search_pattern(regex_t *pattern, char *pattern_str, + size_t pattern_str_size, char *buf) { int err, val_len; regmatch_t pmatch[2]; @@ -242,8 +250,14 @@ static int search_pattern(regex_t *pattern, char *pattern_str, char *buf) return -1; } val_len = pmatch[1].rm_eo - pmatch[1].rm_so; + if ((size_t)val_len >= pattern_str_size) { + if (debug_on) + fprintf(stderr, "pattern too long in %s\n", buf); + return -1; + } memcpy(pattern_str, buf + pmatch[1].rm_so, val_len); + pattern_str[val_len] = '\0'; return 0; } @@ -300,7 +314,8 @@ static int get_page_num(char *buf) char order_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&order_pattern, order_str, buf); + if (search_pattern(&order_pattern, order_str, sizeof(order_str), buf) < 0) + return 0; errno = 0; order_val = strtol(order_str, &endptr, 10); if (order_val > 64 || errno != 0 || endptr == order_str || *endptr != '\0') { @@ -318,7 +333,8 @@ static pid_t get_pid(char *buf) char pid_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&pid_pattern, pid_str, buf); + if (search_pattern(&pid_pattern, pid_str, sizeof(pid_str), buf) < 0) + return -1; errno = 0; pid = strtol(pid_str, &endptr, 10); if (errno != 0 || endptr == pid_str || *endptr != '\0') { @@ -337,7 +353,8 @@ static pid_t get_tgid(char *buf) char tgid_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&tgid_pattern, tgid_str, buf); + if (search_pattern(&tgid_pattern, tgid_str, sizeof(tgid_str), buf) < 0) + return -1; errno = 0; tgid = strtol(tgid_str, &endptr, 10); if (errno != 0 || endptr == tgid_str || *endptr != '\0') { @@ -356,7 +373,9 @@ static __u64 get_ts_nsec(char *buf) char ts_nsec_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&ts_nsec_pattern, ts_nsec_str, buf); + if (search_pattern(&ts_nsec_pattern, ts_nsec_str, + sizeof(ts_nsec_str), buf) < 0) + return -1; errno = 0; ts_nsec = strtoull(ts_nsec_str, &endptr, 10); if (errno != 0 || endptr == ts_nsec_str || *endptr != '\0') { @@ -372,9 +391,15 @@ static char *get_comm(char *buf) { char *comm_str = malloc(TASK_COMM_LEN); + if (!comm_str) + return NULL; + memset(comm_str, 0, TASK_COMM_LEN); - search_pattern(&comm_pattern, comm_str, buf); + if (search_pattern(&comm_pattern, comm_str, TASK_COMM_LEN, buf) < 0) { + free(comm_str); + return NULL; + } errno = 0; if (errno != 0) { if (debug_on) @@ -386,6 +411,12 @@ static char *get_comm(char *buf) return comm_str; } +static void free_block_list(struct block_list *block) +{ + free(block->comm); + free(block->txt); +} + static int get_arg_type(const char *arg) { if (!strcmp(arg, "pid") || !strcmp(arg, "p")) @@ -450,39 +481,56 @@ static bool match_str_list(const char *str, char **list, int list_size) return false; } -static bool is_need(char *buf) +static enum FILTER_RESULT filter_record(char *buf) { + char *comm; + if ((filter & FILTER_PID) && !match_num_list(get_pid(buf), fc.pids, fc.pids_size)) - return false; + return FILTER_SKIP; if ((filter & FILTER_TGID) && !match_num_list(get_tgid(buf), fc.tgids, fc.tgids_size)) - return false; + return FILTER_SKIP; + if (!(filter & FILTER_COMM)) + return FILTER_MATCH; - char *comm = get_comm(buf); + comm = get_comm(buf); + if (!comm) + return FILTER_ERROR; - if ((filter & FILTER_COMM) && - !match_str_list(comm, fc.comms, fc.comms_size)) { + if (!match_str_list(comm, fc.comms, fc.comms_size)) { free(comm); - return false; + return FILTER_SKIP; } free(comm); - return true; + return FILTER_MATCH; } static bool add_list(char *buf, int len, char *ext_buf) { + enum FILTER_RESULT filter_result; + if (list_size == max_size) { fprintf(stderr, "max_size too small??\n"); return false; } - if (!is_need(buf)) + filter_result = filter_record(buf); + if (filter_result == FILTER_ERROR) { + fprintf(stderr, "Out of memory\n"); + return false; + } + if (filter_result == FILTER_SKIP) return true; list[list_size].pid = get_pid(buf); list[list_size].tgid = get_tgid(buf); list[list_size].comm = get_comm(buf); - list[list_size].txt = malloc(len+1); + if (!list[list_size].comm) { + fprintf(stderr, "Out of memory\n"); + return false; + } + list[list_size].txt = malloc(len + 1); if (!list[list_size].txt) { fprintf(stderr, "Out of memory\n"); + free(list[list_size].comm); return false; } memcpy(list[list_size].txt, buf, len); @@ -841,8 +889,10 @@ int main(int argc, char **argv) } else { list[count-1].num += list[i].num; list[count-1].page_num += list[i].page_num; + free_block_list(&list[i]); } } + list_size = count; qsort(list, count, sizeof(list[0]), compare_sort_condition); @@ -876,8 +926,11 @@ out_free: free(ext_buf); if (buf) free(buf); - if (list) + if (list) { + for (i = 0; i < list_size; i++) + free_block_list(&list[i]); free(list); + } out_ts: regfree(&ts_nsec_pattern); out_comm: diff --git a/tools/testing/memblock/internal.h b/tools/testing/memblock/internal.h index b6b1d147fd75..4168e646af09 100644 --- a/tools/testing/memblock/internal.h +++ b/tools/testing/memblock/internal.h @@ -28,22 +28,6 @@ static inline struct page *virt_to_page(void *virt) return virt; } -void memblock_free_pages(unsigned long pfn, unsigned int order) -{ -} - -static inline void accept_memory(phys_addr_t start, unsigned long size) -{ -} - -unsigned long free_reserved_area(void *start, void *end, int poison, const char *s); -void free_reserved_page(struct page *page); - -static inline bool deferred_pages_enabled(void) -{ - return false; -} - #define for_each_valid_pfn(pfn, start_pfn, end_pfn) \ for ((pfn) = (start_pfn); (pfn) < (end_pfn); (pfn)++) @@ -60,10 +44,6 @@ static inline bool __is_kernel(unsigned long addr) #define for_each_valid_pfn(pfn, start_pfn, end_pfn) \ for ((pfn) = (start_pfn); (pfn) < (end_pfn); (pfn)++) -static inline void init_deferred_page(unsigned long pfn, int nid) -{ -} - #define __SetPageReserved(p) ((void)(p)) #endif diff --git a/tools/testing/memblock/mm_init.h b/tools/testing/memblock/mm_init.h new file mode 100644 index 000000000000..95bc5f2e8ed8 --- /dev/null +++ b/tools/testing/memblock/mm_init.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __MM_MM_INIT_H +#define __MM_MM_INIT_H + +void memblock_free_pages(unsigned long pfn, unsigned int order) +{ +} + +static inline void accept_memory(phys_addr_t start, unsigned long size) +{ +} + +unsigned long free_reserved_area(void *start, void *end, int poison, const char *s); +void free_reserved_page(struct page *page); + +static inline bool deferred_pages_enabled(void) +{ + return false; +} + +static inline void init_deferred_page(unsigned long pfn, int nid) +{ +} +#endif /* __MM_MM_INIT_H */ diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c index 0607913a3022..d967e76a3c06 100644 --- a/tools/testing/radix-tree/maple.c +++ b/tools/testing/radix-tree/maple.c @@ -35234,7 +35234,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt) mt_set_non_kernel(1); /* Spanning store */ mas_set_range(&mas, 1, 100); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0); allocated = mas_allocated(&mas); height = mas_mt_height(&mas); MT_BUG_ON(mt, allocated != 0); @@ -35257,7 +35257,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt) MT_BUG_ON(mt, mas_allocated(&mas) != 0); mas_set_range(&mas, 0, 200); mt_set_non_kernel(1); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0); allocated = mas_allocated(&mas); height = mas_mt_height(&mas); MT_BUG_ON(mt, allocated != 0); diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 8d4db2241cc2..2cc63e4134fb 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 TARGETS += acct +TARGETS += alloc_tag TARGETS += alsa TARGETS += amd-pstate TARGETS += arm64 @@ -14,6 +15,7 @@ TARGETS += core TARGETS += cpufreq TARGETS += cpu-hotplug TARGETS += damon +TARGETS += dax TARGETS += devices/error_logs TARGETS += devices/probe TARGETS += dmabuf-heaps diff --git a/tools/testing/selftests/alloc_tag/Makefile b/tools/testing/selftests/alloc_tag/Makefile new file mode 100644 index 000000000000..f41d92d06b18 --- /dev/null +++ b/tools/testing/selftests/alloc_tag/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 + +TEST_GEN_PROGS := allocinfo_ioctl_test + +CFLAGS += -Wall +CFLAGS += $(KHDR_INCLUDES) + +include ../lib.mk + diff --git a/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c b/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c new file mode 100644 index 000000000000..7d94abb785a1 --- /dev/null +++ b/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* kselftest for allocinfo ioctl + * allocinfo ioctl retrives allocinfo data through ioctl + * Copyright (C) 2026 Google, Inc. + */ + +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdbool.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <linux/types.h> +#include <linux/alloc_tag.h> +#include "../kselftest.h" + +#define MAX_LINE_LEN 512 +#define ALLOCINFO_PROC "/proc/allocinfo" + +enum ioctl_ret { + IOCTL_SUCCESS = 0, + IOCTL_FAILURE = 1, + IOCTL_INVALID_DATA = 2, +}; + +#define VEC_MAX_ENTRIES 32 + +struct allocinfo_tag_data_vec { + struct allocinfo_tag_data tag[VEC_MAX_ENTRIES]; + __u64 count; +}; + +static inline int __allocinfo_get_content_id(int dev_fd, struct allocinfo_content_id *params) +{ + return ioctl(dev_fd, ALLOCINFO_IOC_CONTENT_ID, params); +} + +static inline int __allocinfo_get_at(int dev_fd, struct allocinfo_get_at *params) +{ + return ioctl(dev_fd, ALLOCINFO_IOC_GET_AT, params); +} + +static inline int __allocinfo_get_next(int dev_fd, struct allocinfo_tag_data *params) +{ + return ioctl(dev_fd, ALLOCINFO_IOC_GET_NEXT, params); +} + +static bool match_entry(const struct allocinfo_tag_data *procfs_entry, + const struct allocinfo_tag_data *tag_data, + bool match_bytes, bool match_calls, bool match_lineno, + bool match_function, bool match_filename) +{ + if (match_bytes && tag_data->counter.bytes != procfs_entry->counter.bytes) { + ksft_print_msg("size retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_calls && tag_data->counter.calls != procfs_entry->counter.calls) { + ksft_print_msg("call count retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_lineno && tag_data->tag.lineno != procfs_entry->tag.lineno) { + ksft_print_msg("lineno retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_function && + strncmp(tag_data->tag.function, procfs_entry->tag.function, ALLOCINFO_STR_SIZE)) { + ksft_print_msg("function retrieved through ioctl does not match procfs\n"); + return false; + } + + if (match_filename && + strncmp(tag_data->tag.filename, procfs_entry->tag.filename, ALLOCINFO_STR_SIZE)) { + ksft_print_msg("filename retrieved through ioctl does not match procfs\n"); + return false; + } + return true; +} + +static bool match_entries(const struct allocinfo_tag_data_vec *procfs_entries, + const struct allocinfo_tag_data_vec *tags, + bool match_bytes, bool match_calls, bool match_lineno, + bool match_function, bool match_filename) +{ + __u64 i; + + if (procfs_entries->count != tags->count) { + ksft_print_msg("Entry count mismatch. ioctl entries: %llu, proc entries: %llu\n", + tags->count, procfs_entries->count); + return false; + } + for (i = 0; i < procfs_entries->count; i++) { + if (!match_entry(&procfs_entries->tag[i], &tags->tag[i], + match_bytes, match_calls, match_lineno, + match_function, match_filename)) { + ksft_print_msg("%lluth entry does not match.\n", i); + return false; + } + } + return true; +} + +static const char *allocinfo_str(const char *str) +{ + size_t len = strlen(str); + + if (len >= ALLOCINFO_STR_SIZE) + str += (len - ALLOCINFO_STR_SIZE) + 1; + return str; +} + +static void allocinfo_copy_str(char *dest, const char *src) +{ + strncpy(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE - 1); + dest[ALLOCINFO_STR_SIZE - 1] = '\0'; +} + +static int get_filtered_procfs_entries(struct allocinfo_tag_data_vec *procfs_entries, + const struct allocinfo_filter *filter) +{ + FILE *fp = fopen(ALLOCINFO_PROC, "r"); + char line[MAX_LINE_LEN]; + int matches; + struct allocinfo_tag_data procfs_entry; + + if (!fp) { + ksft_print_msg("Failed to open " ALLOCINFO_PROC " for reading\n"); + return 1; + } + memset(procfs_entries, 0, sizeof(*procfs_entries)); + while (fgets(line, sizeof(line), fp) && procfs_entries->count < VEC_MAX_ENTRIES) { + char filename[MAX_LINE_LEN]; + char function[MAX_LINE_LEN]; + + memset(&procfs_entry, 0, sizeof(procfs_entry)); + matches = sscanf(line, "%llu %llu %[^:]:%llu func:%s", + &procfs_entry.counter.bytes, + &procfs_entry.counter.calls, + filename, + &procfs_entry.tag.lineno, + function); + + if (matches != 5) + continue; + + allocinfo_copy_str(procfs_entry.tag.filename, filename); + allocinfo_copy_str(procfs_entry.tag.function, function); + + if (filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) { + if (strncmp(procfs_entry.tag.filename, + filter->fields.filename, ALLOCINFO_STR_SIZE)) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) { + if (strncmp(procfs_entry.tag.function, + filter->fields.function, ALLOCINFO_STR_SIZE)) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_LINENO) { + if (procfs_entry.tag.lineno != filter->fields.lineno) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) { + if (procfs_entry.counter.bytes < filter->min_size) + continue; + } + if (filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) { + if (procfs_entry.counter.bytes > filter->max_size) + continue; + } + + memcpy(&procfs_entries->tag[procfs_entries->count++], &procfs_entry, + sizeof(procfs_entry)); + } + fclose(fp); + return 0; +} + +static enum ioctl_ret get_filtered_ioctl_entries(struct allocinfo_tag_data_vec *tags, + const struct allocinfo_filter *filter, + __u64 start_pos) +{ + int fd = open(ALLOCINFO_PROC, O_RDONLY); + + if (fd < 0) { + ksft_print_msg("Failed to open " ALLOCINFO_PROC " for IOCTL\n"); + return IOCTL_FAILURE; + } + + struct allocinfo_content_id start_cont_id, end_cont_id; + struct allocinfo_get_at get_at_params; + const int max_retries = 10; + int retry_count = 0; + int status; + + /* + * __allocinfo_get_content_id may return different values if a kernel module was loaded + * between the two calls. If that happens, the data gathered cannot be considered consistent + * and hence needs to be fetched again to avoid flakiness. + */ + do { + if (__allocinfo_get_content_id(fd, &start_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + status = IOCTL_FAILURE; + break; + } + + memset(tags, 0, sizeof(*tags)); + memset(&get_at_params, 0, sizeof(get_at_params)); + memcpy(&get_at_params.filter, filter, sizeof(*filter)); + get_at_params.pos = start_pos; + if (__allocinfo_get_at(fd, &get_at_params)) { + ksft_print_msg("allocinfo_get_at failed\n"); + status = IOCTL_FAILURE; + break; + } + memcpy(&tags->tag[tags->count++], &get_at_params.data, sizeof(get_at_params.data)); + + while (tags->count < VEC_MAX_ENTRIES && + __allocinfo_get_next(fd, &tags->tag[tags->count]) == 0) + tags->count++; + + if (__allocinfo_get_content_id(fd, &end_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + status = IOCTL_FAILURE; + break; + } + + if (start_cont_id.id == end_cont_id.id) { + status = IOCTL_SUCCESS; + } else { + ksft_print_msg("allocinfo_get_content_id mismatch, retrying...\n"); + status = IOCTL_INVALID_DATA; + } + } while (status == IOCTL_INVALID_DATA && retry_count++ < max_retries); + + close(fd); + return status; +} + +static int run_filter_test(const struct allocinfo_filter *filter) +{ + struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags)); + struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries)); + int ioctl_status; + int ret = KSFT_PASS; + + if (!tags || !procfs_entries) { + ksft_print_msg("Memory allocation failed.\n"); + ret = KSFT_FAIL; + goto exit; + } + + if (get_filtered_procfs_entries(procfs_entries, filter)) { + ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n"); + ret = KSFT_SKIP; + goto exit; + } + + if (procfs_entries->count == 0) { + ksft_print_msg("No entries found in " ALLOCINFO_PROC ", skipping test\n"); + ret = KSFT_SKIP; + goto exit; + } + + ioctl_status = get_filtered_ioctl_entries(tags, filter, 0); + if (ioctl_status == IOCTL_INVALID_DATA) { + ksft_print_msg("Trouble retrieving valid IOCTL entries, skipping.\n"); + ret = KSFT_SKIP; + goto exit; + } + if (ioctl_status == IOCTL_FAILURE) { + ksft_print_msg("Error retrieving IOCTL entries.\n"); + ret = KSFT_FAIL; + goto exit; + } + + if (!match_entries(procfs_entries, tags, false, false, true, true, true)) + ret = KSFT_FAIL; + +exit: + free(tags); + free(procfs_entries); + return ret; +} + +static int test_filename_filter(void) +{ + struct allocinfo_filter filter; + const char *target_filename = "mm/memory.c"; + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_FILENAME; + strncpy(filter.fields.filename, target_filename, ALLOCINFO_STR_SIZE); + + return run_filter_test(&filter); +} + +static int test_function_filter(void) +{ + struct allocinfo_filter filter; + const char *target_function = "dup_mm"; + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_FUNCTION; + strncpy(filter.fields.function, target_function, ALLOCINFO_STR_SIZE); + + return run_filter_test(&filter); +} + +static int test_size_filter(void) +{ + int fd; + struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags)); + struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries)); + struct allocinfo_filter filter; + int ret = KSFT_PASS; + __u64 target_size, i, pos; + struct allocinfo_tag_data *found_tag = NULL; + const char *target_function = "do_init_module"; + struct allocinfo_content_id start_cont_id, end_cont_id; + int retry = 0; + const int max_retries = 10; + + if (!tags || !procfs_entries) { + ksft_print_msg("Memory allocation failed.\n"); + ret = KSFT_FAIL; + goto freemem; + } + + fd = open(ALLOCINFO_PROC, O_RDONLY); + if (fd < 0) { + ksft_print_msg("Failed to open " ALLOCINFO_PROC ": %s\n", strerror(errno)); + ret = KSFT_SKIP; + goto freemem; + } + + do { + found_tag = NULL; + pos = 0; + + if (__allocinfo_get_content_id(fd, &start_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + ret = KSFT_FAIL; + goto exit; + } + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_FUNCTION; + strncpy(filter.fields.function, target_function, ALLOCINFO_STR_SIZE); + + if (get_filtered_procfs_entries(procfs_entries, &filter)) { + ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n"); + ret = KSFT_SKIP; + goto exit; + } + + if (procfs_entries->count == 0) { + ksft_print_msg("Function %s not found in procfs\n", target_function); + ret = KSFT_SKIP; + goto exit; + } + + target_size = procfs_entries->tag[0].counter.bytes; + + memset(&filter, 0, sizeof(filter)); + filter.mask |= ALLOCINFO_FILTER_MASK_MIN_SIZE | ALLOCINFO_FILTER_MASK_MAX_SIZE; + filter.min_size = target_size; + filter.max_size = target_size; + + while (1) { + struct allocinfo_get_at get_at_params; + + memset(&get_at_params, 0, sizeof(get_at_params)); + memcpy(&get_at_params.filter, &filter, sizeof(filter)); + get_at_params.pos = pos; + + if (__allocinfo_get_at(fd, &get_at_params)) + break; + + tags->count = 0; + memcpy(&tags->tag[tags->count++], &get_at_params.data, + sizeof(get_at_params.data)); + + while (tags->count < VEC_MAX_ENTRIES && + __allocinfo_get_next(fd, &tags->tag[tags->count]) == 0) + tags->count++; + + for (i = 0; i < tags->count; i++) { + if (strcmp(tags->tag[i].tag.function, target_function) == 0) { + found_tag = &tags->tag[i]; + break; + } + } + + if (found_tag || tags->count < VEC_MAX_ENTRIES) + break; + + pos += tags->count; + } + + if (__allocinfo_get_content_id(fd, &end_cont_id)) { + ksft_print_msg("allocinfo_get_content_id failed\n"); + ret = KSFT_FAIL; + goto exit; + } + + if (start_cont_id.id == end_cont_id.id) + break; + + ksft_print_msg("Module load detected during size verification, retrying...\n"); + } while (retry++ < max_retries); + + if (start_cont_id.id == end_cont_id.id && !found_tag) { + ksft_print_msg("Entry with function %s not found in IOCTL results\n", + target_function); + ret = KSFT_FAIL; + } else if (start_cont_id.id != end_cont_id.id) { + ksft_print_msg("Failed to match content_ids for procfs and IOCTL, skipping...\n"); + ret = KSFT_SKIP; + } else if (found_tag && found_tag->counter.bytes != target_size) { + ksft_print_msg("IOCTL entry size %llu does not match target size %llu\n", + found_tag->counter.bytes, target_size); + ret = KSFT_FAIL; + } + +exit: + close(fd); +freemem: + free(tags); + free(procfs_entries); + return ret; +} + +static int test_lineno_filter(void) +{ + struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags)); + struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries)); + struct allocinfo_filter filter; + enum ioctl_ret ioctl_status; + int ret = KSFT_PASS; + __u64 target_lineno, i; + struct allocinfo_tag_data *target_tag; + bool found = false; + + if (!tags || !procfs_entries) { + ksft_print_msg("Memory allocation failed.\n"); + ret = KSFT_FAIL; + goto exit; + } + + memset(&filter, 0, sizeof(filter)); + + if (get_filtered_procfs_entries(procfs_entries, &filter)) { + ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n"); + ret = KSFT_SKIP; + goto exit; + } + if (procfs_entries->count == 0) { + ksft_print_msg("Could not retrieve procfs entries\n"); + ret = KSFT_SKIP; + goto exit; + } + /* + * We depend on the procfs results to determine the line number for the filter before + * making the ioctl query. Hence, we cannot reuse run_filter_test here. + */ + target_tag = &procfs_entries->tag[0]; + target_lineno = target_tag->tag.lineno; + + filter.mask |= ALLOCINFO_FILTER_MASK_LINENO; + filter.fields.lineno = target_lineno; + + ioctl_status = get_filtered_ioctl_entries(tags, &filter, 0); + if (ioctl_status == IOCTL_INVALID_DATA) { + ksft_print_msg("Trouble retrieving valid IOCTL entries, skipping.\n"); + ret = KSFT_SKIP; + goto exit; + } + if (ioctl_status == IOCTL_FAILURE) { + ksft_print_msg("Error retrieving IOCTL entries.\n"); + ret = KSFT_FAIL; + goto exit; + } + + for (i = 0; i < tags->count; i++) { + if (tags->tag[i].tag.lineno != target_lineno) { + ksft_print_msg("IOCTL entry %llu has incorrect lineno %llu.\n", + i, tags->tag[i].tag.lineno); + ret = KSFT_FAIL; + goto exit; + } + + if (strncmp(tags->tag[i].tag.function, target_tag->tag.function, + ALLOCINFO_STR_SIZE) == 0 && + strncmp(tags->tag[i].tag.filename, target_tag->tag.filename, + ALLOCINFO_STR_SIZE) == 0) + found = true; + } + + if (!found) { + ksft_print_msg("Original procfs entry not found in IOCTL lineno filter results.\n"); + ret = KSFT_FAIL; + } + +exit: + free(tags); + free(procfs_entries); + return ret; +} + +int main(int argc, char *argv[]) +{ + int ret; + + ksft_set_plan(4); + + ret = test_filename_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_filename_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_filename_filter\n"); + + ret = test_function_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_function_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_function_filter\n"); + + ret = test_size_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_size_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_size_filter\n"); + + ret = test_lineno_filter(); + if (ret == KSFT_SKIP) + ksft_test_result_skip("Skipping test_lineno_filter\n"); + else + ksft_test_result(ret == KSFT_PASS, "test_lineno_filter\n"); + + ksft_finished(); +} diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile index 2180c328a825..ece244e5c5b9 100644 --- a/tools/testing/selftests/damon/Makefile +++ b/tools/testing/selftests/damon/Makefile @@ -13,6 +13,7 @@ TEST_PROGS += sysfs.py TEST_PROGS += sysfs_update_schemes_tried_regions_wss_estimation.py TEST_PROGS += damos_quota.py damos_quota_goal.py damos_apply_interval.py TEST_PROGS += damos_tried_regions.py damon_nr_regions.py +TEST_PROGS += sysfs_refresh.py TEST_PROGS += reclaim.sh lru_sort.sh # regression tests (reproducers of previously found bugs) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index 8b12cc048440..e6a2265d721e 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -271,7 +271,7 @@ class DamosFilter: self.type_ = type_ self.matching = matching self.allow = allow - self.memcg_path = memcg_path, + self.memcg_path = memcg_path self.addr_start = addr_start self.addr_end = addr_end self.target_idx = target_idx @@ -624,17 +624,23 @@ class DamonCtx: pause = None idx = None - def __init__(self, ops='paddr', monitoring_attrs=DamonAttrs(), targets=[], - schemes=[], pause=False): + def __init__(self, ops='paddr', monitoring_attrs=None, targets=None, + schemes=None, pause=False): self.ops = ops + if monitoring_attrs is None: + monitoring_attrs = DamonAttrs() self.monitoring_attrs = monitoring_attrs self.monitoring_attrs.context = self + if targets is None: + targets = [] self.targets = targets for idx, target in enumerate(self.targets): target.idx = idx target.context = self + if schemes is None: + schemes = [] self.schemes = schemes for idx, scheme in enumerate(self.schemes): scheme.idx = idx @@ -692,12 +698,14 @@ class DamonCtx: class Kdamond: state = None pid = None + refresh_ms = None contexts = None idx = None # index of this kdamond between siblings kdamonds = None # parent - def __init__(self, contexts=[]): + def __init__(self, contexts=[], refresh_ms=None): self.contexts = contexts + self.refresh_ms = refresh_ms for idx, context in enumerate(self.contexts): context.idx = idx context.kdamond = self @@ -720,6 +728,11 @@ class Kdamond: err = context.stage() if err is not None: return err + if self.refresh_ms is not None: + err = write_file(os.path.join(self.sysfs_dir(), 'refresh_ms'), + '%d' % self.refresh_ms) + if err is not None: + return err err = write_file(os.path.join(self.sysfs_dir(), 'state'), 'on') if err is not None: return err @@ -831,7 +844,7 @@ class Kdamond: for goal in scheme.quota.goals: err = goal.stage() if err is not None: - print('commit_schemes_quota_goals failed stagign: %s'% + print('commit_schemes_quota_goals failed staging: %s'% err) exit(1) return write_file(os.path.join(self.sysfs_dir(), 'state'), diff --git a/tools/testing/selftests/damon/damos_apply_interval.py b/tools/testing/selftests/damon/damos_apply_interval.py index f04d43702481..0f2f36584e48 100755 --- a/tools/testing/selftests/damon/damos_apply_interval.py +++ b/tools/testing/selftests/damon/damos_apply_interval.py @@ -56,7 +56,7 @@ def main(): # Because the second scheme was having the apply interval that is ten times # lower than that of the first scheme, the second scheme should be tried # about ten times more frequently than the first scheme. For possible - # timing errors, check if it was at least nine times more freuqnetly tried. + # timing errors, check if it was at least nine times more frequently tried. ratio = nr_tried_stats[1] / nr_tried_stats[0] if ratio < 9: print('%d / %d = %f (< 9)' % diff --git a/tools/testing/selftests/damon/damos_quota_goal.py b/tools/testing/selftests/damon/damos_quota_goal.py index f76e0412b564..661e4ba4765a 100755 --- a/tools/testing/selftests/damon/damos_quota_goal.py +++ b/tools/testing/selftests/damon/damos_quota_goal.py @@ -66,7 +66,7 @@ def main(): # effective quota was already minimum that cannot be more reduced if expect_increase is False and last_effective_bytes == 1: continue - print('efective bytes not changed: %d' % goal.effective_bytes) + print('effective bytes not changed: %d' % goal.effective_bytes) exit(1) increased = last_effective_bytes < goal.effective_bytes diff --git a/tools/testing/selftests/damon/damos_tried_regions.py b/tools/testing/selftests/damon/damos_tried_regions.py index 3b347eb28bd2..d6472e6a6e08 100755 --- a/tools/testing/selftests/damon/damos_tried_regions.py +++ b/tools/testing/selftests/damon/damos_tried_regions.py @@ -55,10 +55,10 @@ def main(): collected_nr_regions.sort() sample = collected_nr_regions[4] print('50-th percentile nr_regions: %d' % sample) - print('expectation (>= 14) is %s' % 'met' if sample >= 14 else 'not met') + print('expectation (>= 14) is %s' % ('met' if sample >= 14 else 'not met')) if collected_nr_regions[4] < 14: print('full nr_regions:') - print('\n'.join(collected_nr_regions)) + print('\n'.join(['%d' % x for x in collected_nr_regions])) exit(1) if __name__ == '__main__': diff --git a/tools/testing/selftests/damon/drgn_dump_damon_status.py b/tools/testing/selftests/damon/drgn_dump_damon_status.py index 972948e6215f..09552e91bc78 100755 --- a/tools/testing/selftests/damon/drgn_dump_damon_status.py +++ b/tools/testing/selftests/damon/drgn_dump_damon_status.py @@ -59,7 +59,6 @@ def region_to_dict(region): ['ar', addr_range_to_dict], ['sampling_addr', int], ['nr_accesses', int], - ['nr_accesses_bp', int], ['age', int], ]) @@ -163,7 +162,7 @@ def damos_filter_to_dict(damos_filter): int(damos_filter.addr_range.end)] elif type_ == 'target': dict_['target_idx'] = int(damos_filter.target_idx) - elif type_ == 'hugeapge_size': + elif type_ == 'hugepage_size': dict_['sz_range'] = [int(damos_filter.sz_range.min), int(damos_filter.sz_range.max)] return dict_ diff --git a/tools/testing/selftests/damon/sysfs.py b/tools/testing/selftests/damon/sysfs.py index aa03a1187489..3ffa054b6386 100755 --- a/tools/testing/selftests/damon/sysfs.py +++ b/tools/testing/selftests/damon/sysfs.py @@ -119,7 +119,7 @@ def assert_access_pattern_committed(pattern, dump): 'max_nr_accesses', dump) assert_true(dump['min_age_region'] == pattern.age[0], 'min_age_region', dump) - assert_true(dump['max_age_region'] == pattern.age[1], 'miaxage_region', + assert_true(dump['max_age_region'] == pattern.age[1], 'max_age_region', dump) def assert_scheme_committed(scheme, dump): @@ -129,7 +129,7 @@ def assert_scheme_committed(scheme, dump): 'cold': 1, 'pageout': 2, 'hugepage': 3, - 'nohugeapge': 4, + 'nohugepage': 4, 'collapse': 5, 'lru_prio': 6, 'lru_deprio': 7, @@ -250,6 +250,35 @@ def assert_ctxs_committed(kdamonds): if ctx in ctxs_paused_for_dump: ctx.pause = False +def test_memcg_filter_memcg_path_staging(): + global kdamonds + memcg_filter = _damon_sysfs.DamosFilter( + type_='memcg', matching=True, allow=True, memcg_path='/') + kdamonds = _damon_sysfs.Kdamonds( + [_damon_sysfs.Kdamond( + contexts=[_damon_sysfs.DamonCtx( + targets=[_damon_sysfs.DamonTarget(pid=-1)], + schemes=[_damon_sysfs.Damos( + ops_filters=[memcg_filter])], + )])]) + kdamonds.start() + + shown, rd_err = _damon_sysfs.read_file( + os.path.join(memcg_filter.sysfs_dir(), 'memcg_path')) + if rd_err is not None: + print('memcg_path staging: sysfs read (%s)' % rd_err) + kdamonds.stop() + exit(1) + if shown.rstrip('\n') != memcg_filter.memcg_path: + print('memcg_path staging: memcg_path readback ' + '(shown=%s, expected=%s)' % + (shown.rstrip('\n'), memcg_filter.memcg_path)) + kdamonds.stop() + exit(1) + + kdamonds.stop() + kdamonds = None + def main(): global kdamonds kdamonds = _damon_sysfs.Kdamonds( @@ -356,5 +385,7 @@ def main(): assert_ctxs_committed(kdamonds) kdamonds.stop() + test_memcg_filter_memcg_path_staging() + if __name__ == '__main__': main() diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index 78f4badb5beb..f7fb94b84e71 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -3,7 +3,7 @@ source _common.sh -# Kselftest frmework requirement - SKIP code is 4. +# Kselftest framework requirement - SKIP code is 4. ksft_skip=4 ensure_write_succ() @@ -28,7 +28,7 @@ ensure_write_fail() if (echo "$content" > "$file") 2> /dev/null then - echo "writing $content to $file succeed ($fail_reason)" + echo "writing $content to $file succeeded ($reason)" echo "expected failure because $reason" exit 1 fi @@ -67,7 +67,7 @@ ensure_file() echo "$file permission: expected $permission but $perm" exit 1 fi - elif [ "$to_ensure" = "not_exist" ] && [ -f "$dir" ] + elif [ "$to_ensure" = "not_exist" ] && [ -f "$file" ] then echo "$file is not expected but found" exit 1 @@ -99,14 +99,55 @@ test_stats() done } +test_dest() +{ + dest_dir=$1 + ensure_file "$dest_dir/id" "exist" "600" + ensure_file "$dest_dir/weight" "exist" "600" +} + +test_dests() +{ + dests_dir=$1 + ensure_file "$dests_dir/nr_dests" "exist" "600" + ensure_write_succ "$dests_dir/nr_dests" "1" "valid input" + test_dest "$dests_dir/0" + + ensure_write_succ "$dests_dir/nr_dests" "2" "valid input" + test_dest "$dests_dir/0" + test_dest "$dests_dir/1" + + ensure_write_succ "$dests_dir/nr_dests" "0" "valid input" + ensure_dir "$dests_dir/0" "not_exist" + ensure_dir "$dests_dir/1" "not_exist" +} + test_filter() { filter_dir=$1 ensure_file "$filter_dir/type" "exist" "600" - ensure_write_succ "$filter_dir/type" "anon" "valid input" - ensure_write_succ "$filter_dir/type" "memcg" "valid input" - ensure_write_succ "$filter_dir/type" "addr" "valid input" - ensure_write_succ "$filter_dir/type" "target" "valid input" + + local dir_name=$(basename "$(dirname "$filter_dir")") + if [ "$dir_name" = "filters" ] || [ "$dir_name" = "ops_filters" ] + then + ensure_write_succ "$filter_dir/type" "anon" "valid input" + ensure_write_succ "$filter_dir/type" "memcg" "valid input" + fi + if [ "$dir_name" = "filters" ] || [ "$dir_name" = "core_filters" ] + then + ensure_write_succ "$filter_dir/type" "addr" "valid input" + ensure_write_succ "$filter_dir/type" "target" "valid input" + fi + if [ "$dir_name" = "core_filters" ] + then + ensure_write_fail "$filter_dir/type" "anon" "ops type" + ensure_write_fail "$filter_dir/type" "memcg" "ops type" + fi + if [ "$dir_name" = "ops_filters" ] + then + ensure_write_fail "$filter_dir/type" "addr" "core type" + ensure_write_fail "$filter_dir/type" "target" "core type" + fi ensure_write_fail "$filter_dir/type" "foo" "invalid input" ensure_file "$filter_dir/matching" "exist" "600" ensure_file "$filter_dir/memcg_path" "exist" "600" @@ -158,6 +199,20 @@ test_goal() ensure_dir "$goal_dir" "exist" ensure_file "$goal_dir/target_value" "exist" "600" ensure_file "$goal_dir/current_value" "exist" "600" + ensure_file "$goal_dir/target_metric" "exist" "600" + local fpath="$goal_dir/target_metric" + ensure_write_succ "$fpath" "user_input" "valid input" + ensure_write_succ "$fpath" "some_mem_psi_us" "valid input" + ensure_write_succ "$fpath" "node_mem_used_bp" "valid input" + ensure_write_succ "$fpath" "node_mem_free_bp" "valid input" + ensure_write_succ "$fpath" "node_memcg_used_bp" "valid input" + ensure_write_succ "$fpath" "node_memcg_free_bp" "valid input" + ensure_write_succ "$fpath" "active_mem_bp" "valid input" + ensure_write_succ "$fpath" "inactive_mem_bp" "valid input" + ensure_write_succ "$fpath" "node_eligible_mem_bp" "valid input" + ensure_write_fail "$fpath" "foo" "invalid input" + ensure_file "$goal_dir/nid" "exist" "600" + ensure_file "$goal_dir/path" "exist" "600" } test_goals() @@ -207,7 +262,10 @@ test_scheme() ensure_file "$scheme_dir/apply_interval_us" "exist" "600" test_quotas "$scheme_dir/quotas" test_watermarks "$scheme_dir/watermarks" + test_dests "$scheme_dir/dests" test_filters "$scheme_dir/filters" + test_filters "$scheme_dir/core_filters" + test_filters "$scheme_dir/ops_filters" test_stats "$scheme_dir/stats" test_tried_regions "$scheme_dir/tried_regions" } @@ -346,8 +404,13 @@ test_probes() ensure_write_succ "$probes_dir/nr_probes" "1" "valid input" test_probe "$probes_dir/0" + ensure_write_succ "$probes_dir/nr_probes" "2" "valid input" + test_probe "$probes_dir/0" + test_probe "$probes_dir/1" + ensure_write_succ "$probes_dir/nr_probes" "0" "valid input" ensure_dir "$probes_dir/0" "not_exist" + ensure_dir "$probes_dir/1" "not_exist" } test_monitoring_attrs() @@ -363,7 +426,7 @@ test_context() { context_dir=$1 ensure_dir "$context_dir" "exist" - ensure_file "$context_dir/avail_operations" "exit" 400 + ensure_file "$context_dir/avail_operations" "exist" 400 ensure_file "$context_dir/operations" "exist" 600 ensure_file "$context_dir/addr_unit" "exist" 600 ensure_file "$context_dir/pause" "exist" 600 diff --git a/tools/testing/selftests/damon/sysfs_refresh.py b/tools/testing/selftests/damon/sysfs_refresh.py new file mode 100755 index 000000000000..012b7e8f509f --- /dev/null +++ b/tools/testing/selftests/damon/sysfs_refresh.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +import os +import subprocess +import time + +import _damon_sysfs + +def main(): + # Continuously access a memory region for far longer than the test needs, + # so the kdamond always has a live target to monitor while we poll. + sz_region = 10 * 1024 * 1024 + proc = subprocess.Popen( + ['./access_memory', '1', '%d' % sz_region, '60000', 'repeat']) + + # A 'stat' scheme with the default (maximally wide) access pattern matches + # every monitored region, so its 'nr_tried' stat increases as the kdamond + # runs. refresh_ms should make DAMON update the schemes' stats files under + # sysfs on its own, without a manual 'update_schemes_stats' request. + kdamond = _damon_sysfs.Kdamond( + refresh_ms=100, + contexts=[_damon_sysfs.DamonCtx( + ops='vaddr', + targets=[_damon_sysfs.DamonTarget(pid=proc.pid)], + schemes=[_damon_sysfs.Damos(action='stat')], + )]) + kdamonds = _damon_sysfs.Kdamonds([kdamond]) + + err = kdamonds.start() + if err is not None: + # Kernels older than the refresh_ms feature have no such file; treat + # that as unsupported rather than a failure. + if not os.path.exists(os.path.join(kdamond.sysfs_dir(), 'refresh_ms')): + proc.terminate() + proc.wait() + print('kdamond has no refresh_ms file; skipping') + exit(_damon_sysfs.ksft_skip) + proc.terminate() + proc.wait() + print('kdamond start failed: %s' % err) + exit(1) + + scheme = kdamond.contexts[0].schemes[0] + nr_tried_path = os.path.join(scheme.sysfs_dir(), 'stats', 'nr_tried') + + try: + # Poll the stat file directly. We never request an update (e.g. + # 'update_schemes_stats'), so 'nr_tried' can become non-zero only + # through the periodic refresh that refresh_ms enables. + nr_tried = 0 + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if proc.poll() is not None: + print('the access_memory target exited unexpectedly') + exit(1) + content, err = _damon_sysfs.read_file(nr_tried_path) + if err is not None: + print('reading %s failed: %s' % (nr_tried_path, err)) + exit(1) + nr_tried = int(content) + if nr_tried > 0: + break + time.sleep(0.1) + finally: + kdamonds.stop() + proc.terminate() + proc.wait() + + if nr_tried == 0: + print('refresh_ms did not auto-update the schemes stats') + exit(1) + +if __name__ == '__main__': + main() diff --git a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py index 35c724a63f6c..16fdc6e7fc56 100755 --- a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py +++ b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py @@ -7,7 +7,7 @@ import time import _damon_sysfs def pass_wss_estimation(sz_region): - # access two regions of given size, 2 seocnds per each region + # access two regions of given size, 2 seconds per each region proc = subprocess.Popen( ['./access_memory', '2', '%d' % sz_region, '2000', 'repeat']) kdamonds = _damon_sysfs.Kdamonds([_damon_sysfs.Kdamond( diff --git a/tools/testing/selftests/dax/Makefile b/tools/testing/selftests/dax/Makefile new file mode 100644 index 000000000000..25a4f3d73a5b --- /dev/null +++ b/tools/testing/selftests/dax/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +all: + +TEST_PROGS := dax-kmem-hotplug.sh + +include ../lib.mk diff --git a/tools/testing/selftests/dax/config b/tools/testing/selftests/dax/config new file mode 100644 index 000000000000..4c9aaeb6ceb4 --- /dev/null +++ b/tools/testing/selftests/dax/config @@ -0,0 +1,4 @@ +CONFIG_DEV_DAX=m +CONFIG_DEV_DAX_KMEM=m +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTREMOVE=y diff --git a/tools/testing/selftests/dax/dax-kmem-hotplug.sh b/tools/testing/selftests/dax/dax-kmem-hotplug.sh new file mode 100755 index 000000000000..9299120ee509 --- /dev/null +++ b/tools/testing/selftests/dax/dax-kmem-hotplug.sh @@ -0,0 +1,317 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Exercise the dax/kmem "state" sysfs attribute: +# /sys/bus/dax/devices/daxX.Y/state -> unplugged | online | online_kernel | online_movable +# +# The test needs a dax device already bound to the kmem driver. +# +# This test mutates a device's memory: online/offline cycles migrate any +# in-use pages, and the optional unbind subtest wedges the device until +# reboot. The tester must identify the target device and opt into the +# destructive unbind tests. +# +# DAX_KMEM_TEST_DEV=daxX.Y test this specific device +# DAX_KMEM_TEST_DEV=auto auto-discover the first kmem-bound dax device +# (best-effort: it may be a device in use!) +# DAX_KMEM_TEST_UNBIND=1 also run the destructive unbind-while-online test +# +# If DAX_KMEM_TEST_DEV is unset the whole test SKIPs. +# +# A dax device can be provisioned with the memmap= boot param, e.g.: +# memmap=2G!4G +# +# then, in the booted system: +# +# ndctl create-namespace -m devdax -e namespace0.0 -f +# daxctl reconfigure-device -N -m system-ram dax0.0 # bind kmem +# DAX_KMEM_TEST_DEV=auto ./dax-kmem-hotplug.sh + +# shellcheck disable=SC1091 +DIR="$(dirname "$(readlink -f "$0")")" +. "$DIR"/../kselftest/ktap_helpers.sh + +DAX_BASE=/sys/bus/dax/devices +MEM_BASE=/sys/devices/system/memory + +memtotal_kb() { awk '/^MemTotal:/ {print $2}' /proc/meminfo; } +get_state() { cat "$HP" 2>/dev/null; } +# set_state STATE -- write a state to the state attribute; returns the +# write's exit status (0 = accepted by the kernel) +set_state() { echo "$1" > "$HP" 2>/dev/null; } + +is_kmem_dax() { + local drv + [ -e "$DAX_BASE/$1/state" ] || return 1 + drv=$(readlink "$DAX_BASE/$1/driver" 2>/dev/null) + [ "$(basename "${drv:-}")" = kmem ] +} + +find_kmem_dax() { + local d + for d in "$DAX_BASE"/dax*; do + is_kmem_dax "$(basename "$d")" || continue + basename "$d" + return 0 + done + return 1 +} + +# find_device_blocks -- print every memoryN block backing this dax device. +# The blocks are derived from the device's own range(s) in /proc/iomem (the +# reserved resource is named after the device), so we act on *its* blocks +# rather than guessing by NUMA node - the target node may also hold unrelated +# (and non-offlineable) memory. +find_device_blocks() { + local bs + bs=$(cat "$MEM_BASE/block_size_bytes" 2>/dev/null) # hex, no leading 0x + [ -n "$bs" ] || return 1 + grep -E " : ${DAX}\$" /proc/iomem | while read -r line; do + local range s e i + range=${line%% :*}; range=${range// /} + s=${range%-*}; e=${range#*-} + for (( i = 0x$s / 0x$bs; i <= 0x$e / 0x$bs; i++ )); do + echo "memory$i" + done + done +} + +# find_device_block -- print the first online block backing this dax device. +find_device_block() { + local b + for b in $(find_device_blocks); do + [ -f "$MEM_BASE/$b/state" ] || continue + [ "$(cat "$MEM_BASE/$b/state")" = online ] || continue + echo "$b" + return 0 + done + return 1 +} + +ktap_print_header + +if [ "$UID" != 0 ]; then + ktap_skip_all "must be run as root" + exit "$KSFT_SKIP" +fi + +# Device selection is opt-in - see the header for why. +DEV_SEL=${DAX_KMEM_TEST_DEV:-} +if [ -z "$DEV_SEL" ]; then + ktap_skip_all "set DAX_KMEM_TEST_DEV=<daxX.Y|auto> to opt in (mutates device memory)" + exit "$KSFT_SKIP" +fi +if [ "$DEV_SEL" = auto ]; then + DAX=$(find_kmem_dax) +else + DAX=$DEV_SEL +fi +if [ -z "$DAX" ] || ! is_kmem_dax "$DAX"; then + ktap_skip_all "no kmem-bound dax device with a state attribute (${DEV_SEL})" + exit "$KSFT_SKIP" +fi +HP=$DAX_BASE/$DAX/state +ORIG=$(get_state) + +# A failure to reach the baseline is environmental (memory in use), not an +# interface failure, so skip rather than fail. +set_state unplugged; rc=$? +if [ "$rc" != 0 ] || [ "$(get_state)" != unplugged ]; then + ktap_skip_all "$DAX: cannot reach 'unplugged' baseline (memory in use?)" + [ -n "$ORIG" ] && set_state "$ORIG" + exit "$KSFT_SKIP" +fi +mt_unplugged=$(memtotal_kb) + +DRV=/sys/bus/dax/drivers/kmem +AOB=$MEM_BASE/auto_online_blocks + +ktap_print_msg "using $DAX (initial state was: $ORIG)" +ktap_set_plan 10 + +# A public (N_MEMORY) kmem node onlined into a kernel zone (online/online_kernel) +# collects unmovable allocations and can then never be offlined, which would +# wedge the device for the rest of this test. So this test only ever +# successfully onlines online_movable, the one mode that is reliably unpluggable. + +set_state online_movable; rc=$? +mt_online=$(memtotal_kb) +if [ "$rc" = 0 ] && [ "$(get_state)" = online_movable ] && [ "$mt_online" -gt "$mt_unplugged" ]; then + ktap_test_pass "online_movable: state=online_movable, MemTotal $mt_unplugged -> $mt_online kB" +else + ktap_test_fail "online_movable: rc=$rc state=$(get_state) MemTotal $mt_unplugged -> $mt_online" +fi + +set_state online_movable; rc=$? +if [ "$rc" = 0 ] && [ "$(get_state)" = online_movable ]; then + ktap_test_pass "online_movable idempotent" +else + ktap_test_fail "online_movable idempotent: rc=$rc state=$(get_state)" +fi + +# A different online type is rejected without an intervening unplug. The write +# is refused before any hotplug, so this never actually onlines a kernel zone. +set_state online_kernel; rc=$? +if [ "$rc" != 0 ] && [ "$(get_state)" = online_movable ]; then + ktap_test_pass "reject online_kernel without intervening unplug (no kernel-zone online)" +else + ktap_test_fail "online_movable->online_kernel not rejected: rc=$rc state=$(get_state)" +fi + +set_state unplugged; rc=$? +mt=$(memtotal_kb) +if [ "$rc" = 0 ] && [ "$(get_state)" = unplugged ] && [ "$mt" -lt "$mt_online" ]; then + ktap_test_pass "unplug from online_movable: MemTotal $mt_online -> $mt kB" +else + ktap_test_fail "unplug from online_movable: rc=$rc state=$(get_state) MemTotal $mt_online -> $mt" +fi + +before=$(get_state) +set_state bogus_state; rc=$? +if [ "$rc" != 0 ] && [ "$(get_state)" = "$before" ]; then + ktap_test_pass "reject invalid state string" +else + ktap_test_fail "invalid state not rejected: rc=$rc state=$(get_state)" +fi + +# An online_movable -> unplug cycle must re-acquire the per-range resources on +# each online and release them on each unplug. Assert every iteration grows +# MemTotal past the baseline and returns exactly to it; memory left online after +# unplug (off > baseline) is a partial-free failure. +set_state unplugged +cycle_ok=1; fail_i=0; on=0; off=0 +for i in 1 2 3; do + if ! set_state online_movable; then cycle_ok=0; fail_i=$i; break; fi + on=$(memtotal_kb) + if ! set_state unplugged; then cycle_ok=0; fail_i=$i; break; fi + off=$(memtotal_kb) + # online must grow past baseline, and unplug must return to it - a + # partial free (memory left online) is a failure, not just off == on. + if [ "$on" -le "$mt_unplugged" ] || [ "$off" -gt "$mt_unplugged" ]; then + cycle_ok=0; fail_i=$i; break + fi +done +if [ "$cycle_ok" = 1 ]; then + ktap_test_pass "online_movable/unplug cycle re-acquires resources (3x: added and freed each time)" +else + ktap_test_fail "online_movable/unplug cycle regressed at iteration $fail_i (on=$on off=$off baseline=$mt_unplugged)" +fi + +# Desync: toggle a block through the legacy per-block memoryN/state interface +# behind the driver's back, then unplug the whole device via daxX.Y/state. +# +# The driver only updates daxX.Y/state on its own writes, so it still reports +# online_movable while a block underneath is already offline. +# +# Whole-device unplug must still succeed (within reason, an actor changing a +# device from online_movable to online_kernel can no longer guarantee unplug). +# At the very least, an already-offline block should not produce an error. +set_state unplugged +set_state online_movable +blk=$(find_device_block) +if [ -n "$blk" ] && echo offline > "$MEM_BASE/$blk/state" 2>/dev/null; then + # daxX.Y/state is now stale (still online_movable); unplug the device. + set_state unplugged; rc=$? + mt=$(memtotal_kb) + if [ "$rc" = 0 ] && [ "$(get_state)" = unplugged ] && [ "$mt" -le "$mt_unplugged" ]; then + ktap_test_pass "unplug tolerates a block pre-offlined via memoryN/state ($blk)" + else + ktap_test_fail "desync unplug: rc=$rc state=$(get_state) MemTotal=$mt baseline=$mt_unplugged" + fi +else + set_state unplugged 2>/dev/null + ktap_test_skip "could not locate a device block to offline for desync test" +fi + +# change system default online policy while the device is unbound, and show +# the new system default policy is utilized across bindings. +set_state unplugged +if [ -w "$AOB" ] && [ -w "$DRV/unbind" ] && [ -w "$DRV/bind" ]; then + orig_aob=$(cat "$AOB") + echo "$DAX" > "$DRV/unbind" 2>/dev/null + echo offline > "$AOB" 2>/dev/null + echo "$DAX" > "$DRV/bind" 2>/dev/null + sleep 1 + st=$(get_state) + echo "$orig_aob" > "$AOB" 2>/dev/null # restore system policy + if [ "$st" = offline ]; then + ktap_test_pass "online policy resolved at bind: auto_online_blocks=offline -> state=offline" + else + ktap_test_fail "bind-time policy not honored: state=$st (expected offline)" + fi + set_state unplugged 2>/dev/null +else + ktap_test_skip "auto_online_blocks or driver bind/unbind not writable" +fi + +# Blocks offlined out-of-band (via memoryN/state) leave daxX.Y/state stale +# (still online_movable) while every block is actually offline. A driver unbind +# must still hot-remove the offline memory and free its resources rather than +# trust the stale state and leak until reboot. Unbind uses remove_memory(), +# which never offlines, so removing already-offline blocks is non-destructive and +# the device rebinds cleanly afterwards. +if [ -w "$DRV/unbind" ] && [ -w "$DRV/bind" ]; then + set_state unplugged + set_state online_movable + offl_ok=1 + for b in $(find_device_blocks); do + [ -f "$MEM_BASE/$b/state" ] || continue + [ "$(cat "$MEM_BASE/$b/state")" = online ] || continue + echo offline > "$MEM_BASE/$b/state" 2>/dev/null || offl_ok=0 + done + # daxX.Y/state is now stale (still online_movable) while all blocks are + # offline; the unbind must hot-remove them anyway. + if [ "$offl_ok" = 1 ] && [ "$(get_state)" = online_movable ]; then + echo "$DAX" > "$DRV/unbind" 2>/dev/null + mt_after=$(memtotal_kb) + leaked=$(grep -cE " : ${DAX}\$" /proc/iomem) # before rebind + echo "$DAX" > "$DRV/bind" 2>/dev/null # restore for later steps + sleep 1 + if [ "$mt_after" -le "$mt_unplugged" ] && [ "$leaked" = 0 ]; then + ktap_test_pass "unbind with stale online state hot-removes offlined blocks (no leak)" + else + ktap_test_fail "desync unbind leaked: MemTotal=$mt_after baseline=$mt_unplugged iomem_left=$leaked" + fi + set_state unplugged 2>/dev/null + else + ktap_test_skip "could not offline all device blocks for desync-unbind test" + fi +else + ktap_test_skip "driver bind/unbind not writable for desync-unbind test" +fi + +[ -n "$ORIG" ] && set_state "$ORIG" + +# DESTRUCTIVE and opt-in only (DAX_KMEM_TEST_UNBIND=1): +# +# unbinding the driver while memory is online causes the resources to leak - but +# the unbind should not deadlock. Instead the driver leaks it with a warning. + +# This leaves the memory online and the device unbound until reboot, so it runs +# last and only when explicitly requested. online_movable only: this test +# never onlines a public node into a kernel zone. + +if [ "${DAX_KMEM_TEST_UNBIND:-}" = 1 ] && [ -w "$DRV/unbind" ]; then + set_state unplugged; set_state online_movable +fi +if [ "${DAX_KMEM_TEST_UNBIND:-}" = 1 ] && [ "$(get_state)" = online_movable ] && + [ -w "$DRV/unbind" ]; then + mt_on=$(memtotal_kb) + dmesg -C 2>/dev/null + echo "$DAX" > "$DRV/unbind" 2>/dev/null + mt_after=$(memtotal_kb) + # The leaked "System RAM (kmem)" regions stay in the iomem tree; reading + # their names dereferences res_name, which a buggy unbind already freed. + # Walk /proc/iomem to provoke that use-after-free (caught by KASAN). + cat /proc/iomem > /dev/null 2>&1 + splat=$(dmesg 2>/dev/null | grep -ciE "KASAN|BUG:|use-after-free|general protection|Oops|refcount_t") + if [ "$splat" = 0 ] && [ "$mt_after" -ge "$mt_on" ]; then + ktap_test_pass "unbind while online: memory left online, no UAF/oops (MemTotal $mt_on -> $mt_after kB)" + else + ktap_test_fail "unbind while online regressed: splat=$splat MemTotal $mt_on -> $mt_after kB" + fi +else + ktap_test_skip "destructive unbind-while-online test (set DAX_KMEM_TEST_UNBIND=1)" +fi + +ktap_finished diff --git a/tools/testing/selftests/dax/settings b/tools/testing/selftests/dax/settings new file mode 100644 index 000000000000..ba4d85f74cd6 --- /dev/null +++ b/tools/testing/selftests/dax/settings @@ -0,0 +1 @@ +timeout=90 diff --git a/tools/testing/selftests/mincore/mincore_selftest.c b/tools/testing/selftests/mincore/mincore_selftest.c index cdd022c1c497..18dfbf318b1f 100644 --- a/tools/testing/selftests/mincore/mincore_selftest.c +++ b/tools/testing/selftests/mincore/mincore_selftest.c @@ -242,8 +242,10 @@ TEST(check_file_mmap) } /* - * Touch a page in the middle of the mapping. We expect the next - * few pages (the readahead window) to be populated too. + * Touch a page in the middle of the mapping. We expect some + * surrounding pages (the readahead window) to be populated too. + * Depending on the page size and readahead setting, the pages may + * land before the faulted page rather than after it. */ addr[FILE_SIZE / 2] = 1; retval = mincore(addr, FILE_SIZE, vec); @@ -252,6 +254,12 @@ TEST(check_file_mmap) TH_LOG("Page not found in memory after use"); } + i = FILE_SIZE / 2 / page_size - 1; + while (i >= 0 && vec[i]) { + ra_pages++; + i--; + } + i = FILE_SIZE / 2 / page_size + 1; while (i < vec_size && vec[i]) { ra_pages++; diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index e6df968f0971..eae504bd94c8 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -66,7 +66,6 @@ TEST_GEN_FILES += hugetlb-mremap TEST_GEN_FILES += hugetlb-read-hwpoison TEST_GEN_FILES += hugetlb-shm TEST_GEN_FILES += hugetlb-soft-offline -TEST_GEN_FILES += hugetlb-vmemmap TEST_GEN_FILES += khugepaged TEST_GEN_FILES += madv_populate TEST_GEN_FILES += map_fixed_noreplace @@ -150,6 +149,7 @@ TEST_PROGS += ksft_gup_test.sh TEST_PROGS += ksft_hmm.sh TEST_PROGS += ksft_hugetlb.sh TEST_PROGS += ksft_hugevm.sh +TEST_PROGS += ksft_kmemleak_confirm.sh TEST_PROGS += ksft_kmemleak_dedup.sh TEST_PROGS += ksft_ksm.sh TEST_PROGS += ksft_ksm_numa.sh @@ -174,6 +174,10 @@ TEST_PROGS += ksft_userfaultfd.sh TEST_PROGS += ksft_vma_merge.sh TEST_PROGS += ksft_vmalloc.sh +# Destructive: every successful run panics the kernel. Installed and +# kept executable, but not run from a default kselftest invocation. +TEST_PROGS_EXTENDED += hwpoison-panic.sh + TEST_FILES := test_vmalloc.sh TEST_FILES += test_hmm.sh TEST_FILES += va_high_addr_switch.sh diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 2f2b9879d100..e2642eca0d02 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -29,6 +29,10 @@ #include <sys/mman.h> #include <sys/ioctl.h> #include <sys/time.h> +#include <sys/syscall.h> +#include <sys/eventfd.h> +#include <linux/userfaultfd.h> +#include <poll.h> /* * This is a private UAPI to the kernel test module so it isn't exported @@ -225,6 +229,53 @@ static void hmm_buffer_free(struct hmm_buffer *buffer) } /* + * Allocate a buffer structure with memory mapping and mirror. + * + * @mmap_size: total size of the mmap region (may differ from @mirror_size + * for alignment padding in THP tests). + * @mirror_size: size of the mirror data buffer (the actual working set). + * @prot: protection flags for the mmap (e.g. PROT_READ | PROT_WRITE). + * @flags: flags for the mmap (e.g. MAP_PRIVATE, MAP_SHARED, + * MAP_ANONYMOUS, MAP_HUGETLB). + * @fd: file descriptor for the mmap; pass -1 for MAP_ANONYMOUS. + * + * All internal allocations are checked; returns NULL and cleans up on any + * failure. Caller must ASSERT_NE or otherwise check the return value. + */ +static struct hmm_buffer *hmm_buffer_alloc(unsigned long mmap_size, + unsigned long mirror_size, + int prot, int flags, + int fd) +{ + struct hmm_buffer *buffer; + + buffer = malloc(sizeof(*buffer)); + if (!buffer) { + perror("malloc buffer"); + return NULL; + } + + buffer->fd = fd; + buffer->size = mmap_size; + buffer->mirror = malloc(mirror_size); + if (!buffer->mirror) { + perror("malloc mirror"); + free(buffer); + return NULL; + } + + buffer->ptr = mmap(NULL, mmap_size, prot, flags, fd, 0); + if (buffer->ptr == MAP_FAILED) { + perror("mmap"); + free(buffer->mirror); + free(buffer); + return NULL; + } + + return buffer; +} + +/* * Create a temporary file that will be deleted on close. */ static int hmm_create_file(unsigned long size) @@ -314,20 +365,11 @@ TEST_F(hmm, anon_read) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* * Initialize buffer in system memory but leave the first two pages * zero (pte_none and pfn_zero). @@ -377,20 +419,11 @@ TEST_F(hmm, anon_read_prot) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -436,20 +469,11 @@ TEST_F(hmm, anon_write) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -484,20 +508,11 @@ TEST_F(hmm, anon_write_prot) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Simulate a device reading a zero page of memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); ASSERT_EQ(ret, 0); @@ -559,20 +574,10 @@ TEST_F(hmm, anon_write_child) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size * 2, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size * 2; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size * 2, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - old_ptr = buffer->ptr; if (use_thp) { map = (void *)ALIGN((uintptr_t)buffer->ptr, size); @@ -661,20 +666,11 @@ TEST_F(hmm, anon_write_child_shared) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer->ptr so we can tell if it is written. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -736,20 +732,10 @@ TEST_F(hmm, anon_write_huge) size = 2 * read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - size /= 2; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); @@ -795,22 +781,11 @@ TEST_F(hmm, anon_write_hugetlbfs) size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; - buffer = malloc(sizeof(*buffer)); - ASSERT_NE(buffer, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, - -1, 0); - if (buffer->ptr == MAP_FAILED) { - free(buffer); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1); + if (!buffer) SKIP(return, "Huge page could not be allocated"); - } - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) @@ -852,14 +827,11 @@ TEST_F(hmm, file_read) fd = hmm_create_file(size); ASSERT_GE(fd, 0); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ, + MAP_SHARED, fd); ASSERT_NE(buffer, NULL); - buffer->fd = fd; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - /* Write initial contents of the file. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -867,12 +839,6 @@ TEST_F(hmm, file_read) ASSERT_EQ(len, size); memset(buffer->mirror, 0, size); - buffer->ptr = mmap(NULL, size, - PROT_READ, - MAP_SHARED, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Simulate a device reading system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); ASSERT_EQ(ret, 0); @@ -907,20 +873,11 @@ TEST_F(hmm, file_write) fd = hmm_create_file(size); ASSERT_GE(fd, 0); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_SHARED, fd); ASSERT_NE(buffer, NULL); - buffer->fd = fd; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_SHARED, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -960,20 +917,11 @@ TEST_F(hmm, migrate) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1010,20 +958,11 @@ TEST_F(hmm, migrate_file_private) fd = hmm_create_file(size); ASSERT_GE(fd, 0); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE, fd); ASSERT_NE(buffer, NULL); - buffer->fd = fd; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1058,20 +997,11 @@ TEST_F(hmm, migrate_fault) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1114,18 +1044,11 @@ TEST_F(hmm, migrate_release) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1164,20 +1087,11 @@ TEST_F(hmm, migrate_shared) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, -ENOENT); @@ -1201,20 +1115,10 @@ TEST_F(hmm2, migrate_mixed) npages = 6; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); p = buffer->ptr; /* Migrating a protected area should be an error. */ @@ -1295,20 +1199,11 @@ TEST_F(hmm, migrate_multiple) size = npages << self->page_shift; for (c = 0; c < NTIMES; c++) { - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1354,20 +1249,11 @@ TEST_F(hmm, anon_read_multiple) size = npages << self->page_shift; for (c = 0; c < NTIMES; c++) { - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i + c; @@ -1420,20 +1306,11 @@ TEST_F(hmm, anon_teardown) int *ptr; int rc; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i + c; @@ -1474,22 +1351,11 @@ TEST_F(hmm, mixedmap) npages = 1; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, npages, + PROT_READ | PROT_WRITE, + MAP_PRIVATE, self->fd); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(npages); - ASSERT_NE(buffer->mirror, NULL); - - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE, - self->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Simulate a device snapshotting CPU pagetables. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); ASSERT_EQ(ret, 0); @@ -1519,20 +1385,10 @@ TEST_F(hmm2, snapshot) npages = 7; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, npages, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(npages); - ASSERT_NE(buffer->mirror, NULL); - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); p = buffer->ptr; /* Punch a hole after the first page address. */ @@ -1626,21 +1482,11 @@ TEST_F(hmm, compound) size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; - buffer = malloc(sizeof(*buffer)); - ASSERT_NE(buffer, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, - -1, 0); - if (buffer->ptr == MAP_FAILED) { - free(buffer); - return; - } - - buffer->size = size; - buffer->mirror = malloc(npages); - ASSERT_NE(buffer->mirror, NULL); + buffer = hmm_buffer_alloc(size, npages, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1); + if (!buffer) + SKIP(return, "Huge page could not be allocated"); /* Initialize the pages the device will snapshot in buffer->ptr. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) @@ -1707,21 +1553,11 @@ TEST_F(hmm2, double_map) npages = 6; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - /* Reserve a range of addresses. */ - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1783,20 +1619,11 @@ TEST_F(hmm, exclusive) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1837,20 +1664,11 @@ TEST_F(hmm, exclusive_mprotect) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1892,20 +1710,11 @@ TEST_F(hmm, exclusive_cow) ASSERT_NE(npages, 0); size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -1993,20 +1802,11 @@ TEST_F(hmm, hmm_gup_test) npages = 4; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -2079,20 +1879,11 @@ TEST_F(hmm, hmm_cow_in_device) npages = 4; size = npages << self->page_shift; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); - - buffer->ptr = mmap(NULL, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; @@ -2152,21 +1943,12 @@ TEST_F(hmm, migrate_anon_huge_empty) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -2204,21 +1986,12 @@ TEST_F(hmm, migrate_anon_huge_zero) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -2267,21 +2040,12 @@ TEST_F(hmm, migrate_anon_huge_free) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -2329,21 +2093,12 @@ TEST_F(hmm, migrate_anon_huge_fault) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); old_ptr = buffer->ptr; @@ -2425,21 +2180,11 @@ TEST_F(hmm, migrate_partial_unmap_fault) for (use_thp = 0; use_thp < 2; ++use_thp) { for (j = 0; j < ARRAY_SIZE(offsets); ++j) { - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - - buffer->ptr = mmap(NULL, 2 * size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); if (use_thp) @@ -2505,21 +2250,11 @@ TEST_F(hmm, migrate_remap_fault) if (dont_unmap) flags |= MREMAP_DONTUNMAP; - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(8 * size, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 8 * size; - buffer->mirror = malloc(size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, size); - - buffer->ptr = mmap(NULL, buffer->size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - buffer->fd, 0); - ASSERT_NE(buffer->ptr, MAP_FAILED); - npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); if (use_thp) @@ -2590,18 +2325,13 @@ TEST_F(hmm, migrate_anon_huge_err) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, 2 * size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(2 * size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, 2 * size); - old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); - ASSERT_NE(old_ptr, MAP_FAILED); + old_ptr = buffer->ptr; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)old_ptr, size); @@ -2685,18 +2415,13 @@ TEST_F(hmm, migrate_anon_huge_zero_err) size = read_pmd_pagesize(); - buffer = malloc(sizeof(*buffer)); + buffer = hmm_buffer_alloc(2 * size, 2 * size, + PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1); ASSERT_NE(buffer, NULL); - - buffer->fd = -1; - buffer->size = 2 * size; - buffer->mirror = malloc(2 * size); - ASSERT_NE(buffer->mirror, NULL); memset(buffer->mirror, 0xFF, 2 * size); - old_ptr = mmap(NULL, 2 * size, PROT_READ, - MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); - ASSERT_NE(old_ptr, MAP_FAILED); + old_ptr = buffer->ptr; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)old_ptr, size); @@ -2766,19 +2491,6 @@ static double get_time_ms(void) return (tv.tv_sec * 1000.0) + (tv.tv_usec / 1000.0); } -static inline struct hmm_buffer *hmm_buffer_alloc(unsigned long size) -{ - struct hmm_buffer *buffer; - - buffer = malloc(sizeof(*buffer)); - - buffer->fd = -1; - buffer->size = size; - buffer->mirror = malloc(size); - memset(buffer->mirror, 0xFF, size); - return buffer; -} - static void print_benchmark_results(const char *test_name, size_t buffer_size, struct benchmark_results *thp, struct benchmark_results *regular) @@ -2823,14 +2535,14 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz int ret, i; int *ptr; - buffer = hmm_buffer_alloc(buffer_size); - - /* Map memory */ - buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - - if (buffer->ptr == MAP_FAILED) - return -1; + buffer = hmm_buffer_alloc(buffer_size, buffer_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1); + if (!buffer) { + ret = -1; + goto cleanup; + } + memset(buffer->mirror, 0xFF, buffer_size); /* Apply THP hint if requested */ if (use_thp) @@ -2839,7 +2551,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE); if (ret) - return ret; + goto cleanup; /* Initialize memory to make sure pages are allocated */ ptr = (int *)buffer->ptr; @@ -2849,11 +2561,11 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz /* Warmup iteration */ ret = hmm_migrate_sys_to_dev(fd, buffer, npages); if (ret) - return ret; + goto cleanup; ret = hmm_migrate_dev_to_sys(fd, buffer, npages); if (ret) - return ret; + goto cleanup; /* Benchmark iterations */ for (i = 0; i < iterations; i++) { @@ -2862,7 +2574,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = hmm_migrate_sys_to_dev(fd, buffer, npages); if (ret) - return ret; + goto cleanup; end = get_time_ms(); s2d_total += (end - start); @@ -2872,7 +2584,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz ret = hmm_migrate_dev_to_sys(fd, buffer, npages); if (ret) - return ret; + goto cleanup; end = get_time_ms(); d2s_total += (end - start); @@ -2886,9 +2598,9 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) / (results->dev_to_sys_time / 1000.0); - /* Cleanup */ +cleanup: hmm_buffer_free(buffer); - return 0; + return ret; } /* @@ -2949,4 +2661,150 @@ TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) &thp_results, ®ular_results); } } +/* + * Test that HMM can fault in pages backed by userfaultfd using the + * hmm_range_fault_unlocked_timeout() path with no timeout. This exercises + * the lock-drop retry logic in the HMM framework. + */ +struct uffd_thread_args { + int uffd; + int stop_fd; + void *page_buffer; + unsigned long page_size; +}; + +static void *uffd_handler_thread(void *arg) +{ + struct uffd_thread_args *args = arg; + struct uffd_msg msg; + struct uffdio_copy copy; + struct pollfd pollfd[2]; + int ret; + + pollfd[0].fd = args->uffd; + pollfd[0].events = POLLIN; + pollfd[1].fd = args->stop_fd; + pollfd[1].events = POLLIN; + + while (1) { + ret = poll(pollfd, 2, -1); + if (ret <= 0) + break; + if (pollfd[1].revents) + break; + if (!(pollfd[0].revents & POLLIN)) + break; + + ret = read(args->uffd, &msg, sizeof(msg)); + if (ret != sizeof(msg)) + break; + + if (msg.event != UFFD_EVENT_PAGEFAULT) + break; + + /* Fill the page with a known pattern */ + memset(args->page_buffer, 0xAB, args->page_size); + + copy.dst = msg.arg.pagefault.address & ~(args->page_size - 1); + copy.src = (unsigned long)args->page_buffer; + copy.len = args->page_size; + copy.mode = 0; + copy.copy = 0; + + ret = ioctl(args->uffd, UFFDIO_COPY, ©); + if (ret < 0) + break; + } + + return NULL; +} + +TEST_F(hmm, userfaultfd_read) +{ + struct hmm_buffer *buffer; + struct uffd_thread_args uffd_args; + unsigned long npages; + unsigned long size; + unsigned long i; + unsigned char *ptr; + pthread_t thread; + int uffd; + int stop_fd; + int ret; + struct uffdio_api api; + struct uffdio_register reg; + uint64_t stop = 1; + ssize_t nwrite; + + npages = 4; + size = npages << self->page_shift; + + /* Create userfaultfd */ + uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); + if (uffd < 0) + SKIP(return, "userfaultfd not available"); + + api.api = UFFD_API; + api.features = 0; + ret = ioctl(uffd, UFFDIO_API, &api); + ASSERT_EQ(ret, 0); + + buffer = malloc(sizeof(*buffer)); + ASSERT_NE(buffer, NULL); + + buffer->fd = -1; + buffer->size = size; + buffer->mirror = malloc(size); + ASSERT_NE(buffer->mirror, NULL); + + /* Create anonymous mapping */ + buffer->ptr = mmap(NULL, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + ASSERT_NE(buffer->ptr, MAP_FAILED); + + /* Register the region with userfaultfd */ + reg.range.start = (unsigned long)buffer->ptr; + reg.range.len = size; + reg.mode = UFFDIO_REGISTER_MODE_MISSING; + ret = ioctl(uffd, UFFDIO_REGISTER, ®); + ASSERT_EQ(ret, 0); + + /* Set up the handler thread */ + uffd_args.uffd = uffd; + stop_fd = eventfd(0, EFD_CLOEXEC); + ASSERT_GE(stop_fd, 0); + uffd_args.stop_fd = stop_fd; + uffd_args.page_buffer = malloc(self->page_size); + ASSERT_NE(uffd_args.page_buffer, NULL); + uffd_args.page_size = self->page_size; + + ret = pthread_create(&thread, NULL, uffd_handler_thread, &uffd_args); + ASSERT_EQ(ret, 0); + + /* + * Use the unlocked read path which allows the mmap lock to be + * dropped during the fault, enabling userfaultfd resolution. + */ + ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ_UNLOCKED, + buffer, npages); + ASSERT_EQ(ret, 0); + ASSERT_EQ(buffer->cpages, npages); + + /* Verify the device read the data filled by the uffd handler */ + ptr = buffer->mirror; + for (i = 0; i < size; ++i) + ASSERT_EQ(ptr[i], (unsigned char)0xAB); + + nwrite = write(stop_fd, &stop, sizeof(stop)); + ASSERT_EQ(nwrite, sizeof(stop)); + pthread_join(thread, NULL); + close(stop_fd); + free(uffd_args.page_buffer); + close(uffd); + hmm_buffer_free(buffer); +} + + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/mm/hugepage_settings.c b/tools/testing/selftests/mm/hugepage_settings.c index 2eab2110ac6a..d7917dce3aba 100644 --- a/tools/testing/selftests/mm/hugepage_settings.c +++ b/tools/testing/selftests/mm/hugepage_settings.c @@ -437,7 +437,7 @@ void hugetlb_set_nr_pages(unsigned long size, unsigned long nr) hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); - write_num(path, nr); + write_num_ignore_einval(path, nr); } unsigned long hugetlb_free_pages(unsigned long size) diff --git a/tools/testing/selftests/mm/hugetlb-vmemmap.c b/tools/testing/selftests/mm/hugetlb-vmemmap.c deleted file mode 100644 index 507df78a158d..000000000000 --- a/tools/testing/selftests/mm/hugetlb-vmemmap.c +++ /dev/null @@ -1,132 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * A test case of using hugepage memory in a user application using the - * mmap system call with MAP_HUGETLB flag. Before running this program - * make sure the administrator has allocated enough default sized huge - * pages to cover the 2 MB allocation. - */ -#include <stdlib.h> -#include <stdio.h> -#include <unistd.h> -#include <sys/mman.h> -#include <fcntl.h> -#include "vm_util.h" -#include "hugepage_settings.h" - -#define PAGE_COMPOUND_HEAD (1UL << 15) -#define PAGE_COMPOUND_TAIL (1UL << 16) -#define PAGE_HUGE (1UL << 17) - -#define HEAD_PAGE_FLAGS (PAGE_COMPOUND_HEAD | PAGE_HUGE) -#define TAIL_PAGE_FLAGS (PAGE_COMPOUND_TAIL | PAGE_HUGE) - -#define PM_PFRAME_BITS 55 -#define PM_PFRAME_MASK ~((1UL << PM_PFRAME_BITS) - 1) - -static size_t pagesize; -static size_t maplength; - -static void write_bytes(char *addr, size_t length) -{ - unsigned long i; - - for (i = 0; i < length; i++) - *(addr + i) = (char)i; -} - -static unsigned long virt_to_pfn(void *addr) -{ - int fd; - unsigned long pagemap; - - fd = open("/proc/self/pagemap", O_RDONLY); - if (fd < 0) - return -1UL; - - lseek(fd, (unsigned long)addr / pagesize * sizeof(pagemap), SEEK_SET); - read(fd, &pagemap, sizeof(pagemap)); - close(fd); - - return pagemap & ~PM_PFRAME_MASK; -} - -static int check_page_flags(unsigned long pfn) -{ - int fd, i; - unsigned long pageflags; - - fd = open("/proc/kpageflags", O_RDONLY); - if (fd < 0) - return -1; - - lseek(fd, pfn * sizeof(pageflags), SEEK_SET); - - read(fd, &pageflags, sizeof(pageflags)); - if ((pageflags & HEAD_PAGE_FLAGS) != HEAD_PAGE_FLAGS) { - close(fd); - ksft_print_msg("Head page flags (%lx) is invalid\n", pageflags); - return -1; - } - - /* - * pages other than the first page must be tail and shouldn't be head; - * this also verifies kernel has correctly set the fake page_head to tail - * while hugetlb_free_vmemmap is enabled. - */ - for (i = 1; i < maplength / pagesize; i++) { - read(fd, &pageflags, sizeof(pageflags)); - if ((pageflags & TAIL_PAGE_FLAGS) != TAIL_PAGE_FLAGS || - (pageflags & HEAD_PAGE_FLAGS) == HEAD_PAGE_FLAGS) { - close(fd); - ksft_print_msg("Tail page flags (%lx) is invalid\n", pageflags); - return -1; - } - } - - close(fd); - - return 0; -} - -int main(int argc, char **argv) -{ - void *addr; - unsigned long pfn; - int ret; - - ksft_print_header(); - ksft_set_plan(1); - - if (!hugetlb_setup_default(1)) - ksft_exit_skip("Not enough free huge pages\n"); - - pagesize = psize(); - maplength = default_huge_page_size(); - if (!maplength) - ksft_exit_skip("Unable to determine huge page size\n"); - - addr = mmap(NULL, maplength, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) - ksft_exit_fail_perror("mmap"); - - /* Trigger allocation of HugeTLB page. */ - write_bytes(addr, maplength); - - pfn = virt_to_pfn(addr); - if (pfn == -1UL) { - ksft_perror("virt_to_pfn"); - munmap(addr, maplength); - ksft_exit_fail(); - } - - ksft_print_msg("Returned address is %p whose pfn is %lx\n", addr, pfn); - - ret = check_page_flags(pfn); - - if (munmap(addr, maplength)) - ksft_exit_fail_perror("munmap"); - - ksft_test_result(!ret, "HugeTLB vmemmap page flags\n"); - ksft_finished(); -} diff --git a/tools/testing/selftests/mm/hwpoison-panic.sh b/tools/testing/selftests/mm/hwpoison-panic.sh new file mode 100755 index 000000000000..d953d1367332 --- /dev/null +++ b/tools/testing/selftests/mm/hwpoison-panic.sh @@ -0,0 +1,255 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Verify vm.panic_on_unrecoverable_memory_failure by injecting a hwpoison +# error on a kernel-owned page and confirming the kernel panics. +# +# Three "kinds" of kernel-owned page can be targeted, selectable via the +# first positional argument (default: rodata): +# +# rodata - a PG_reserved page in the kernel rodata range +# (sourced from /proc/iomem "Kernel rodata"). Exercises +# memory_failure() -> get_any_page() on a PageReserved page. +# +# slab - a slab page found via /proc/kpageflags (KPF_SLAB). +# Exercises memory_failure() -> get_any_page() on a non +# PG_reserved kernel-owned page. This path is what catches +# regressions where get_any_page() collapses kernel-owned +# pages into a transient -EIO instead of -ENOTRECOVERABLE. +# +# pgtable - a page-table page found via /proc/kpageflags (KPF_PGTABLE). +# Same path as slab, different page type. +# +# This test is DESTRUCTIVE: a successful run crashes the kernel. It is +# meant to be executed inside a disposable VM (e.g. virtme-ng) with a +# serial console captured by the harness. It is skipped unless the +# caller opts in via RUN_DESTRUCTIVE=1. +# +# Test passes externally: the kernel must panic with +# "Memory failure: <pfn>: unrecoverable page" +# A return from the inject means no panic fired: that is a failure, +# unless the target PFN raced to a different page type before injection, +# in which case the run is inconclusive and is skipped. +# +# Author: Breno Leitao <leitao@debian.org> + +set -u + +# KTAP output helpers (ktap_print_msg, ktap_skip_all, ktap_exit_fail_msg, ...). +DIR="$(dirname "$(readlink -f "$0")")" +# shellcheck source=../kselftest/ktap_helpers.sh +source "${DIR}"/../kselftest/ktap_helpers.sh + +sysctl_path=/proc/sys/vm/panic_on_unrecoverable_memory_failure +inject_path=/sys/devices/system/memory/hard_offline_page +kpageflags_path=/proc/kpageflags +unpoison_path=/sys/kernel/debug/hwpoison/unpoison-pfn + +# /proc/kpageflags bit positions (see include/uapi/linux/kernel-page-flags.h) +KPF_SLAB=7 +KPF_COMPOUND_TAIL=16 +KPF_HWPOISON=19 +KPF_NOPAGE=20 +KPF_PGTABLE=26 +KPF_RESERVED=32 + +pagesize=$(getconf PAGE_SIZE) + +kind=${1:-rodata} + +if [ "$(id -u)" -ne 0 ]; then + ktap_skip_all "must run as root" + exit "$KSFT_SKIP" +fi + +if [ ! -w "$sysctl_path" ]; then + ktap_skip_all "$sysctl_path not present (kernel without the sysctl?)" + exit "$KSFT_SKIP" +fi + +if [ ! -w "$inject_path" ]; then + ktap_skip_all "$inject_path not present (no MEMORY_HOTPLUG?)" + exit "$KSFT_SKIP" +fi + +if [ "${RUN_DESTRUCTIVE:-0}" != "1" ]; then + ktap_skip_all "destructive test; re-run with RUN_DESTRUCTIVE=1 inside a disposable VM" + exit "$KSFT_SKIP" +fi + +# Pick a PFN inside the kernel image rodata region of /proc/iomem. +# This is preferred over a top-level "Reserved" entry because top-level +# Reserved ranges are often firmware holes that have no backing struct +# page; pfn_to_online_page() returns NULL on those and memory_failure() +# bails out with -ENXIO before reaching the panic path. +# +# "Kernel rodata" is reported as a sub-resource of "System RAM" on every +# major architecture, which guarantees: +# - the PFN is backed by struct page (within an online memory range); +# - PG_reserved is set on the page (kernel image area); +# - the memory is read-only, so setting PG_hwpoison on it does not +# corrupt writable kernel state if the panic somehow does not fire. +# +# /proc/iomem entries look like (indented for sub-resources): +# " 02500000-02ffffff : Kernel rodata" +pick_rodata_phys_addr() { + awk -v pagesize="$(getconf PAGE_SIZE)" ' + # Convert a hex string to a number without relying on the gawk-only + # strtonum(). mawk lacks it and would otherwise spuriously skip + # this test on distros that ship mawk as /usr/bin/awk. + function hex2num(s, n, i, c, v) { + n = 0 + for (i = 1; i <= length(s); i++) { + c = tolower(substr(s, i, 1)) + v = index("0123456789abcdef", c) - 1 + if (v < 0) + return -1 + n = n * 16 + v + } + return n + } + /: Kernel rodata[[:space:]]*$/ { + sub(/^[[:space:]]+/, "") + n = split($0, a, /[- ]/) + start = hex2num(a[1]) + end = hex2num(a[2]) + if (end <= start) + next + # Page-align upward and emit the first byte of that page. + pfn = int((start + pagesize - 1) / pagesize) + printf "0x%x\n", pfn * pagesize + exit 0 + } + ' /proc/iomem +} + +# Walk /proc/kpageflags and return the phys addr of the first PFN that +# has bit $1 set, with KPF_HWPOISON, KPF_NOPAGE and KPF_COMPOUND_TAIL +# all clear (so we attack a real, non-tail, not-already-poisoned page). +# +# We skip the first 16 MiB of PFNs to step past low-memory special +# ranges (BIOS/EFI/ACPI/etc.) that often are PG_reserved and would not +# exhibit the slab/pgtable type we are looking for. +pick_kpageflags_phys_addr() { + local want_bit=$1 + local pagesize skip_pfn + + [ -r "$kpageflags_path" ] || return + + pagesize=$(getconf PAGE_SIZE) + skip_pfn=$(((16 * 1024 * 1024) / pagesize)) + + od -An -tx8 -v -w8 -j "$((skip_pfn * 8))" "$kpageflags_path" 2>/dev/null | \ + awk -v want_bit="$want_bit" \ + -v hwp_bit="$KPF_HWPOISON" \ + -v nopage_bit="$KPF_NOPAGE" \ + -v tail_bit="$KPF_COMPOUND_TAIL" \ + -v base_pfn="$skip_pfn" \ + -v pagesize="$pagesize" ' + # Test whether bit "b" is set in the 16-hex-digit value "hex". + # Done with substring + per-digit lookup so we never rely on awk + # bitwise operators (mawk lacks them), 64-bit FP precision or the + # gawk-only strtonum(). + function bit_set(hex, b, di, bi, c, v) { + di = int(b / 4) + bi = b - di * 4 + c = substr(hex, length(hex) - di, 1) + v = index("0123456789abcdef", tolower(c)) - 1 + if (bi == 0) return (v % 2) == 1 + if (bi == 1) return int(v / 2) % 2 == 1 + if (bi == 2) return int(v / 4) % 2 == 1 + return int(v / 8) % 2 == 1 + } + { + gsub(/^[[:space:]]+/, "") + h = $1 + if (bit_set(h, want_bit) && + !bit_set(h, hwp_bit) && + !bit_set(h, nopage_bit) && + !bit_set(h, tail_bit)) { + pfn = base_pfn + NR - 1 + printf "0x%x\n", pfn * pagesize + exit 0 + } + } + ' +} + +# Return 0 if /proc/kpageflags bit $2 is set for PFN $1, 1 if it is +# clear, or 2 if the word cannot be read. Used to re-confirm the target +# page type after a non-panicking inject. +kpageflags_bit_set() { + local word + + word=$(od -An -tx8 -v -j "$(($1 * 8))" -N 8 "$kpageflags_path" 2>/dev/null | tr -d '[:space:]') + [ -n "$word" ] || return 2 + (( (16#$word >> $2) & 1 )) +} + +# Best-effort: drop the PG_hwpoison marker set by the inject so a failed +# run does not leave a poisoned page behind. hard_offline_page() injects +# with MF_SW_SIMULATED, so the page stays unpoisonable through the +# hwpoison debugfs interface (needs CONFIG_HWPOISON_INJECT + debugfs). +try_unpoison() { + [ -w "$unpoison_path" ] || return 0 + echo "$1" > "$unpoison_path" 2>/dev/null || true +} + +case "$kind" in +rodata) + phys_addr=$(pick_rodata_phys_addr) + recheck_bit=$KPF_RESERVED + missing_msg='no "Kernel rodata" entry in /proc/iomem' + ;; +slab) + phys_addr=$(pick_kpageflags_phys_addr "$KPF_SLAB") + recheck_bit=$KPF_SLAB + missing_msg="no usable slab PFN found in $kpageflags_path" + ;; +pgtable) + phys_addr=$(pick_kpageflags_phys_addr "$KPF_PGTABLE") + recheck_bit=$KPF_PGTABLE + missing_msg="no usable page-table PFN found in $kpageflags_path" + ;; +*) + ktap_exit_fail_msg "unknown kind '$kind' (expected: rodata|slab|pgtable)" + ;; +esac + +if [ -z "$phys_addr" ]; then + ktap_skip_all "$missing_msg" + exit "$KSFT_SKIP" +fi + +ktap_print_msg "enabling $sysctl_path" +prior=$(cat "$sysctl_path") +echo 1 > "$sysctl_path" || ktap_exit_fail_msg "failed to enable sysctl" + +pfn=$((phys_addr / pagesize)) +ktap_print_msg "injecting hwpoison at phys 0x$(printf '%x' "$phys_addr") (pfn 0x$(printf '%x' "$pfn"), kind=$kind)" +ktap_print_msg "expecting kernel panic: 'Memory failure: <pfn>: unrecoverable page'" + +# A successful run never returns from the inject -- it panics the kernel. +# Reaching the code below therefore means no panic fired. Note whether +# the write itself succeeded, then put the machine back: restore the +# sysctl and best-effort unpoison the page we just marked. +if echo "$phys_addr" > "$inject_path"; then + verdict="inject returned without panic; sysctl ineffective" +else + verdict="inject failed before reaching the panic path" +fi + +echo "$prior" > "$sysctl_path" +try_unpoison "$pfn" + +# The page type can change between selection and injection (e.g. a slab +# or page-table page is freed and reused). Only treat a missing panic as +# a failure if the target PFN is still the kernel-owned type we aimed at; +# if it raced to another type the run is inconclusive, so skip instead. +kpageflags_bit_set "$pfn" "$recheck_bit" +case $? in +0) ktap_exit_fail_msg "$verdict (page still $kind)" ;; +1) ktap_skip_all "target PFN no longer $kind; raced before inject, inconclusive" + exit "$KSFT_SKIP" ;; +*) ktap_exit_fail_msg "$verdict (could not reconfirm page type via $kpageflags_path)" ;; +esac diff --git a/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh b/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh new file mode 100755 index 000000000000..3a8576e835c8 --- /dev/null +++ b/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Functional test for kmemleak's N-consecutive-scan leak confirmation +# (the min_unref_scans module parameter). +# +# kmemleak only reports an object once it has stayed unreferenced for +# min_unref_scans consecutive scans. The default of 1 reports on the first +# scan (historical behaviour); higher values filter transient false +# positives where a live object's only reference is briefly invisible to a +# single scan (e.g. an RCU tree update in flight while the scan runs). The +# test loads samples/kmemleak's helper module to create orphan allocations +# and, counting only those orphans (matched by their [kmemleak_test] +# backtrace so unrelated leaks already present on the system are ignored), +# checks that: +# - a freshly allocated object is greyed on its first scan (its checksum +# settles then), so nothing can be reported before that priming scan; +# each case below primes once first, +# - with the default threshold (min_unref_scans=1) one scan after priming +# reports the orphans, +# - raising the threshold to 2 needs two scans after priming: one is not +# enough, the second reports, +# - the parameter reads back what was written. +# +# The "one post-prime scan is not enough at min_unref_scans=2" check is the +# core regression test: raising min_unref_scans must push the report +# strictly later. Like ksft_kmemleak_dedup.sh, if the module yields no +# detectable orphan at all in the running environment the test skips rather +# than failing. +# +# Author: Breno Leitao <leitao@debian.org> + +# KTAP output helpers (ktap_skip_all, ktap_exit_fail_msg, ktap_test_pass, ...). +DIR="$(dirname "$(readlink -f "$0")")" +# shellcheck source=../kselftest/ktap_helpers.sh +source "${DIR}"/../kselftest/ktap_helpers.sh + +KMEMLEAK=/sys/kernel/debug/kmemleak +PARAM=/sys/module/kmemleak/parameters/min_unref_scans +MODULE=kmemleak-test +AGE=6 # seconds; must exceed kmemleak's 5s minimum object age + +ktap_print_header + +[ "$(id -u)" -eq 0 ] || { ktap_skip_all "must run as root"; exit "$KSFT_SKIP"; } +[ -r "$KMEMLEAK" ] || + { ktap_skip_all "no kmemleak debugfs (CONFIG_DEBUG_KMEMLEAK)"; exit "$KSFT_SKIP"; } +[ -w "$PARAM" ] || + { ktap_skip_all "min_unref_scans module parameter not present"; exit "$KSFT_SKIP"; } +modinfo "$MODULE" >/dev/null 2>&1 || + { ktap_skip_all "$MODULE not built (CONFIG_SAMPLE_KMEMLEAK)"; exit "$KSFT_SKIP"; } + +# kmemleak can be present but disabled at runtime (kmemleak=off boot arg, +# or it self-disabled after an internal error); a "scan" then returns +# EPERM. Probe once and skip if so. +echo scan > "$KMEMLEAK" 2>/dev/null || + { ktap_skip_all "kmemleak is disabled (check dmesg or kmemleak= boot arg)"; exit "$KSFT_SKIP"; } + +prev=$(cat "$PARAM") +# shellcheck disable=SC2317 # invoked indirectly via trap +cleanup() { + echo "$prev" > "$PARAM" 2>/dev/null # restore the parameter + echo scan=on > "$KMEMLEAK" 2>/dev/null # re-enable auto scan + rmmod "$MODULE" 2>/dev/null + echo clear > "$KMEMLEAK" 2>/dev/null +} +trap cleanup EXIT + +# Stop the automatic scan thread: only our manual scans should advance an +# object's consecutive-unreferenced run. An auto scan landing between two +# manual scans would change the result and make the test flaky. +echo scan=off > "$KMEMLEAK" 2>/dev/null + +# Create a fresh, aged set of orphan objects from the helper module's init +# path (its kmalloc/vmalloc/percpu allocations are dropped right away). +# Pre-existing reported leaks are greyed first ("clear") so only our +# orphans are counted. The module is left loaded on purpose: once it is +# unloaded its symbols are gone, so the orphan backtraces no longer resolve +# to [kmemleak_test] and could not be matched below. +gen_orphans() { + rmmod "$MODULE" 2>/dev/null + echo clear > "$KMEMLEAK" + modprobe "$MODULE" || + { ktap_skip_all "failed to load $MODULE"; exit "$KSFT_SKIP"; } + sleep "$AGE" +} + +scan() { echo scan > "$KMEMLEAK"; } + +# Number of helper-module orphans currently reported by kmemleak. Matching +# the module's own backtrace ([kmemleak_test]) keeps the count immune to +# unrelated leaks on the running system. kmemleak only lists an object here +# once it has been reported, so this reflects the confirmation gating. +count_orphans() { + c=$(grep -c '\[kmemleak_test\]' "$KMEMLEAK" 2>/dev/null) + echo "${c:-0}" +} + +# 0) the parameter reads back what was written. +echo 3 > "$PARAM" +[ "$(cat "$PARAM")" = "3" ] || ktap_exit_fail_msg "min_unref_scans did not read back as 3" + +# Priming scan: kmemleak greys a freshly allocated object on its first scan +# (its checksum settles then), so nothing can be reported until a second +# scan. Every case below runs this priming scan before counting. +prime() { scan; } + +# 1) min_unref_scans=1 (default): one scan after priming reports the +# orphans. This also establishes that the helper produces detectable +# orphans here. +echo 1 > "$PARAM" +gen_orphans +prime +scan +first=$(count_orphans) +[ "$first" -gt 0 ] || + { ktap_skip_all "$MODULE produced no detectable orphans (cannot test min_unref_scans)"; exit "$KSFT_SKIP"; } + +# 2) min_unref_scans=2: after priming, one scan is not enough (still +# gated), the second reports. The gated-scan-zero check is the core +# regression. +echo 2 > "$PARAM" +gen_orphans +prime +scan; s1=$(count_orphans) +scan; s2=$(count_orphans) +[ "$s1" -eq 0 ] || ktap_exit_fail_msg "min_unref_scans=2: $s1 orphan(s) after 1 post-prime scan (must be 0)" +[ "$s2" -gt 0 ] || ktap_exit_fail_msg "min_unref_scans=2: no report after 2 post-prime scans (false negative)" + +ktap_set_plan 1 +ktap_test_pass "min_unref_scans=1 reported $first orphan(s) one scan after priming; =2 held them one scan longer ($s1 after one scan, $s2 after two); param read-back ok" +ktap_finished diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index a050f4840cfa..5fd7792a0d47 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -288,8 +288,8 @@ static bool assert_ksm_pages_count(long dupl_page_count) static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) { if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes)) || ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || @@ -304,8 +304,8 @@ static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) static int ksm_restore(struct ksm_sysfs *ksm_sysfs) { if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || @@ -440,9 +440,9 @@ static int get_next_mem_node(int node) mem_node = i % (max_node + 1); node_size = numa_node_size(mem_node, NULL); if (node_size > 0) - break; + return mem_node; } - return mem_node; + return -ENODEV; } static int get_first_mem_node(void) @@ -455,8 +455,8 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo { void *numa1_map_ptr, *numa2_map_ptr; struct timespec start_time; + int first_node, second_node; int page_count = 2; - int first_node; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { ksft_perror("clock_gettime"); @@ -467,17 +467,19 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo ksft_print_msg("NUMA support not enabled\n"); return KSFT_SKIP; } - if (numa_num_configured_nodes() <= 1) { - ksft_print_msg("At least 2 NUMA nodes must be available\n"); + first_node = get_first_mem_node(); + second_node = get_next_mem_node(first_node); + + if (second_node < 0) { + ksft_print_msg("At least 2 NUMA nodes with memory must be available\n"); return KSFT_SKIP; } if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) return KSFT_FAIL; /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ - first_node = get_first_mem_node(); numa1_map_ptr = numa_alloc_onnode(page_size, first_node); - numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); + numa2_map_ptr = numa_alloc_onnode(page_size, second_node); if (!numa1_map_ptr || !numa2_map_ptr) { ksft_perror("numa_alloc_onnode"); return KSFT_FAIL; @@ -844,8 +846,8 @@ int main(int argc, char *argv[]) if (ksm_write_sysfs(KSM_FP("run"), 2) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) ksft_exit_fail_msg("Cannot set up KSM tunables\n"); diff --git a/tools/testing/selftests/mm/merge.c b/tools/testing/selftests/mm/merge.c index 519e5ac02db7..edfbd39ae58e 100644 --- a/tools/testing/selftests/mm/merge.c +++ b/tools/testing/selftests/mm/merge.c @@ -1305,6 +1305,167 @@ TEST_F(merge, merge_vmas_with_mseal) ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 2 * page_size); } +TEST_F(merge, virt_and_page_offset_mismatch_memfd) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd; + + /* Create a 10 page memfd descriptor. */ + fd = memfd_create("virt_page_offset_test", MFD_CLOEXEC); + ASSERT_NE(fd, -1); + ASSERT_EQ(ftruncate(fd, 10 * page_size), 0); + + /* Map a region using the memfd at page offset 0. */ + ptr = mmap(carveout, 5 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd, 0); + ASSERT_NE(ptr, MAP_FAILED); + + /* + * Map another separately, and fault in, at page offset 5: + * + * |-----------| |---------| + * | unfaulted | | faulted | + * |-----------| |---------| + */ + ptr2 = mmap(&carveout[10 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, + fd, 5 * page_size); + ASSERT_NE(ptr2, MAP_FAILED); + ptr2[0] = 'x'; + + /* + * Now move it in place: + * + * |----------| + * | | + * v | + * |-----------| |---------| + * | unfaulted | | faulted | + * |-----------| |---------| + * + * Because virtual page offset of the faulted region is now + * &carveout[10 * page_size], despite the two regions being mergeable + * due to file page offset, they are NOT mergeable due to virtual page + * offset. + */ + ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size, + MREMAP_MAYMOVE | MREMAP_FIXED, + &carveout[5 * page_size]); + ASSERT_NE(ptr2, MAP_FAILED); + + /* Assert that they did not merge. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 5 * page_size); +} + +TEST_F(merge, merge_map_private_dev_zero_unfaulted) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd_zero; + + if (access("/dev/zero", F_OK)) + SKIP(return, "No /dev/zero."); + fd_zero = open("/dev/zero", O_RDWR); + ASSERT_NE(fd_zero, -1); + + /* + * Map two MAP_PRIVATE-/dev/zero VMAs next to one another with offset 0 + * each. + * + * With these being made truly anonymous upon mapping, they will + * merge. If they were file-backed VMAs the page offsets would prevent + * merge: + * + * |-----||------| |-------------| + * | ptr || ptr2 | -> | ptr | + * |-----||------| |-------------| + */ + ptr = mmap(carveout, 5 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + if (ptr == MAP_FAILED) { + close(fd_zero); + ASSERT_TRUE(false); + } + ptr2 = mmap(&carveout[5 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + if (ptr2 == MAP_FAILED) { + close(fd_zero); + ASSERT_TRUE(false); + } + close(fd_zero); + + /* Assert that they merged. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 10 * page_size); +} + +TEST_F(merge, merge_map_private_dev_zero_faulted_unfaulted) +{ + struct procmap_fd *procmap = &self->procmap; + unsigned int page_size = self->page_size; + char *carveout = self->carveout; + char *ptr, *ptr2; + int fd_zero; + + if (access("/dev/zero", F_OK)) + SKIP(return, "No /dev/zero."); + fd_zero = open("/dev/zero", O_RDWR); + ASSERT_NE(fd_zero, -1); + + /* + * Map a MAP_PRIVATE mapping of /dev/zero with page offset 0, then fault + * it in: + * + * |-------------------------------| + * | faulted | + * |-------------------------------| + */ + ptr = mmap(carveout, 15 * page_size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd_zero, 0); + if (ptr == MAP_FAILED) { + close(fd_zero); + ASSERT_TRUE(false); + } + memset(ptr, 'x', 15 * page_size); + + /* + * Unmap the middle: + * + * |---------| |---------| + * | faulted | | faulted | + * |---------| |---------| + */ + ASSERT_EQ(munmap(&ptr[5 * page_size], 5 * page_size), 0); + + /* + * Map in a new unfaulted mapping in the middle with page offset 0 - + * this should merge and would not if it were treated as a file rather + * than pure anon: + * + * |---------|-----------|---------| + * | faulted | unfaulted | faulted | + * |---------|-----------|---------| + */ + ptr2 = mmap(&carveout[5 * page_size], 5 * page_size, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE, + fd_zero, 0); + close(fd_zero); + ASSERT_NE(ptr2, MAP_FAILED); + + /* Assert that they merged. */ + ASSERT_TRUE(find_vma_procmap(procmap, ptr)); + ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr); + ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 15 * page_size); +} + TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev) { struct procmap_fd *procmap = &self->procmap; diff --git a/tools/testing/selftests/mm/mremap_dontunmap.c b/tools/testing/selftests/mm/mremap_dontunmap.c index a4f75d836733..96ba537facf7 100644 --- a/tools/testing/selftests/mm/mremap_dontunmap.c +++ b/tools/testing/selftests/mm/mremap_dontunmap.c @@ -313,7 +313,7 @@ static void mremap_dontunmap_partial_mapping_overwrite(void) mremap(source_mapping, 5 * page_size, 5 * page_size, MREMAP_DONTUNMAP | MREMAP_MAYMOVE | MREMAP_FIXED, dest_mapping); - BUG_ON(dest_mapping == MAP_FAILED, "mremap"); + BUG_ON(remapped_mapping == MAP_FAILED, "mremap"); BUG_ON(dest_mapping != remapped_mapping, "expected to remap to dest_mapping"); BUG_ON(check_region_contains_byte(source_mapping, 5 * page_size, 0) != diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index f9bcff8e78fa..f6ab626a90b4 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -8,6 +8,7 @@ #include <errno.h> #include <malloc.h> #include <linux/types.h> +#include <linux/mman.h> #include <linux/memfd.h> #include <linux/userfaultfd.h> #include <linux/fs.h> @@ -1058,50 +1059,96 @@ static void test_simple(void) * the generic path (reached e.g. via category_anyof_mask) must report every * page written. */ -static void unpopulated_scan_test(void) +/* + * Populate @mem (optionally collapsing it into a THP first), drop it with + * MADV_DONTNEED, then check PAGEMAP_SCAN reports the whole range written via + * both the fast and generic query paths. A dropped THP leaves a pmd_none hole + * with no page table, exercising pagemap_scan_pte_hole(); a base-page range + * leaves pte_none entries. + */ +static void unpopulated_written_test(const char *name, char *mem, long size, + bool use_thp) { - int npages = 16, i; - long mem_size = npages * page_size; + long npages = size / page_size, fast = 0, slow = 0, ret; struct page_region regions[16]; - long fast = 0, slow = 0, ret; - char *mem; - - mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED) - ksft_exit_fail_msg("%s mmap failed\n", __func__); + int i; - wp_init(mem, mem_size); + wp_init(mem, size); - /* Populate, then drop: the ptes become pte_none without a marker. */ - memset(mem, 1, mem_size); - if (madvise(mem, mem_size, MADV_DONTNEED)) - ksft_exit_fail_msg("%s MADV_DONTNEED failed\n", __func__); + /* Populate, optionally collapse to a THP, then drop it. */ + memset(mem, 1, size); + if (use_thp && + (madvise(mem, size, MADV_COLLAPSE) || + !check_huge_anon(mem, size / hpage_size, hpage_size))) { + ksft_test_result_skip("%s could not form a THP\n", name); + goto out; + } + if (madvise(mem, size, MADV_DONTNEED)) { + ksft_test_result_fail("%s MADV_DONTNEED failed\n", name); + goto out; + } /* Fast path: category_mask == return_mask == PAGE_IS_WRITTEN. */ - ret = pagemap_ioctl(mem, mem_size, regions, npages, 0, 0, + ret = pagemap_ioctl(mem, size, regions, ARRAY_SIZE(regions), 0, 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("%s fast scan failed\n", __func__); - for (i = 0; i < ret; i++) + for (i = 0; ret > 0 && i < ret; i++) fast += LEN(regions[i]); /* Generic path: same query expressed via category_anyof_mask. */ - ret = pagemap_ioctl(mem, mem_size, regions, npages, 0, 0, + ret = pagemap_ioctl(mem, size, regions, ARRAY_SIZE(regions), 0, 0, 0, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN); - if (ret < 0) - ksft_exit_fail_msg("%s generic scan failed\n", __func__); - for (i = 0; i < ret; i++) + for (i = 0; ret > 0 && i < ret; i++) slow += LEN(regions[i]); ksft_test_result(fast == npages && slow == npages, - "%s unpopulated ptes reported written by both paths (%ld, %ld of %d)\n", - __func__, fast, slow, npages); + "%s unpopulated range reported written by both paths (%ld, %ld of %ld)\n", + name, fast, slow, npages); +out: + wp_free(mem, size); +} - wp_free(mem, mem_size); +static void unpopulated_scan_test(void) +{ + long mem_size = 16 * page_size; + char *mem; + + mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + ksft_test_result_skip("%s mmap failed\n", __func__); + return; + } + + unpopulated_written_test(__func__, mem, mem_size, false); munmap(mem, mem_size); } +/* + * Same as unpopulated_scan_test(), but the range is a THP: a full-PMD + * MADV_DONTNEED leaves a pmd_none hole with no page table. + */ +static void unpopulated_thp_scan_test(void) +{ + char *area, *mem; + + if (!hpage_size) { + ksft_test_result_skip("%s THP not supported\n", __func__); + return; + } + + /* Over-allocate so a PMD-aligned, THP-sized range fits inside. */ + area = mmap(NULL, 2 * hpage_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (area == MAP_FAILED) { + ksft_test_result_skip("%s mmap failed\n", __func__); + return; + } + mem = (char *)(((unsigned long)area + hpage_size - 1) & ~(hpage_size - 1)); + + unpopulated_written_test(__func__, mem, hpage_size, true); + munmap(area, 2 * hpage_size); +} + int sanity_tests(void) { unsigned long long mem_size, vec_size; @@ -1610,7 +1657,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) if (!hugetlb_setup_default(4)) ksft_print_msg("HugeTLB test will be skipped\n"); - ksft_set_plan(118); + ksft_set_plan(119); page_size = getpagesize(); hpage_size = read_pmd_pagesize(); @@ -1790,6 +1837,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) /* 18. Unpopulated pte scan-path consistency */ unpopulated_scan_test(); + unpopulated_thp_scan_test(); close(pagemap_fd); ksft_finished(); diff --git a/tools/testing/selftests/mm/pkey-helpers.h b/tools/testing/selftests/mm/pkey-helpers.h index 2c377f4e9df1..46a8a1878dc1 100644 --- a/tools/testing/selftests/mm/pkey-helpers.h +++ b/tools/testing/selftests/mm/pkey-helpers.h @@ -68,7 +68,9 @@ static inline void sigsafe_printf(const char *format, ...) #define dprintf3(args...) dprintf_level(3, args) #define dprintf4(args...) dprintf_level(4, args) -extern void abort_hooks(void); +void tracing_on(void); +void tracing_off(void); +void abort_hooks(void); #define pkey_assert(condition) do { \ if (!(condition)) { \ dprintf0("# assert() at %s::%d test_nr: %d iteration: %d\n", \ diff --git a/tools/testing/selftests/mm/pkey-powerpc.h b/tools/testing/selftests/mm/pkey-powerpc.h index 17bf2d1b0192..2ce85580b404 100644 --- a/tools/testing/selftests/mm/pkey-powerpc.h +++ b/tools/testing/selftests/mm/pkey-powerpc.h @@ -126,7 +126,7 @@ static inline void *malloc_pkey_with_mprotect_subpage(long size, int prot, u16 p size, prot, pkey); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); ret = syscall(__NR_subpage_prot, ptr, size, NULL); if (ret) { diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index 302fef54049c..c218d0510a2a 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -19,7 +19,6 @@ #include <stdint.h> #include <stdbool.h> #include <signal.h> -#include <assert.h> #include <stdlib.h> #include <sys/mman.h> #include <sys/types.h> @@ -207,22 +206,25 @@ static void test_sigsegv_handler_with_pkey0_disabled(void) struct sigaction sa; pthread_attr_t attr; pthread_t thr; + int ret; sa.sa_flags = SA_SIGINFO; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thr, &attr, thread_segv_with_pkey0_disabled, NULL); + ret = pthread_create(&thr, &attr, thread_segv_with_pkey0_disabled, NULL); + if (ret) { + errno = ret; + pkey_assert(0); + } pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) @@ -247,22 +249,25 @@ static void test_sigsegv_handler_cannot_access_stack(void) struct sigaction sa; pthread_attr_t attr; pthread_t thr; + int ret; sa.sa_flags = SA_SIGINFO; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thr, &attr, thread_segv_pkuerr_stack, NULL); + ret = pthread_create(&thr, &attr, thread_segv_pkuerr_stack, NULL); + if (ret) { + errno = ret; + pkey_assert(0); + } pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) @@ -285,24 +290,22 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) static stack_t sigstack; void *stack; int pkey; - int parent_pid = 0; int child_pid = 0; u64 pkey_reg; + long ret; sa.sa_flags = SA_SIGINFO | SA_ONSTACK; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); stack = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - assert(stack != MAP_FAILED); + pkey_assert(stack != MAP_FAILED); /* Allow access to MPK 0 and MPK 1 */ pkey_reg = pkey_reg_restrictive_default(); @@ -317,33 +320,41 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) /* Set up alternate signal stack that will use the default MPK */ sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + pkey_assert(sigstack.ss_sp != MAP_FAILED); sigstack.ss_flags = 0; sigstack.ss_size = STACK_SIZE; memset(&siginfo, 0, sizeof(siginfo)); /* Use clone to avoid newer glibcs using rseq on new threads */ - long ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | - CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | - CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | - CLONE_DETACHED, - stack + STACK_SIZE, - &parent_pid, - &child_pid); + ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | + CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | + CLONE_DETACHED, + stack + STACK_SIZE, + NULL, + NULL); if (ret < 0) { errno = -ret; - perror("clone"); + pkey_assert(0); } else if (ret == 0) { thread_segv_maperr_ptr(&sigstack); syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0); } + child_pid = ret; + pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) pthread_cond_wait(&cond, &mutex); pthread_mutex_unlock(&mutex); + /* Wait for child to exit before returning */ + do { + sched_yield(); + ret = syscall_raw(SYS_tkill, child_pid, 0, 0, 0, 0, 0); + } while (ret != -ESRCH && ret != -EINVAL); + ksft_test_result(siginfo.si_signo == SIGSEGV && siginfo.si_code == SEGV_MAPERR && siginfo.si_addr == NULL, @@ -358,6 +369,7 @@ static void test_pkru_preserved_after_sigusr1(void) { struct sigaction sa; u64 pkey_reg; + int ret; /* Allow access to MPK 0 and an arbitrary set of keys */ pkey_reg = pkey_reg_restrictive_default(); @@ -369,10 +381,8 @@ static void test_pkru_preserved_after_sigusr1(void) sa.sa_sigaction = sigusr1_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGUSR1, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGUSR1, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); @@ -441,9 +451,15 @@ static void test_pkru_sigreturn(void) static stack_t sigstack; void *stack; int pkey; - int parent_pid = 0; int child_pid = 0; u64 pkey_reg; + long ret; + + /* + * SIGSEGV handler is reset to SIG_DFL below; turn tracing off first + * so a crash does not leave ftrace enabled. + */ + tracing_off(); sa.sa_handler = SIG_DFL; sa.sa_flags = 0; @@ -453,24 +469,20 @@ static void test_pkru_sigreturn(void) * For this testcase, we do not want to handle SIGSEGV. Reset handler * to default so that the application can crash if it receives SIGSEGV. */ - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); sa.sa_flags = SA_SIGINFO | SA_ONSTACK; sa.sa_sigaction = sigusr2_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGUSR2, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGUSR2, &sa, NULL); + pkey_assert(ret == 0); stack = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - assert(stack != MAP_FAILED); + pkey_assert(stack != MAP_FAILED); /* * Allow access to MPK 0 and MPK 2. The child thread (to be created @@ -490,21 +502,21 @@ static void test_pkru_sigreturn(void) /* Set up alternate signal stack that will use the default MPK */ sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + pkey_assert(sigstack.ss_sp != MAP_FAILED); sigstack.ss_flags = 0; sigstack.ss_size = STACK_SIZE; /* Use clone to avoid newer glibcs using rseq on new threads */ - long ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | - CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | - CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | - CLONE_DETACHED, - stack + STACK_SIZE, - &parent_pid, - &child_pid); + ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | + CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | + CLONE_DETACHED, + stack + STACK_SIZE, + NULL, + NULL); if (ret < 0) { errno = -ret; - perror("clone"); + pkey_assert(0); } else if (ret == 0) { thread_sigusr2_self(&sigstack); syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0); @@ -530,16 +542,17 @@ static void (*pkey_tests[])(void) = { int main(int argc, char *argv[]) { - int i; - ksft_print_header(); ksft_set_plan(ARRAY_SIZE(pkey_tests)); if (!is_pkeys_supported()) ksft_exit_skip("pkeys not supported\n"); - for (i = 0; i < ARRAY_SIZE(pkey_tests); i++) - (*pkey_tests[i])(); + for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) { + tracing_on(); + (*pkey_tests[test_nr])(); + tracing_off(); + } ksft_finished(); return 0; diff --git a/tools/testing/selftests/mm/pkey_util.c b/tools/testing/selftests/mm/pkey_util.c index 255b332f7a08..fbef3cd45447 100644 --- a/tools/testing/selftests/mm/pkey_util.c +++ b/tools/testing/selftests/mm/pkey_util.c @@ -2,9 +2,99 @@ #define __SANE_USERSPACE_TYPES__ #include <sys/syscall.h> #include <unistd.h> +#include <fcntl.h> +#include <stdio.h> +#include <string.h> #include "pkey-helpers.h" +int iteration_nr = 1; +int test_nr; +int dprint_in_signal; + +#if CONTROL_TRACING > 0 +static void cat_into_file(char *str, char *file) +{ + int fd = open(file, O_RDWR); + int ret; + + dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file); + /* + * these need to be raw because they are called under + * pkey_assert() + */ + if (fd < 0) { + fprintf(stderr, "error opening '%s'\n", file); + perror("error: "); + exit(__LINE__); + } + + ret = write(fd, str, strlen(str)); + if (ret != strlen(str)) { + perror("write to file failed"); + fprintf(stderr, "filename: '%s' str: '%s'\n", file, str); + exit(__LINE__); + } + close(fd); +} + +static int warned_tracing; +static int tracing_root_ok(void) +{ + if (geteuid() != 0) { + if (!warned_tracing) + fprintf(stderr, "WARNING: not run as root, " + "can not do tracing control\n"); + warned_tracing = 1; + return 0; + } + return 1; +} +#endif + +void tracing_on(void) +{ +#if CONTROL_TRACING > 0 +#define TRACEDIR "/sys/kernel/tracing" + char pidstr[32]; + + if (!tracing_root_ok()) + return; + + sprintf(pidstr, "%d", getpid()); + cat_into_file("0", TRACEDIR "/tracing_on"); + cat_into_file("\n", TRACEDIR "/trace"); + if (1) { + cat_into_file("function_graph", TRACEDIR "/current_tracer"); + cat_into_file("1", TRACEDIR "/options/funcgraph-proc"); + } else { + cat_into_file("nop", TRACEDIR "/current_tracer"); + } + cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid"); + cat_into_file("1", TRACEDIR "/tracing_on"); + dprintf1("enabled tracing\n"); +#endif +} + +void tracing_off(void) +{ +#if CONTROL_TRACING > 0 + if (!tracing_root_ok()) + return; + cat_into_file("0", "/sys/kernel/tracing/tracing_on"); +#endif +} + +void abort_hooks(void) +{ + fflush(stdout); + fprintf(stderr, "running %s()...\n", __func__); + tracing_off(); +#ifdef SLEEP_ON_ABORT + sleep(SLEEP_ON_ABORT); +#endif +} + int sys_pkey_alloc(unsigned long flags, unsigned long init_val) { int ret = syscall(SYS_pkey_alloc, flags, init_val); diff --git a/tools/testing/selftests/mm/protection_keys.c b/tools/testing/selftests/mm/protection_keys.c index 9a6d954ee371..ae6e1530b354 100644 --- a/tools/testing/selftests/mm/protection_keys.c +++ b/tools/testing/selftests/mm/protection_keys.c @@ -49,11 +49,7 @@ #include "hugepage_settings.h" #include "pkey-helpers.h" -int iteration_nr = 1; -int test_nr; - u64 shadow_pkey_reg; -int dprint_in_signal; noinline int read_ptr(int *ptr) { @@ -62,89 +58,6 @@ noinline int read_ptr(int *ptr) return *ptr; } -#if CONTROL_TRACING > 0 -static void cat_into_file(char *str, char *file) -{ - int fd = open(file, O_RDWR); - int ret; - - dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file); - /* - * these need to be raw because they are called under - * pkey_assert() - */ - if (fd < 0) { - fprintf(stderr, "error opening '%s'\n", str); - perror("error: "); - exit(__LINE__); - } - - ret = write(fd, str, strlen(str)); - if (ret != strlen(str)) { - perror("write to file failed"); - fprintf(stderr, "filename: '%s' str: '%s'\n", file, str); - exit(__LINE__); - } - close(fd); -} - -static int warned_tracing; -static int tracing_root_ok(void) -{ - if (geteuid() != 0) { - if (!warned_tracing) - fprintf(stderr, "WARNING: not run as root, " - "can not do tracing control\n"); - warned_tracing = 1; - return 0; - } - return 1; -} -#endif - -static void tracing_on(void) -{ -#if CONTROL_TRACING > 0 -#define TRACEDIR "/sys/kernel/tracing" - char pidstr[32]; - - if (!tracing_root_ok()) - return; - - sprintf(pidstr, "%d", getpid()); - cat_into_file("0", TRACEDIR "/tracing_on"); - cat_into_file("\n", TRACEDIR "/trace"); - if (1) { - cat_into_file("function_graph", TRACEDIR "/current_tracer"); - cat_into_file("1", TRACEDIR "/options/funcgraph-proc"); - } else { - cat_into_file("nop", TRACEDIR "/current_tracer"); - } - cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid"); - cat_into_file("1", TRACEDIR "/tracing_on"); - dprintf1("enabled tracing\n"); -#endif -} - -static void tracing_off(void) -{ -#if CONTROL_TRACING > 0 - if (!tracing_root_ok()) - return; - cat_into_file("0", "/sys/kernel/tracing/tracing_on"); -#endif -} - -void abort_hooks(void) -{ - fflush(stdout); - fprintf(stderr, "running %s()...\n", __func__); - tracing_off(); -#ifdef SLEEP_ON_ABORT - sleep(SLEEP_ON_ABORT); -#endif -} - /* * This attempts to have roughly a page of instructions followed by a few * instructions that do a write, and another page of instructions. That @@ -669,7 +582,7 @@ static void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey) size, prot, pkey); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey); pkey_assert(!ret); record_pkey_malloc(ptr, size, prot); @@ -692,7 +605,7 @@ static void *malloc_pkey_anon_huge(long size, int prot, u16 pkey) */ size = ALIGN_UP(size, HPAGE_SIZE * 2); ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); record_pkey_malloc(ptr, size, prot); mprotect_pkey(ptr, size, prot, pkey); @@ -750,7 +663,7 @@ static void *malloc_pkey_hugetlb(long size, int prot, u16 pkey) size = ALIGN_UP(size, HPAGE_SIZE * 2); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); mprotect_pkey(ptr, size, prot, pkey); record_pkey_malloc(ptr, size, prot); @@ -779,7 +692,7 @@ static void *malloc_pkey(long size, int prot, u16 pkey) pkey_assert(malloc_type < nr_malloc_types); ret = pkey_malloc[malloc_type](size, prot, pkey); - pkey_assert(ret != (void *)-1); + pkey_assert(ret != MAP_FAILED); malloc_type++; if (malloc_type >= nr_malloc_types) @@ -1197,6 +1110,7 @@ static void arch_force_pkey_reg_init(void) * doing the XSAVE size enumeration dance. */ buf = mmap(NULL, 1*MB, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); + pkey_assert(buf != MAP_FAILED); /* These __builtins require compiling with -mxsave */ @@ -1763,7 +1677,8 @@ int main(void) ksft_print_msg("running PKEY tests for unsupported CPU/OS\n"); ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - assert(ptr != (void *)-1); + if (ptr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); test_mprotect_pkey_on_unsupported_cpu(ptr, 1); ksft_test_result_pass("pkey on unsupported CPU/OS\n"); ksft_finished(); diff --git a/tools/testing/selftests/mm/rmap.c b/tools/testing/selftests/mm/rmap.c index 53f2058b0ef2..1c293ad3f8b8 100644 --- a/tools/testing/selftests/mm/rmap.c +++ b/tools/testing/selftests/mm/rmap.c @@ -430,4 +430,85 @@ TEST_F(migrate, ksm) propagate_children(_metadata, data); } +static bool range_maps_the_same_pfn(int pagemap_fd, void *region, int nr_pages) +{ + int i; + int retries = 0; + unsigned long first_pfn; + +retry: + if (retries > 10) + return false; + + first_pfn = pagemap_get_pfn(pagemap_fd, region); + for (i = 0; i < nr_pages; i++) { + if (pagemap_get_pfn(pagemap_fd, region + i * getpagesize()) != first_pfn) { + /* + * Retry up to 10 times at most in case of the low chance of page + * compaction migrating the page while we check for pfn. + */ + retries++; + goto retry; + } + } + + return true; +} + +TEST_F(migrate, ksm_and_mremap) +{ + unsigned long old_pfn, new_pfn; + void *region, *mremap_region; + const int nr_pages = 16; + size_t mmap_size; + int pagemap_fd; + + /* Skip if KSM is not available */ + if (ksm_stop() < 0) + SKIP(return, "accessing \"/sys/kernel/mm/ksm/run\" failed"); + if (ksm_get_full_scans() < 0) + SKIP(return, "accessing \"/sys/kernel/mm/ksm/full_scan\" failed"); + + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + SKIP(return, "opening pagemap failed"); + + /* Allocate and populate twice the anon pages initially. */ + mmap_size = 2 * nr_pages * getpagesize(); + region = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + ASSERT_NE(region, MAP_FAILED); + memset(region, 0x77, mmap_size); + + /* mremap the second half over the first half, to stress rmap handling */ + mmap_size /= 2; + mremap_region = mremap(region + mmap_size, mmap_size, mmap_size, + MREMAP_MAYMOVE | MREMAP_FIXED, region); + ASSERT_EQ(mremap_region, region); + + /* Merge all pages into a single KSM page. */ + madvise(region, mmap_size, MADV_MERGEABLE); + ASSERT_EQ(ksm_start(), 0); + + /* The whole range should map the same KSM page. */ + old_pfn = pagemap_get_pfn(pagemap_fd, region); + if (old_pfn == -1ul) + SKIP(return, "Obtaining PFN failed"); + ksm_start(); + ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages)); + + /* + * Migrate the KSM page; the whole range should map the new (migrated) + * KSM page. + */ + ASSERT_EQ(try_to_move_page(region), 0); + + new_pfn = pagemap_get_pfn(pagemap_fd, region); + if (new_pfn == -1ul) + SKIP(return, "Obtaining PFN failed"); + ASSERT_NE(new_pfn, old_pfn); + ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages)); +} + + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 8c296dedf047..687d115e3bd8 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -262,7 +262,6 @@ echo "TAP version 13" | tap_output CATEGORY="hugetlb" run_test ./hugetlb-mmap CATEGORY="hugetlb" run_test ./hugetlb-shm CATEGORY="hugetlb" run_test ./hugetlb-mremap -CATEGORY="hugetlb" run_test ./hugetlb-vmemmap CATEGORY="hugetlb" run_test ./hugetlb-madvise CATEGORY="hugetlb" run_test ./hugetlb_dio CATEGORY="hugetlb" run_test ./hugetlb_fault_after_madv @@ -303,7 +302,7 @@ CATEGORY="compaction" run_test ./compaction_test if command -v sudo &> /dev/null && sudo -u nobody ls ./on-fault-limit >/dev/null; then - CATEGORY="mlock" run_test sudo -u nobody ./on-fault-limit + CATEGORY="mlock" run_test sudo -u nobody sh -c ./on-fault-limit else echo "# SKIP ./on-fault-limit" fi diff --git a/tools/testing/selftests/mm/transhuge-stress.c b/tools/testing/selftests/mm/transhuge-stress.c index 8eb0c5630e7e..a0f9fc900360 100644 --- a/tools/testing/selftests/mm/transhuge-stress.c +++ b/tools/testing/selftests/mm/transhuge-stress.c @@ -35,6 +35,7 @@ int main(int argc, char **argv) size_t map_len; int pagemap_fd; int duration = 0; + double elapsed = 0; ksft_print_header(); @@ -60,8 +61,6 @@ int main(int argc, char **argv) len = atoll(argv[i]) << 20; } - ksft_set_plan(1); - if (name) { backing_fd = open(name, O_RDWR); if (backing_fd == -1) @@ -78,6 +77,10 @@ int main(int argc, char **argv) ksft_exit_fail_msg("open pagemap\n"); len -= len % HPAGE_SIZE; + if (len == 0) + ksft_exit_fail_msg("len must be at least %d MiB\n", + HPAGE_SIZE >> 20); + ptr = mmap(NULL, len + HPAGE_SIZE, PROT_RW, mmap_flags, backing_fd, 0); if (ptr == MAP_FAILED) ksft_exit_fail_msg("initial mmap"); @@ -91,6 +94,7 @@ int main(int argc, char **argv) if (!map) ksft_exit_fail_msg("map malloc\n"); + ksft_set_plan(1); clock_gettime(CLOCK_MONOTONIC, &start); while (1) { @@ -125,16 +129,28 @@ int main(int argc, char **argv) /* split transhuge page, keep last page */ if (madvise(p, HPAGE_SIZE - psize(), MADV_DONTNEED)) ksft_exit_fail_msg("MADV_DONTNEED"); + + if (duration > 0) { + clock_gettime(CLOCK_MONOTONIC, &b); + elapsed = b.tv_sec - start.tv_sec + + (b.tv_nsec - start.tv_nsec) / 1000000000.; + if (elapsed >= duration) + break; + } } clock_gettime(CLOCK_MONOTONIC, &b); s = b.tv_sec - a.tv_sec + (b.tv_nsec - a.tv_nsec) / 1000000000.; ksft_print_msg("%.3f s/loop, %.3f ms/page, %10.3f MiB/s\t" - "%4d succeed, %4d failed, %4d different pages\n", - s, s * 1000 / (len >> HPAGE_SHIFT), len / s / (1 << 20), - nr_succeed, nr_failed, nr_pages); - - if (duration > 0 && b.tv_sec - start.tv_sec >= duration) { + "%4d succeed, %4d failed, %4d different pages\n", + s, s * 1000 / (nr_failed + nr_succeed), + s > 0 ? + (double)(nr_failed + nr_succeed) * + (HPAGE_SIZE >> 20) / s : + 0.0, + nr_succeed, nr_failed, nr_pages); + + if (duration > 0 && elapsed >= duration) { ksft_test_result_pass("Completed\n"); ksft_finished(); } diff --git a/tools/testing/selftests/mm/uffd-common.c b/tools/testing/selftests/mm/uffd-common.c index edd02328f77b..f48f5d4594ab 100644 --- a/tools/testing/selftests/mm/uffd-common.c +++ b/tools/testing/selftests/mm/uffd-common.c @@ -639,8 +639,13 @@ int __copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool retry uffdio_copy.mode = 0; uffdio_copy.copy = 0; if (ioctl(gopts->uffd, UFFDIO_COPY, &uffdio_copy)) { - /* real retval in ufdio_copy.copy */ - if (uffdio_copy.copy != -EEXIST) + /* + * real retval in uffdio_copy.copy + * + * -EEXIST: the page was faulted in concurrently + * -ENOENT: the destination range was concurrently removed + */ + if (uffdio_copy.copy != -EEXIST && uffdio_copy.copy != -ENOENT) err("UFFDIO_COPY error: %"PRId64, (int64_t)uffdio_copy.copy); wake_range(gopts->uffd, uffdio_copy.dst, gopts->page_size); diff --git a/tools/testing/selftests/mm/uffd-unit-tests.c b/tools/testing/selftests/mm/uffd-unit-tests.c index a6c14109e818..ef9b3956bdcf 100644 --- a/tools/testing/selftests/mm/uffd-unit-tests.c +++ b/tools/testing/selftests/mm/uffd-unit-tests.c @@ -7,6 +7,8 @@ #include "uffd-common.h" +#include <linux/fs.h> +#include <sys/uio.h> #include "../../../../mm/gup_test.h" #ifdef __NR_userfaultfd @@ -109,6 +111,10 @@ static void uffd_test_skip(const char *message) static void test_uffd_api(bool use_dev) { + const uint64_t expected_ioctls = + BIT_ULL(_UFFDIO_REGISTER) | + BIT_ULL(_UFFDIO_UNREGISTER) | + BIT_ULL(_UFFDIO_API); struct uffdio_api uffdio_api; int uffd; @@ -148,6 +154,15 @@ static void test_uffd_api(bool use_dev) goto out; } + /* Verify returned fd-level ioctls bitmask */ + if ((uffdio_api.ioctls & expected_ioctls) != expected_ioctls) { + uffd_test_fail("UFFDIO_API missing expected ioctls: " + "got=0x%"PRIx64", expected=0x%"PRIx64, + (uint64_t)uffdio_api.ioctls, + expected_ioctls); + goto out; + } + /* Test double requests of UFFDIO_API with a random feature set */ uffdio_api.features = BIT_ULL(0); if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { @@ -602,6 +617,814 @@ void uffd_minor_collapse_test(uffd_global_test_opts_t *gopts, uffd_test_args_t * uffd_minor_test_common(gopts, true, false); } +static int uffd_register_rwp(int uffd, void *addr, uint64_t len) +{ + struct uffdio_register reg = { + .range = { .start = (unsigned long)addr, .len = len }, + .mode = UFFDIO_REGISTER_MODE_RWP, + }; + + if (ioctl(uffd, UFFDIO_REGISTER, ®) == -1) + return -errno; + return 0; +} + +static void rwprotect_range(int uffd, __u64 start, __u64 len, bool protect) +{ + struct uffdio_rwprotect rwp = { + .range = { .start = start, .len = len }, + .mode = protect ? UFFDIO_RWPROTECT_MODE_RWP : 0, + }; + + if (ioctl(uffd, UFFDIO_RWPROTECT, &rwp)) + err("UFFDIO_RWPROTECT failed"); +} + +static void set_async_mode(int uffd, bool enable) +{ + struct uffdio_set_mode mode = { }; + + if (enable) + mode.enable = UFFD_FEATURE_RWP_ASYNC; + else + mode.disable = UFFD_FEATURE_RWP_ASYNC; + + if (ioctl(uffd, UFFDIO_SET_MODE, &mode)) + err("UFFDIO_SET_MODE failed"); +} + +/* + * Test async RWP faults on anonymous memory. + * Populate pages, register MODE_RWP with RWP_ASYNC, + * RW-protect, re-access, verify content preserved and no faults delivered. + */ +static void uffd_rwp_async_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + + /* Populate all pages with known content */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + /* Register MODE_RWP */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + /* RW-protect all pages (sets protnone) */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Access all pages — should auto-resolve, no faults */ + for (p = 0; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + unsigned char expected = p % 255 + 1; + + if (page[0] != expected) { + uffd_test_fail("page %lu content mismatch: %u != %u", + p, page[0], expected); + return; + } + } + + uffd_test_pass(); +} + +/* + * Fault handler for RWP — unprotect the page via UFFDIO_RWPROTECT. + */ +static void uffd_handle_rwp_fault(uffd_global_test_opts_t *gopts, + struct uffd_msg *msg, + struct uffd_args *uargs) +{ + if (!(msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_RWP)) + err("expected RWP fault, got 0x%llx", + msg->arg.pagefault.flags); + + rwprotect_range(gopts->uffd, msg->arg.pagefault.address, + gopts->page_size, false); + uargs->minor_faults++; +} + +/* + * Test sync RWP faults on anonymous memory. + * Populate pages, register MODE_RWP (sync), RW-protect, + * access from worker thread, verify fault delivered, UFFDIO_RWPROTECT resolves. + */ +static void uffd_rwp_sync_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + pthread_t uffd_mon; + struct uffd_args uargs = { }; + bool failed = false; + char c = '\0'; + unsigned long p; + + uargs.gopts = gopts; + uargs.handle_fault = uffd_handle_rwp_fault; + + /* Populate all pages */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + /* Register MODE_RWP */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + /* RW-protect all pages */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Start fault handler thread */ + if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs)) + err("uffd_poll_thread create"); + + /* Access all pages — triggers sync RWP faults, handler unprotects */ + for (p = 0; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + + if (page[0] != (p % 255 + 1)) { + uffd_test_fail("page %lu content mismatch", p); + failed = true; + goto out; + } + } + +out: + /* + * Stop the handler before reading minor_faults: the last fault + * resolution rwprotect_range()s before incrementing the counter, + * so the main thread can race ahead of the increment. + */ + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join() failed"); + + if (failed) + return; + if (uargs.minor_faults == 0) + uffd_test_fail("expected RWP faults, got 0"); + else + uffd_test_pass(); +} + +/* + * Test PAGEMAP_SCAN working-set discovery via the "hot" (accessed) scan. + * + * The working-set primitive is to find pages that were accessed: scan for + * PAGE_IS_ACCESSED, which is set once an access clears the protnone+uffd + * marker. A VMM treats every access fault as "hot" (RWP here; MINOR/MISSING + * for non-resident pages) and reclaims the rest from the backing file. + * + * We deliberately do NOT use an inverted "cold" scan: that only sees + * VMA-resident ptes, so for a file mapping it misses cached-but-unmapped (and + * never-faulted, pre-populated) pages, which are pte_none and thus invisible. + * Hot tracking + file-level reclaim covers them; a cold pte scan cannot. + */ +static void uffd_rwp_pagemap_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + struct page_region regions[16]; + struct pm_scan_arg pm_arg; + int pagemap_fd; + long ret; + + /* Need at least 4 pages */ + if (nr_pages < 4) { + uffd_test_skip("need at least 4 pages"); + return; + } + + /* Populate all pages */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, 0xab, page_size); + + /* Register and RW-protect */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Touch first half of pages to re-activate them (async auto-resolve) */ + for (p = 0; p < nr_pages / 2; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; + } + + uint64_t start = (uint64_t)gopts->area_dst; + uint64_t boundary = start + (nr_pages / 2) * page_size; + uint64_t end = start + nr_pages * page_size; + + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + err("open pagemap"); + + /* + * Hot scan: report the pages that were accessed. PAGE_IS_ACCESSED is + * set once the protnone+uffd marker is cleared (by the access, async + * auto-resolve here). The touched first half must come back as exactly + * one hot region [start, boundary); the untouched second half must not + * appear. + */ + memset(&pm_arg, 0, sizeof(pm_arg)); + pm_arg.size = sizeof(pm_arg); + pm_arg.start = start; + pm_arg.end = end; + pm_arg.vec = (uint64_t)regions; + pm_arg.vec_len = ARRAY_SIZE(regions); + pm_arg.category_mask = PAGE_IS_ACCESSED; + pm_arg.return_mask = PAGE_IS_ACCESSED; + + ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg); + close(pagemap_fd); + + if (ret < 0) { + uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno)); + return; + } + + if (ret != 1 || regions[0].start != start || + regions[0].end != boundary) { + uffd_test_fail("hot set wrong: got %ld regions [0x%lx,0x%lx), expected 1 [0x%lx,0x%lx)", + ret, (unsigned long)regions[0].start, + (unsigned long)regions[0].end, + (unsigned long)start, (unsigned long)boundary); + return; + } + + uffd_test_pass(); +} + +/* + * Test that RWP protection survives a mprotect(PROT_NONE) -> + * mprotect(PROT_READ|PROT_WRITE) round-trip. The uffd-wp bit on a + * VM_UFFD_RWP VMA must continue to carry PROT_NONE semantics after + * mprotect() changes the base protection; otherwise accesses would + * silently succeed and the pagemap bit would stick without a fault + * ever clearing it. + */ +static void uffd_rwp_mprotect_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + struct page_region regions[16]; + struct pm_scan_arg pm_arg; + int pagemap_fd; + uint64_t value; + long ret; + + /* Populate all pages */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, 0xab, page_size); + + /* Register and RW-protect the whole range */ + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Round-trip mprotect(): PROT_NONE -> PROT_READ|PROT_WRITE */ + if (mprotect(gopts->area_dst, nr_pages * page_size, PROT_NONE)) + err("mprotect() PROT_NONE"); + if (mprotect(gopts->area_dst, nr_pages * page_size, + PROT_READ | PROT_WRITE)) + err("mprotect() PROT_READ|PROT_WRITE"); + + /* + * The marker must survive the round-trip; if mprotect() dropped it, + * the touches below would not fault and the scan would pass + * vacuously. + */ + pagemap_fd = pagemap_open(); + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); + close(pagemap_fd); + if (!(value & PM_UFFD_WP)) { + uffd_test_fail("RWP marker lost across mprotect()"); + return; + } + + /* Touch every page. Async RWP must auto-resolve each fault. */ + for (p = 0; p < nr_pages; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; + } + + /* + * After touching, no page should remain RW-protected. A stuck + * uffd-wp bit would mean mprotect() silently dropped PROT_NONE and + * the access never faulted. + */ + pagemap_fd = open("/proc/self/pagemap", O_RDONLY); + if (pagemap_fd < 0) + err("open pagemap"); + + memset(&pm_arg, 0, sizeof(pm_arg)); + pm_arg.size = sizeof(pm_arg); + pm_arg.start = (uint64_t)gopts->area_dst; + pm_arg.end = (uint64_t)gopts->area_dst + nr_pages * page_size; + pm_arg.vec = (uint64_t)regions; + pm_arg.vec_len = ARRAY_SIZE(regions); + pm_arg.category_mask = PAGE_IS_ACCESSED; + pm_arg.category_inverted = PAGE_IS_ACCESSED; + pm_arg.return_mask = PAGE_IS_ACCESSED; + + ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg); + close(pagemap_fd); + + if (ret < 0) { + uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno)); + return; + } + if (ret != 0) { + uffd_test_fail("expected no cold pages after mprotect()+touch, got %ld regions", + ret); + return; + } + + uffd_test_pass(); +} + +/* + * Test that GUP resolves through protnone PTEs (async mode). + * vmsplice() into a pipe pins user pages via get_user_pages_fast() -- + * unlike write(), which goes through copy_from_user() and ordinary + * hardware page faults -- so it exercises gup_can_follow_protnone() on + * the RW-protected PTE. In async mode the kernel auto-restores + * permissions and GUP returns the page. + */ +static void uffd_rwp_gup_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + struct iovec iov; + char buf; + int pipefd[2]; + + /* Populate first page with known content */ + memset(gopts->area_dst, 0xCD, gopts->page_size); + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, gopts->page_size)) + err("register failure"); + + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + gopts->page_size, true); + + if (pipe(pipefd)) + err("pipe"); + + /* + * One byte's worth of iov is enough to GUP the containing page and + * keeps the pipe transfer well under any pipe-capacity limit even on + * hugetlb-backed runs. + */ + iov.iov_base = gopts->area_dst; + iov.iov_len = 1; + if (vmsplice(pipefd[1], &iov, 1, 0) != 1) { + uffd_test_fail("vmsplice from RW-protected page failed: %s", + strerror(errno)); + goto out; + } + + if (read(pipefd[0], &buf, 1) != 1) { + uffd_test_fail("read from pipe failed"); + goto out; + } + + if (buf != (char)0xCD) { + uffd_test_fail("content mismatch: got 0x%02x, expected 0xCD", + (unsigned char)buf); + goto out; + } + + uffd_test_pass(); +out: + close(pipefd[0]); + close(pipefd[1]); +} + +/* + * Test runtime toggle between async and sync modes. + * Start in async mode (detection), flip to sync (eviction), verify faults + * block, resolve them, flip back to async. + */ +static void uffd_rwp_async_toggle_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + struct uffd_args uargs = { }; + pthread_t uffd_mon; + char c = '\0'; + unsigned long p; + + uargs.gopts = gopts; + uargs.handle_fault = uffd_handle_rwp_fault; + + /* Populate */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + /* Phase 1: async detection — RW-protect, access first half */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + for (p = 0; p < nr_pages / 2; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; /* auto-resolves in async mode */ + } + + /* Phase 2: flip to sync for eviction */ + set_async_mode(gopts->uffd, false); + + /* Start handler — will receive faults for cold pages */ + if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs)) + err("uffd_poll_thread create"); + + /* Access second half (cold pages) — should trigger sync faults */ + for (p = nr_pages / 2; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + if (page[0] != (p % 255 + 1)) { + uffd_test_fail("page %lu content mismatch", p); + goto out; + } + } + + /* + * Stop the handler before reading minor_faults: the last fault + * resolution rwprotect_range()s before incrementing the counter, + * so the main thread can race ahead of the increment. Stopping + * here also makes Phase 3 a clean async-only test -- with the + * handler still running it would silently resolve any sync fault + * the kernel erroneously delivers, masking a regression. + */ + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join() failed"); + + if (uargs.minor_faults == 0) { + uffd_test_fail("expected sync faults, got 0"); + return; + } + + /* Phase 3: flip back to async */ + set_async_mode(gopts->uffd, true); + + /* RW-protect and access again — should auto-resolve */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + for (p = 0; p < nr_pages; p++) { + volatile char *page = gopts->area_dst + p * page_size; + (void)*page; + } + + uffd_test_pass(); + return; +out: + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join() failed"); +} + +/* + * Test that RW-protected pages become accessible after closing uffd. + */ +static void uffd_rwp_close_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + unsigned long p; + + /* Populate */ + for (p = 0; p < nr_pages; p++) + memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size); + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failure"); + + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + nr_pages * page_size, true); + + /* Close uffd — should restore protnone PTEs */ + close(gopts->uffd); + gopts->uffd = -1; + + /* All pages should be accessible with original content */ + for (p = 0; p < nr_pages; p++) { + unsigned char *page = (unsigned char *)gopts->area_dst + + p * page_size; + unsigned char expected = p % 255 + 1; + + if (page[0] != expected) { + uffd_test_fail("page %lu not accessible after close", p); + return; + } + } + + uffd_test_pass(); +} + +/* + * Test that RWP protection is preserved across fork() when + * UFFD_FEATURE_EVENT_FORK is enabled. Without preservation, the child's + * PTEs would lose the uffd-wp marker and RWP-protected accesses would + * silently fall through to do_numa_page(). + */ +static void uffd_rwp_fork_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + int pagemap_fd; + uint64_t value; + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, + nr_pages * page_size)) + err("register failed"); + + /* Populate + RWP-protect */ + *gopts->area_dst = 1; + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, + page_size, true); + + /* Parent: verify uffd-wp bit is set before fork */ + pagemap_fd = pagemap_open(); + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); + pagemap_check_wp(value, true); + + /* + * Fork with EVENT_FORK: child inherits VM_UFFD_RWP. Child reads + * its own pagemap and must still see the uffd-wp bit set. + */ + if (pagemap_test_fork(gopts, true, false)) { + uffd_test_fail("RWP marker lost in child after fork"); + goto out; + } + + uffd_test_pass(); +out: + close(pagemap_fd); +} + +/* + * Test that RWP protection on a pinned anon page is preserved across fork(). + * Pinning forces copy_present_page() in the child path, which must restore + * PAGE_NONE on top of the uffd bit. Using async mode, a read in the child + * auto-resolves if — and only if — the PTE was actually protnone+uffd; the + * cleared uffd bit afterward proves the fault path ran. + */ +static void uffd_rwp_fork_pin_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long page_size = gopts->page_size; + fork_event_args fevent_args = { .gopts = gopts, .child_uffd = -1 }; + pin_args pin_args = {}; + int pagemap_fd, status; + pthread_t fevent_thread; + uint64_t value; + pid_t child; + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size)) + err("register failed"); + + /* Populate. */ + *gopts->area_dst = 1; + + /* RO-longterm pin so fork() takes copy_present_page() for this PTE. */ + if (pin_pages(&pin_args, gopts->area_dst, page_size)) { + uffd_test_skip("Possibly CONFIG_GUP_TEST missing or unprivileged"); + uffd_unregister(gopts->uffd, gopts->area_dst, page_size); + return; + } + + /* RWP-protect: PTE is now PAGE_NONE + uffd bit. */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true); + + pagemap_fd = pagemap_open(); + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); + pagemap_check_wp(value, true); + + /* + * UFFD_FEATURE_EVENT_FORK is required so the child inherits + * VM_UFFD_RWP and the marker; without it dup_userfaultfd() resets + * the child VMA and the test would pass for the wrong reason. + * dup_userfaultfd() blocks until the EVENT_FORK message is consumed, + * so spawn a reader before the fork(). + */ + gopts->ready_for_fork = false; + if (pthread_create(&fevent_thread, NULL, fork_event_consumer, + &fevent_args)) + err("pthread_create() for fork event consumer"); + while (!gopts->ready_for_fork) + ; /* Wait for consumer to start polling. */ + + child = fork(); + if (child < 0) + err("fork"); + if (child == 0) { + volatile char c; + int cfd; + + /* + * Precondition: the child must have inherited the marker. + * If copy_present_page() dropped it together with PAGE_NONE, + * the read below would succeed without the fault path and + * the after-read check would pass for the wrong reason. + */ + cfd = pagemap_open(); + value = pagemap_get_entry(cfd, gopts->area_dst); + if (!(value & PM_UFFD_WP)) { + close(cfd); + _exit(2); + } + + /* + * Read the pinned page. Only reaches the fault path if the + * child PTE is protnone + uffd; async mode auto-resolves and + * clears the uffd bit. If copy_present_page() dropped + * PAGE_NONE, the read would silently succeed and the bit + * would still be set. + */ + c = *(volatile char *)gopts->area_dst; + (void)c; + + value = pagemap_get_entry(cfd, gopts->area_dst); + close(cfd); + _exit((value & PM_UFFD_WP) ? 1 : 0); + } + if (waitpid(child, &status, 0) < 0) + err("waitpid"); + if (pthread_join(fevent_thread, NULL)) + err("pthread_join() for fork event consumer"); + if (fevent_args.child_uffd >= 0) + close(fevent_args.child_uffd); + + unpin_pages(&pin_args); + close(pagemap_fd); + if (uffd_unregister(gopts->uffd, gopts->area_dst, page_size)) + err("unregister failed"); + + if (WIFEXITED(status) && WEXITSTATUS(status) == 2) { + uffd_test_fail("RWP marker not inherited by child"); + return; + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + uffd_test_fail("RWP not enforced in child after pinned fork"); + return; + } + + uffd_test_pass(); +} + +/* + * A non-exclusive (forked, COW-shared) anon page that is RWP-protected and + * then swapped out must keep tracking across swap-in. On the write that + * swaps it back in, do_swap_page() restores PAGE_NONE and the access retries + * through the RWP fault path, instead of being COWed straight to a fresh + * accessible page -- which would silently drop the marker for a non-exclusive + * folio. Sync mode lets us observe the fault directly: with the bug, the + * write COWs without delivering any RWP fault. + * + * Needs a swap device; skipped if MADV_PAGEOUT cannot evict the page. + */ +static void uffd_rwp_swap_cow_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long page_size = gopts->page_size; + struct uffd_args uargs = { }; + int pagemap_fd, go[2], i; + pthread_t uffd_mon; + char c = '\0'; + pid_t child; + + uargs.gopts = gopts; + uargs.handle_fault = uffd_handle_rwp_fault; + + if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size)) + err("register failed"); + + /* Populate one page (exclusive at this point). */ + *gopts->area_dst = 0x11; + + /* RWP-protect: PTE becomes PAGE_NONE + uffd bit (still exclusive). */ + rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true); + + /* + * Swap the page out while it is still exclusive: a shared (forked) + * folio does not get reclaimed by MADV_PAGEOUT. Retry, since a hot + * page may just be rotated on the first reclaim pass. + */ + pagemap_fd = pagemap_open(); + for (i = 0; i < 100; i++) { + if (madvise(gopts->area_dst, page_size, MADV_PAGEOUT)) + err("MADV_PAGEOUT"); + if (pagemap_is_swapped(pagemap_fd, gopts->area_dst)) + break; + usleep(10000); + } + if (!pagemap_is_swapped(pagemap_fd, gopts->area_dst)) { + uffd_test_skip("MADV_PAGEOUT did not swap the page; is swap enabled?"); + close(pagemap_fd); + uffd_unregister(gopts->uffd, gopts->area_dst, page_size); + return; + } + + /* + * fork() now: the child duplicates the swap entry, so the slot becomes + * non-exclusive. The child parks (keeping the reference) until the + * parent has faulted the page back in. + */ + if (pipe(go)) + err("pipe"); + child = fork(); + if (child < 0) + err("fork"); + if (child == 0) { + close(go[1]); + read(go[0], &c, 1); + _exit(0); + } + close(go[0]); + + if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs)) + err("uffd_poll_thread create"); + + /* + * Write the page: swaps it back in (do_swap_page) on a non-exclusive + * folio with FAULT_FLAG_WRITE. The marker must survive and deliver an + * RWP fault rather than COW silently. + */ + *gopts->area_dst = 0x22; + + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) + err("pipe write"); + if (pthread_join(uffd_mon, NULL)) + err("join failed"); + + if (uargs.minor_faults == 0) + uffd_test_fail("no RWP fault on swapped-in non-exclusive page"); + else + uffd_test_pass(); + + close(pagemap_fd); + if (write(go[1], &c, 1) != 1) + err("child release"); + close(go[1]); + waitpid(child, NULL, 0); +} + +/* + * WP and RWP share the uffd-wp PTE bit and cannot coexist in the same VMA. + * Registration requesting both modes must be rejected. + */ +static void uffd_rwp_wp_exclusive_test(uffd_global_test_opts_t *gopts, + uffd_test_args_t *args) +{ + unsigned long nr_pages = gopts->nr_pages; + unsigned long page_size = gopts->page_size; + struct uffdio_register reg = { }; + + reg.range.start = (unsigned long)gopts->area_dst; + reg.range.len = nr_pages * page_size; + reg.mode = UFFDIO_REGISTER_MODE_WP | UFFDIO_REGISTER_MODE_RWP; + + if (ioctl(gopts->uffd, UFFDIO_REGISTER, ®) == 0) { + uffd_test_fail("register with WP|RWP unexpectedly succeeded"); + return; + } + if (errno != EINVAL) { + uffd_test_fail("register with WP|RWP: expected EINVAL, got %d", + errno); + return; + } + uffd_test_pass(); +} + static sigjmp_buf jbuf, *sigbuf; static void sighndl(int sig, siginfo_t *siginfo, void *ptr) @@ -1605,6 +2428,83 @@ uffd_test_case_t uffd_tests[] = { .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM, }, { + .name = "rwp-async", + .uffd_fn = uffd_rwp_async_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-sync", + .uffd_fn = uffd_rwp_sync_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = UFFD_FEATURE_RWP, + }, + { + .name = "rwp-pagemap", + .uffd_fn = uffd_rwp_pagemap_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-mprotect", + .uffd_fn = uffd_rwp_mprotect_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-gup", + .uffd_fn = uffd_rwp_gup_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-async-toggle", + .uffd_fn = uffd_rwp_async_toggle_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC, + }, + { + .name = "rwp-close", + .uffd_fn = uffd_rwp_close_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = UFFD_FEATURE_RWP, + }, + { + .name = "rwp-fork", + .uffd_fn = uffd_rwp_fork_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_EVENT_FORK, + }, + { + .name = "rwp-fork-pin", + .uffd_fn = uffd_rwp_fork_pin_test, + .mem_targets = MEM_ANON, + .uffd_feature_required = + UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC | + UFFD_FEATURE_EVENT_FORK, + }, + { + .name = "rwp-swap-cow", + .uffd_fn = uffd_rwp_swap_cow_test, + .mem_targets = MEM_ANON, + .uffd_feature_required = UFFD_FEATURE_RWP, + }, + { + .name = "rwp-wp-exclusive", + .uffd_fn = uffd_rwp_wp_exclusive_test, + .mem_targets = MEM_ALL, + .uffd_feature_required = + UFFD_FEATURE_RWP | + UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_WP_HUGETLBFS_SHMEM, + }, + { .name = "sigbus", .uffd_fn = uffd_sigbus_test, .mem_targets = MEM_ALL, @@ -1814,6 +2714,25 @@ int main(int argc, char *argv[]) uffd_test_skip(errmsg); continue; } + /* + * RWP tracks protection on ptes; a THP-backed shmem/anon + * range (e.g. shmem_enabled=always) would split on + * rwprotect and change behaviour under the test. Keep + * such ranges off THP. hugetlb is huge by definition and + * rejects MADV_NOHUGEPAGE, so skip it. + */ + if ((test->uffd_feature_required & UFFD_FEATURE_RWP) && + !(mem_type->mem_flag & (MEM_HUGETLB | MEM_HUGETLB_PRIVATE))) { + unsigned long len = gopts.nr_pages * gopts.page_size; + + /* + * EINVAL means CONFIG_TRANSPARENT_HUGEPAGE=n: + * nothing to opt out of. + */ + if (madvise(gopts.area_dst, len, MADV_NOHUGEPAGE) && + errno != EINVAL) + err("madvise(MADV_NOHUGEPAGE)"); + } test->uffd_fn(&gopts, &args); uffd_test_ctx_clear(&gopts); } diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 311fc5b4513e..ef1ea11981a7 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -719,7 +719,7 @@ int read_file(const char *path, char *buf, size_t buflen) return (unsigned int) numread; } -void write_file(const char *path, const char *buf, size_t buflen) +static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval) { int fd, saved_errno; ssize_t numwritten; @@ -735,14 +735,22 @@ void write_file(const char *path, const char *buf, size_t buflen) saved_errno = errno; close(fd); errno = saved_errno; - if (numwritten < 0) + if (numwritten < 0) { + if (ignore_einval && errno == EINVAL) + return; ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1), buf, strerror(errno)); + } if (numwritten != buflen - 1) ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n", path, (int)(buflen - 1), buf, buflen - 1, numwritten); } +void write_file(const char *path, const char *buf, size_t buflen) +{ + __write_file(path, buf, buflen, /* ignore_einval = */ false); +} + unsigned long read_num(const char *path) { char buf[21]; @@ -753,12 +761,22 @@ unsigned long read_num(const char *path) return strtoul(buf, NULL, 10); } -void write_num(const char *path, unsigned long num) +static void __write_num(const char *path, unsigned long num, bool ignore_einval) { char buf[21]; sprintf(buf, "%lu", num); - write_file(path, buf, strlen(buf) + 1); + __write_file(path, buf, strlen(buf) + 1, ignore_einval); +} + +void write_num(const char *path, unsigned long num) +{ + return __write_num(path, num, /* ignore_einval = */ false); +} + +void write_num_ignore_einval(const char *path, unsigned long num) +{ + return __write_num(path, num, /* ignore_einval = */ true); } static unsigned long shmall, shmmax; diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index ea8fc8fdf0eb..7799154b67ee 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -168,6 +168,7 @@ void write_file(const char *path, const char *buf, size_t buflen); int read_file(const char *path, char *buf, size_t buflen); unsigned long read_num(const char *path); void write_num(const char *path, unsigned long num); +void write_num_ignore_einval(const char *path, unsigned long num); void shm_limits_prepare(unsigned long length); void __shm_limits_restore(void); diff --git a/tools/testing/shared/linux/kmemleak.h b/tools/testing/shared/linux/kmemleak.h index 155f112786c4..65a383090fb5 100644 --- a/tools/testing/shared/linux/kmemleak.h +++ b/tools/testing/shared/linux/kmemleak.h @@ -1 +1,2 @@ static inline void kmemleak_update_trace(const void *ptr) { } +static inline void kmemleak_transient_leak(const void *ptr) { } diff --git a/tools/testing/vma/Makefile b/tools/testing/vma/Makefile index e72b45dedda5..ef6cc558afe1 100644 --- a/tools/testing/vma/Makefile +++ b/tools/testing/vma/Makefile @@ -10,7 +10,7 @@ OFILES = $(SHARED_OFILES) main.o shared.o maple-shim.o TARGETS = vma # These can be varied to test different sizes. -CFLAGS += -DNUM_VMA_FLAG_BITS=128 -DNUM_MM_FLAG_BITS=128 +CFLAGS += -DNUM_VMA_FLAG_BITS=64 -DNUM_MM_FLAG_BITS=64 main.o: main.c shared.c shared.h vma_internal.h tests/merge.c tests/mmap.c tests/vma.c ../../../mm/vma.c ../../../mm/vma_init.c ../../../mm/vma_exec.c ../../../mm/vma.h include/custom.h include/dup.h include/stubs.h diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index bf26b3f48d3a..800e5fa02d78 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -15,6 +15,21 @@ struct task_struct *get_current(void); #define MMF_HAS_MDWE 28 #define current get_current() +#define MINORBITS 20 +#define MINORMASK ((1U << MINORBITS) - 1) + +#define MAJOR(dev) ((unsigned int) ((dev) >> MINORBITS)) +#define MINOR(dev) ((unsigned int) ((dev) & MINORMASK)) +#define MKDEV(ma, mi) (((ma) << MINORBITS) | (mi)) + +#define S_IFMT 00170000 +#define S_IFCHR 0020000 + +#define S_ISCHR(m) (((m) & S_IFMT) == S_IFCHR) + +#define MEM_MAJOR 1 +#define DEVZERO_MINOR 5 + /* * Define the task command name length as enum, then it can be visible to * BPF programs. @@ -23,6 +38,8 @@ enum { TASK_COMM_LEN = 16, }; +typedef unsigned short umode_t; + /* PARTIALLY implemented types. */ struct mm_struct { struct maple_tree mm_mt; @@ -45,6 +62,10 @@ struct address_space { unsigned long flags; atomic_t i_mmap_writable; }; +struct inode { + umode_t i_mode; + dev_t i_rdev; +}; struct file_operations { int (*mmap)(struct file *, struct vm_area_struct *); int (*mmap_prepare)(struct vm_area_desc *); @@ -52,6 +73,7 @@ struct file_operations { struct file { struct address_space *f_mapping; const struct file_operations *f_op; + struct inode *f_inode; }; struct anon_vma_chain { struct anon_vma *anon_vma; @@ -245,8 +267,10 @@ enum { #define VM_STACK INIT_VM_FLAG(STACK) #ifdef CONFIG_STACK_GROWS_UP #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) +#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT) #else #define VM_STACK_EARLY VM_NONE +#define VMA_STACK_EARLY EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARCH_HAS_PKEYS #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT) @@ -315,6 +339,8 @@ enum { /* Bits set in the VMA until the stack is in its final location */ #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) +#define VMA_STACK_INCOMPLETE_SETUP append_vma_flags( \ + VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT) #define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ VM_EXEC_BIT : VM_READ_BIT) @@ -573,6 +599,7 @@ struct vm_area_struct { */ unsigned int vm_lock_seq; #endif + unsigned int __vm_virt_pgoff_lo; /* * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma @@ -609,6 +636,9 @@ struct vm_area_struct { /* Unstable RCU readers are allowed to read this. */ refcount_t vm_refcnt; #endif +#ifdef CONFIG_64BIT + unsigned int __vm_virt_pgoff_hi; +#endif /* * For areas with an address space and backing store, * linkage into the address_space->i_mmap interval tree. @@ -1163,6 +1193,11 @@ static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) return mas_find(&vmi->mas, ULONG_MAX); } +static inline bool vma_is_attached(struct vm_area_struct *vma) +{ + return refcount_read(&vma->vm_refcnt); +} + /* * WARNING: to avoid racing with vma_mark_attached()/vma_mark_detached(), these * assertions should be made either under mmap_write_lock or when the object @@ -1170,12 +1205,12 @@ static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) */ static inline void vma_assert_attached(struct vm_area_struct *vma) { - WARN_ON_ONCE(!refcount_read(&vma->vm_refcnt)); + WARN_ON_ONCE(!vma_is_attached(vma)); } static inline void vma_assert_detached(struct vm_area_struct *vma) { - WARN_ON_ONCE(refcount_read(&vma->vm_refcnt)); + WARN_ON_ONCE(vma_is_attached(vma)); } static inline void vma_assert_write_locked(struct vm_area_struct *); @@ -1301,6 +1336,38 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; } +static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) +{ + return vma->vm_pgoff; +} + +static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_pgoff(vma) + vma_pages(vma); +} + +static inline pgoff_t vma_start_virt_pgoff(const struct vm_area_struct *vma) +{ + pgoff_t pgoff = 0; + +#ifdef CONFIG_64BIT + pgoff += vma->__vm_virt_pgoff_hi; + pgoff <<= 32; +#endif + pgoff += vma->__vm_virt_pgoff_lo; + return pgoff; +} + +static inline pgoff_t vma_end_virt_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_virt_pgoff(vma) + vma_pages(vma); +} + +static inline pgoff_t vma_last_virt_pgoff(const struct vm_area_struct *vma) +{ + return vma_end_virt_pgoff(vma) - 1; +} + static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc) { return file->f_op->mmap_prepare(desc); @@ -1372,7 +1439,7 @@ static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr) mas_set(&vmi->mas, addr); } -static inline bool vma_is_anonymous(struct vm_area_struct *vma) +static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; } @@ -1532,9 +1599,73 @@ static inline int get_sysctl_max_map_count(void) #define pgtable_supports_soft_dirty() IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) #endif -static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags) +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) { const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); return vm_get_page_prot(vm_flags); } + +static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma, + const unsigned long address) +{ + return (address - vma->vm_start) >> PAGE_SHIFT; +} + +static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + pgoff_t pgoff; + + pgoff = linear_page_delta(vma, address); + pgoff += vma_start_pgoff(vma); + return pgoff; +} + +static inline void vma_assert_can_modify(struct vm_area_struct *vma) +{ + if (vma_is_attached(vma)) + vma_assert_write_locked(vma); +} + +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) +{ + return vma_flags_to_page_prot(vma->flags); +} + +static inline pgoff_t __linear_virt_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + pgoff_t pgoff; + + pgoff = linear_page_delta(vma, address); + pgoff += vma_start_virt_pgoff(vma); + return pgoff; +} + +static inline pgoff_t linear_virt_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + const pgoff_t pgoff = __linear_virt_page_index(vma, address); + + VM_WARN_ON_ONCE(vma_test(vma, VMA_SHARED_BIT)); + if (vma_is_anonymous(vma)) + VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); + + return pgoff; +} + +static inline struct inode *file_inode(const struct file *f) +{ + return f->f_inode; +} + +static inline unsigned iminor(const struct inode *inode) +{ + return MINOR(inode->i_rdev); +} + +static inline unsigned imajor(const struct inode *inode) +{ + return MAJOR(inode->i_rdev); +} diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index 64164e25658f..d6136e19a8af 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -257,13 +257,13 @@ static inline void vm_acct_memory(long pages) { } -static inline void vma_interval_tree_insert(struct vm_area_struct *vma, - struct rb_root_cached *rb) +static inline void mapping_rmap_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping) { } -static inline void vma_interval_tree_remove(struct vm_area_struct *vma, - struct rb_root_cached *rb) +static inline void mapping_rmap_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping) { } @@ -271,13 +271,13 @@ static inline void flush_dcache_mmap_unlock(struct address_space *mapping) { } -static inline void anon_vma_interval_tree_insert(struct anon_vma_chain *avc, - struct rb_root_cached *rb) +static inline void anon_rmap_tree_insert(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { } -static inline void anon_vma_interval_tree_remove(struct anon_vma_chain *avc, - struct rb_root_cached *rb) +static inline void anon_rmap_tree_remove(struct anon_vma_chain *avc, + struct anon_vma *anon_vma) { } diff --git a/tools/testing/vma/shared.c b/tools/testing/vma/shared.c index 2565a5aecb80..f410bb6f858e 100644 --- a/tools/testing/vma/shared.c +++ b/tools/testing/vma/shared.c @@ -23,7 +23,8 @@ struct vm_area_struct *alloc_vma(struct mm_struct *mm, vma->vm_start = start; vma->vm_end = end; - vma->vm_pgoff = pgoff; + vma_set_pgoff(vma, pgoff); + vma_set_virt_pgoff(vma, start >> PAGE_SHIFT); vma->flags = vma_flags; vma_assert_detached(vma); @@ -120,12 +121,3 @@ unsigned long rlimit(unsigned int limit) { return (unsigned long)-1; } - -void vma_set_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - pgoff_t pgoff) -{ - vma->vm_start = start; - vma->vm_end = end; - vma->vm_pgoff = pgoff; -} diff --git a/tools/testing/vma/shared.h b/tools/testing/vma/shared.h index 8b9e3b11c3cb..97cd7a679dc1 100644 --- a/tools/testing/vma/shared.h +++ b/tools/testing/vma/shared.h @@ -21,19 +21,35 @@ } \ } while (0) -#define ASSERT_TRUE(_expr) \ - do { \ - if (!(_expr)) { \ - fprintf(stderr, \ - "Assert FAILED at %s:%d:%s(): %s is FALSE.\n", \ - __FILE__, __LINE__, __FUNCTION__, #_expr); \ - return false; \ - } \ +#define __ASSERT_TRUE(_expr, _fmt, ...) \ + do { \ + if (!(_expr)) { \ + fprintf(stderr, \ + "Assert FAILED at %s:%d:%s(): %s is FALSE" \ + _fmt ".\n", \ + __FILE__, __LINE__, __FUNCTION__, #_expr \ + __VA_OPT__(,) __VA_ARGS__); \ + return false; \ + } \ } while (0) +#define __TO_SCALAR(x) ((unsigned long long)(uintptr_t)(x)) + +#define ASSERT_TRUE(_expr) __ASSERT_TRUE(_expr, "") #define ASSERT_FALSE(_expr) ASSERT_TRUE(!(_expr)) -#define ASSERT_EQ(_val1, _val2) ASSERT_TRUE((_val1) == (_val2)) -#define ASSERT_NE(_val1, _val2) ASSERT_TRUE((_val1) != (_val2)) +#define ASSERT_EQ(_val1, _val2) do { \ + __typeof__(_val1) __val1 = (_val1); \ + __typeof__(_val2) __val2 = (_val2); \ + __ASSERT_TRUE(__val1 == __val2, " (0x%llx != 0x%llx)", \ + __TO_SCALAR(__val1), __TO_SCALAR(__val2)); \ + } while (0) + +#define ASSERT_NE(_val1, _val2) do { \ + __typeof__(_val1) __val1 = (_val1); \ + __typeof__(_val2) __val2 = (_val2); \ + __ASSERT_TRUE(__val1 != __val2, " (0x%llx == 0x%llx)", \ + __TO_SCALAR(__val1), __TO_SCALAR(__val2)); \ + } while (0) #define ASSERT_FLAGS_SAME_MASK(_flags, _flags_other) \ ASSERT_TRUE(vma_flags_same_mask((_flags), (_flags_other))) @@ -53,8 +69,6 @@ #define ASSERT_FLAGS_NONEMPTY(_flags) \ ASSERT_FALSE(vma_flags_empty(_flags)) -#define IS_SET(_val, _flags) ((_val & _flags) == _flags) - extern bool fail_prealloc; /* Override vma_iter_prealloc() so we can choose to fail it. */ @@ -125,8 +139,3 @@ void __vma_set_dummy_anon_vma(struct vm_area_struct *vma, /* Provide a simple dummy VMA/anon_vma dummy setup for testing. */ void vma_set_dummy_anon_vma(struct vm_area_struct *vma, struct anon_vma_chain *avc); - -/* Helper function to specify a VMA's range. */ -void vma_set_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - pgoff_t pgoff); diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index 03b6f9820e0a..89ba273afe4b 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -45,6 +45,7 @@ void vmg_set_range(struct vma_merge_struct *vmg, unsigned long start, vmg->start = start; vmg->end = end; vmg->pgoff = pgoff; + vmg->anon_pgoff = start >> PAGE_SHIFT; vmg->vma_flags = vma_flags; vmg->just_expand = false; @@ -108,6 +109,7 @@ static bool test_simple_merge(void) .end = 0x2000, .vma_flags = vma_flags, .pgoff = 1, + .anon_pgoff = 1, }; ASSERT_FALSE(attach_vma(&mm, vma_left)); @@ -118,7 +120,8 @@ static bool test_simple_merge(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_FLAGS_SAME_MASK(&vma->flags, vma_flags); detach_free_vma(vma); @@ -150,7 +153,8 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x1000); ASSERT_EQ(vma->vm_end, 0x2000); - ASSERT_EQ(vma->vm_pgoff, 1); + ASSERT_EQ(vma_start_pgoff(vma), 1); + ASSERT_EQ(vma_start_virt_pgoff(vma), 1); /* * Now walk through the three split VMAs and make sure they are as @@ -162,7 +166,8 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); detach_free_vma(vma); vma_iter_clear(&vmi); @@ -171,7 +176,8 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x1000); ASSERT_EQ(vma->vm_end, 0x2000); - ASSERT_EQ(vma->vm_pgoff, 1); + ASSERT_EQ(vma_start_pgoff(vma), 1); + ASSERT_EQ(vma_start_virt_pgoff(vma), 1); detach_free_vma(vma); vma_iter_clear(&vmi); @@ -180,7 +186,8 @@ static bool test_simple_modify(void) ASSERT_EQ(vma->vm_start, 0x2000); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 2); + ASSERT_EQ(vma_start_pgoff(vma), 2); + ASSERT_EQ(vma_start_virt_pgoff(vma), 2); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -209,7 +216,8 @@ static bool test_simple_expand(void) ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -227,11 +235,12 @@ static bool test_simple_shrink(void) ASSERT_FALSE(attach_vma(&mm, vma)); - ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, 0)); + ASSERT_FALSE(vma_shrink(&vmi, vma, 0x1000)); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); detach_free_vma(vma); mtree_destroy(&mm.mm_mt); @@ -324,7 +333,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x4000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -343,7 +352,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x5000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -364,7 +374,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0x6000); ASSERT_EQ(vma->vm_end, 0x9000); - ASSERT_EQ(vma->vm_pgoff, 6); + ASSERT_EQ(vma_start_pgoff(vma), 6); + ASSERT_EQ(vma_start_virt_pgoff(vma), 6); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 3); @@ -384,7 +395,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x9000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -404,7 +416,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0xa000); ASSERT_EQ(vma->vm_end, 0xc000); - ASSERT_EQ(vma->vm_pgoff, 0xa); + ASSERT_EQ(vma_start_pgoff(vma), 0xa); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0xa); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -423,7 +436,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_TRUE(merged); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0xc000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 1); @@ -443,7 +457,8 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky, ASSERT_NE(vma, NULL); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0xc000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_EQ(vma->anon_vma, &dummy_anon_vma); detach_free_vma(vma); @@ -805,7 +820,8 @@ static bool test_vma_merge_new_with_close(void) ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x5000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_EQ(vma->vm_ops, &vm_ops); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -861,11 +877,13 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_next->vm_start, 0x3000); ASSERT_EQ(vma_next->vm_end, 0x9000); - ASSERT_EQ(vma_next->vm_pgoff, 3); + ASSERT_EQ(vma_start_pgoff(vma_next), 3); + ASSERT_EQ(vma_start_virt_pgoff(vma_next), 3); ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma); ASSERT_EQ(vma->vm_start, 0x2000); ASSERT_EQ(vma->vm_end, 0x3000); - ASSERT_EQ(vma->vm_pgoff, 2); + ASSERT_EQ(vma_start_pgoff(vma), 2); + ASSERT_EQ(vma_start_virt_pgoff(vma), 2); ASSERT_TRUE(vma_write_started(vma)); ASSERT_TRUE(vma_write_started(vma_next)); ASSERT_EQ(mm.map_count, 2); @@ -927,11 +945,13 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x6000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma_prev), 0); ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma); ASSERT_EQ(vma->vm_start, 0x6000); ASSERT_EQ(vma->vm_end, 0x7000); - ASSERT_EQ(vma->vm_pgoff, 6); + ASSERT_EQ(vma_start_pgoff(vma), 6); + ASSERT_EQ(vma_start_virt_pgoff(vma), 6); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 2); @@ -962,7 +982,8 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x7000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma_prev), 0); ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_EQ(mm.map_count, 1); @@ -994,7 +1015,8 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma_prev->vm_start, 0); ASSERT_EQ(vma_prev->vm_end, 0x9000); - ASSERT_EQ(vma_prev->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma_prev), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma_prev), 0); ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma); ASSERT_TRUE(vma_write_started(vma_prev)); ASSERT_EQ(mm.map_count, 1); @@ -1416,7 +1438,8 @@ static bool test_merge_extend(void) ASSERT_EQ(vma_merge_extend(&vmi, vma, 0x2000), vma); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x4000); - ASSERT_EQ(vma->vm_pgoff, 0); + ASSERT_EQ(vma_start_pgoff(vma), 0); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(mm.map_count, 1); @@ -1431,7 +1454,7 @@ static bool test_expand_only_mode(void) struct mm_struct mm = {}; VMA_ITERATOR(vmi, &mm, 0); struct vm_area_struct *vma_prev, *vma; - VMG_STATE(vmg, &mm, &vmi, 0x5000, 0x9000, vma_flags, 5); + VMG_STATE(vmg, &mm, &vmi, 0x5000, 0x9000, vma_flags, 5, 5); /* * Place a VMA prior to the one we're expanding so we assert that we do @@ -1456,7 +1479,8 @@ static bool test_expand_only_mode(void) ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS); ASSERT_EQ(vma->vm_start, 0x3000); ASSERT_EQ(vma->vm_end, 0x9000); - ASSERT_EQ(vma->vm_pgoff, 3); + ASSERT_EQ(vma_start_pgoff(vma), 3); + ASSERT_EQ(vma_start_virt_pgoff(vma), 3); ASSERT_TRUE(vma_write_started(vma)); ASSERT_EQ(vma_iter_addr(&vmi), 0x3000); vma_assert_attached(vma); diff --git a/tools/testing/vma/tests/mmap.c b/tools/testing/vma/tests/mmap.c index c85bc000d1cb..b720deb4227d 100644 --- a/tools/testing/vma/tests/mmap.c +++ b/tools/testing/vma/tests/mmap.c @@ -45,7 +45,57 @@ static bool test_mmap_region_basic(void) return true; } +static int dummy_mmap_prepare(struct vm_area_desc *desc) +{ + return 0; +} + +static bool test_pure_anon_dev_zero(void) +{ + const vma_flags_t vma_flags = mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, + VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT); + const struct file_operations f_op = { + .mmap_prepare = dummy_mmap_prepare, + }; + struct inode inode = { + .i_mode = S_IFCHR, + .i_rdev = MKDEV(MEM_MAJOR, DEVZERO_MINOR), + }; + struct file file = { + .f_inode = &inode, + .f_op = &f_op, + }; + struct mm_struct mm = {}; + struct vm_area_struct *vma; + unsigned long addr; + VMA_ITERATOR(vmi, &mm, 0); + + current->mm = &mm; + + /* + * Map a MAP_PRIVATE-/dev/zero mapping at address 0x300000 with a page + * offset of 0x10, which we expect to be reset to the virtual page + * offset. + */ + addr = __mmap_region(&file, 0x300000, 0x3000, vma_flags, 0x10, NULL); + ASSERT_EQ(addr, 0x300000); + + /* Assert that it truly is an anonymous mapping. */ + vma = vma_lookup(&mm, addr); + ASSERT_NE(vma, NULL); + ASSERT_TRUE(vma_is_anonymous(vma)); + ASSERT_EQ(vma->vm_file, NULL); + ASSERT_EQ(vma->vm_private_data, NULL); + /* Expect virtual page offsets. */ + ASSERT_EQ(vma->vm_pgoff, 0x300); + ASSERT_EQ(vma_start_virt_pgoff(vma), 0x300); + + cleanup_mm(&mm, &vmi); + return true; +} + static void run_mmap_tests(int *num_tests, int *num_fail) { TEST(mmap_region_basic); + TEST(pure_anon_dev_zero); } diff --git a/tools/testing/vma/tests/vma.c b/tools/testing/vma/tests/vma.c index 754a2da06321..7ca5289e0f95 100644 --- a/tools/testing/vma/tests/vma.c +++ b/tools/testing/vma/tests/vma.c @@ -38,7 +38,7 @@ static bool test_copy_vma(void) /* Move backwards and do not merge. */ vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags); - vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks); + vma_new = copy_vma(&vma, 0, 0x2000, 0, 3, &need_locks); ASSERT_NE(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0); ASSERT_EQ(vma_new->vm_end, 0x2000); @@ -51,7 +51,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags); vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags); - vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks); + vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, 4, &need_locks); vma_assert_attached(vma_new); ASSERT_EQ(vma_new, vma_next); diff --git a/tools/testing/vma/vma_internal.h b/tools/testing/vma/vma_internal.h index e12ab2c80f95..8a48b231aa7a 100644 --- a/tools/testing/vma/vma_internal.h +++ b/tools/testing/vma/vma_internal.h @@ -14,8 +14,8 @@ #include <stdlib.h> -#define CONFIG_MMU -#define CONFIG_PER_VMA_LOCK +#define CONFIG_MMU 1 +#define CONFIG_PER_VMA_LOCK 1 #ifdef __CONCAT #undef __CONCAT @@ -53,6 +53,7 @@ typedef __bitwise unsigned int vm_fault_t; #define VM_WARN_ON(_expr) (WARN_ON(_expr)) #define VM_WARN_ON_ONCE(_expr) (WARN_ON_ONCE(_expr)) +#define VM_WARN_ON_ONCE_VMA(_expr, _vma) (WARN_ON_ONCE(_expr)) #define VM_WARN_ON_VMG(_expr, _vmg) (WARN_ON(_expr)) #define VM_BUG_ON(_expr) (BUG_ON(_expr)) #define VM_BUG_ON_VMA(_expr, _vma) (BUG_ON(_expr)) |