+#ifndef __ASSEMBLER__

+

+#endif /* __ASSEMBLER__ */

+- Return thunk untraining validation -- validate that all entry paths

+ its ELF function annotation by using SYM_CODE_{START,END} and use the

+ manual unwind hint macros in asm/unwind_hints.h.

+ placed in a SYM_CODE_{START,END} block with unwind hints.

+ cases it's probably appropriate to use SYM_CODE_{START,END} with unwind

+ ERROR("unexpected ELF machine type %d", elf->ehdr.e_machine);

+ if (len != LOONGARCH_INSN_SIZE) {

+ ERROR("invalid NOP size: %d\n", len);

+ return NULL;

+ }

+ if (len != LOONGARCH_INSN_SIZE) {

+ ERROR("invalid RET size: %d\n", len);

+ return NULL;

+ }

+ ERROR_INSN(insn, "unknown unwind hint type %d", cfi->type);

+ ERROR_INSN(insn, "unknown CFA base reg %d", cfi->cfa.base);

+ ERROR_INSN(insn, "unknown FP base reg %d", fp->base);

+ ERROR_INSN(insn, "unknown RA base reg %d", ra->base);

+ ERROR("unexpected ELF machine type %d", elf->ehdr.e_machine);

+ ERROR("can't decode instruction at %s:0x%lx", sec->name, offset);

+ /* ERROR ? */

+ WARN("ENQCMD instruction at %s:%lx", sec->name, offset);

+ ERROR("no func for pv_ops[]");

+ ERROR("invalid NOP size: %d\n", len);

+ ERROR("invalid RET size: %d\n", len);

+ ERROR_INSN(insn, "unknown unwind hint type %d", cfi->type);

+ ERROR_INSN(insn, "unknown CFA base reg %d", cfi->cfa.base);

+ ERROR_INSN(insn, "unknown BP base reg %d", bp->base);

+#include <objtool/warn.h>

+void arch_handle_alternative(struct special_alt *alt)

+ if (reloc_type(text_reloc) == R_X86_64_PC32) {

+ WARN_INSN(insn, "ignoring unreachables due to jump table quirk");

+ }

+#include <errno.h>

+#include <objtool/warn.h>

+#define ORIG_SUFFIX ".orig"

+int orig_argc;

+static char **orig_argv;

+ ERROR("can't open %s for reading: %s", src, strerror(errno));

+ ERROR("can't open %s for writing: %s", dst, strerror(errno));

+ ERROR_GLIBC("fstat");

+ ERROR_GLIBC("fchmod");

+ ERROR_GLIBC("sendfile");

+static void save_argv(int argc, const char **argv)

+ ERROR_GLIBC("calloc");

+ exit(1);

+ ERROR_GLIBC("strdup(%s)", argv[i]);

+ exit(1);

+void print_args(void)

+{

+ char *backup = NULL;

+

+ if (opts.output || opts.dryrun)

+ goto print;

+

+ /*

+ * Make a backup before kbuild deletes the file so the error

+ * can be recreated without recompiling or relinking.

+ */

+ backup = malloc(strlen(objname) + strlen(ORIG_SUFFIX) + 1);

+ if (!backup) {

+ ERROR_GLIBC("malloc");

+ goto print;

+ }

+

+ strcpy(backup, objname);

+ strcat(backup, ORIG_SUFFIX);

+ if (copy_file(objname, backup)) {

+ backup = NULL;

+ goto print;

+ }

+

+print:

+ /*

+ * Print the cmdline args to make it easier to recreate. If '--output'

+ * wasn't used, add it to the printed args with the backup as input.

+ */

+ fprintf(stderr, "%s", orig_argv[0]);

+

+ for (int i = 1; i < orig_argc; i++) {

+ char *arg = orig_argv[i];

+

+ if (backup && !strcmp(arg, objname))

+ fprintf(stderr, " %s -o %s", backup, objname);

+ else

+ fprintf(stderr, " %s", arg);

+ }

+

+ fprintf(stderr, "\n");

+}

+ orig_argc = argc;

+ save_argv(argc, argv);

+ return 1;

+ return 1;

+ return ret;

+ return 0;

+ if (opts.noinstr && sec)

+ ERROR_GLIBC("calloc");

+ ERROR_GLIBC("mmap fail cfi_hash");

+ ERROR_GLIBC("calloc");

+ ERROR("%s(): STT_FUNC at end of section", func->name);

+ ERROR("%s(): can't find starting instruction", func->name);

+ idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);

+

+ if (!func) {

+ ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc),

+ "can't find func at %s[%d]", symname, idx);

+ return -1;

+ }

+ if (objtool_pv_add(file, idx, func))

+ return -1;

+ int idx, nr, ret;

+ if (!file->pv_ops) {

+ ERROR_GLIBC("calloc");

+ }

+ for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) {

+ ret = add_pv_ops(file, pv_ops);

+ if (ret)

+ return ret;

+ }

+ ERROR_GLIBC("strdup");

+ ERROR("static_call: trampoline name malformed: %s", key_name);

+ ERROR("static_call: can't find static_call_key symbol: %s", tmp);

+ !strcmp(sym->name, "cleanup_module"))) {

+ ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead",

+ sym->name);

+ return -1;

+ }

+static int add_ignores(struct objtool_file *file)

+ return 0;

+ ERROR("unexpected relocation symbol type in %s: %d",

+ rsec->name, reloc->sym->type);

+ return -1;

+ func->ignore = true;

+ if (func->cfunc)

+ func->cfunc->ignore = true;

+

+ return 0;

+ /* TRACE_BRANCH_PROFILING */

+ "ftrace_likely_update",

+ /* STACKPROTECTOR */

+ "__stack_chk_fail",

+static int annotate_call_site(struct objtool_file *file,

+ return 0;

+ return 0;

+ if (elf_write_insn(file->elf, insn->sec,

+ insn->offset, insn->len,

+ sibling ? arch_ret_insn(insn->len)

+ : arch_nop_insn(insn->len))) {

+ return -1;

+ }

+ return 0;

+ if (elf_write_insn(file->elf, insn->sec,

+ insn->offset, insn->len,

+ arch_nop_insn(insn->len))) {

+ return -1;

+ }

+ return 0;

+

+ return 0;

+static int add_call_dest(struct objtool_file *file, struct instruction *insn,

+ return 0;

+ return annotate_call_site(file, insn, sibling);

+static int add_retpoline_call(struct objtool_file *file, struct instruction *insn)

+ return 0;

+ return annotate_call_site(file, insn, false);

+ int ret;

+ struct symbol *func = insn_func(insn);

+

+ ret = add_retpoline_call(file, insn);

+ if (ret)

+ return ret;

+ } else if (func) {

+ ret = add_call_dest(file, insn, reloc->sym, true);

+ if (ret)

+ return ret;

+ /*

+ * GCOV/KCOV dead code can jump to the end of the

+ * function/section.

+ */

+ if (file->ignore_unreachables && func &&

+ dest_sec == insn->sec &&

+ dest_off == func->offset + func->len)

+ continue;

+

+ ERROR_INSN(insn, "can't find jump dest instruction at %s+0x%lx",

+ dest_sec->name, dest_off);

+ ret = add_retpoline_call(file, insn);

+ if (ret)

+ return ret;

+ if (func && insn_func(jump_dest) && func != insn_func(jump_dest)) {

+ if (!strstr(func->name, ".cold") &&

+ func->cfunc = insn_func(jump_dest);

+ insn_func(jump_dest)->pfunc = func;

+ ret = add_call_dest(file, insn, insn_func(jump_dest), true);

+ if (ret)

+ return ret;

+ int ret;

+ struct symbol *func = insn_func(insn);

+ ret = add_call_dest(file, insn, dest, false);

+ if (ret)

+ return ret;

+ if (func && func->ignore)

+ ERROR_INSN(insn, "unannotated intra-function call");

+ if (func && insn_call_dest(insn)->type != STT_FUNC) {

+ ERROR_INSN(insn, "unsupported call to non-function");

+ ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx",

+ reloc->sym->sec->name, dest_off);

+ ret = add_call_dest(file, insn, dest, false);

+ if (ret)

+ return ret;

+ ret = add_retpoline_call(file, insn);

+ if (ret)

+ return ret;

+ } else {

+ ret = add_call_dest(file, insn, reloc->sym, false);

+ if (ret)

+ return ret;

+ }

+ orig_alt_group = calloc(1, sizeof(*orig_alt_group));

+ ERROR_GLIBC("calloc");

+ ERROR_GLIBC("calloc");

+ orig_alt_group->ignore = orig_insn->ignore_alts;

+ ERROR_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",

+ orig_alt_group->last_insn->offset +

+ orig_alt_group->last_insn->len -

+ orig_alt_group->first_insn->offset,

+ special_alt->orig_len);

+ new_alt_group = calloc(1, sizeof(*new_alt_group));

+ ERROR_GLIBC("calloc");

+ nop = calloc(1, sizeof(*nop));

+ ERROR_GLIBC("calloc");

+ ERROR_INSN(insn, "unsupported relocation in alternatives section");

+ ERROR_INSN(insn, "can't find alternative jump destination");

+ ERROR_FUNC(special_alt->new_sec, special_alt->new_off,

+ "can't find last new alternative instruction");

+ new_alt_group->ignore = (*new_insn)->ignore_alts;

+ ERROR_INSN(orig_insn, "unsupported instruction at jump label");

+

+ if (elf_write_insn(file->elf, orig_insn->sec,

+ orig_insn->offset, orig_insn->len,

+ arch_nop_insn(orig_insn->len))) {

+ return -1;

+ }

+

+ if (special_get_alts(file->elf, &special_alts))

+ return -1;

+ ERROR_FUNC(special_alt->orig_sec, special_alt->orig_off,

+ "special: can't find orig instruction");

+ return -1;

+ ERROR_FUNC(special_alt->new_sec, special_alt->new_off,

+ "special: can't find new instruction");

+ return -1;

+ ERROR_INSN(orig_insn, "empty alternative entry");

+ return ret;

+

+ return ret;

+ alt = calloc(1, sizeof(*alt));

+ ERROR_GLIBC("calloc");

+ return -1;

+ return 0;

+static int add_jump_table(struct objtool_file *file, struct instruction *insn)

+ if (reloc != table && is_jump_table(reloc))

+ alt = calloc(1, sizeof(*alt));

+ ERROR_GLIBC("calloc");

+ ERROR_INSN(insn, "can't find switch jump table");

+ break;

+ set_jump_table(table_reloc);

+

+ struct instruction *insn;

+ int ret;

+ ret = add_jump_table(file, insn);

+ return 0;

+ ERROR("missing .rela.discard.unwind_hints section");

+ ERROR("struct unwind_hint size mismatch");

+ ERROR("can't find reloc for unwind_hints[%d]", i);

+ ERROR("unexpected relocation symbol type in %s", sec->rsec->name);

+ ERROR("can't find insn for unwind_hints[%d]", i);

+ ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");

+ return -1;

+ ERROR_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);

+ ERROR("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type);

+

+ /* Must be before add_special_section_alts() */

+ ERROR_INSN(insn, "intra_function_call not a direct call");

+ ERROR_INSN(insn, "can't find call dest at %s+0x%lx",

+ insn->sec->name, dest_off);

+ ERROR_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");

+ ERROR_INSN(insn, "Unknown annotation type: %d", type);

+ return -1;

+ ret = add_ignores(file);

+ if (ret)

+ return ret;

+

+ return 1;

+ WARN_INSN(orig, "stack layout conflict in alternatives: %s",

+ offstr(insn->sec, insn->offset));

+ int ret;

+ ret = update_cfi_state(insn, next_insn, &state->cfi, op);

+ if (ret)

+ return ret;

+ if (!opts.uaccess || !insn->alt_group)

+ return false;

+ }

+

+ if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {

+

+ if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], sizeof(struct cfi_reg)))

+ return false;

+ }

+ if (cfi1->type != cfi2->type) {

+ return false;

+ }

+ if (cfi1->drap != cfi2->drap ||

+ return false;

+ }

+ return true;

+ if (func && func->ignore)

+ return 0;

+

+ if (file->ignore_unreachables)

+ return 0;

+

+ func->warned = 1;

+ if (insn->alts) {

+ if (insn->alt_group && insn->alt_group->ignore)

+ return 0;

+

+ if (!opts.uaccess)

+ break;

+

+ if (!opts.uaccess)

+ break;

+

+ if (file->ignore_unreachables)

+ return 0;

+

+ WARN("%s%sunexpected end of section %s",

+ func ? func->name : "", func ? "(): " : "",

+ sec->name);

+ if (insn->hint && !insn->visited) {

+ if (insn->alts) {

+ return 1;

+ return 1;

+ return 1;

+ int warnings = 0;

+ warnings += validate_unret(file, insn);

+ warnings++;

+ }

+ continue;

+ WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",

+ insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");

+ struct symbol *func = insn_func(insn);

+ int i;

+ if (insn->type == INSN_NOP || insn->type == INSN_TRAP || (func && func->ignore))

+ if (opts.link && !func) {

+ insn_func(insn->jump_dest)->ignore = true;

+ if (!func)

+ if (func->static_call_tramp)

+ if (prev_insn && prev_insn->dead_end &&

+ insn_func(insn->jump_dest) == func) {

+ if (insn->offset + insn->len >= func->offset + func->len)

+ if (!insn || insn->visited)

+ if (opts.uaccess)

+ state->uaccess = sym->uaccess_safe;

+ WARN_FUNC(reloc->sec->base, reloc_offset(reloc),

+ "data relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));

+static void disas_funcs(const char *funcs)

+ if (!cross_compile)

+ cross_compile = "";

+ return;

+ return;

+static void disas_warned_funcs(struct objtool_file *file)

+ if (sym->warned) {

+ if (!funcs) {

+ ERROR_GLIBC("malloc");

+ return;

+ }

+ if (!tmp) {

+ ERROR_GLIBC("malloc");

+ return;

+ }

+ int ret = 0, warnings = 0;

+ if (ret)

+ if (opts.retpoline)

+ warnings += validate_retpoline(file);

+ int w = 0;

+ w += validate_functions(file);

+ w += validate_unwind_hints(file, NULL);

+ if (!w)

+ w += validate_reachable_instructions(file);

+ warnings += w;

+ warnings += validate_noinstr_sections(file);

+ warnings += validate_unrets(file);

+ if (opts.ibt)

+ warnings += validate_ibt(file);

+ if (opts.sls)

+ warnings += validate_sls(file);

+ if (ret)

+ if (ret)

+ if (ret)

+ if (ret)

+ if (ret)

+ if (ret)

+ if (ret)

+ if (ret)

+ if (ret)

+ if (!ret && !warnings)

+ return 0;

+

+ if (opts.werror && warnings)

+ ret = 1;

+

+ if (opts.verbose) {

+ if (opts.werror && warnings)

+ print_args();

+ disas_warned_funcs(file);

+ return ret;

+#define elf_alloc_hash(name, size) \

+({ \

+ __elf_bits(name) = max(10, ilog2(size)); \

+ PROT_READ|PROT_WRITE, \

+ MAP_PRIVATE|MAP_ANON, -1, 0); \

+ if (__elf_table(name) == (void *)-1L) { \

+ ERROR_GLIBC("mmap fail " #name); \

+ __elf_table(name) = NULL; \

+ } \

+ __elf_table(name); \

+ ERROR_ELF("elf_getshdrnum");

+ ERROR_ELF("elf_getshdrstrndx");

+ ERROR_GLIBC("calloc");

+ ERROR_ELF("elf_getscn");

+ ERROR_ELF("gelf_getshdr");

+ ERROR_ELF("elf_strptr");

+ ERROR_ELF("elf_getdata");

+ ERROR("unexpected data attributes for %s", sec->name);

+ ERROR("section entry mismatch");

+ ERROR_GLIBC("calloc");

+ ERROR_ELF("gelf_getsymshndx");

+ ERROR_ELF("elf_strptr");

+ ERROR("couldn't find section for symbol %s", sym->name);

+ ERROR("%s(): failed to allocate memory", sym->name);

+ ERROR("%s(): can't find parent function", sym->name);

+ for (reloc = sym->relocs; reloc; reloc = sym_next_reloc(reloc))

+ ERROR_ELF("elf_getscn");

+ ERROR_ELF("elf_getscn");

+ ERROR("index out of range");

+ ERROR_GLIBC("calloc");

+ ERROR_GLIBC("calloc");

+ ERROR("zero size data");

+ ERROR("negative index");

+ ERROR("no .symtab_shndx");

+ ERROR_ELF("gelf_update_symshndx");

+ ERROR("no .symtab");

+ ERROR("elf_update_symbol move");

+ ERROR("elf_update_symbol");

+ ERROR_GLIBC("malloc");

+ ERROR_GLIBC("malloc");

+ ERROR("%s: bad reloc_idx %u for %s with %d relocs",

+ __func__, reloc_idx, rsec->name, sec_num_entries(rsec));

+ ERROR("%s: %s: reloc %d already initialized!",

+ __func__, rsec->name, reloc_idx);

+ set_sym_next_reloc(reloc, sym->relocs);

+ ERROR("bad call to %s() for data symbol %s", __func__, sym->name);

+ ERROR("bad call to %s() for text symbol %s", __func__, sym->name);

+ ERROR("can't find base section for reloc section %s", rsec->name);

+ ERROR_GLIBC("calloc");

+ ERROR("can't find reloc entry symbol %d for %s", symndx, rsec->name);

+ set_sym_next_reloc(reloc, sym->relocs);

+ ERROR_GLIBC("malloc");

+ ERROR_ELF("elf_begin");

+ ERROR_ELF("gelf_getehdr");

+ ERROR("can't find .strtab section");

+ ERROR_ELF("elf_getscn");

+ ERROR_ELF("elf_newdata");

+ ERROR_GLIBC("malloc");

+ ERROR_ELF("elf_newscn");

+ ERROR_GLIBC("strdup");

+ ERROR_ELF("elf_newdata");

+ ERROR_GLIBC("malloc");

+ ERROR_ELF("gelf_getshdr");

+ ERROR("can't find .shstrtab or .strtab section");

+ ERROR_GLIBC("malloc");

+ ERROR_GLIBC("calloc");

+ ERROR("write to unexpected data for section: %s", sec->name);

+ ERROR_ELF("elf_getscn");

+ ERROR("end of section data but non-zero size left\n");

+ ERROR("truncated; but more data\n");

+ ERROR("zero size data");

+ ERROR_ELF("elf_getscn");

+ ERROR_ELF("gelf_update_shdr");

+ ERROR_ELF("elf_update");

+int cmd_parse_options(int argc, const char **argv, const char * const usage[]);

+int objtool_run(int argc, const char **argv);

+

+void print_args(void);

+

+ bool ignore;

+ u8 warned : 1;

+ u8 ignore : 1;

+ unsigned long _sym_next_reloc;

+#define RELOC_JUMP_TABLE_BIT 1UL

+

+/* Does reloc mark the beginning of a jump table? */

+static inline bool is_jump_table(struct reloc *reloc)

+{

+ return reloc->_sym_next_reloc & RELOC_JUMP_TABLE_BIT;

+}

+

+static inline void set_jump_table(struct reloc *reloc)

+{

+ reloc->_sym_next_reloc |= RELOC_JUMP_TABLE_BIT;

+}

+

+static inline struct reloc *sym_next_reloc(struct reloc *reloc)

+{

+ return (struct reloc *)(reloc->_sym_next_reloc & ~RELOC_JUMP_TABLE_BIT);

+}

+

+static inline void set_sym_next_reloc(struct reloc *reloc, struct reloc *next)

+{

+ unsigned long bit = reloc->_sym_next_reloc & RELOC_JUMP_TABLE_BIT;

+

+ reloc->_sym_next_reloc = (unsigned long)next | bit;

+}

+

+int objtool_pv_add(struct objtool_file *file, int idx, struct symbol *func);

+void arch_handle_alternative(struct special_alt *alt);

+#include <errno.h>

+#define ___WARN(severity, extra, format, ...) \

+ fprintf(stderr, \

+ "%s%s%s: objtool" extra ": " format "\n", \

+ objname ?: "", \

+ objname ? ": " : "", \

+ severity, \

+#define __WARN(severity, format, ...) \

+ ___WARN(severity, "", format, ##__VA_ARGS__)

+

+#define __WARN_LINE(severity, format, ...) \

+ ___WARN(severity, " [%s:%d]", format, __FILE__, __LINE__, ##__VA_ARGS__)

+

+#define __WARN_ELF(severity, format, ...) \

+ __WARN_LINE(severity, "%s: " format " failed: %s", __func__, ##__VA_ARGS__, elf_errmsg(-1))

+

+#define __WARN_GLIBC(severity, format, ...) \

+ __WARN_LINE(severity, "%s: " format " failed: %s", __func__, ##__VA_ARGS__, strerror(errno))

+

+#define __WARN_FUNC(severity, sec, offset, format, ...) \

+({ \

+ char *_str = offstr(sec, offset); \

+ __WARN(severity, "%s: " format, _str, ##__VA_ARGS__); \

+ free(_str); \

+#define WARN_STR (opts.werror ? "error" : "warning")

+

+#define WARN(format, ...) __WARN(WARN_STR, format, ##__VA_ARGS__)

+#define WARN_FUNC(sec, offset, format, ...) __WARN_FUNC(WARN_STR, sec, offset, format, ##__VA_ARGS__)

+ if (!_insn->sym || !_insn->sym->warned) \

+ WARN_FUNC(_insn->sec, _insn->offset, format, \

+ if (_insn->sym) \

+ _insn->sym->warned = 1; \

+#define ERROR_STR "error"

+

+#define ERROR(format, ...) __WARN(ERROR_STR, format, ##__VA_ARGS__)

+#define ERROR_ELF(format, ...) __WARN_ELF(ERROR_STR, format, ##__VA_ARGS__)

+#define ERROR_GLIBC(format, ...) __WARN_GLIBC(ERROR_STR, format, ##__VA_ARGS__)

+#define ERROR_FUNC(sec, offset, format, ...) __WARN_FUNC(ERROR_STR, sec, offset, format, ##__VA_ARGS__)

+#define ERROR_INSN(insn, format, ...) WARN_FUNC(insn->sec, insn->offset, format, ##__VA_ARGS__)

+ ERROR("won't handle more than one file at a time");

+int objtool_pv_add(struct objtool_file *f, int idx, struct symbol *func)

+ return 0;

+ ERROR("paravirt confusion");

+ return -1;

+ return 0;

+ return 0;

+ return 0;

+ ERROR_ELF("elf_begin");

+ ERROR_ELF("elf64_getehdr");

+ ERROR_ELF("elf_getshdrnum");

+ ERROR_ELF("elf_getshdrstrndx");

+ ERROR_ELF("elf_getscn");

+ ERROR_ELF("gelf_getshdr");

+ ERROR_ELF("elf_strptr");

+ ERROR_ELF("elf_getdata");

+ ERROR("bad .orc_unwind section size");

+ ERROR_ELF("gelf_getrela");

+ ERROR_ELF("gelf_getsym");

+ ERROR_ELF("elf_getscn");

+ ERROR_ELF("gelf_getshdr");

+ ERROR_ELF("elf_strptr");

+ ERROR_ELF("elf_strptr");

+void __weak arch_handle_alternative(struct special_alt *alt)

+ ERROR_FUNC(sec, offset + entry->orig, "can't find orig reloc");

+ arch_handle_alternative(alt);

+ ERROR_FUNC(sec, offset + entry->new, "can't find new reloc");

+ ERROR_FUNC(sec, offset + entry->key, "can't find key reloc");

+ ERROR("%s size not a multiple of %d", sec->name, entry->size);

+ ERROR_GLIBC("malloc failed");

+non_syscall="timer:hrtimer_setup,timer:hrtimer_start"

+\fB--show CATEGORY --hide CATEGORY\fP Show and hide also accept a single CATEGORY of columns: "all", "topology", "idle", "frequency", "power", "cpuidle", "hwidle", "swidle", "other". "idle" (enabled by default), includes "hwidle" and "idle_pct". "cpuidle" (default disabled) includes cpuidle software invocation counters. "swidle" includes "cpuidle" plus "idle_pct". "hwidle" includes only hardware based idle residency counters. Older versions of turbostat used the term "sysfs" for what is now "swidle".

+\fBC1, C2, C3...\fP The number times Linux requested the C1, C2, C3 idle state during the measurement interval. The system summary line shows the sum for all CPUs. These are C-state names as exported in /sys/devices/system/cpu/cpu*/cpuidle/state*/name. While their names are generic, their attributes are processor specific. They the system description section of output shows what MWAIT sub-states they are mapped to on each system. These counters are in the "cpuidle" group, which is disabled, by default.

+\fBC1+, C2+, C3+...\fP The idle governor idle state misprediction statistics. Inidcates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a deeper idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/below file. These counters are in the "cpuidle" group, which is disabled, by default.

+\fBC1-, C2-, C3-...\fP The idle governor idle state misprediction statistics. Inidcates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a shallower idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/above file. These counters are in the "cpuidle" group, which is disabled, by default.

+.PP

+\fBC1%, C2%, C3%\fP The residency percentage that Linux requested C1, C2, C3.... The system summary is the average of all CPUs in the system. Note that these are software, reflecting what was requested. The hardware counters reflect what was actually achieved. These counters are in the "pct_idle" group, which is enabled by default.

+.PP

+\fBCPU%c1, CPU%c3, CPU%c6, CPU%c7\fP show the percentage residency in hardware core idle states. These numbers are from hardware residency counters and are in the "hwidle" group, which is enabled, by default.

+\fBCoreThr\fP Core Thermal Throttling events during the measurement interval. Note that events since boot can be find in /sys/devices/system/cpu/cpu*/thermal_throttle/*

+.PP

+Intel Granite Rapids systems use domains 0-2 for CPUs, and 3-4 for IO, with cluster always 0.

+For the "--show" and "--hide" options, use "UncMHz" to operate on all UMHz*.* as a group.

+ { 0x0, "cpuidle", NULL, 0, 0, 0, NULL, 0 },

+ { 0x0, "pct_idle", NULL, 0, 0, 0, NULL, 0 },

+#define BIC_cpuidle (1ULL << 10)

+#define BIC_pct_idle (1ULL << 63)

+

+#define BIC_GROUP_TOPOLOGY (BIC_Package | BIC_Node | BIC_CoreCnt | BIC_PkgCnt | BIC_Core | BIC_CPU | BIC_Die)

+#define BIC_GROUP_THERMAL_PWR (BIC_CoreTmp | BIC_PkgTmp | BIC_PkgWatt | BIC_CorWatt | BIC_GFXWatt | BIC_RAMWatt | BIC_PKG__ | BIC_RAM__ | BIC_SysWatt)

+#define BIC_GROUP_FREQUENCY (BIC_Avg_MHz | BIC_Busy | BIC_Bzy_MHz | BIC_TSC_MHz | BIC_GFXMHz | BIC_GFXACTMHz | BIC_SAMMHz | BIC_SAMACTMHz | BIC_UNCORE_MHZ)

+#define BIC_GROUP_HW_IDLE (BIC_Busy | BIC_CPU_c1 | BIC_CPU_c3 | BIC_CPU_c6 | BIC_CPU_c7 | BIC_GFX_rc6 | BIC_Pkgpc2 | BIC_Pkgpc3 | BIC_Pkgpc6 | BIC_Pkgpc7 | BIC_Pkgpc8 | BIC_Pkgpc9 | BIC_Pkgpc10 | BIC_CPU_LPI | BIC_SYS_LPI | BIC_Mod_c6 | BIC_Totl_c0 | BIC_Any_c0 | BIC_GFX_c0 | BIC_CPUGFX | BIC_SAM_mc6 | BIC_Diec6)

+#define BIC_GROUP_SW_IDLE (BIC_Busy | BIC_cpuidle | BIC_pct_idle )

+#define BIC_GROUP_IDLE (BIC_GROUP_HW_IDLE | BIC_pct_idle)

+#define BIC_DISABLED_BY_DEFAULT (BIC_USEC | BIC_TOD | BIC_APIC | BIC_X2APIC | BIC_cpuidle)

+unsigned long long bic_present = BIC_USEC | BIC_TOD | BIC_cpuidle | BIC_pct_idle | BIC_APIC | BIC_X2APIC;

+#define CPU_SUBSET_MAXCPUS 8192 /* need to use before probe... */

+int probe_rapl_msr(int cpu, off_t offset, int index)

+ /* if the read failed, the probe fails */

+ if (retval != sizeof(value))

+ /* If an Energy Status Counter MSR returns 0, the probe fails */

+ switch (index) {

+ case RAPL_RCI_INDEX_ENERGY_PKG:

+ case RAPL_RCI_INDEX_ENERGY_CORES:

+ case RAPL_RCI_INDEX_DRAM:

+ case RAPL_RCI_INDEX_GFX:

+ case RAPL_RCI_INDEX_ENERGY_PLATFORM:

+ if (value == 0)

+ return 1;

+ }

+

+ /* PKG,DRAM_PERF_STATUS MSRs, can return any value */

+ retval |= BIC_GROUP_TOPOLOGY;

+ retval |= BIC_GROUP_THERMAL_PWR;

+ retval |= BIC_GROUP_IDLE;

+ break;

+ } else if (!strcmp(name_list, "swidle")) {

+ retval |= BIC_GROUP_SW_IDLE;

+ break;

+ } else if (!strcmp(name_list, "sysfs")) { /* legacy compatibility */

+ retval |= BIC_GROUP_SW_IDLE;

+ break;

+ } else if (!strcmp(name_list, "hwidle")) {

+ retval |= BIC_GROUP_HW_IDLE;

+ retval |= BIC_GROUP_FREQUENCY;

+ fprintf(stderr, "deferred %s\n", name_list);

+ old->core_throt_cnt = new->core_throt_cnt - old->core_throt_cnt;

+ fflush(gfx_info[idx].fp);

+ /*

+ * Once add_couter() is called, that counter is always read

+ * and reported -- So it is effectively (enabled & present).

+ * Only call add_counter() here if legacy BIC_UNCORE_MHZ (UncMHz)

+ * is (enabled). Since we are in this routine, we

+ * know we will not probe and set (present) the legacy counter.

+ *

+ * This allows "--show/--hide UncMHz" to be effective for

+ * the clustered MHz counters, as a group.

+ */

+ if BIC_IS_ENABLED(BIC_UNCORE_MHZ)

+ add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, package_id);

+ } else if (!no_msr && cai->msr && probe_rapl_msr(cpu, cai->msr, cai->rci_index) == 0) {

+ } else if (!no_msr && cai->msr && probe_rapl_msr(cpu, cai->msr, cai->rci_index) == 0) {

+ && probe_rapl_msr(cpu, cai->msr, cai->rci_index) == 0) {

+ fprintf(outf, "turbostat version 2025.04.06 - Len Brown <lenb@kernel.org>\n");

+ if (!strcmp(name, mp->name))

+void probe_cpuidle_residency(void)

+ int min_state = 1024, max_state = 0;

+ if (!DO_BIC(BIC_pct_idle))

+ return;

+

+ if (!DO_BIC(BIC_pct_idle) && !is_deferred_add(name_buf))

+

+ if (state > max_state)

+ max_state = state;

+ if (state < min_state)

+ min_state = state;

+}

+

+void probe_cpuidle_counts(void)

+{

+ char path[64];

+ char name_buf[16];

+ FILE *input;

+ int state;

+ int min_state = 1024, max_state = 0;

+ char *sp;

+

+ if (!DO_BIC(BIC_cpuidle))

+ return;

+ if (!DO_BIC(BIC_cpuidle) && !is_deferred_add(name_buf))

+ /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */

+ sp = strchr(name_buf, '-');

+ if (!sp)

+ sp = strchrnul(name_buf, '\n');

+

+ /*

+ * The 'below' sysfs file always contains 0 for the deepest state (largest index),

+ * do not add it.

+ */

+ if (state != max_state) {

+ /*

+ * Add 'C1+' for C1, and so on. The 'below' sysfs file always contains 0 for

+ * the last state, so do not add it.

+ */

+

+ *sp = '+';

+ *(sp + 1) = '\0';

+ sprintf(path, "cpuidle/state%d/below", state);

+ add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);

+ }

+

+ *sp = '\0';

+ sprintf(path, "cpuidle/state%d/usage", state);

+ /*

+ * The 'above' sysfs file always contains 0 for the shallowest state (smallest

+ * index), do not add it.

+ */

+ if (state != min_state) {

+ *sp = '-';

+ *(sp + 1) = '\0';

+ sprintf(path, "cpuidle/state%d/above", state);

+ add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);

+ }

+ }

+ probe_cpuidle_residency();

+ probe_cpuidle_counts();

+/*

+ * __unqual_typeof(x) - Declare an unqualified scalar type, leaving

+ * non-scalar types unchanged,

+ *

+ * Prefer C11 _Generic for better compile-times and simpler code. Note: 'char'

+ * is not type-compatible with 'signed char', and we define a separate case.

+ *

+ * This is copied verbatim from kernel's include/linux/compiler_types.h, but

+ * with default expression (for pointers) changed from (x) to (typeof(x)0).

+ *

+ * This is because LLVM has a bug where for lvalue (x), it does not get rid of

+ * an extra address_space qualifier, but does in case of rvalue (typeof(x)0).

+ * Hence, for pointers, we need to create an rvalue expression to get the

+ * desired type. See https://github.com/llvm/llvm-project/issues/53400.

+ */

+#define __scalar_type_to_expr_cases(type) \

+ unsigned type : (unsigned type)0, signed type : (signed type)0

+

+#define __unqual_typeof(x) \

+ typeof(_Generic((x), \

+ char: (char)0, \

+ __scalar_type_to_expr_cases(char), \

+ __scalar_type_to_expr_cases(short), \

+ __scalar_type_to_expr_cases(int), \

+ __scalar_type_to_expr_cases(long), \

+ __scalar_type_to_expr_cases(long long), \

+ default: (typeof(x))0))

+

+#define READ_ONCE(x) \

+({ \

+ union { __unqual_typeof(x) __val; char __c[1]; } __u = \

+ { .__c = { 0 } }; \

+ __read_once_size((__unqual_typeof(x) *)&(x), __u.__c, sizeof(x)); \

+ __u.__val; \

+#define WRITE_ONCE(x, val) \

+({ \

+ union { __unqual_typeof(x) __val; char __c[1]; } __u = \

+ { .__val = (val) }; \

+ __write_once_size((__unqual_typeof(x) *)&(x), __u.__c, sizeof(x)); \

+ __u.__val; \

+/*

+ * Return a value proportionally scaled to the task's weight.

+ */

+static inline u64 scale_by_task_weight(const struct task_struct *p, u64 value)

+{

+ return (value * p->scx.weight) / 100;

+}

+

+/*

+ * Return a value inversely proportional to the task's weight.

+ */

+static inline u64 scale_by_task_weight_inverse(const struct task_struct *p, u64 value)

+{

+ return value * 100 / p->scx.weight;

+}

+

+

+#define HAVE_SCX_OPS_ENQ_MIGRATION_DISABLED

+#define HAVE_SCX_OPS_ALLOW_QUEUED_WAKEUP

+#define HAVE_SCX_RQ_CLK_VALID

+const volatile u64 __SCX_RQ_ONLINE __weak;

+#define SCX_RQ_ONLINE __SCX_RQ_ONLINE

+

+const volatile u64 __SCX_RQ_CAN_STOP_TICK __weak;

+#define SCX_RQ_CAN_STOP_TICK __SCX_RQ_CAN_STOP_TICK

+

+const volatile u64 __SCX_RQ_BAL_PENDING __weak;

+#define SCX_RQ_BAL_PENDING __SCX_RQ_BAL_PENDING

+

+const volatile u64 __SCX_RQ_BAL_KEEP __weak;

+#define SCX_RQ_BAL_KEEP __SCX_RQ_BAL_KEEP

+

+const volatile u64 __SCX_RQ_BYPASSING __weak;

+#define SCX_RQ_BYPASSING __SCX_RQ_BYPASSING

+

+const volatile u64 __SCX_RQ_CLK_VALID __weak;

+#define SCX_RQ_CLK_VALID __SCX_RQ_CLK_VALID

+

+const volatile u64 __SCX_RQ_IN_WAKEUP __weak;

+#define SCX_RQ_IN_WAKEUP __SCX_RQ_IN_WAKEUP

+

+const volatile u64 __SCX_RQ_IN_BALANCE __weak;

+#define SCX_RQ_IN_BALANCE __SCX_RQ_IN_BALANCE

+

+ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_ONLINE); \

+ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CAN_STOP_TICK); \

+ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_PENDING); \

+ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_KEEP); \

+ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BYPASSING); \

+ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CLK_VALID); \

+ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_WAKEUP); \

+ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_BALANCE); \

+ if (!res)

+ *val = 0;

+cxl_core-y += $(CXL_CORE_SRC)/ras.o

+cxl_core-y += $(CXL_CORE_SRC)/acpi.o

+cxl_core-$(CONFIG_CXL_MCE) += $(CXL_CORE_SRC)/mce.o

+ u32 target[3];

+ .interleave_ways = 8,

+ .granularity = 1,

+ .window_size = SZ_512M * 6UL,

+ .target = { 0, 1, 2, },

+ for (int i = 0; i < cxlds->nr_partitions; i++) {

+ struct resource *res = &cxlds->part[i].res;

+ struct cxl_dpa_perf *perf = &cxlds->part[i].perf;

+ struct range range = {

+ .start = res->start,

+ .end = res->end,

+ };

+ dpa_perf_setup(port, &range, perf);

+ }

+ .opcode = cpu_to_le16(CXL_MBOX_OP_SET_SHUTDOWN_STATE),

+ .effect = POLICY_CHANGE_IMMEDIATE,

+ },

+ {

+ u8 shutdown_state;

+static int mock_set_shutdown_state(struct cxl_mockmem_data *mdata,

+ struct cxl_mbox_cmd *cmd)

+{

+ struct cxl_mbox_set_shutdown_state_in *ss = cmd->payload_in;

+

+ if (cmd->size_in != sizeof(*ss))

+ return -EINVAL;

+

+ if (cmd->size_out != 0)

+ return -EINVAL;

+

+ mdata->shutdown_state = ss->state;

+ return 0;

+}

+

+ case CXL_MBOX_OP_SET_SHUTDOWN_STATE:

+ rc = mock_set_shutdown_state(mdata, cmd);

+ break;

+ struct cxl_dpa_info range_info = { 0 };

+ cxlds->serial = pdev->id + 1;

+ rc = cxl_mem_dpa_fetch(mds, &range_info);

+ if (rc)

+ return rc;

+

+ rc = cxl_dpa_setup(cxlds, &range_info);

+TARGETS += mseal_system_mappings

+ *(volatile int *)ret;

+ *(volatile int *)ret;

+ *(volatile int *)ret;

+ *(volatile int *)ret;

+__xlated("5: r0 = ")

+__xlated("3: r0 =")

+__xlated("3: r0 =")

+__jited(" addq %gs:{{.*}}, %r9")

+__jited(" addq %gs:{{.*}}, %r9")

+__jited(" addq %gs:{{.*}}, %r9")

+#define __NR_clone3 435

+ prog = cfg.net_lib_dir / "xdp_dummy.bpf.o"

+ cfg.bin_local = cfg.net_lib_dir / "csum"

+ cfg.require_ipver('6')

+ tx_cmd = f"{cfg.bin_local} -c -h {cfg.remote_addr_v['6']} -p 9999 -l 12840"

+ cfg.require_ipver('6')

+ tx_cmd = f"{cfg.bin_local} -c -h {cfg.remote_addr_v['6']} -p 9999 -l 4096 -z 16384"

+ (cfg.ifname, cfg.net_lib_dir / "xdp_dummy.bpf.o"))

+

+ Attributes:

+ test_dir: Path to the source directory of the test

+ net_lib_dir: Path to the net/lib directory

+ self.src_path = Path(src_path)

+ self.test_dir = self.src_path.parent.resolve()

+ self.net_lib_dir = (Path(__file__).parent / "../../../../net/lib").resolve()

+ self.env = self._load_env_file()

+ prog = cfg.net_lib_dir / "xdp_dummy.bpf.o"

+ prog = cfg.net_lib_dir / "xdp_dummy.bpf.o"

+ prog = cfg.net_lib_dir / "xdp_dummy.bpf.o"

+ prog = cfg.net_lib_dir / "xdp_dummy.bpf.o"

+ prog = cfg.net_lib_dir / "xdp_dummy.bpf.o"

+ xdp = cmd(f'{cfg.test_dir / "xdp_helper"} - -', fail=False)

+ with bkg(f'{cfg.test_dir / "xdp_helper"} {cfg.ifindex} {xdp_queue_id}',

+ uint32_t device_pasid_id;

+ bool pasid_capable;

+

+ if (variant->pasid_capable)

+ test_cmd_mock_domain_flags(self->ioas_id,

+ MOCK_FLAGS_DEVICE_PASID,

+ NULL, NULL,

+ &self->device_pasid_id);

+ .pasid_capable = true,

+ test_err_hwpt_alloc(EOPNOTSUPP, self->device_id, self->ioas_id,

+ IOMMU_HWPT_ALLOC_NEST_PARENT |

+ IOMMU_HWPT_FAULT_ID_VALID,

+ &test_hwpt_id);

+ uint8_t max_pasid = 0;

+

+ test_cmd_get_hw_info_pasid(self->device_id, &max_pasid);

+ ASSERT_EQ(0, max_pasid);

+ if (variant->pasid_capable) {

+ test_cmd_get_hw_info_pasid(self->device_pasid_id,

+ &max_pasid);

+ ASSERT_EQ(MOCK_PASID_WIDTH, max_pasid);

+ }

+ uint32_t vdev_id;

+ /* Must allocate vdevice before attaching to a nested hwpt */

+ test_err_mock_domain_replace(ENOENT, self->stdev_id,

+ iopf_hwpt_id);

+ test_cmd_vdevice_alloc(viommu_id, dev_id, 0x99, &vdev_id);

+ uint32_t veventq_id;

+ uint32_t veventq_fd;

+ int prev_seq = -1;

+ /* Must allocate vdevice before attaching to a nested hwpt */

+ test_err_mock_domain_replace(ENOENT, self->stdev_id,

+ self->nested_hwpt_id);

+

+ /* Allocate a vEVENTQ with veventq_depth=2 */

+ test_cmd_veventq_alloc(viommu_id, IOMMU_VEVENTQ_TYPE_SELFTEST,

+ &veventq_id, &veventq_fd);

+ test_err_veventq_alloc(EEXIST, viommu_id,

+ IOMMU_VEVENTQ_TYPE_SELFTEST, NULL, NULL);

+ test_cmd_mock_domain_replace(self->stdev_id,

+ self->nested_hwpt_id);

+ test_cmd_trigger_vevents(dev_id, 1);

+ test_cmd_read_vevents(veventq_fd, 1, 0x99, &prev_seq);

+ test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id);

+

+ /* Try again with 0x88 */

+ test_cmd_mock_domain_replace(self->stdev_id,

+ self->nested_hwpt_id);

+ /* Trigger an overflow with three events */

+ test_cmd_trigger_vevents(dev_id, 3);

+ test_err_read_vevents(EOVERFLOW, veventq_fd, 3, 0x88,

+ &prev_seq);

+ /* Overflow must be gone after the previous reads */

+ test_cmd_trigger_vevents(dev_id, 1);

+ test_cmd_read_vevents(veventq_fd, 1, 0x88, &prev_seq);

+ close(veventq_fd);

+ test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id);

+ test_ioctl_destroy(veventq_id);

+FIXTURE(iommufd_device_pasid)

+{

+ int fd;

+ uint32_t ioas_id;

+ uint32_t hwpt_id;

+ uint32_t stdev_id;

+ uint32_t device_id;

+ uint32_t no_pasid_stdev_id;

+ uint32_t no_pasid_device_id;

+};

+

+FIXTURE_VARIANT(iommufd_device_pasid)

+{

+ bool pasid_capable;

+};

+

+FIXTURE_SETUP(iommufd_device_pasid)

+{

+ self->fd = open("/dev/iommu", O_RDWR);

+ ASSERT_NE(-1, self->fd);

+ test_ioctl_ioas_alloc(&self->ioas_id);

+

+ test_cmd_mock_domain_flags(self->ioas_id,

+ MOCK_FLAGS_DEVICE_PASID,

+ &self->stdev_id, &self->hwpt_id,

+ &self->device_id);

+ if (!variant->pasid_capable)

+ test_cmd_mock_domain_flags(self->ioas_id, 0,

+ &self->no_pasid_stdev_id, NULL,

+ &self->no_pasid_device_id);

+}

+

+FIXTURE_TEARDOWN(iommufd_device_pasid)

+{

+ teardown_iommufd(self->fd, _metadata);

+}

+

+FIXTURE_VARIANT_ADD(iommufd_device_pasid, no_pasid)

+{

+ .pasid_capable = false,

+};

+

+FIXTURE_VARIANT_ADD(iommufd_device_pasid, has_pasid)

+{

+ .pasid_capable = true,

+};

+

+TEST_F(iommufd_device_pasid, pasid_attach)

+{

+ struct iommu_hwpt_selftest data = {

+ .iotlb = IOMMU_TEST_IOTLB_DEFAULT,

+ };

+ uint32_t nested_hwpt_id[3] = {};

+ uint32_t parent_hwpt_id = 0;

+ uint32_t fault_id, fault_fd;

+ uint32_t s2_hwpt_id = 0;

+ uint32_t iopf_hwpt_id;

+ uint32_t pasid = 100;

+ uint32_t viommu_id;

+

+ /*

+ * Negative, detach pasid without attaching, this is not expected.

+ * But it should not result in failure anyway.

+ */

+ test_cmd_pasid_detach(pasid);

+

+ /* Allocate two nested hwpts sharing one common parent hwpt */

+ test_cmd_hwpt_alloc(self->device_id, self->ioas_id,

+ IOMMU_HWPT_ALLOC_NEST_PARENT,

+ &parent_hwpt_id);

+ test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id,

+ IOMMU_HWPT_ALLOC_PASID,

+ &nested_hwpt_id[0],

+ IOMMU_HWPT_DATA_SELFTEST,

+ &data, sizeof(data));

+ test_cmd_hwpt_alloc_nested(self->device_id, parent_hwpt_id,

+ IOMMU_HWPT_ALLOC_PASID,

+ &nested_hwpt_id[1],

+ IOMMU_HWPT_DATA_SELFTEST,

+ &data, sizeof(data));

+

+ /* Fault related preparation */

+ test_ioctl_fault_alloc(&fault_id, &fault_fd);

+ test_cmd_hwpt_alloc_iopf(self->device_id, parent_hwpt_id, fault_id,

+ IOMMU_HWPT_FAULT_ID_VALID | IOMMU_HWPT_ALLOC_PASID,

+ &iopf_hwpt_id,

+ IOMMU_HWPT_DATA_SELFTEST, &data,

+ sizeof(data));

+

+ /* Allocate a regular nested hwpt based on viommu */

+ test_cmd_viommu_alloc(self->device_id, parent_hwpt_id,

+ IOMMU_VIOMMU_TYPE_SELFTEST,

+ &viommu_id);

+ test_cmd_hwpt_alloc_nested(self->device_id, viommu_id,

+ IOMMU_HWPT_ALLOC_PASID,

+ &nested_hwpt_id[2],

+ IOMMU_HWPT_DATA_SELFTEST, &data,

+ sizeof(data));

+

+ test_cmd_hwpt_alloc(self->device_id, self->ioas_id,

+ IOMMU_HWPT_ALLOC_PASID,

+ &s2_hwpt_id);

+

+ /* Attach RID to non-pasid compat domain, */

+ test_cmd_mock_domain_replace(self->stdev_id, parent_hwpt_id);

+ /* then attach to pasid should fail */

+ test_err_pasid_attach(EINVAL, pasid, s2_hwpt_id);

+

+ /* Attach RID to pasid compat domain, */

+ test_cmd_mock_domain_replace(self->stdev_id, s2_hwpt_id);

+ /* then attach to pasid should succeed, */

+ test_cmd_pasid_attach(pasid, nested_hwpt_id[0]);

+ /* but attach RID to non-pasid compat domain should fail now. */

+ test_err_mock_domain_replace(EINVAL, self->stdev_id, parent_hwpt_id);

+ /*

+ * Detach hwpt from pasid 100, and check if the pasid 100

+ * has null domain.

+ */

+ test_cmd_pasid_detach(pasid);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, 0));

+ /* RID is attached to pasid-comapt domain, pasid path is not used */

+

+ if (!variant->pasid_capable) {

+ /*

+ * PASID-compatible domain can be used by non-PASID-capable

+ * device.

+ */

+ test_cmd_mock_domain_replace(self->no_pasid_stdev_id, nested_hwpt_id[0]);

+ test_cmd_mock_domain_replace(self->no_pasid_stdev_id, self->ioas_id);

+ /*

+ * Attach hwpt to pasid 100 of non-PASID-capable device,

+ * should fail, no matter domain is pasid-comapt or not.

+ */

+ EXPECT_ERRNO(EINVAL,

+ _test_cmd_pasid_attach(self->fd, self->no_pasid_stdev_id,

+ pasid, parent_hwpt_id));

+ EXPECT_ERRNO(EINVAL,

+ _test_cmd_pasid_attach(self->fd, self->no_pasid_stdev_id,

+ pasid, s2_hwpt_id));

+ }

+

+ /*

+ * Attach non pasid compat hwpt to pasid-capable device, should

+ * fail, and have null domain.

+ */

+ test_err_pasid_attach(EINVAL, pasid, parent_hwpt_id);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, 0));

+

+ /*

+ * Attach ioas to pasid 100, should fail, domain should

+ * be null.

+ */

+ test_err_pasid_attach(EINVAL, pasid, self->ioas_id);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, 0));

+

+ /*

+ * Attach the s2_hwpt to pasid 100, should succeed, domain should

+ * be valid.

+ */

+ test_cmd_pasid_attach(pasid, s2_hwpt_id);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, s2_hwpt_id));

+

+ /*

+ * Try attach pasid 100 with another hwpt, should FAIL

+ * as attach does not allow overwrite, use REPLACE instead.

+ */

+ test_err_pasid_attach(EBUSY, pasid, nested_hwpt_id[0]);

+

+ /*

+ * Detach hwpt from pasid 100 for next test, should succeed,

+ * and have null domain.

+ */

+ test_cmd_pasid_detach(pasid);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, 0));

+

+ /*

+ * Attach nested hwpt to pasid 100, should succeed, domain

+ * should be valid.

+ */

+ test_cmd_pasid_attach(pasid, nested_hwpt_id[0]);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, nested_hwpt_id[0]));

+

+ /* Attach to pasid 100 which has been attached, should fail. */

+ test_err_pasid_attach(EBUSY, pasid, nested_hwpt_id[0]);

+

+ /* cleanup pasid 100 */

+ test_cmd_pasid_detach(pasid);

+

+ /* Replace tests */

+

+ pasid = 200;

+ /*

+ * Replace pasid 200 without attaching it, should fail

+ * with -EINVAL.

+ */

+ test_err_pasid_replace(EINVAL, pasid, s2_hwpt_id);

+

+ /*

+ * Attach the s2 hwpt to pasid 200, should succeed, domain should

+ * be valid.

+ */

+ test_cmd_pasid_attach(pasid, s2_hwpt_id);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, s2_hwpt_id));

+

+ /*

+ * Replace pasid 200 with self->ioas_id, should fail

+ * and domain should be the prior s2 hwpt.

+ */

+ test_err_pasid_replace(EINVAL, pasid, self->ioas_id);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, s2_hwpt_id));

+

+ /*

+ * Replace a nested hwpt for pasid 200, should succeed,

+ * and have valid domain.

+ */

+ test_cmd_pasid_replace(pasid, nested_hwpt_id[0]);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, nested_hwpt_id[0]));

+

+ /*

+ * Replace with another nested hwpt for pasid 200, should

+ * succeed, and have valid domain.

+ */

+ test_cmd_pasid_replace(pasid, nested_hwpt_id[1]);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, nested_hwpt_id[1]));

+

+ /* cleanup pasid 200 */

+ test_cmd_pasid_detach(pasid);

+

+ /* Negative Tests for pasid replace, use pasid 1024 */

+

+ /*

+ * Attach the s2 hwpt to pasid 1024, should succeed, domain should

+ * be valid.

+ */

+ pasid = 1024;

+ test_cmd_pasid_attach(pasid, s2_hwpt_id);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, s2_hwpt_id));

+

+ /*

+ * Replace pasid 1024 with nested_hwpt_id[0], should fail,

+ * but have the old valid domain. This is a designed

+ * negative case. Normally, this shall succeed.

+ */

+ test_err_pasid_replace(ENOMEM, pasid, nested_hwpt_id[0]);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, s2_hwpt_id));

+

+ /* cleanup pasid 1024 */

+ test_cmd_pasid_detach(pasid);

+

+ /* Attach to iopf-capable hwpt */

+

+ /*

+ * Attach an iopf hwpt to pasid 2048, should succeed, domain should

+ * be valid.

+ */

+ pasid = 2048;

+ test_cmd_pasid_attach(pasid, iopf_hwpt_id);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, iopf_hwpt_id));

+

+ test_cmd_trigger_iopf_pasid(self->device_id, pasid, fault_fd);

+

+ /*

+ * Replace with s2_hwpt_id for pasid 2048, should

+ * succeed, and have valid domain.

+ */

+ test_cmd_pasid_replace(pasid, s2_hwpt_id);

+ ASSERT_EQ(0,

+ test_cmd_pasid_check_hwpt(self->fd, self->stdev_id,

+ pasid, s2_hwpt_id));

+

+ /* cleanup pasid 2048 */

+ test_cmd_pasid_detach(pasid);

+

+ test_ioctl_destroy(iopf_hwpt_id);

+ close(fault_fd);

+ test_ioctl_destroy(fault_id);

+

+ /* Detach the s2_hwpt_id from RID */

+ test_cmd_mock_domain_replace(self->stdev_id, self->ioas_id);

+}

+

+ uint32_t stdev_id;

+ uint32_t pasid;

+ self->stdev_id = 0;

+ self->pasid = 0; //test should use a non-zero value

+ if (self->pasid && self->stdev_id)

+ _test_cmd_pasid_detach(self->fd, self->stdev_id, self->pasid);

+ uint32_t veventq_id, veventq_fd;

+ uint32_t test_hwpt_id;

+ if (_test_cmd_mock_domain_flags(self->fd, ioas_id,

+ MOCK_FLAGS_DEVICE_PASID,

+ &self->stdev_id, NULL, &idev_id))

+ if (_test_cmd_get_hw_info(self->fd, idev_id, &info,

+ sizeof(info), NULL, NULL))

+ if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, 0,

+ IOMMU_HWPT_ALLOC_PASID, &hwpt_id,

+ if (_test_cmd_mock_domain_replace(self->fd, self->stdev_id, ioas_id2, NULL))

+ if (_test_cmd_mock_domain_replace(self->fd, self->stdev_id, hwpt_id, NULL))

+ IOMMU_HWPT_ALLOC_NEST_PARENT |

+ IOMMU_HWPT_ALLOC_PASID,

+ &hwpt_id,

+ if (_test_cmd_veventq_alloc(self->fd, viommu_id,

+ IOMMU_VEVENTQ_TYPE_SELFTEST, &veventq_id,

+ &veventq_fd))

+ return -1;

+ close(veventq_fd);

+

+ if (_test_cmd_hwpt_alloc(self->fd, idev_id, ioas_id, 0,

+ IOMMU_HWPT_ALLOC_PASID,

+ &test_hwpt_id,

+ IOMMU_HWPT_DATA_NONE, 0, 0))

+ return -1;

+

+ /* Tests for pasid attach/replace/detach */

+

+ self->pasid = 200;

+

+ if (_test_cmd_pasid_attach(self->fd, self->stdev_id,

+ self->pasid, hwpt_id)) {

+ self->pasid = 0;

+ return -1;

+ }

+

+ if (_test_cmd_pasid_replace(self->fd, self->stdev_id,

+ self->pasid, test_hwpt_id))

+ return -1;

+

+ if (_test_cmd_pasid_detach(self->fd, self->stdev_id, self->pasid))

+ return -1;

+

+ self->pasid = 0;

+

+#include <poll.h>

+ size_t data_len, uint32_t *capabilities,

+ uint8_t *max_pasid)

+ if (max_pasid)

+ *max_pasid = cmd.out_max_pasid_log2;

+

+ data_len, NULL, NULL))

+ data_len, NULL, NULL))

+ ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, NULL, \

+ 0, &caps, NULL))

+

+#define test_cmd_get_hw_info_pasid(device_id, max_pasid) \

+ ASSERT_EQ(0, _test_cmd_get_hw_info(self->fd, device_id, NULL, \

+ 0, NULL, max_pasid))

+static int _test_cmd_trigger_iopf(int fd, __u32 device_id, __u32 pasid,

+ __u32 fault_fd)

+ .pasid = pasid,

+ ASSERT_EQ(0, _test_cmd_trigger_iopf(self->fd, device_id, 0x1, fault_fd))

+#define test_cmd_trigger_iopf_pasid(device_id, pasid, fault_fd) \

+ ASSERT_EQ(0, _test_cmd_trigger_iopf(self->fd, device_id, \

+ pasid, fault_fd))

+

+static int _test_cmd_veventq_alloc(int fd, __u32 viommu_id, __u32 type,

+ __u32 *veventq_id, __u32 *veventq_fd)

+{

+ struct iommu_veventq_alloc cmd = {

+ .size = sizeof(cmd),

+ .type = type,

+ .veventq_depth = 2,

+ .viommu_id = viommu_id,

+ };

+ int ret;

+

+ ret = ioctl(fd, IOMMU_VEVENTQ_ALLOC, &cmd);

+ if (ret)

+ return ret;

+ if (veventq_id)

+ *veventq_id = cmd.out_veventq_id;

+ if (veventq_fd)

+ *veventq_fd = cmd.out_veventq_fd;

+ return 0;

+}

+

+#define test_cmd_veventq_alloc(viommu_id, type, veventq_id, veventq_fd) \

+ ASSERT_EQ(0, _test_cmd_veventq_alloc(self->fd, viommu_id, type, \

+ veventq_id, veventq_fd))

+#define test_err_veventq_alloc(_errno, viommu_id, type, veventq_id, \

+ veventq_fd) \

+ EXPECT_ERRNO(_errno, \

+ _test_cmd_veventq_alloc(self->fd, viommu_id, type, \

+ veventq_id, veventq_fd))

+

+static int _test_cmd_trigger_vevents(int fd, __u32 dev_id, __u32 nvevents)

+{

+ struct iommu_test_cmd trigger_vevent_cmd = {

+ .size = sizeof(trigger_vevent_cmd),

+ .op = IOMMU_TEST_OP_TRIGGER_VEVENT,

+ .trigger_vevent = {

+ .dev_id = dev_id,

+ },

+ };

+ int ret;

+

+ while (nvevents--) {

+ ret = ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_TRIGGER_VEVENT),

+ &trigger_vevent_cmd);

+ if (ret < 0)

+ return -1;

+ }

+ return ret;

+}

+

+#define test_cmd_trigger_vevents(dev_id, nvevents) \

+ ASSERT_EQ(0, _test_cmd_trigger_vevents(self->fd, dev_id, nvevents))

+

+static int _test_cmd_read_vevents(int fd, __u32 event_fd, __u32 nvevents,

+ __u32 virt_id, int *prev_seq)

+{

+ struct pollfd pollfd = { .fd = event_fd, .events = POLLIN };

+ struct iommu_viommu_event_selftest *event;

+ struct iommufd_vevent_header *hdr;

+ ssize_t bytes;

+ void *data;

+ int ret, i;

+

+ ret = poll(&pollfd, 1, 1000);

+ if (ret < 0)

+ return -1;

+

+ data = calloc(nvevents, sizeof(*hdr) + sizeof(*event));

+ if (!data) {

+ errno = ENOMEM;

+ return -1;

+ }

+

+ bytes = read(event_fd, data,

+ nvevents * (sizeof(*hdr) + sizeof(*event)));

+ if (bytes <= 0) {

+ errno = EFAULT;

+ ret = -1;

+ goto out_free;

+ }

+

+ for (i = 0; i < nvevents; i++) {

+ hdr = data + i * (sizeof(*hdr) + sizeof(*event));

+

+ if (hdr->flags & IOMMU_VEVENTQ_FLAG_LOST_EVENTS ||

+ hdr->sequence - *prev_seq > 1) {

+ *prev_seq = hdr->sequence;

+ errno = EOVERFLOW;

+ ret = -1;

+ goto out_free;

+ }

+ *prev_seq = hdr->sequence;

+ event = data + sizeof(*hdr);

+ if (event->virt_id != virt_id) {

+ errno = EINVAL;

+ ret = -1;

+ goto out_free;

+ }

+ }

+

+ ret = 0;

+out_free:

+ free(data);

+ return ret;

+}

+

+#define test_cmd_read_vevents(event_fd, nvevents, virt_id, prev_seq) \

+ ASSERT_EQ(0, _test_cmd_read_vevents(self->fd, event_fd, nvevents, \

+ virt_id, prev_seq))

+#define test_err_read_vevents(_errno, event_fd, nvevents, virt_id, prev_seq) \

+ EXPECT_ERRNO(_errno, \

+ _test_cmd_read_vevents(self->fd, event_fd, nvevents, \

+ virt_id, prev_seq))

+

+static int _test_cmd_pasid_attach(int fd, __u32 stdev_id, __u32 pasid,

+ __u32 pt_id)

+{

+ struct iommu_test_cmd test_attach = {

+ .size = sizeof(test_attach),

+ .op = IOMMU_TEST_OP_PASID_ATTACH,

+ .id = stdev_id,

+ .pasid_attach = {

+ .pasid = pasid,

+ .pt_id = pt_id,

+ },

+ };

+

+ return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_ATTACH),

+ &test_attach);

+}

+

+#define test_cmd_pasid_attach(pasid, hwpt_id) \

+ ASSERT_EQ(0, _test_cmd_pasid_attach(self->fd, self->stdev_id, \

+ pasid, hwpt_id))

+

+#define test_err_pasid_attach(_errno, pasid, hwpt_id) \

+ EXPECT_ERRNO(_errno, \

+ _test_cmd_pasid_attach(self->fd, self->stdev_id, \

+ pasid, hwpt_id))

+

+static int _test_cmd_pasid_replace(int fd, __u32 stdev_id, __u32 pasid,

+ __u32 pt_id)

+{

+ struct iommu_test_cmd test_replace = {

+ .size = sizeof(test_replace),

+ .op = IOMMU_TEST_OP_PASID_REPLACE,

+ .id = stdev_id,

+ .pasid_replace = {

+ .pasid = pasid,

+ .pt_id = pt_id,

+ },

+ };

+

+ return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_REPLACE),

+ &test_replace);

+}

+

+#define test_cmd_pasid_replace(pasid, hwpt_id) \

+ ASSERT_EQ(0, _test_cmd_pasid_replace(self->fd, self->stdev_id, \

+ pasid, hwpt_id))

+

+#define test_err_pasid_replace(_errno, pasid, hwpt_id) \

+ EXPECT_ERRNO(_errno, \

+ _test_cmd_pasid_replace(self->fd, self->stdev_id, \

+ pasid, hwpt_id))

+

+static int _test_cmd_pasid_detach(int fd, __u32 stdev_id, __u32 pasid)

+{

+ struct iommu_test_cmd test_detach = {

+ .size = sizeof(test_detach),

+ .op = IOMMU_TEST_OP_PASID_DETACH,

+ .id = stdev_id,

+ .pasid_detach = {

+ .pasid = pasid,

+ },

+ };

+

+ return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_DETACH),

+ &test_detach);

+}

+

+#define test_cmd_pasid_detach(pasid) \

+ ASSERT_EQ(0, _test_cmd_pasid_detach(self->fd, self->stdev_id, pasid))

+

+static int test_cmd_pasid_check_hwpt(int fd, __u32 stdev_id, __u32 pasid,

+ __u32 hwpt_id)

+{

+ struct iommu_test_cmd test_pasid_check = {

+ .size = sizeof(test_pasid_check),

+ .op = IOMMU_TEST_OP_PASID_CHECK_HWPT,

+ .id = stdev_id,

+ .pasid_check = {

+ .pasid = pasid,

+ .hwpt_id = hwpt_id,

+ },

+ };

+

+ return ioctl(fd, _IOMMU_TEST_CMD(IOMMU_TEST_OP_PASID_CHECK_HWPT),

+ &test_pasid_check);

+}

+ case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZAAMO:

+ case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZALRSC:

+ KVM_ISA_EXT_ARR(ZAAMO),

+ KVM_ISA_EXT_ARR(ZALRSC),

+KVM_ISA_EXT_SIMPLE_CONFIG(zaamo, ZAAMO);

+KVM_ISA_EXT_SIMPLE_CONFIG(zalrsc, ZALRSC);

+ &config_zaamo,

+ &config_zalrsc,

+check_supported_ppc64()

+{

+ local config="/proc/config.gz"

+ [[ -f "${config}" ]] || config="/boot/config-$(uname -r)"

+ [[ -f "${config}" ]] || fail "Cannot find kernel config in /proc or /boot"

+

+ local pg_table_levels=$(gzip -dcfq "${config}" | grep PGTABLE_LEVELS | cut -d'=' -f 2)

+ if [[ "${pg_table_levels}" -lt 5 ]]; then

+ echo "$0: PGTABLE_LEVELS=${pg_table_levels}, must be >= 5 to run this test"

+ exit $ksft_skip

+ fi

+

+ local mmu_support=$(grep -m1 "mmu" /proc/cpuinfo | awk '{print $3}')

+ if [[ "$mmu_support" != "radix" ]]; then

+ echo "$0: System does not use Radix MMU, required for 5-level paging"

+ exit $ksft_skip

+ fi

+

+ local hugepages_total=$(awk '/HugePages_Total/ {print $2}' /proc/meminfo)

+ if [[ "${hugepages_total}" -eq 0 ]]; then

+ echo "$0: HugePages are not enabled, required for some tests"

+ exit $ksft_skip

+ fi

+}

+

+ "ppc64le"|"ppc64")

+ check_supported_ppc64

+ ;;

+# SPDX-License-Identifier: GPL-2.0-only

+sysmap_is_sealed

+# SPDX-License-Identifier: GPL-2.0-only

+CFLAGS += -std=c99 -pthread -Wall $(KHDR_INCLUDES)

+

+TEST_GEN_PROGS := sysmap_is_sealed

+

+include ../lib.mk

+CONFIG_MSEAL_SYSTEM_MAPPINGS=y

+// SPDX-License-Identifier: GPL-2.0-only

+/*

+ * test system mappings are sealed when

+ * KCONFIG_MSEAL_SYSTEM_MAPPINGS=y

+ */

+

+#define _GNU_SOURCE

+#include <stdio.h>

+#include <errno.h>

+#include <unistd.h>

+#include <string.h>

+#include <stdbool.h>

+

+#include "../kselftest.h"

+#include "../kselftest_harness.h"

+

+#define VMFLAGS "VmFlags:"

+#define MSEAL_FLAGS "sl"

+#define MAX_LINE_LEN 512

+

+bool has_mapping(char *name, FILE *maps)

+{

+ char line[MAX_LINE_LEN];

+

+ while (fgets(line, sizeof(line), maps)) {

+ if (strstr(line, name))

+ return true;

+ }

+

+ return false;

+}

+

+bool mapping_is_sealed(char *name, FILE *maps)

+{

+ char line[MAX_LINE_LEN];

+

+ while (fgets(line, sizeof(line), maps)) {

+ if (!strncmp(line, VMFLAGS, strlen(VMFLAGS))) {

+ if (strstr(line, MSEAL_FLAGS))

+ return true;

+

+ return false;

+ }

+ }

+

+ return false;

+}

+

+FIXTURE(basic) {

+ FILE *maps;

+};

+

+FIXTURE_SETUP(basic)

+{

+ self->maps = fopen("/proc/self/smaps", "r");

+ if (!self->maps)

+ SKIP(return, "Could not open /proc/self/smap, errno=%d",

+ errno);

+};

+

+FIXTURE_TEARDOWN(basic)

+{

+ if (self->maps)

+ fclose(self->maps);

+};

+

+FIXTURE_VARIANT(basic)

+{

+ char *name;

+ bool sealed;

+};

+

+FIXTURE_VARIANT_ADD(basic, vdso) {

+ .name = "[vdso]",

+ .sealed = true,

+};

+

+FIXTURE_VARIANT_ADD(basic, vvar) {

+ .name = "[vvar]",

+ .sealed = true,

+};

+

+FIXTURE_VARIANT_ADD(basic, vvar_vclock) {

+ .name = "[vvar_vclock]",

+ .sealed = true,

+};

+

+FIXTURE_VARIANT_ADD(basic, sigpage) {

+ .name = "[sigpage]",

+ .sealed = true,

+};

+

+FIXTURE_VARIANT_ADD(basic, vectors) {

+ .name = "[vectors]",

+ .sealed = true,

+};

+

+FIXTURE_VARIANT_ADD(basic, uprobes) {

+ .name = "[uprobes]",

+ .sealed = true,

+};

+

+FIXTURE_VARIANT_ADD(basic, stack) {

+ .name = "[stack]",

+ .sealed = false,

+};

+

+TEST_F(basic, check_sealed)

+{

+ if (!has_mapping(variant->name, self->maps)) {

+ SKIP(return, "could not find the mapping, %s",

+ variant->name);

+ }

+

+ EXPECT_EQ(variant->sealed,

+ mapping_is_sealed(variant->name, self->maps));

+};

+

+TEST_HARNESS_MAIN

+ for i in `seq 10`; do

+ ip netns exec "${SOURCE}" bash -c \

+ 'cat /dev/urandom | head -c 100M | nc -w 1 -u 239.0.0.1 4001'

+ echo -n "."

+ done

+ for i in `seq 10`; do

+ ip netns exec "${SOURCE}" bash -c \

+ 'cat /dev/urandom | head -c 100M | nc -w 1 -u ff0e::5:6 6001'

+ echo -n "."

+ done

+printf "TEST: %-50s" "IPv4 amt traffic forwarding torture"

+printf " [ OK ]\n"

+printf "TEST: %-50s" "IPv6 amt traffic forwarding torture"

+printf " [ OK ]\n"

+# Create netdevsim with given id and net namespace.

+create_netdevsim() {

+ local id="$1"

+ local ns="$2"

+

+ modprobe netdevsim &> /dev/null

+ udevadm settle

+

+ echo "$id 1" | ip netns exec $ns tee /sys/bus/netdevsim/new_device >/dev/null

+ local dev=$(ip netns exec $ns ls /sys/bus/netdevsim/devices/netdevsim$id/net)

+ ip -netns $ns link set dev $dev name nsim$id

+ ip -netns $ns link set dev nsim$id up

+

+ echo nsim$id

+}

+

+# Remove netdevsim with given id.

+cleanup_netdevsim() {

+ local id="$1"

+

+ if [ -d "/sys/bus/netdevsim/devices/netdevsim$id/net" ]; then

+ echo "$id" > /sys/bus/netdevsim/del_device

+ fi

+}

+

+mptcp_diag

+ if (fd_in < 0)

+ if (fd_in < 0)

+ goto out;

+out:

+ if (cfg_input)

+ close(fd_in);

+ return ret;

+NSIM_ADDR=2025

+ cleanup_netdevsim $NSIM_ADDR

+# Use netdevsim because it has extra asserts for notifiers.

+

+nsim=$(create_netdevsim $NSIM_ADDR $NS)

+ip -netns $NS link set dev $nsim netns $test_ns ||

+ip -netns $test_ns link show dev $nsim >> /dev/null ||

+ fail "Device not found after move"

+cleanup_netdevsim $NSIM_ADDR

+ addresses = rtnl.getmulticast({"ifa-family": socket.AF_INET}, dump=True)

+ addr for addr in addresses if addr['multicast'] == IPV4_ALL_HOSTS_MULTICAST

+

+ if (ip_route_add(lo_intf, TEST_FAMILY, local_addr, local_addr))

+ test_error("Failed to add a local route %s", lo_intf);

+BPF_FILE="lib/xdp_dummy.bpf.o"

+BPF_FILE="lib/xdp_dummy.bpf.o"

+BPF_FILE="lib/xdp_dummy.bpf.o"

+BPF_FILE="lib/xdp_dummy.bpf.o"

+#define __NR_pidfd_open 434

+#define __NR_pidfd_send_signal 424

+#define __NR_clone3 435

+#define __NR_pidfd_getfd 438

+static void test_no_cbo_inval(void *arg)

+{

+ ksft_print_msg("Testing cbo.inval instruction remain privileged\n");

+ illegal_insn = false;

+ cbo_inval(&mem[0]);

+ ksft_test_result(illegal_insn, "No cbo.inval\n");

+}

+

+static void test_zicbom(void *arg)

+{

+ struct riscv_hwprobe pair = {

+ .key = RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE,

+ };

+ cpu_set_t *cpus = (cpu_set_t *)arg;

+ __u64 block_size;

+ long rc;

+

+ rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0);

+ block_size = pair.value;

+ ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE &&

+ is_power_of_2(block_size), "Zicbom block size\n");

+ ksft_print_msg("Zicbom block size: %llu\n", block_size);

+

+ illegal_insn = false;

+ cbo_clean(&mem[block_size]);

+ ksft_test_result(!illegal_insn, "cbo.clean\n");

+

+ illegal_insn = false;

+ cbo_flush(&mem[block_size]);

+ ksft_test_result(!illegal_insn, "cbo.flush\n");

+}

+

+static void check_no_zicbo_cpus(cpu_set_t *cpus, __u64 cbo)

+ char *cbostr;

+ cbostr = cbo == RISCV_HWPROBE_EXT_ZICBOZ ? "Zicboz" : "Zicbom";

+

+ if (pair.value & cbo)

+ ksft_exit_fail_msg("%s is only present on a subset of harts.\n"

+ "Use taskset to select a set of harts where %s\n"

+ "presence (present or not) is consistent for each hart\n",

+ cbostr, cbostr);

+ TEST_ZICBOM,

+ TEST_NO_CBO_INVAL,

+ [TEST_ZICBOM] = { .nr_tests = 3, test_zicbom },

+ [TEST_NO_ZICBOM] = { .nr_tests = 2, test_no_zicbom },

+ [TEST_NO_CBO_INVAL] = { .nr_tests = 1, test_no_cbo_inval },

+ tests[TEST_NO_CBO_INVAL].enabled = true;

+ check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOZ);

+ }

+

+ if (pair.value & RISCV_HWPROBE_EXT_ZICBOM) {

+ tests[TEST_ZICBOM].enabled = true;

+ tests[TEST_NO_ZICBOM].enabled = false;

+ } else {

+ check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOM);

+ * To be "clean" all values must be all zeroes.

+ char value = 0;

+ if (value != 0x00) { \

+ RTC_ALARM_RES_MINUTE,

+static enum rtc_alarm_state get_rtc_alarm_state(int fd, int need_seconds)

+ /* Check if alarm has desired granularity */

+ if (need_seconds && (param.uvalue & _BITUL(RTC_FEATURE_ALARM_RES_MINUTE)))

+ return RTC_ALARM_RES_MINUTE;

+

+ alarm_state = get_rtc_alarm_state(self->fd, 1);

+ if (alarm_state == RTC_ALARM_RES_MINUTE)

+ SKIP(return, "Skipping test since alarms has only minute granularity.");

+ alarm_state = get_rtc_alarm_state(self->fd, 1);

+ if (alarm_state == RTC_ALARM_RES_MINUTE)

+ SKIP(return, "Skipping test since alarms has only minute granularity.");

+ alarm_state = get_rtc_alarm_state(self->fd, 0);

+ alarm_state = get_rtc_alarm_state(self->fd, 0);

+ "matchPattern": "action order [0-9]+: nat ingress 0.0.0.0/0 10.10.10.1 pass.*index 12 ref",

+ "matchPattern": "action order [0-9]+: nat ingress 0.0.0.0/0 10.10.10.1 pass.*index 12 ref",

+ "matchPattern": "action order [0-9]+: nat ingress 0.0.0.0/0 10.10.10.1 pass.*index 12 ref",

+ "matchPattern": "action order [0-9]+: nat egress 0.0.0.0/0 20.20.20.1 pipe.*index 10 ref",

+ "matchPattern": "action order [0-9]+: nat egress 0.0.0.0/0 20.20.20.1 pipe.*index 10 ref",

+ "matchPattern": "action order [0-9]+: nat egress 0.0.0.0/0 20.20.20.1 pipe.*index 10 ref",

+ "matchPattern": "action order [0-9]+: nat egress 0.0.0.0/0 20.20.20.1 pipe.*index 10 ref.*cookie aa1bc2d3eeff112233445566778800a1",

+ },

+ {

+ "id": "c024",

+ "name": "Test TBF with SKBPRIO - catch qlen corner cases",

+ "category": [

+ "qdisc",

+ "tbf",

+ "skbprio"

+ ],

+ "plugins": {

+ "requires": "nsPlugin"

+ },

+ "setup": [

+ "$IP link set dev $DUMMY up || true",

+ "$IP addr add 10.10.10.10/24 dev $DUMMY || true",

+ "$TC qdisc add dev $DUMMY handle 1: root tbf rate 100bit burst 2000 limit 1000",

+ "$TC qdisc add dev $DUMMY parent 1: handle 10: skbprio limit 1",

+ "ping -c 1 -W 0.1 -Q 0x00 -s 1400 -I $DUMMY 10.10.10.1 > /dev/null || true",

+ "ping -c 1 -W 0.1 -Q 0x1c -s 1400 -I $DUMMY 10.10.10.1 > /dev/null || true",

+ "ping -c 1 -W 0.1 -Q 0x00 -s 1400 -I $DUMMY 10.10.10.1 > /dev/null || true",

+ "ping -c 1 -W 0.1 -Q 0x1c -s 1400 -I $DUMMY 10.10.10.1 > /dev/null || true",

+ "sleep 0.5"

+ ],

+ "cmdUnderTest": "$TC -s qdisc show dev $DUMMY",

+ "expExitCode": "0",

+ "verifyCmd": "$TC -s qdisc show dev $DUMMY | grep -A 5 'qdisc skbprio'",

+ "matchPattern": "dropped [1-9][0-9]*",

+ "matchCount": "1",

+ "teardown": [

+ "$TC qdisc del dev $DUMMY handle 1: root",

+ "$IP addr del 10.10.10.10/24 dev $DUMMY || true"

+ ]

+ }

+TEST_PROGS += test_generic_02.sh

+TEST_PROGS += test_generic_03.sh

+TEST_PROGS += test_loop_05.sh

+TEST_PROGS += test_stripe_03.sh

+TEST_PROGS += test_stripe_04.sh

+ .cmd_op = UBLK_U_CMD_STOP_DEV,

+ .cmd_op = UBLK_U_CMD_GET_PARAMS,

+ ublk_err("failed to get params %d %s\n", ret, strerror(-ret));

+ q->io_cmd_buf = mmap(0, cmd_buf_size, PROT_READ,

+ struct ublksrv_io_desc *io_cmd_buf;

+ return &q->io_cmd_buf[tag];

+ .types = UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DMA_ALIGN |

+ UBLK_PARAM_TYPE_SEGMENT,

+ .dma = {

+ .alignment = 4095,

+ },

+ .seg = {

+ .seg_boundary_mask = 4095,

+ .max_segment_size = 32 << 10,

+ .max_segments = 32,

+ },

+static inline enum io_uring_op stripe_to_uring_op(

+ const struct ublksrv_io_desc *iod, int zc)

+ return zc ? IORING_OP_READV_FIXED : IORING_OP_READV;

+ return zc ? IORING_OP_WRITEV_FIXED : IORING_OP_WRITEV;

+ int zc = !!(ublk_queue_use_zc(q) != 0);

+ enum io_uring_op op = stripe_to_uring_op(iod, zc);

+ int i, extra = zc ? 2 : 0;

+ ublk_queue_alloc_sqes(q, sqe, s->nr + extra);

+

+ if (zc) {

+ io_uring_prep_buf_register(sqe[0], 0, tag, q->q_id, tag);

+ sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK;

+ sqe[0]->user_data = build_user_data(tag,

+ ublk_cmd_op_nr(sqe[0]->cmd_op), 0, 1);

+ }

+

+ for (i = zc; i < s->nr + extra - zc; i++) {

+ struct stripe *t = &s->s[i - zc];

+ if (zc) {

+ sqe[i]->buf_index = tag;

+ io_uring_sqe_set_flags(sqe[i],

+ IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK);

+ } else {

+ io_uring_sqe_set_flags(sqe[i], IOSQE_FIXED_FILE);

+ }

+ sqe[i]->user_data = build_user_data(tag, ublksrv_get_op(iod), i - zc, 1);

+ }

+ if (zc) {

+ struct io_uring_sqe *unreg = sqe[s->nr + 1];

+

+ io_uring_prep_buf_unregister(unreg, 0, tag, q->q_id, tag);

+ unreg->user_data = build_user_data(tag, ublk_cmd_op_nr(unreg->cmd_op), 0, 1);

+

+ /* register buffer is skip_success */

+ return s->nr + zc;

+ if (res < 0 || op != ublk_cmd_op_nr(UBLK_U_IO_UNREGISTER_IO_BUF)) {

+ if (res < 0)

+ ublk_err("%s: io failure %d tag %u\n", __func__, res, tag);

+ /* buffer register op is IOSQE_CQE_SKIP_SUCCESS */

+ if (op == ublk_cmd_op_nr(UBLK_U_IO_REGISTER_IO_BUF))

+ io->tgt_ios += 1;

+

+ if (res < s->s[seq].nr_sects << 9) {

+ ublk_err("%s: short rw op %u res %d exp %u tag %u\n",

+ __func__, op, res, s->s[seq].vec->iov_len, tag);

+ }

+ int ret, i, mul = 1;

+

+ if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY)

+ mul = 2;

+ dev->tgt.sq_depth = mul * dev->dev_info.queue_depth * conf->nr_files;

+ dev->tgt.cq_depth = mul * dev->dev_info.queue_depth * conf->nr_files;

+_run_fio_verify_io() {

+ fio --name=verify --rw=randwrite --direct=1 --ioengine=libaio \

+ --bs=8k --iodepth=32 --verify=crc32c --do_verify=1 \

+ --verify_state_save=0 "$@" > /dev/null

+}

+

+#!/bin/bash

+# SPDX-License-Identifier: GPL-2.0

+

+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh

+

+TID="generic_02"

+ERR_CODE=0

+

+if ! _have_program bpftrace; then

+ exit "$UBLK_SKIP_CODE"

+fi

+

+_prep_test "null" "sequential io order for MQ"

+

+dev_id=$(_add_ublk_dev -t null -q 2)

+_check_add_dev $TID $?

+

+dev_t=$(_get_disk_dev_t "$dev_id")

+bpftrace trace/seq_io.bt "$dev_t" "W" 1 > "$UBLK_TMP" 2>&1 &

+btrace_pid=$!

+sleep 2

+

+if ! kill -0 "$btrace_pid" > /dev/null 2>&1; then

+ _cleanup_test "null"

+ exit "$UBLK_SKIP_CODE"

+fi

+

+# run fio over this ublk disk

+fio --name=write_seq \

+ --filename=/dev/ublkb"${dev_id}" \

+ --ioengine=libaio --iodepth=16 \

+ --rw=write \

+ --size=512M \

+ --direct=1 \

+ --bs=4k > /dev/null 2>&1

+ERR_CODE=$?

+kill "$btrace_pid"

+wait

+if grep -q "io_out_of_order" "$UBLK_TMP"; then

+ cat "$UBLK_TMP"

+ ERR_CODE=255

+fi

+_cleanup_test "null"

+_show_result $TID $ERR_CODE

+#!/bin/bash

+# SPDX-License-Identifier: GPL-2.0

+

+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh

+

+TID="generic_03"

+ERR_CODE=0

+

+_prep_test "null" "check dma & segment limits for zero copy"

+

+dev_id=$(_add_ublk_dev -t null -z)

+_check_add_dev $TID $?

+

+sysfs_path=/sys/block/ublkb"${dev_id}"

+dma_align=$(cat "$sysfs_path"/queue/dma_alignment)

+max_segments=$(cat "$sysfs_path"/queue/max_segments)

+max_segment_size=$(cat "$sysfs_path"/queue/max_segment_size)

+if [ "$dma_align" != "4095" ]; then

+ ERR_CODE=255

+fi

+if [ "$max_segments" != "32" ]; then

+ ERR_CODE=255

+fi

+if [ "$max_segment_size" != "32768" ]; then

+ ERR_CODE=255

+fi

+_cleanup_test "null"

+_show_result $TID $ERR_CODE

+if ! _have_program fio; then

+ exit "$UBLK_SKIP_CODE"

+fi

+

+_run_fio_verify_io --filename=/dev/ublkb"${dev_id}" --size=256M

+if ! _have_program fio; then

+ exit "$UBLK_SKIP_CODE"

+fi

+

+_run_fio_verify_io --filename=/dev/ublkb"${dev_id}" --size=256M