CFLAGS_REMOVE_irq_work.o = $(CC_FLAGS_FTRACE)

endif

# Prevents flicker of uninteresting __do_softirq()/__local_bh_disable_ip()

# in coverage traces.

KCOV_INSTRUMENT_softirq.o := n

struct dentry *d = kern_path_locked(watch->path, parent);

if (IS_ERR(d))

return PTR_ERR(d);

inode_unlock(d_backing_inode(parent->dentry));

dput(d);

return 0;

/* caller expects mutex locked */

mutex_lock(&audit_filter_mutex);

audit_put_watch(watch);

return ret;

}

/* either find an old parent or attach a new one */

parent = audit_find_parent(d_backing_inode(parent_path.dentry));

list_for_each_entry(n, &context->names_list, list) {

if (!n->name)

continue;

}

return NULL;

}

n->name = name;

n->name_len = AUDIT_NAME_FULL;

name->aname = n;

}

static inline int audit_copy_fcaps(struct audit_names *name,

return;

if (name) {

n->name = name;

}

out:

if (found_parent) {

found_child->name = found_parent->name;

found_child->name_len = AUDIT_NAME_FULL;

}

}

atomic_set(&t->cancelling, 0);

INIT_WORK(&t->cb.delete_work, bpf_timer_delete_work);

cb->value = (void *)async - map->record->timer_off;

break;

case BPF_ASYNC_TYPE_WQ:

inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));

}

{

struct inode *inode;

inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFDIR);

if (IS_ERR(inode))

inode->i_op = &bpf_dir_iops;

inode->i_fop = &simple_dir_operations;

inc_nlink(dir);

bpf_dentry_finalize(dentry, inode, dir);

}

struct map_iter {

if (insn->imm == BPF_FUNC_get_smp_processor_id &&

verifier_inlines_helper_call(env, insn->imm)) {

/* BPF_FUNC_get_smp_processor_id inlining is an

* changed in some incompatible and hard to support

* way, it's fine to back out this inlining logic

*/

#ifdef CONFIG_SMP

insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);

insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0);

cnt = 3;

#include <linux/cfi.h>

enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr,

unsigned long *target, u32 type)

{

pr_err("CFI failure at %pS (no target information)\n",

(void *)addr);

__warn(NULL, 0, (void *)addr, 0, regs, NULL);

return BUG_TRAP_TYPE_WARN;

}

extern struct cgroup_subsys *cgroup_subsys[];

extern struct list_head cgroup_roots;

/* iterate across the hierarchies */

#define for_each_root(root) \

int proc_cgroupstats_show(struct seq_file *m, void *v)

{

struct cgroup_subsys *ss;

int i;

seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");

for_each_subsys(ss, i) {

if (cgroup1_subsys_absent(ss))

continue;

seq_printf(m, "%s\t%d\t%d\t%d\n",

ss->legacy_name, ss->root->hierarchy_id,

atomic_read(&ss->root->nr_cgrps),

cgroup_ssid_enabled(i));

}

return 0;

}

* The default hierarchy always exists but is hidden until mounted for the

* first time. This is for backward compatibility.

*/

/* some controllers are not supported in the default hierarchy */

static u16 cgrp_dfl_inhibit_ss_mask;

* function currently returns 0 as long as @cfts registration is successful

* even if some file creation attempts on existing cgroups fail.

*/

{

int ret;

}

/*

* that @cgrp->kn is always accessible.

*/

kernfs_get(cgrp->kn);

// SPDX-License-Identifier: GPL-2.0-or-later

#include "cpuset-internal.h"

/*

switch (type) {

case FILE_SCHED_RELAX_DOMAIN_LEVEL:

retval = update_relax_domain_level(cs, val);

break;

default:

return ret;

}

static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)

{

struct cpuset *cs = css_cs(css);

retval = cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, val);

break;

case FILE_MEM_EXCLUSIVE:

retval = cpuset_update_flag(CS_MEM_EXCLUSIVE, cs, val);

break;

case FILE_MEM_HARDWALL:

retval = cpuset_update_flag(CS_MEM_HARDWALL, cs, val);

break;

case FILE_SCHED_LOAD_BALANCE:

retval = cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, val);

break;

case FILE_MEMORY_MIGRATE:

retval = cpuset_update_flag(CS_MEMORY_MIGRATE, cs, val);

break;

case FILE_MEMORY_PRESSURE_ENABLED:

cpuset_memory_pressure_enabled = !!val;

break;

case FILE_SPREAD_PAGE:

retval = cpuset_update_flag(CS_SPREAD_PAGE, cs, val);

break;

case FILE_SPREAD_SLAB:

retval = cpuset_update_flag(CS_SPREAD_SLAB, cs, val);

break;

default:

* License. See the file COPYING in the main directory of the Linux

* distribution for more details.

*/

#include "cpuset-internal.h"

#include <linux/init.h>

css_task_iter_end(&it);

}

{

struct cpuset *cs = NULL;

struct cgroup_subsys_state *pos_css;

lockdep_assert_held(&cpuset_mutex);

lockdep_assert_cpus_held();

rcu_read_lock();

cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {

rcu_read_unlock();

}

/*

* Rebuild scheduler domains.

*

ndoms = generate_sched_domains(&doms, &attr);

/* Have scheduler rebuild the domains */

}

#else /* !CONFIG_SMP */

void rebuild_sched_domains_locked(void)

cpus_read_unlock();

}

/**

* cpuset_update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.

* @cs: the cpuset in which each task's cpus_allowed mask needs to be changed

rcu_read_unlock();

}

/* Display task mems_allowed in /proc/<pid>/status file. */

void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task)

{

if (strcmp(buf, freezer_state_strs(0)) == 0)

freeze = false;

freeze = true;

return -EINVAL;

freezer_change_state(css_freezer(of_css(of)), freeze);

}

/**

* misc_cg_set_capacity() - Set the capacity of the misc cgroup res.

* @type: Type of the misc res.

* @capacity: Supported capacity of the misc res on the host.

spin_unlock_irq(&cgroup_rstat_lock);

}

/**

* cgroup_rstat_flush - flush stats in @cgrp's subtree

* @cgrp: target cgroup

*/

__bpf_kfunc void cgroup_rstat_flush(struct cgroup *cgrp)

{

might_sleep();

}

int cgroup_rstat_init(struct cgroup *cgrp)

void cgroup_base_stat_cputime_show(struct seq_file *seq)

{

struct cgroup *cgrp = seq_css(seq)->cgroup;

if (cgroup_parent(cgrp)) {

cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime,

} else {

}

seq_printf(seq, "usage_usec %llu\n"

"user_usec %llu\n"

"system_usec %llu\n"

"nice_usec %llu\n",

}

/* Add bpf kfuncs for cgroup_rstat_updated() and cgroup_rstat_flush() */

# CONFIG_UBSAN_SHIFT is not set

# CONFIG_UBSAN_DIV_ZERO is not set

# CONFIG_UBSAN_UNREACHABLE is not set

# CONFIG_UBSAN_BOOL is not set

# CONFIG_UBSAN_ENUM is not set

# CONFIG_UBSAN_ALIGNMENT is not set

*/

static noinstr void ct_kernel_exit_state(int offset)

{

/*

* CPUs seeing atomic_add_return() must see prior RCU read-side

* critical sections, and we also must force ordering with the

* next idle sojourn.

*/

rcu_task_trace_heavyweight_enter(); // Before CT state update!

}

/*

percpu_rwsem_assert_held(&cpu_hotplug_lock);

}

#ifdef CONFIG_LOCKDEP

int lockdep_is_cpus_held(void)

out:

cpus_write_unlock();

arch_smt_update();

return ret;

}

memset(&prstatus, 0, sizeof(prstatus));

prstatus.common.pr_pid = current->pid;

elf_core_copy_regs(&prstatus.pr_reg, regs);

&prstatus, sizeof(prstatus));

final_note(buf);

}

UBSAN_SANITIZE := n

KCOV_INSTRUMENT := n

CFLAGS_REMOVE_common.o = -fstack-protector -fstack-protector-strong

CFLAGS_common.o += -fno-stack-protector

/* Do seccomp after ptrace, to catch any tracer changes. */

if (work & SYSCALL_WORK_SECCOMP) {

if (ret == -1L)

return ret;

}

int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;

int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;

static inline size_t perf_callchain_entry__sizeof(void)

{

return entry;

}

{

int *value = table->data;

int new_value = *value, ret;

return ret;

}

#include <linux/pgtable.h>

#include <linux/buildid.h>

#include <linux/task_work.h>

#include "internal.h"

*/

int sysctl_perf_event_paranoid __read_mostly = 2;

/*

* max perf event sample rate

#define DEFAULT_CPU_TIME_MAX_PERCENT 25

int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;

static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);

static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS;

static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc);

void *buffer, size_t *lenp, loff_t *ppos)

{

int ret;

return 0;

}

void *buffer, size_t *lenp, loff_t *ppos)

{

int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

return 0;

}

/*

* perf samples are done in some very critical code paths (NMIs).

* If they take too much CPU time, the system can lock up and not

cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);

raw_spin_lock_init(&cpc->hrtimer_lock);

}

static int perf_mux_hrtimer_restart(struct perf_cpu_pmu_context *cpc)

return perf_mux_hrtimer_restart(arg);

}

void perf_pmu_disable(struct pmu *pmu)

{

if (!(*count)++)

pmu->pmu_disable(pmu);

}

void perf_pmu_enable(struct pmu *pmu)

{

if (!--(*count))

pmu->pmu_enable(pmu);

}

static void perf_assert_pmu_disabled(struct pmu *pmu)

{

}

{

}

{

}

static void free_ctx(struct rcu_head *head)

event_sched_out(struct perf_event *event, struct perf_event_context *ctx)

{

struct perf_event_pmu_context *epc = event->pmu_ctx;

enum perf_event_state state = PERF_EVENT_STATE_INACTIVE;

// XXX cpc serialization, probably per-cpu IRQ disabled

pmu_ctx->rotate_necessary = 0;

if (ctx->task && ctx->is_active) {

WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);

cpc->task_epc = NULL;

}

event_sched_in(struct perf_event *event, struct perf_event_context *ctx)

{

struct perf_event_pmu_context *epc = event->pmu_ctx;

int ret = 0;

WARN_ON_ONCE(event->ctx != ctx);

static int group_can_go_on(struct perf_event *event, int can_add_hw)

{

struct perf_event_pmu_context *epc = event->pmu_ctx;

/*

* Groups consisting entirely of software events can always go on.

struct pmu *pmu = pmu_ctx->pmu;

if (ctx->task && !(ctx->is_active & EVENT_ALL)) {

WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);

cpc->task_epc = NULL;

}

* we know the event must be on the current CPU, therefore we

* don't need to use it.

*/

perf_event_update_time(event);

}

}

{

struct perf_event_pmu_context *pmu_ctx;

struct perf_cpu_pmu_context *cpc;

list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {

if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)

}

}

WRITE_ONCE(ctx->task, next);

WRITE_ONCE(next_ctx->task, task);

perf_ctx_enable(ctx, false);

/*

* RCU_INIT_POINTER here is safe because we've not

* modified the ctx and the above modification of

*/

RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx);

RCU_INIT_POINTER(next->perf_event_ctxp, ctx);

perf_ctx_disable(ctx, false);

inside_switch:

task_ctx_sched_out(ctx, NULL, EVENT_ALL);

perf_ctx_enable(ctx, false);

void perf_sched_cb_dec(struct pmu *pmu)

{

this_cpu_dec(perf_sched_cb_usages);

barrier();

void perf_sched_cb_inc(struct pmu *pmu)

{

if (!cpc->sched_cb_usage++)

list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list));

* PEBS requires this to provide PID/TID information. This requires we flush

* all queued PEBS records before we context switch to a new task.

*/

{

struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);

struct pmu *pmu;

perf_ctx_lock(cpuctx, cpuctx->task_ctx);

perf_pmu_disable(pmu);

perf_pmu_enable(pmu);

perf_ctx_unlock(cpuctx, cpuctx->task_ctx);

return;

list_for_each_entry(cpc, this_cpu_ptr(&sched_cb_list), sched_cb_entry)

}

static void perf_event_switch(struct task_struct *task,

if (!pmu_ctx->ctx->task)

return;

WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);

cpc->task_epc = pmu_ctx;

}

if (event->attr.pinned) {

perf_cgroup_event_disable(event, ctx);

perf_event_set_state(event, PERF_EVENT_STATE_ERROR);

} else {

event->pmu_ctx->rotate_necessary = 1;

perf_mux_hrtimer_restart(cpc);

group_update_userpage(event);

}

perf_ctx_lock(cpuctx, ctx);

perf_ctx_disable(ctx, false);

perf_ctx_enable(ctx, false);

perf_ctx_unlock(cpuctx, ctx);

perf_event_sched_in(cpuctx, ctx, NULL);

if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))

perf_ctx_enable(&cpuctx->ctx, false);

pmu->read(event);

data->ret = pmu->commit_txn(pmu);

if (!task) {

/* Must be root to operate on a CPU event: */

if (err)

return ERR_PTR(err);

struct perf_event *event)

{

struct perf_event_pmu_context *new = NULL, *pos = NULL, *epc;

if (!ctx->task) {

/*

*/

struct perf_cpu_pmu_context *cpc;

epc = &cpc->epc;

raw_spin_lock_irq(&ctx->lock);

if (!epc->ctx) {

epc->embedded = 1;

list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);

epc->ctx = ctx;

if (!new)

return ERR_PTR(-ENOMEM);

__perf_init_event_pmu_context(new, pmu);

/*

epc->ctx = ctx;

found_epc:

raw_spin_unlock_irq(&ctx->lock);

kfree(new);

return epc;

WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount));

}

static void free_epc_rcu(struct rcu_head *head)

{

struct perf_event_pmu_context *epc = container_of(head, typeof(*epc), rcu_head);

kfree(epc);

}

raw_spin_unlock_irqrestore(&ctx->lock, flags);

return;

call_rcu(&epc->rcu_head, free_epc_rcu);

}

atomic_dec(&nr_freq_events);

}

static void unaccount_event(struct perf_event *event)

{

bool dec = false;

return -EBUSY;

}

return 0;

}

{

struct pmu *pmu = event->pmu;

/* see comment in exclusive_event_init() */

if (event->attach_state & PERF_ATTACH_TASK)

atomic_dec(&pmu->exclusive_cnt);

else

atomic_inc(&pmu->exclusive_cnt);

}

static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2)

return true;

}

static void perf_pending_task_sync(struct perf_event *event)

{

rcuwait_wait_event(&event->pending_work_wait, !event->pending_work, TASK_UNINTERRUPTIBLE);

}

{

if (is_cgroup_event(event))

perf_detach_cgroup(event);

if (event->destroy)

event->destroy(event);

if (event->hw.target)

put_task_struct(event->hw.target);

put_pmu_ctx(event->pmu_ctx);

/*

*/

if (event->ctx)

put_ctx(event->ctx);

call_rcu(&event->rcu_head, free_event_rcu);

}

/*

* Used to free events which have a known refcount of 1, such as in error paths

* where the event isn't exposed yet and inherited events.

if (is_event_hup(event))

return events;

/*

* Pin the event->rb by taking event->mmap_mutex; otherwise

* perf_event_set_output() can swizzle our rb and make us miss wakeups.

unsigned long vma_size;

unsigned long nr_pages;

long user_extra = 0, extra = 0;

/*

* Don't allow mmap() of inherited per-task counters. This would

return ret;

vma_size = vma->vm_end - vma->vm_start;

if (vma->vm_pgoff == 0) {

} else {

/*

* AUX area mapping: if rb->aux_nr_pages != 0, it's already

*/

u64 aux_offset, aux_size;

rb = event->rb;

if (!rb)

goto aux_unlock;

}

atomic_set(&rb->aux_mmap_count, 1);

}

user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);

/*

rb->aux_mmap_locked = extra;

}

unlock:

if (!ret) {

atomic_long_add(user_extra, &user->locked_vm);

if (!ret)

ret = map_range(rb, vma);

event->pmu->event_mapped(event, vma->vm_mm);

return ret;

if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)

values[n++] = running;

values[n++] = perf_event_count(leader, self);

if (read_format & PERF_FORMAT_ID)

for_each_sibling_event(sub, leader) {

n = 0;

values[n++] = perf_event_count(sub, self);

if (read_format & PERF_FORMAT_ID)

{

u64 sample_type = data->type;

perf_output_put(handle, *header);

if (sample_type & PERF_SAMPLE_IDENTIFIER)

task_ctx);

}

void perf_event_fork(struct task_struct *task)

{

perf_event_task(task, NULL, 1);

perf_event_namespaces(task);

}

/*

unsigned int size;

memset(comm, 0, sizeof(comm));

size = ALIGN(strlen(comm)+1, sizeof(u64));

comm_event->comm = comm;

}

cpy_name:

name = tmp;

got_name:

/*

ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN)

goto err;

name_len = strlen(name) + 1;

while (!IS_ALIGNED(name_len, sizeof(u64)))

name[name_len++] = '\0';

void perf_event_free_bpf_prog(struct perf_event *event)

{

if (!perf_event_is_tracing(event)) {

perf_event_free_bpf_handler(event);

return;

free_filters_list(&list);

}

/*

* Scan through mm's vmas and see if one of them matches the

* @filter; if so, adjust filter's address range.

if (!is_sampling_event(event))

return;

/*

* Since hrtimers have a fixed rate, we can do a static freq->period

return 0;

}

/*

* Let userspace know that this PMU supports address range filtering:

*/

{

struct pmu *pmu = dev_get_drvdata(dev);

}

DEVICE_ATTR_RO(nr_addr_filters);

{

struct pmu *pmu = dev_get_drvdata(dev);

}

static DEVICE_ATTR_RO(type);

{

struct pmu *pmu = dev_get_drvdata(dev);

}

static DEFINE_MUTEX(mux_interval_mutex);

cpus_read_lock();

for_each_online_cpu(cpu) {

struct perf_cpu_pmu_context *cpc;

cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);

cpu_function_call(cpu, perf_mux_hrtimer_restart_ipi, cpc);

free_dev:

put_device(pmu->dev);

goto out;

}

return true;

}

{

}

pmu->name = name;

if (type >= 0)

max = type;

atomic_set(&pmu->exclusive_cnt, 0);

if (pmu_bus_running && !pmu->dev) {

if (ret)

}

if (!pmu->cpu_pmu_context)

for_each_possible_cpu(cpu) {

__perf_init_event_pmu_context(&cpc->epc, pmu);

__perf_mux_hrtimer_init(cpc, cpu);

}

* Now that the PMU is complete, make it visible to perf_try_init_event().

*/

if (!idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu))

list_add_rcu(&pmu->entry, &pmus);

}

EXPORT_SYMBOL_GPL(perf_pmu_register);

void perf_pmu_unregister(struct pmu *pmu)

{

/*

* We dereference the pmu list under both SRCU and regular RCU, so

synchronize_srcu(&pmus_srcu);

synchronize_rcu();

}

EXPORT_SYMBOL_GPL(perf_pmu_unregister);

if (ctx)

perf_event_ctx_unlock(event->group_leader, ctx);

}

return ret;

}

static struct pmu *perf_init_event(struct perf_event *event)

{

bool extended_type = false;

struct pmu *pmu;

/*

* Save original type before calling pmu->event_init() since certain

pmu = event->parent->pmu;

ret = perf_try_init_event(pmu, event);

if (!ret)

}

/*

}

again:

if (pmu) {

if (event->attr.type != type && type != PERF_TYPE_RAW &&

!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE))

ret = perf_try_init_event(pmu, event);

if (ret == -ENOENT && event->attr.type != type && !extended_type) {

}

if (ret)

}

list_for_each_entry_rcu(pmu, &pmus, entry, lockdep_is_held(&pmus_srcu)) {

ret = perf_try_init_event(pmu, event);

if (!ret)

}

}

static void attach_sb_event(struct perf_event *event)

void *context, int cgroup_fd)

{

struct pmu *pmu;

struct hw_perf_event *hwc;

long err = -EINVAL;

int node;

}

node = (cpu >= 0) ? cpu_to_node(cpu) : -1;

if (!event)

return ERR_PTR(-ENOMEM);

* See perf_output_read().

*/

if (has_inherit_and_sample_read(attr) && !(attr->sample_type & PERF_SAMPLE_TID))

if (!has_branch_stack(event))

event->attr.branch_sample_type = 0;

pmu = perf_init_event(event);

}

/*

* events (they don't make sense as the cgroup will be different

* on other CPUs in the uncore mask).

*/

if (event->attr.aux_output &&

(!(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT) ||

if (event->attr.aux_start_paused) {

event->hw.aux_paused = 1;

}

if (cgroup_fd != -1) {

err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader);

if (err)

}

err = exclusive_event_init(event);

if (err)

if (has_addr_filter(event)) {

event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters,

sizeof(struct perf_addr_filter_range),

GFP_KERNEL);

/*

* Clone the parent's vma offsets: they are valid until exec()

if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {

err = get_callchain_buffers(attr->sample_max_stack);

if (err)

}

}

err = security_perf_event_alloc(event);

if (err)

/* symmetric to unaccount_event() in _free_event() */

account_event(event);

}

static int perf_copy_attr(struct perf_event_attr __user *uattr,

}

/* privileged levels capture (kernel, hv): check permissions */

if (mask & PERF_SAMPLE_BRANCH_PERM_PLM) {

if (ret)

return ret;

}

return err;

/* Do we allow access to perf_event_open(2) ? */

if (err)

return err;

if (!attr.exclude_kernel) {

if (err)

return err;

}

/* Only privileged users can get physical addresses */

if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR)) {

if (err)

return err;

}

* At this point we need to send EXIT events to cpu contexts.

*/

perf_event_task(child, NULL, 0);

}

static void perf_free_event(struct perf_event *event,

return &event->attr;

}

{

if (sysctl_perf_event_paranoid > 1 && !perfmon_capable())

return -EACCES;

}

EXPORT_SYMBOL_GPL(perf_allow_kernel);

if (is_orphaned_event(parent_event) ||

!atomic_long_inc_not_zero(&parent_event->refcount)) {

mutex_unlock(&parent_event->child_mutex);

free_event(child_event);

return NULL;

}

child->perf_event_ctxp = NULL;

mutex_init(&child->perf_event_mutex);

INIT_LIST_HEAD(&child->perf_event_list);

ret = perf_event_init_context(child, clone_flags);

if (ret) {

return -ENOENT;

/*

*/

return -EOPNOTSUPP;

err = register_perf_hw_breakpoint(bp);

static void perf_output_wakeup(struct perf_output_handle *handle)

{

handle->event->pending_wakeup = 1;

handle->rb = rb;

handle->event = event;

have_lost = local_read(&rb->lost);

if (unlikely(have_lost)) {

*/

unsigned long uprobe_get_trampoline_vaddr(void)

{

struct xol_area *area;

/* Pairs with xol_add_vma() smp_store_release() */

area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */

WARN_ON_ONCE(utask->state != UTASK_SSTEP);

if (task_sigpending(t)) {

clear_tsk_thread_flag(t, TIF_SIGPENDING);

if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {

utask->state = UTASK_SSTEP_TRAPPED;

utask->state = UTASK_RUNNING;

xol_free_insn_slot(utask);

if (unlikely(err)) {

uprobe_warn(current, "execute the probed insn, sending SIGILL.");

#include <linux/sysfs.h>

#include <linux/user_events.h>

#include <linux/uaccess.h>

#include <uapi/linux/wait.h>

late_initcall(kernel_exit_sysfs_init);

#endif

{

nr_threads--;

if (group_dead) {

list_del_rcu(&p->tasks);

list_del_init(&p->sibling);

/*

* This function expects the tasklist_lock write-locked.

*/

{

struct signal_struct *sig = tsk->signal;

bool group_dead = thread_group_leader(tsk);

sig->curr_target = next_thread(tsk);

}

/*

* Accumulate here the counters for all threads as they die. We could

* skip the group leader because it is the last user of signal_struct,

task_io_accounting_add(&sig->ioac, &tsk->ioac);

sig->sum_sched_runtime += tsk->se.sum_exec_runtime;

sig->nr_threads--;

write_sequnlock(&sig->stats_lock);

tsk->sighand = NULL;

spin_unlock(&sighand->siglock);

__cleanup_sighand(sighand);

tty_kref_put(tty);

}

static void delayed_put_task_struct(struct rcu_head *rhp)

void release_task(struct task_struct *p)

{

struct task_struct *leader;

struct pid *thread_pid;

int zap_leader;

repeat:

/* don't need to get the RCU readlock here - the process is dead and

* can't be modifying its own credentials. But shut RCU-lockdep up */

rcu_read_lock();

dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);

rcu_read_unlock();

cgroup_release(p);

write_lock_irq(&tasklist_lock);

ptrace_release_task(p);

/*

* If we are the last non-leader member of the thread

write_unlock_irq(&tasklist_lock);

proc_flush_pid(thread_pid);

put_pid(thread_pid);

release_thread(p);

put_task_struct_rcu_user(p);

p = leader;

tsk->exit_state = EXIT_ZOMBIE;

/*

*/

do_notify_pidfd(tsk);

if (unlikely(tsk->ptrace)) {

#ifdef CONFIG_POSIX_TIMERS

INIT_HLIST_HEAD(&sig->posix_timers);

INIT_HLIST_HEAD(&sig->ignored_posix_timers);

#endif

task_lock(current->group_leader);

*/

static int __pidfd_prepare(struct pid *pid, unsigned int flags, struct file **ret)

{

struct file *pidfd_file;

if (pidfd < 0)

return pidfd;

pidfd_file = pidfs_alloc_file(pid, flags | O_RDWR);

return PTR_ERR(pidfd_file);

*ret = pidfd_file;

}

/**

if (clone_flags & CLONE_PIDFD) {

int flags = (clone_flags & CLONE_THREAD) ? PIDFD_THREAD : 0;

if (retval < 0)

goto bad_fork_free_pid;

pidfd = retval;

*/

static struct {

struct futex_hash_bucket *queues;

} __futex_data __read_mostly __aligned(2*sizeof(long));

#define futex_queues (__futex_data.queues)

/*

u32 hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / 4,

key->both.offset);

}

static int __init futex_init(void)

{

unsigned int futex_shift;

#ifdef CONFIG_BASE_SMALL

#else

#endif

futex_queues = alloc_large_system_hash("futex", sizeof(*futex_queues),

&futex_shift, NULL,

atomic_set(&futex_queues[i].waiters, 0);

plist_head_init(&futex_queues[i].chain);

spin_lock_init(&futex_queues[i].lock);

}

return 0;

}

core_initcall(futex_init);

#include <linux/ioremap.h>

#ifndef arch_memremap_wb

{

#ifdef ioremap_cache

return (__force void *)ioremap_cache(offset, size);

if (is_ram == REGION_INTERSECTS)

addr = try_ram_remap(offset, size, flags);

if (!addr)

}

/*

}

#endif

static int __irq_startup(struct irq_desc *desc)

{

struct irq_data *d = irq_desc_get_irq_data(desc);

irq_domain_deactivate_irq(&desc->irq_data);

}

static void __irq_disable(struct irq_desc *desc, bool mask)

{

if (irqd_irq_disabled(&desc->irq_data)) {

extern void irq_shutdown(struct irq_desc *desc);

extern void irq_shutdown_and_deactivate(struct irq_desc *desc);

extern void irq_disable(struct irq_desc *desc);

extern void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu);

extern void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu);

extern void unmask_irq(struct irq_desc *desc);

extern void unmask_threaded_irq(struct irq_desc *desc);

#ifdef CONFIG_SPARSE_IRQ

static inline void irq_mark_irq(unsigned int irq) { }

#else

extern void irq_mark_irq(unsigned int irq);

#endif

irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc);

irqreturn_t handle_irq_event_percpu(struct irq_desc *desc);

irqreturn_t handle_irq_event(struct irq_desc *desc);

extern bool irq_can_set_affinity_usr(unsigned int irq);

extern int irq_do_set_affinity(struct irq_data *data,

const struct cpumask *dest, bool force);

return desc->pending_mask;

}

bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear);

#else /* CONFIG_GENERIC_PENDING_IRQ */

static inline bool irq_can_move_pcntxt(struct irq_data *data)

{

{

return false;

}

#endif /* !CONFIG_GENERIC_PENDING_IRQ */

#if !defined(CONFIG_IRQ_DOMAIN) || !defined(CONFIG_IRQ_DOMAIN_HIERARCHY)

return desc && desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, cpu) : 0;

}

{

unsigned int sum = 0;

int cpu;

}

}

{

if (!domain->ops->alloc) {

pr_debug("domain->ops->alloc() is NULL\n");

early_param("threadirqs", setup_forced_irqthreads);

#endif

static void __synchronize_hardirq(struct irq_desc *desc, bool sync_chip)

{

struct irq_data *irqd = irq_desc_get_irq_data(desc);

* set_cpus_allowed_ptr() here as we hold desc->lock and this

* code can be called from hard interrupt context.

*/

{

struct irqaction *action;

irq_put_desc_unlock(desc, flags);

}

{

struct irq_chip *chip;

int err = -EINVAL;

return true;

}

void irq_move_masked_irq(struct irq_data *idata)

{

struct irq_desc *desc = irq_data_to_desc(idata);

if (!masked)

idata->chip->irq_unmask(idata);

}

#include <linux/mutex.h>

#include <linux/pci.h>

#include <linux/slab.h>

#include <linux/sysfs.h>

#include <linux/types.h>

#include <linux/xarray.h>

return ret;

}

void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg)

{

struct msi_desc *entry = irq_get_msi_desc(irq);

}

EXPORT_SYMBOL_GPL(get_cached_msi_msg);

}

/**

* @dev: Device to operate on

*/

{

mutex_lock(&dev->msi.data->mutex);

}

/**

* @dev: Device to operate on

*/

{

/* Invalidate the index which was cached by the iterator */

dev->msi.data->__iter_idx = MSI_XA_MAX_INDEX;

mutex_unlock(&dev->msi.data->mutex);

}

static struct msi_desc *msi_find_desc(struct msi_device_data *md, unsigned int domid,

enum msi_desc_filter filter)

unsigned int msi_domain_get_virq(struct device *dev, unsigned int domid, unsigned int index)

{

struct msi_desc *desc;

bool pcimsi = false;

struct xarray *xa;

if (dev_is_pci(dev) && domid == MSI_DEFAULT_DOMAIN)

pcimsi = to_pci_dev(dev)->msi_enabled;

xa = &dev->msi.data->__domains[domid].store;

desc = xa_load(xa, pcimsi ? 0 : index);

if (desc && desc->irq) {

* PCI-MSIX and platform MSI use a descriptor per

* interrupt.

*/

}

}

EXPORT_SYMBOL_GPL(msi_domain_get_virq);

return info->ops->msi_translate(domain, fwspec, hwirq, type);

}

static const struct irq_domain_ops msi_domain_ops = {

.alloc = msi_domain_alloc,

.free = msi_domain_free,

.activate = msi_domain_activate,

.deactivate = msi_domain_deactivate,

.translate = msi_domain_translate,

};

static irq_hw_number_t msi_domain_ops_get_hwirq(struct msi_domain_info *info,

void *chip_data)

{

struct irq_domain *domain, *parent = dev->msi.domain;

const struct msi_parent_ops *pops;

if (!irq_domain_is_msi_parent(parent))

return false;

if (domid >= MSI_MAX_DEVICE_IRQDOMAINS)

return false;

if (!bundle)

return false;

* node as they are not guaranteed to have a fwnode. They are never

* looked up and always handled in the context of the device.

*/

else

if (!fwnode)

if (msi_setup_device_data(dev))

if (WARN_ON_ONCE(msi_get_device_domain(dev, domid)))

if (!pops->init_dev_msi_info(dev, parent, parent, &bundle->info))

domain = __msi_create_irq_domain(fwnode, &bundle->info, IRQ_DOMAIN_FLAG_MSI_DEVICE, parent);

if (!domain)

domain->dev = dev;

dev->msi.data->__domains[domid].domain = domain;

return true;

}

/**

struct msi_domain_info *info;

struct irq_domain *domain;

domain = msi_get_device_domain(dev, domid);

if (!domain || !irq_domain_is_msi_device(domain))

dev->msi.data->__domains[domid].domain = NULL;

info = domain->host_data;

irq_domain_remove(domain);

irq_domain_free_fwnode(fwnode);

kfree(container_of(info, struct msi_domain_template, info));

}

/**

{

struct msi_domain_info *info;

struct irq_domain *domain;

domain = msi_get_device_domain(dev, domid);

if (domain && irq_domain_is_msi_device(domain)) {

info = domain->host_data;

}

}

static int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev,

if (!(info->flags & MSI_FLAG_MUST_REACTIVATE))

return false;

return false;

/*

}

/**

* @dev: Pointer to device struct of the device for which the interrupts

* are allocated

* @domid: Id of the interrupt domain to operate on

* @first: First index to allocate (inclusive)

* @last: Last index to allocate (inclusive)

*

* Return: %0 on success or an error code.

*/

{

struct msi_ctrl ctrl = {

.domid = domid,

.nirqs = last + 1 - first,

};

return msi_domain_alloc_locked(dev, &ctrl);

}

EXPORT_SYMBOL_GPL(msi_domain_alloc_irqs_range);

/**

const struct irq_affinity_desc *affdesc,

union msi_instance_cookie *icookie)

{

}

/**

icookie.value = ((u64)type << 32) | hwirq;

if (WARN_ON_ONCE(msi_get_device_domain(dev, domid) != domain))

map.index = -EINVAL;

else

map = __msi_domain_alloc_irq_at(dev, domid, MSI_ANY_INDEX, NULL, &icookie);

return map.index >= 0 ? map.virq : map.index;

}

* @first: First index to free (inclusive)

* @last: Last index to free (inclusive)

*/

{

struct msi_ctrl ctrl = {

.domid = domid,

void msi_domain_free_irqs_range(struct device *dev, unsigned int domid,

unsigned int first, unsigned int last)

{

msi_domain_free_irqs_range_locked(dev, domid, first, last);

}

EXPORT_SYMBOL_GPL(msi_domain_free_irqs_all);

*/

void msi_domain_free_irqs_all(struct device *dev, unsigned int domid)

{

msi_domain_free_irqs_all_locked(dev, domid);

}

/**

if (WARN_ON_ONCE(!dev || !desc || domain->bus_token != DOMAIN_BUS_WIRED_TO_MSI))

return;

}

/**

unsigned long kallsyms_sym_address(int idx)

{

}

static unsigned int get_symbol_seq(int index)

*/

task1 = find_task_by_vpid(pid1);

task2 = find_task_by_vpid(pid2);

goto err_no_task;

get_task_struct(task1);

obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o

KCSAN_SANITIZE_lockdep.o := n

ifdef CONFIG_FUNCTION_TRACER

LOCK_EVENT(rwsem_wlock) /* # of write locks acquired */

LOCK_EVENT(rwsem_wlock_fail) /* # of failed write lock acquisitions */

LOCK_EVENT(rwsem_wlock_handoff) /* # of write lock handoffs */

#include <linux/lockdep.h>

#include <linux/context_tracking.h>

#include <linux/console.h>

#include <asm/sections.h>

#include "lockdep_internals.h"

#include <trace/events/lock.h>

static int graph_lock(void)

{

lockdep_lock();

/*

* Make sure that if another CPU detected a bug while

* walking the graph we dont change it (while the other

if (unlikely(lock->key == &__lockdep_no_track__))

return 0;

check = 0;

if (subclass < NR_LOCKDEP_CACHING_CLASSES)

class = lock->class_cache[subclass];

if (!debug_locks)

return;

if (unlikely(!lockdep_enabled())) {

/* XXX allow trylock from NMI ?!? */

if (lockdep_nmi() && !trylock) {

unsigned long curr = (unsigned long)current;

unsigned long zero = 0UL;

if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))

return true;

#include <trace/events/lock.h>

#include "rtmutex_common.h"

#ifndef WW_RT

# define build_ww_mutex() (false)

struct task_struct *owner;

int ret = 0;

for (;;) {

/* Try to acquire the lock: */

break;

if (timeout && !timeout->task) {

ret = -ETIMEDOUT;

owner = NULL;

raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q);

rt_mutex_schedule();

raw_spin_lock_irq(&lock->wait_lock);

set_current_state(state);

int ret;

lockdep_assert_held(&lock->wait_lock);

/* Try to acquire the lock again: */

if (try_to_take_rt_mutex(lock, current, NULL)) {

__ww_mutex_check_waiters(rtm, ww_ctx, wake_q);

ww_mutex_lock_acquired(ww, ww_ctx);

}

return 0;

}

__ww_mutex_check_waiters(rtm, ww_ctx, wake_q);

ww_mutex_lock_acquired(ww, ww_ctx);

}

} else {

__set_current_state(TASK_RUNNING);

remove_waiter(lock, waiter);

rt_mutex_handle_deadlock(ret, chwalk, lock, waiter);

}

/*

&waiter, wake_q);

debug_rt_mutex_free_waiter(&waiter);

return ret;

}

struct task_struct *owner;

lockdep_assert_held(&lock->wait_lock);

return;

rt_mutex_init_rtlock_waiter(&waiter);

for (;;) {

/* Try to acquire the lock again */

break;

if (&waiter == rt_mutex_top_waiter(lock))

owner = rt_mutex_owner(lock);

owner = NULL;

raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q);

schedule_rtlock();

raw_spin_lock_irq(&lock->wait_lock);

set_current_state(TASK_RTLOCK_WAIT);

debug_rt_mutex_free_waiter(&waiter);

trace_contention_end(lock, 0);

}

static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)

{

}

static int module_memory_alloc(struct module *mod, enum mod_mem_type type)

{

unsigned int size = PAGE_ALIGN(mod->mem[type].size);

if (!ptr)

return -ENOMEM;

if (execmem_is_rox(execmem_type)) {

return -ENOMEM;

}

mod->mem[type].is_rox = true;

}

/*

* *do* eventually get freed, but let's just keep things simple

* and avoid *any* false positives.

*/

return 0;

}

static void module_memory_free(struct module *mod, enum mod_mem_type type)

{

struct module_memory *mem = &mod->mem[type];

execmem_free(mem->base);

}

for_each_mod_mem_type(type) {

if (!mod->mem[type].size) {

mod->mem[type].base = NULL;

continue;

}

void *dest;

Elf_Shdr *shdr = &info->sechdrs[i];

const char *sname;

if (!(shdr->sh_flags & SHF_ALLOC))

continue;

ret = PTR_ERR(dest);

goto out_err;

}

codetag_section_found = true;

} else {

enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;

unsigned long offset = shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK;

}

if (shdr->sh_type != SHT_NOBITS) {

* users of info can keep taking advantage and using the newly

* minted official memory area.

*/

pr_debug("\t0x%lx 0x%.8lx %s\n", (long)shdr->sh_addr,

(long)shdr->sh_size, info->secstrings + shdr->sh_name);

}

return 0;

out_err:

for (t--; t >= 0; t--)

module_memory_free(mod, t);

if (codetag_section_found)

return 0;

}

static int post_relocation(struct module *mod, const struct load_info *info)

{

/* Sort exception table now relocations are done. */

sort_extable(mod->extable, mod->extable + mod->num_exentries);

add_kallsyms(mod, info);

/* Arch-specific module finalizing. */

}

/* Call module constructors. */

mod->mem[type].size);

}

module_deallocate(mod, info);

free_copy:

/*

#include <linux/mm.h>

#include <linux/vmalloc.h>

#include <linux/set_memory.h>

#include "internal.h"

static int module_set_memory(const struct module *mod, enum mod_mem_type type,

int module_enable_text_rox(const struct module *mod)

{

for_class_mod_mem_type(type, text) {

int ret;

ret = module_set_memory(mod, type, set_memory_rox);

else

ret = module_set_memory(mod, type, set_memory_x);

if (remove_object) {

list_del_init(&padata->list);

++pd->processed;

}

spin_unlock(&reorder->lock);

};

EXPORT_SYMBOL_GPL(init_pid_ns);

static __cacheline_aligned_in_smp DEFINE_SPINLOCK(pidmap_lock);

seqcount_spinlock_t pidmap_lock_seq = SEQCNT_SPINLOCK_ZERO(pidmap_lock_seq, &pidmap_lock);

void free_pid(struct pid *pid)

{

int i;

for (i = 0; i <= pid->level; i++) {

struct upid *upid = pid->numbers + i;

struct pid_namespace *ns = upid->ns;

idr_remove(&ns->idr, upid->nr);

}

pidfs_remove_pid(pid);

call_rcu(&pid->rcu, delayed_put_pid);

}

struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid,

size_t set_tid_size)

{

}

idr_preload(GFP_KERNEL);

if (tid) {

nr = idr_alloc(&tmp->idr, NULL, tid,

nr = idr_alloc_cyclic(&tmp->idr, NULL, pid_min,

pid_max, GFP_ATOMIC);

}

idr_preload_end();

if (nr < 0) {

upid = pid->numbers + ns->level;

idr_preload(GFP_KERNEL);

if (!(ns->pid_allocated & PIDNS_ADDING))

goto out_unlock;

pidfs_add_pid(pid);

idr_replace(&upid->ns->idr, pid, upid->nr);

upid->ns->pid_allocated++;

}

idr_preload_end();

return pid;

out_unlock:

idr_preload_end();

put_pid_ns(ns);

out_free:

while (++i <= ns->level) {

upid = pid->numbers + i;

idr_remove(&upid->ns->idr, upid->nr);

if (ns->pid_allocated == PIDNS_ADDING)

idr_set_cursor(&ns->idr, 0);

kmem_cache_free(ns->pid_cachep, pid);

return ERR_PTR(retval);

void disable_pid_allocation(struct pid_namespace *ns)

{

ns->pid_allocated &= ~PIDNS_ADDING;

}

struct pid *find_pid_ns(int nr, struct pid_namespace *ns)

*/

void attach_pid(struct task_struct *task, enum pid_type type)

{

hlist_add_head_rcu(&task->pid_links[type], &pid->tasks[type]);

}

{

int tmp;

pid = *pid_ptr;

hlist_del_rcu(&task->pid_links[type]);

if (pid_has_task(pid, tmp))

return;

}

{

}

struct pid *pid)

{

attach_pid(task, type);

}

struct hlist_head *head1 = &pid1->tasks[PIDTYPE_PID];

struct hlist_head *head2 = &pid2->tasks[PIDTYPE_PID];

/* Swap the single entry tid lists */

hlists_swap_heads_rcu(head1, head2);

enum pid_type type)

{

WARN_ON_ONCE(type == PIDTYPE_PID);

hlist_replace_rcu(&old->pid_links[type], &new->pid_links[type]);

}

*/

struct task_struct *pidfd_get_task(int pidfd, unsigned int *flags)

{

struct pid *pid;

struct task_struct *task;

put_pid(pid);

if (!task)

return ERR_PTR(-ESRCH);

config ENERGY_MODEL

bool "Energy Model for devices with DVFS (CPUs, GPUs, etc)"

help

Several subsystems (thermal and/or the task scheduler for example)

can leverage information about the energy consumed by devices to

static void em_debug_remove_pd(struct device *dev) {}

#endif

static void em_release_table_kref(struct kref *kref)

{

/* It was the last owner of this table so we can free */

}

/**

*

* No return values.

*/

{

kref_put(&table->kref, em_release_table_kref);

}

* has a user.

* Returns allocated table or NULL.

*/

{

int table_size;

table_size = sizeof(struct em_perf_state) * pd->nr_perf_states;

}

static int em_compute_costs(struct device *dev, struct em_perf_state *table,

unsigned long flags)

{

unsigned long prev_cost = ULONG_MAX;

* Return 0 on success or an error code on failure.

*/

int em_dev_update_perf_domain(struct device *dev,

{

struct em_perf_domain *pd;

if (!dev)

kref_get(&new_table->kref);

rcu_assign_pointer(pd->em_table, new_table);

em_cpufreq_update_efficiencies(dev, new_table->state);

static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd,

struct em_perf_state *table,

unsigned long flags)

{

unsigned long power, freq, prev_freq = 0;

}

static int em_create_pd(struct device *dev, int nr_states,

unsigned long flags)

{

struct em_perf_domain *pd;

struct device *cpu_dev;

int cpu, ret, num_cpus;

* Return 0 on success

*/

int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states,

{

unsigned long cap, prev_cap = 0;

unsigned long flags = 0;

int cpu, ret;

dev->em_pd->min_perf_state = 0;

dev->em_pd->max_perf_state = nr_states - 1;

em_debug_create_pd(dev);

dev_info(dev, "EM: created perf domain\n");

mutex_lock(&em_pd_mutex);

em_debug_remove_pd(dev);

kfree(dev->em_pd);

dev->em_pd = NULL;

}

EXPORT_SYMBOL_GPL(em_dev_unregister_perf_domain);

{

struct em_perf_state *ps, *new_ps;

int ps_size;

}

static int em_recalc_and_update(struct device *dev, struct em_perf_domain *pd,

{

int ret;

* are correctly calculated.

*/

static void em_adjust_new_capacity(struct device *dev,

{

em_table = em_table_dup(pd);

if (!em_table) {

}

cpufreq_cpu_put(policy);

if (!pd || em_is_artificial(pd))

continue;

pr_debug("updating cpu%d cpu_cap=%lu old capacity=%lu\n",

cpu, cpu_capacity, em_max_perf);

}

free_cpumask_var(cpu_done_mask);

*/

int em_dev_update_chip_binning(struct device *dev)

{

struct em_perf_domain *pd;

int i, ret;

static int hibernate_compressor_param_set(const char *compressor,

const struct kernel_param *kp)

{

int index, ret;

index = sysfs_match_string(comp_alg_enabled, compressor);

if (index >= 0) {

ret = index;

}

if (ret)

pr_debug("Cannot set specified compressor %s\n",

*/

void *kaddr;

copy_page(buffer, kaddr);

handle->buffer = buffer;

} else {

handle->buffer = page_address(page);

if (last_highmem_page) {

void *dst;

copy_page(dst, buffer);

last_highmem_page = NULL;

}

}

{

void *kaddr1, *kaddr2;

copy_page(buf, kaddr1);

copy_page(kaddr1, kaddr2);

copy_page(kaddr2, buf);

}

/**

{

trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, true);

raw_spin_lock_irq(&s2idle_lock);

if (pm_wakeup_pending())

goto out;

s2idle_state = S2IDLE_STATE_ENTER;

raw_spin_unlock_irq(&s2idle_lock);

/* Push all the CPUs into the idle loop. */

wake_up_all_idle_cpus();

/* Make the current CPU wait so it can enter the idle loop too. */

*/

wake_up_all_idle_cpus();

raw_spin_lock_irq(&s2idle_lock);

out:

}

EXPORT_SYMBOL(_printk);

static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);

#else /* CONFIG_PRINTK */

static u64 syslog_seq;

static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }

#endif /* CONFIG_PRINTK */

* Context: Process context. May sleep while acquiring console lock.

* Return: true if all usable printers are caught up.

*/

{

return __pr_flush(NULL, timeout_ms, reset_on_progress);

}

config PREEMPT_RCU

bool

select TREE_RCU

help

This option selects the RCU implementation that is

help

This option selects the full-fledged version of SRCU.

config NEED_SRCU_NMI_SAFE

def_bool HAVE_NMI && !ARCH_HAS_NMI_SAFE_THIS_CPU_OPS && !TINY_SRCU

config TASKS_RCU

bool

select IRQ_WORK

config FORCE_TASKS_RUDE_RCU

depends on RCU_NOCB_CPU

default n

help

config RCU_LAZY_DEFAULT_OFF

bool "Turn RCU lazy invocation off by default"

depends on RCU_LAZY

default n

help

config RCU_DOUBLE_CHECK_CB_TIME

bool "RCU callback-batch backup time check"

Say N if you are unsure.

config RCU_TORTURE_TEST_CHK_RDR_STATE

depends on RCU_TORTURE_TEST

default n

help

Say N if you are unsure.

config RCU_TORTURE_TEST_LOG_CPU

depends on RCU_TORTURE_TEST

default n

help

Say Y here if you want CPU IDs logged.

Say N if you are unsure.

config RCU_REF_SCALE_TEST

tristate "Scalability tests for read-side synchronization (RCU and others)"

depends on DEBUG_KERNEL

{

unsigned long cur_s = READ_ONCE(*sp);

}

/*

#endif

static inline void rcu_gp_set_torture_wait(int duration) { }

#endif

#ifdef CONFIG_TINY_SRCU

static inline bool rcu_watching_zero_in_eqs(int cpu, int *vp) { return false; }

static inline unsigned long rcu_get_gp_seq(void) { return 0; }

static inline unsigned long rcu_exp_batches_completed(void) { return 0; }

static inline void rcu_force_quiescent_state(void) { }

static inline bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) { return true; }

static inline void show_rcu_gp_kthreads(void) { }

bool rcu_watching_zero_in_eqs(int cpu, int *vp);

unsigned long rcu_get_gp_seq(void);

unsigned long rcu_exp_batches_completed(void);

bool rcu_check_boost_fail(unsigned long gp_state, int *cpup);

void show_rcu_gp_kthreads(void);

int rcu_get_gp_kthreads_prio(void);

void rcu_gp_slow_unregister(atomic_t *rgssp);

#endif /* #else #ifdef CONFIG_TINY_RCU */

#ifdef CONFIG_RCU_NOCB_CPU

void rcu_bind_current_to_nocb(void);

#else

torture_param(int, stutter, 5, "Number of seconds to run/halt test");

torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");

torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds.");

torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds.");

torture_param(int, test_nmis, 0, "End-test NMI tests, 0 to disable.");

torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs");

static int nrealnocbers;

static int nrealreaders;

static struct task_struct *writer_task;

static struct task_struct **fakewriter_tasks;

static struct task_struct **reader_tasks;

bool rt_preempted;

int rt_cpu;

int rt_end_cpu;

};

static int err_segs_recorded;

static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS];

void (*gp_slow_register)(atomic_t *rgssp);

void (*gp_slow_unregister)(atomic_t *rgssp);

bool (*reader_blocked)(void);

long cbflood_max;

int irq_capable;

int can_boost;

.reader_blocked = IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)

? has_rcu_reader_blocked

: NULL,

.irq_capable = 1,

.can_boost = IS_ENABLED(CONFIG_RCU_BOOST),

.extendables = RCUTORTURE_MAX_EXTEND,

.sync = synchronize_rcu_busted,

.exp_sync = synchronize_rcu_busted,

.call = call_rcu_busted,

.irq_capable = 1,

.extendables = RCUTORTURE_MAX_EXTEND,

.name = "busted"

static int srcu_torture_read_lock(void)

{

int idx;

int ret = 0;

if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) {

idx = srcu_read_lock(srcu_ctlp);

WARN_ON_ONCE(idx & ~0x1);

WARN_ON_ONCE(idx & ~0x1);

ret += idx << 2;

}

return ret;

}

static void srcu_torture_read_unlock(int idx)

{

WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));

if (reader_flavor & SRCU_READ_FLAVOR_LITE)

srcu_read_unlock_lite(srcu_ctlp, (idx & 0x4) >> 2);

if (reader_flavor & SRCU_READ_FLAVOR_NMI)

.readunlock = srcu_torture_read_unlock,

.readlock_held = torture_srcu_read_lock_held,

.get_gp_seq = srcu_torture_completed,

.deferred_free = srcu_torture_deferred_free,

.sync = srcu_torture_synchronize,

.exp_sync = srcu_torture_synchronize_expedited,

.readunlock = srcu_torture_read_unlock,

.readlock_held = torture_srcu_read_lock_held,

.get_gp_seq = srcu_torture_completed,

.deferred_free = srcu_torture_deferred_free,

.sync = srcu_torture_synchronize,

.exp_sync = srcu_torture_synchronize_expedited,

unsigned long gp_state;

unsigned long gp_state_time;

unsigned long oldstarttime;

/* Set real-time priority. */

sched_set_fifo_low(current);

sched_set_fifo_low(current);

} while (!torture_must_stop());

/* Clean up and exit. */

while (!kthread_should_stop()) {

torture_shutdown_absorb("rcu_torture_boost");

do {

torture_hrtimeout_jiffies(torture_random(&rand) % 10, &rand);

if (cur_ops->cb_barrier != NULL &&

cur_ops->cb_barrier();

} else {

switch (synctype[torture_random(&rand) % nsynctypes]) {

#define ROEC_ARGS "%s %s: Current %#x To add %#x To remove %#x preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count()

static void rcutorture_one_extend_check(char *s, int curstate, int new, int old, bool insoftirq)

{

if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE))

return;

WARN_ONCE(cur_ops->extendables &&

!(curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) &&

(preempt_count() & SOFTIRQ_MASK), ROEC_ARGS);

(preempt_count() & PREEMPT_MASK), ROEC_ARGS);

cur_ops->readlock_nesting() > 0, ROEC_ARGS);

}

rtrsp[-1].rt_preempted = cur_ops->reader_blocked();

}

}

/*

* Next, remove old protection, in decreasing order of strength

"shuffle_interval=%d stutter=%d irqreader=%d "

"fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "

"test_boost=%d/%d test_boost_interval=%d "

"stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d "

"stall_cpu_block=%d stall_cpu_repeat=%d "

"n_barrier_cbs=%d "

"nocbs_nthreads=%d nocbs_toggle=%d "

"test_nmis=%d "

"preempt_duration=%d preempt_interval=%d\n",

stat_interval, verbose, test_no_idle_hz, shuffle_interval,

stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,

test_boost, cur_ops->can_boost,

stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff,

stall_cpu_block, stall_cpu_repeat,

n_barrier_cbs,

int flags = 0;

unsigned long gp_seq = 0;

int i;

if (torture_cleanup_begin()) {

if (cur_ops->cb_barrier != NULL) {

rcu_torture_reader_mbchk = NULL;

if (fakewriter_tasks) {

torture_stop_kthread(rcu_torture_fakewriter,

fakewriter_tasks[i]);

kfree(fakewriter_tasks);

pr_alert("\t: No segments recorded!!!\n");

firsttime = 1;

for (i = 0; i < rt_read_nsegs; i++) {

if (err_segs[i].rt_delay_jiffies != 0) {

pr_cont("%s%ldjiffies", firsttime ? "" : "+",

err_segs[i].rt_delay_jiffies);

else

pr_cont(" ...");

}

if (err_segs[i].rt_delay_ms != 0) {

pr_cont(" %s%ldms", firsttime ? "" : "+",

err_segs[i].rt_delay_ms);

rcu_torture_init_srcu_lockdep();

if (nreaders >= 0) {

nrealreaders = nreaders;

} else {

writer_task);

if (torture_init_error(firsterr))

goto unwind;

sizeof(fakewriter_tasks[0]),

GFP_KERNEL);

if (fakewriter_tasks == NULL) {

goto unwind;

}

}

firsterr = torture_create_kthread(rcu_torture_fakewriter,

NULL, fakewriter_tasks[i]);

if (torture_init_error(firsterr))

.name = "srcu"

};

static void srcu_lite_ref_scale_read_section(const int nloops)

{

int i;

long i;

int firsterr = 0;

static const struct ref_scale_ops *scale_ops[] = {

&refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops,

&acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops,

&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,

#include "rcu_segcblist.h"

#include "rcu.h"

int rcu_scheduler_active __read_mostly;

static LIST_HEAD(srcu_boot_list);

static bool srcu_init_done;

{

int newval;

newval = READ_ONCE(ssp->srcu_lock_nesting[idx]) - 1;

WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);

preempt_enable();

struct srcu_struct *ssp;

ssp = container_of(wp, struct srcu_struct, srcu_work);

if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) {

preempt_enable();

return; /* Already running or nothing to do. */

WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */

preempt_enable();

swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));

WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */

WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);

preempt_enable();

* at interrupt level, but the ->srcu_gp_running checks will

* straighten that out.

*/

WRITE_ONCE(ssp->srcu_gp_running, false);

idx = ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max));

preempt_enable();

{

unsigned long cookie;

cookie = get_state_synchronize_srcu(ssp);

if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) {

preempt_enable();

rhp->func = func;

rhp->next = NULL;

local_irq_save(flags);

*ssp->srcu_cb_tail = rhp;

ssp->srcu_cb_tail = &rhp->next;

{

unsigned long ret;

ret = get_state_synchronize_srcu(ssp);

srcu_gp_start_if_needed(ssp);

preempt_enable();

}

EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);

/* Lockdep diagnostics. */

void __init rcu_scheduler_starting(void)

{

rcu_scheduler_active = RCU_SCHEDULER_RUNNING;

}

/*

* Queue work for srcu_struct structures with early boot callbacks.

/*

* Initialize SRCU per-CPU data. Note that statically allocated

* srcu_struct structures might already have srcu_read_lock() and

*/

static void init_srcu_struct_data(struct srcu_struct *ssp)

{

* Initialize the per-CPU srcu_data array, which feeds into the

* leaves of the srcu_node tree.

*/

for_each_possible_cpu(cpu) {

sdp = per_cpu_ptr(ssp->sda, cpu);

spin_lock_init(&ACCESS_PRIVATE(sdp, lock));

ssp->srcu_sup->node = NULL;

mutex_init(&ssp->srcu_sup->srcu_cb_mutex);

mutex_init(&ssp->srcu_sup->srcu_gp_mutex);

ssp->srcu_sup->srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL;

ssp->srcu_sup->srcu_barrier_seq = 0;

mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);

atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);

INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);

ssp->srcu_sup->sda_is_static = is_static;

ssp->sda = alloc_percpu(struct srcu_data);

if (!ssp->sda)

goto err_free_sup;

init_srcu_struct_data(ssp);

}

/*

*/

static bool srcu_readers_lock_idx(struct srcu_struct *ssp, int idx, bool gp, unsigned long unlocks)

{

for_each_possible_cpu(cpu) {

struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);

if (IS_ENABLED(CONFIG_PROVE_RCU))

mask = mask | READ_ONCE(sdp->srcu_reader_flavor);

}

WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && (mask & (mask - 1)),

"Mixed reader flavors for srcu_struct at %ps.\n", ssp);

return false;

return sum == unlocks;

}

/*

*/

static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx, unsigned long *rdm)

{

for_each_possible_cpu(cpu) {

struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);

mask = mask | READ_ONCE(sdp->srcu_reader_flavor);

}

WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && (mask & (mask - 1)),

unsigned long unlocks;

unlocks = srcu_readers_unlock_idx(ssp, idx, &rdm);

/*

* Make sure that a lock is always counted if the corresponding

* If the locks are the same as the unlocks, then there must have

* been no readers on this index at some point in this function.

* But there might be more readers, as a task might have read

* that most of the tasks have been preempted between fetching

*

*

* It can clearly do so once, given that it has already fetched

*

* However, it is important to note that a given smp_mb() takes

* effect not just for the task executing it, but also for any

* later task running on that same CPU.

*

*

* OK, but what about nesting? This does impose a limit on

* nesting of half of the size of the task_struct structure

for_each_possible_cpu(cpu) {

struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);

}

return sum;

}

unsigned long jbase = SRCU_INTERVAL;

struct srcu_usage *sup = ssp->srcu_sup;

if (srcu_gp_is_expedited(ssp))

jbase = 0;

if (rcu_seq_state(READ_ONCE(sup->srcu_gp_seq))) {

void cleanup_srcu_struct(struct srcu_struct *ssp)

{

int cpu;

struct srcu_usage *sup = ssp->srcu_sup;

return; /* Just leak it! */

if (WARN_ON(srcu_readers_active(ssp)))

return; /* Just leak it! */

*/

int __srcu_read_lock(struct srcu_struct *ssp)

{

smp_mb(); /* B */ /* Avoid leaking the critical section. */

}

EXPORT_SYMBOL_GPL(__srcu_read_lock);

void __srcu_read_unlock(struct srcu_struct *ssp, int idx)

{

smp_mb(); /* C */ /* Avoid leaking the critical section. */

}

EXPORT_SYMBOL_GPL(__srcu_read_unlock);

*/

int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)

{

smp_mb__after_atomic(); /* B */ /* Avoid leaking the critical section. */

}

EXPORT_SYMBOL_GPL(__srcu_read_lock_nmisafe);

*/

void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)

{

smp_mb__before_atomic(); /* C */ /* Avoid leaking the critical section. */

}

EXPORT_SYMBOL_GPL(__srcu_read_unlock_nmisafe);

/*

* Wait until all readers counted by array index idx complete, but

* loop an additional time if there is an expedited grace period pending.

*/

static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)

{

unsigned long curdelay;

curdelay = !srcu_get_delay(ssp);

for (;;) {

if (srcu_readers_active_idx_check(ssp, idx))

}

/*

* use the other rank of the ->srcu_(un)lock_count[] arrays. This allows

* us to wait for pre-existing readers in a starvation-free manner.

*/

static void srcu_flip(struct srcu_struct *ssp)

{

/*

* preceding call to srcu_readers_active_idx_check() found that

*

* This sum-equality check and ordering also ensures that if

* a given call to __srcu_read_lock() uses the new value of

* that reader's increments, which is all to the good, because

* this grace period need not wait on that reader. After all,

* if those earlier scans had seen that reader, there would have

*/

smp_mb(); /* E */ /* Pairs with B and C. */

/*

* Ensure that if the updater misses an __srcu_read_unlock()

check_init_srcu_struct(ssp);

/* If _lite() readers, don't do unsolicited expediting. */

return false;

/* If the local srcu_data structure has callbacks, not idle. */

sdp = raw_cpu_ptr(ssp->sda);

* read-side critical sections are delimited by srcu_read_lock() and

* srcu_read_unlock(), and may be nested.

*

*/

void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,

rcu_callback_t func)

*

* Wait for the count to drain to zero of both indexes. To avoid the

* possible starvation of synchronize_srcu(), it waits for the count of

*

* Can block; must be called from process context.

*

*/

unsigned long srcu_batches_completed(struct srcu_struct *ssp)

{

}

EXPORT_SYMBOL_GPL(srcu_batches_completed);

/*

* Because readers might be delayed for an extended period after

* might well be readers using both idx=0 and idx=1. We therefore

* need to wait for readers to clear from both index values before

* invoking a callback.

}

if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) {

if (!try_check_zero(ssp, idx, 1)) {

mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);

return; /* readers present, retry later. */

* SRCU read-side critical sections are normally short,

* so check at least twice in quick succession after a flip.

*/

if (!try_check_zero(ssp, idx, 2)) {

mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);

return; /* readers present, retry later. */

ssp = sup->srcu_ssp;

srcu_advance_state(ssp);

curdelay = srcu_get_delay(ssp);

if (curdelay) {

WRITE_ONCE(sup->reschedule_count, 0);

} else {

int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state);

int ss_state_idx = ss_state;

if (ss_state < 0 || ss_state >= ARRAY_SIZE(srcu_size_state_name))

ss_state_idx = ARRAY_SIZE(srcu_size_state_name) - 1;

pr_alert("%s%s Tree SRCU g%ld state %d (%s)",

struct srcu_data *sdp;

sdp = per_cpu_ptr(ssp->sda, cpu);

/*

* Make sure that a lock is always counted if the corresponding

*/

smp_rmb();

c0 = l0 - u0;

c1 = l1 - u1;

ssp->sda = alloc_percpu(struct srcu_data);

if (WARN_ON_ONCE(!ssp->sda))

return -ENOMEM;

}

return 0;

}

#endif

}

{

#ifdef CONFIG_TASKS_RCU

rcu_spawn_tasks_kthread();

// Run the self-tests.

rcu_tasks_initiate_self_tests();

}

#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */

static inline void rcu_tasks_bootup_oddness(void) {}

static inline bool rcu_reclaim_tiny(struct rcu_head *head)

{

rcu_callback_t f;

rcu_lock_acquire(&rcu_callback_map);

trace_rcu_invoke_callback("", head);

f = head->func;

}

EXPORT_SYMBOL_GPL(synchronize_rcu);

/*

* Post an RCU callback to be invoked after the end of an RCU grace

* period. But since we have but one CPU, that would be after any

pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func);

mem_dump_obj(head);

}

return;

}

}

EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu);

{

}

#endif

void __init rcu_init(void)

}

EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);

#if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK))

/*

* An empty function that will trigger a reschedule on

/* Handle the ends of any preceding grace periods first. */

if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) ||

if (!offloaded)

ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */

rdp->core_needs_qs = false;

/* Now handle the beginnings of any new-to-this-CPU grace periods. */

if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) ||

/*

* If the current grace period is waiting for this CPU,

* set up to detect a quiescent state, otherwise don't

rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */

if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap)

WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);

WRITE_ONCE(rdp->last_sched_clock, jiffies);

WRITE_ONCE(rdp->gpwrap, false);

rcu_gpnum_ovf(rnp, rdp);

{

struct rcu_synchronize *rs = container_of(

(struct rcu_head *) node, struct rcu_synchronize, head);

WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) &&

/* Finally. */

complete(&rs->completion);

/* Advance to a new grace period and initialize state. */

record_gp_stall_check_time();

/* Record GP times before starting GP, hence rcu_seq_start(). */

rcu_seq_start(&rcu_state.gp_seq);

ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));

rcu_poll_gp_seq_start(&rcu_state.gp_seq_polled_snap);

raw_spin_unlock_irq_rcu_node(rnp);

static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func)

{

rcu_segcblist_enqueue(&rdp->cblist, head);

trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCBQueued"));

}

* critical sections have completed.

*

* Use this API instead of call_rcu() if you don't want the callback to be

* memory pressure and on systems which are lightly loaded or mostly idle.

* This function will cause callbacks to be invoked sooner than later at the

* expense of extra power. Other than that, this function is identical to, and

* might well execute concurrently with RCU read-side critical sections

* that started after call_rcu() was invoked.

*

* RCU read-side critical sections are delimited by rcu_read_lock()

* and rcu_read_unlock(), and may be nested. In addition, but only in

* v5.0 and later, regions of code across which interrupts, preemption,

*

* Implementation of these memory-ordering guarantees is described here:

* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst.

*/

void call_rcu(struct rcu_head *head, rcu_callback_t func)

{

* snapshot before adding a request.

*/

if (IS_ENABLED(CONFIG_PROVE_RCU))

rcu_sr_normal_add_req(&rs);

*/

void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)

{

/*

* Any prior manipulation of RCU-protected data must happen

* before the loads from ->gp_seq and ->expedited_sequence.

*/

smp_mb(); /* ^^^ */

rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence);

}

EXPORT_SYMBOL_GPL(get_state_synchronize_rcu_full);

* specified leaf rcu_node structure, which is acquired by the caller.

*/

static void rcu_report_exp_cpu_mult(struct rcu_node *rnp, unsigned long flags,

__releases(rnp->lock)

{

int cpu;

struct rcu_data *rdp;

raw_lockdep_assert_held_rcu_node(rnp);

raw_spin_unlock_irqrestore_rcu_node(rnp, flags);

return;

}

WRITE_ONCE(rnp->expmask, rnp->expmask & ~mask);

for_each_leaf_node_cpu_mask(rnp, cpu, mask) {

rdp = per_cpu_ptr(&rcu_data, cpu);

/* Dump out nocb kthread state for the specified rcu_data structure. */

static void show_rcu_nocb_state(struct rcu_data *rdp)

{

struct rcu_data *nocb_next_rdp;

struct rcu_segcblist *rsclp = &rdp->cblist;

bool waslocked;

typeof(*rdp),

nocb_entry_rdp);

rdp->cpu, rdp->nocb_gp_rdp->cpu,

nocb_next_rdp ? nocb_next_rdp->cpu : -1,

"kK"[!!rdp->nocb_cb_kthread],

jiffies - rdp->nocb_nobypass_last,

rdp->nocb_nobypass_count,

".D"[rcu_segcblist_ready_cbs(rsclp)],

".W"[!rcu_segcblist_segempty(rsclp, RCU_WAIT_TAIL)],

rcu_segcblist_segempty(rsclp, RCU_WAIT_TAIL) ? "" : bufw,

".R"[!rcu_segcblist_segempty(rsclp, RCU_NEXT_READY_TAIL)],

rcu_segcblist_segempty(rsclp, RCU_NEXT_READY_TAIL) ? "" : bufr,

".N"[!rcu_segcblist_segempty(rsclp, RCU_NEXT_TAIL)],

".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)],

rcu_segcblist_n_cbs(&rdp->cblist),

rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.',

rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1,

{

struct rcu_data *rdp;

return;

rdp = this_cpu_ptr(&rcu_data);

rdp->cpu_no_qs.b.norm = false;

rcu_report_qs_rdp(rdp);

*/

static void rcu_flavor_sched_clock_irq(int user)

{

/*

* Get here if this CPU took its interrupt from user

*

* No memory barrier is required here because rcu_qs()

* references only CPU-local variables that other CPUs

migrate_to_reboot_cpu();

syscore_shutdown();

pr_emerg("Power down\n");

kmsg_dump(KMSG_DUMP_SHUTDOWN);

machine_power_off();

}

return -EFAULT;

}

#else

static int rseq_validate_ro_fields(struct task_struct *t)

{

return 0;

}

#endif

/*

WARN_ON_ONCE((int) mm_cid < 0);

if (!user_write_access_begin(rseq, t->rseq_len))

goto efault;

/*

* Additional feature fields added after ORIG_RSEQ_SIZE

* need to be conditionally updated only if

* t->rseq_len != ORIG_RSEQ_SIZE.

*/

user_write_access_end();

trace_rseq_update(t);

return 0;

static int rseq_reset_rseq_cpu_node_id(struct task_struct *t)

{

u32 cpu_id_start = 0, cpu_id = RSEQ_CPU_ID_UNINITIALIZED, node_id = 0,

mm_cid = 0;

* Validate read-only rseq fields.

*/

if (rseq_validate_ro_fields(t))

/*

*/

/*

* Additional feature fields added after ORIG_RSEQ_SIZE

* need to be conditionally reset only if

* t->rseq_len != ORIG_RSEQ_SIZE.

*/