diff options
Diffstat (limited to 'kernel/stop_machine.c')
| -rw-r--r-- | kernel/stop_machine.c | 212 |
1 files changed, 212 insertions, 0 deletions
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c new file mode 100644 index 000000000000..c39ed70af174 --- /dev/null +++ b/kernel/stop_machine.c @@ -0,0 +1,212 @@ +#include <linux/stop_machine.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/cpu.h> +#include <linux/err.h> +#include <linux/syscalls.h> +#include <asm/atomic.h> +#include <asm/semaphore.h> +#include <asm/uaccess.h> + +/* Since we effect priority and affinity (both of which are visible + * to, and settable by outside processes) we do indirection via a + * kthread. */ + +/* Thread to stop each CPU in user context. */ +enum stopmachine_state { + STOPMACHINE_WAIT, + STOPMACHINE_PREPARE, + STOPMACHINE_DISABLE_IRQ, + STOPMACHINE_EXIT, +}; + +static enum stopmachine_state stopmachine_state; +static unsigned int stopmachine_num_threads; +static atomic_t stopmachine_thread_ack; +static DECLARE_MUTEX(stopmachine_mutex); + +static int stopmachine(void *cpu) +{ + int irqs_disabled = 0; + int prepared = 0; + + set_cpus_allowed(current, cpumask_of_cpu((int)(long)cpu)); + + /* Ack: we are alive */ + mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */ + atomic_inc(&stopmachine_thread_ack); + + /* Simple state machine */ + while (stopmachine_state != STOPMACHINE_EXIT) { + if (stopmachine_state == STOPMACHINE_DISABLE_IRQ + && !irqs_disabled) { + local_irq_disable(); + irqs_disabled = 1; + /* Ack: irqs disabled. */ + mb(); /* Must read state first. */ + atomic_inc(&stopmachine_thread_ack); + } else if (stopmachine_state == STOPMACHINE_PREPARE + && !prepared) { + /* Everyone is in place, hold CPU. */ + preempt_disable(); + prepared = 1; + mb(); /* Must read state first. */ + atomic_inc(&stopmachine_thread_ack); + } + /* Yield in first stage: migration threads need to + * help our sisters onto their CPUs. */ + if (!prepared && !irqs_disabled) + yield(); + else + cpu_relax(); + } + + /* Ack: we are exiting. */ + mb(); /* Must read state first. */ + atomic_inc(&stopmachine_thread_ack); + + if (irqs_disabled) + local_irq_enable(); + if (prepared) + preempt_enable(); + + return 0; +} + +/* Change the thread state */ +static void stopmachine_set_state(enum stopmachine_state state) +{ + atomic_set(&stopmachine_thread_ack, 0); + wmb(); + stopmachine_state = state; + while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) + cpu_relax(); +} + +static int stop_machine(void) +{ + int i, ret = 0; + struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; + mm_segment_t old_fs = get_fs(); + + /* One high-prio thread per cpu. We'll do this one. */ + set_fs(KERNEL_DS); + sys_sched_setscheduler(current->pid, SCHED_FIFO, + (struct sched_param __user *)¶m); + set_fs(old_fs); + + atomic_set(&stopmachine_thread_ack, 0); + stopmachine_num_threads = 0; + stopmachine_state = STOPMACHINE_WAIT; + + for_each_online_cpu(i) { + if (i == _smp_processor_id()) + continue; + ret = kernel_thread(stopmachine, (void *)(long)i,CLONE_KERNEL); + if (ret < 0) + break; + stopmachine_num_threads++; + } + + /* Wait for them all to come to life. */ + while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) + yield(); + + /* If some failed, kill them all. */ + if (ret < 0) { + stopmachine_set_state(STOPMACHINE_EXIT); + up(&stopmachine_mutex); + return ret; + } + + /* Don't schedule us away at this point, please. */ + local_irq_disable(); + + /* Now they are all started, make them hold the CPUs, ready. */ + stopmachine_set_state(STOPMACHINE_PREPARE); + + /* Make them disable irqs. */ + stopmachine_set_state(STOPMACHINE_DISABLE_IRQ); + + return 0; +} + +static void restart_machine(void) +{ + stopmachine_set_state(STOPMACHINE_EXIT); + local_irq_enable(); +} + +struct stop_machine_data +{ + int (*fn)(void *); + void *data; + struct completion done; +}; + +static int do_stop(void *_smdata) +{ + struct stop_machine_data *smdata = _smdata; + int ret; + + ret = stop_machine(); + if (ret == 0) { + ret = smdata->fn(smdata->data); + restart_machine(); + } + + /* We're done: you can kthread_stop us now */ + complete(&smdata->done); + + /* Wait for kthread_stop */ + set_current_state(TASK_INTERRUPTIBLE); + while (!kthread_should_stop()) { + schedule(); + set_current_state(TASK_INTERRUPTIBLE); + } + __set_current_state(TASK_RUNNING); + return ret; +} + +struct task_struct *__stop_machine_run(int (*fn)(void *), void *data, + unsigned int cpu) +{ + struct stop_machine_data smdata; + struct task_struct *p; + + smdata.fn = fn; + smdata.data = data; + init_completion(&smdata.done); + + down(&stopmachine_mutex); + + /* If they don't care which CPU fn runs on, bind to any online one. */ + if (cpu == NR_CPUS) + cpu = _smp_processor_id(); + + p = kthread_create(do_stop, &smdata, "kstopmachine"); + if (!IS_ERR(p)) { + kthread_bind(p, cpu); + wake_up_process(p); + wait_for_completion(&smdata.done); + } + up(&stopmachine_mutex); + return p; +} + +int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) +{ + struct task_struct *p; + int ret; + + /* No CPUs can come up or down during this. */ + lock_cpu_hotplug(); + p = __stop_machine_run(fn, data, cpu); + if (!IS_ERR(p)) + ret = kthread_stop(p); + else + ret = PTR_ERR(p); + unlock_cpu_hotplug(); + + return ret; +} |