423 lines
13 KiB
C
423 lines
13 KiB
C
|
// SPDX-License-Identifier: GPL-2.0
|
||
|
|
/*
|
||
|
|
* Copyright 2018 Linaro Limited
|
||
|
|
*
|
||
|
|
* Author: Daniel Lezcano <daniel.lezcano@linaro.org>
|
||
|
|
*
|
||
|
|
* The idle injection framework provides a way to force CPUs to enter idle
|
||
|
|
* states for a specified fraction of time over a specified period.
|
||
|
|
*
|
||
|
|
* It relies on the smpboot kthreads feature providing common code for CPU
|
||
|
|
* hotplug and thread [un]parking.
|
||
|
|
*
|
||
|
|
* All of the kthreads used for idle injection are created at init time.
|
||
|
|
*
|
||
|
|
* Next, the users of the idle injection framework provide a cpumask via
|
||
|
|
* its register function. The kthreads will be synchronized with respect to
|
||
|
|
* this cpumask.
|
||
|
|
*
|
||
|
|
* The idle + run duration is specified via separate helpers and that allows
|
||
|
|
* idle injection to be started.
|
||
|
|
*
|
||
|
|
* The idle injection kthreads will call play_idle_precise() with the idle
|
||
|
|
* duration and max allowed latency specified as per the above.
|
||
|
|
*
|
||
|
|
* After all of them have been woken up, a timer is set to start the next idle
|
||
|
|
* injection cycle.
|
||
|
|
*
|
||
|
|
* The timer interrupt handler will wake up the idle injection kthreads for
|
||
|
|
* all of the CPUs in the cpumask provided by the user.
|
||
|
|
*
|
||
|
|
* Idle injection is stopped synchronously and no leftover idle injection
|
||
|
|
* kthread activity after its completion is guaranteed.
|
||
|
|
*
|
||
|
|
* It is up to the user of this framework to provide a lock for higher-level
|
||
|
|
* synchronization to prevent race conditions like starting idle injection
|
||
|
|
* while unregistering from the framework.
|
||
|
|
*/
|
||
|
|
#define pr_fmt(fmt) "ii_dev: " fmt
|
||
|
|
|
||
|
|
#include <linux/cpu.h>
|
||
|
|
#include <linux/hrtimer.h>
|
||
|
|
#include <linux/kthread.h>
|
||
|
|
#include <linux/sched.h>
|
||
|
|
#include <linux/slab.h>
|
||
|
|
#include <linux/smpboot.h>
|
||
|
|
#include <linux/idle_inject.h>
|
||
|
|
|
||
|
|
#include <uapi/linux/sched/types.h>
|
||
|
|
|
||
|
|
/**
|
||
|
|
* struct idle_inject_thread - task on/off switch structure
|
||
|
|
* @tsk: task injecting the idle cycles
|
||
|
|
* @should_run: whether or not to run the task (for the smpboot kthread API)
|
||
|
|
*/
|
||
|
|
struct idle_inject_thread {
|
||
|
|
struct task_struct *tsk;
|
||
|
|
int should_run;
|
||
|
|
};
|
||
|
|
|
||
|
|
/**
|
||
|
|
* struct idle_inject_device - idle injection data
|
||
|
|
* @timer: idle injection period timer
|
||
|
|
* @idle_duration_us: duration of CPU idle time to inject
|
||
|
|
* @run_duration_us: duration of CPU run time to allow
|
||
|
|
* @latency_us: max allowed latency
|
||
|
|
* @update: Optional callback deciding whether or not to skip idle
|
||
|
|
* injection in the given cycle.
|
||
|
|
* @cpumask: mask of CPUs affected by idle injection
|
||
|
|
*
|
||
|
|
* This structure is used to define per instance idle inject device data. Each
|
||
|
|
* instance has an idle duration, a run duration and mask of CPUs to inject
|
||
|
|
* idle.
|
||
|
|
*
|
||
|
|
* Actual CPU idle time is injected by calling kernel scheduler interface
|
||
|
|
* play_idle_precise(). There is one optional callback that can be registered
|
||
|
|
* by calling idle_inject_register_full():
|
||
|
|
*
|
||
|
|
* update() - This callback is invoked just before waking up CPUs to inject
|
||
|
|
* idle. If it returns false, CPUs are not woken up to inject idle in the given
|
||
|
|
* cycle. It also allows the caller to readjust the idle and run duration by
|
||
|
|
* calling idle_inject_set_duration() for the next cycle.
|
||
|
|
*/
|
||
|
|
struct idle_inject_device {
|
||
|
|
struct hrtimer timer;
|
||
|
|
unsigned int idle_duration_us;
|
||
|
|
unsigned int run_duration_us;
|
||
|
|
unsigned int latency_us;
|
||
|
|
bool (*update)(void);
|
||
|
|
unsigned long cpumask[];
|
||
|
|
};
|
||
|
|
|
||
|
|
static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
|
||
|
|
static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_wakeup - Wake up idle injection threads
|
||
|
|
* @ii_dev: target idle injection device
|
||
|
|
*
|
||
|
|
* Every idle injection task associated with the given idle injection device
|
||
|
|
* and running on an online CPU will be woken up.
|
||
|
|
*/
|
||
|
|
static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
|
||
|
|
{
|
||
|
|
struct idle_inject_thread *iit;
|
||
|
|
unsigned int cpu;
|
||
|
|
|
||
|
|
for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
|
||
|
|
iit = per_cpu_ptr(&idle_inject_thread, cpu);
|
||
|
|
iit->should_run = 1;
|
||
|
|
wake_up_process(iit->tsk);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_timer_fn - idle injection timer function
|
||
|
|
* @timer: idle injection hrtimer
|
||
|
|
*
|
||
|
|
* This function is called when the idle injection timer expires. It wakes up
|
||
|
|
* idle injection tasks associated with the timer and they, in turn, invoke
|
||
|
|
* play_idle_precise() to inject a specified amount of CPU idle time.
|
||
|
|
*
|
||
|
|
* Return: HRTIMER_RESTART.
|
||
|
|
*/
|
||
|
|
static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
|
||
|
|
{
|
||
|
|
unsigned int duration_us;
|
||
|
|
struct idle_inject_device *ii_dev =
|
||
|
|
container_of(timer, struct idle_inject_device, timer);
|
||
|
|
|
||
|
|
if (!ii_dev->update || (ii_dev->update && ii_dev->update()))
|
||
|
|
idle_inject_wakeup(ii_dev);
|
||
|
|
|
||
|
|
duration_us = READ_ONCE(ii_dev->run_duration_us);
|
||
|
|
duration_us += READ_ONCE(ii_dev->idle_duration_us);
|
||
|
|
|
||
|
|
hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));
|
||
|
|
|
||
|
|
return HRTIMER_RESTART;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_fn - idle injection work function
|
||
|
|
* @cpu: the CPU owning the task
|
||
|
|
*
|
||
|
|
* This function calls play_idle_precise() to inject a specified amount of CPU
|
||
|
|
* idle time.
|
||
|
|
*/
|
||
|
|
static void idle_inject_fn(unsigned int cpu)
|
||
|
|
{
|
||
|
|
struct idle_inject_device *ii_dev;
|
||
|
|
struct idle_inject_thread *iit;
|
||
|
|
|
||
|
|
ii_dev = per_cpu(idle_inject_device, cpu);
|
||
|
|
iit = per_cpu_ptr(&idle_inject_thread, cpu);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Let the smpboot main loop know that the task should not run again.
|
||
|
|
*/
|
||
|
|
iit->should_run = 0;
|
||
|
|
|
||
|
|
play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
|
||
|
|
READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_set_duration - idle and run duration update helper
|
||
|
|
* @ii_dev: idle injection control device structure
|
||
|
|
* @run_duration_us: CPU run time to allow in microseconds
|
||
|
|
* @idle_duration_us: CPU idle time to inject in microseconds
|
||
|
|
*/
|
||
|
|
void idle_inject_set_duration(struct idle_inject_device *ii_dev,
|
||
|
|
unsigned int run_duration_us,
|
||
|
|
unsigned int idle_duration_us)
|
||
|
|
{
|
||
|
|
if (run_duration_us + idle_duration_us) {
|
||
|
|
WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
|
||
|
|
WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
|
||
|
|
}
|
||
|
|
if (!run_duration_us)
|
||
|
|
pr_debug("CPU is forced to 100 percent idle\n");
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, IDLE_INJECT);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_get_duration - idle and run duration retrieval helper
|
||
|
|
* @ii_dev: idle injection control device structure
|
||
|
|
* @run_duration_us: memory location to store the current CPU run time
|
||
|
|
* @idle_duration_us: memory location to store the current CPU idle time
|
||
|
|
*/
|
||
|
|
void idle_inject_get_duration(struct idle_inject_device *ii_dev,
|
||
|
|
unsigned int *run_duration_us,
|
||
|
|
unsigned int *idle_duration_us)
|
||
|
|
{
|
||
|
|
*run_duration_us = READ_ONCE(ii_dev->run_duration_us);
|
||
|
|
*idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, IDLE_INJECT);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_set_latency - set the maximum latency allowed
|
||
|
|
* @ii_dev: idle injection control device structure
|
||
|
|
* @latency_us: set the latency requirement for the idle state
|
||
|
|
*/
|
||
|
|
void idle_inject_set_latency(struct idle_inject_device *ii_dev,
|
||
|
|
unsigned int latency_us)
|
||
|
|
{
|
||
|
|
WRITE_ONCE(ii_dev->latency_us, latency_us);
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, IDLE_INJECT);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_start - start idle injections
|
||
|
|
* @ii_dev: idle injection control device structure
|
||
|
|
*
|
||
|
|
* The function starts idle injection by first waking up all of the idle
|
||
|
|
* injection kthreads associated with @ii_dev to let them inject CPU idle time
|
||
|
|
* sets up a timer to start the next idle injection period.
|
||
|
|
*
|
||
|
|
* Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
|
||
|
|
*/
|
||
|
|
int idle_inject_start(struct idle_inject_device *ii_dev)
|
||
|
|
{
|
||
|
|
unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
|
||
|
|
unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);
|
||
|
|
|
||
|
|
if (!(idle_duration_us + run_duration_us))
|
||
|
|
return -EINVAL;
|
||
|
|
|
||
|
|
pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
|
||
|
|
cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
|
||
|
|
|
||
|
|
idle_inject_wakeup(ii_dev);
|
||
|
|
|
||
|
|
hrtimer_start(&ii_dev->timer,
|
||
|
|
ns_to_ktime((idle_duration_us + run_duration_us) *
|
||
|
|
NSEC_PER_USEC),
|
||
|
|
HRTIMER_MODE_REL);
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL_NS_GPL(idle_inject_start, IDLE_INJECT);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_stop - stops idle injections
|
||
|
|
* @ii_dev: idle injection control device structure
|
||
|
|
*
|
||
|
|
* The function stops idle injection and waits for the threads to finish work.
|
||
|
|
* If CPU idle time is being injected when this function runs, then it will
|
||
|
|
* wait until the end of the cycle.
|
||
|
|
*
|
||
|
|
* When it returns, there is no more idle injection kthread activity. The
|
||
|
|
* kthreads are scheduled out and the periodic timer is off.
|
||
|
|
*/
|
||
|
|
void idle_inject_stop(struct idle_inject_device *ii_dev)
|
||
|
|
{
|
||
|
|
struct idle_inject_thread *iit;
|
||
|
|
unsigned int cpu;
|
||
|
|
|
||
|
|
pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
|
||
|
|
cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
|
||
|
|
|
||
|
|
hrtimer_cancel(&ii_dev->timer);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Stopping idle injection requires all of the idle injection kthreads
|
||
|
|
* associated with the given cpumask to be parked and stay that way, so
|
||
|
|
* prevent CPUs from going online at this point. Any CPUs going online
|
||
|
|
* after the loop below will be covered by clearing the should_run flag
|
||
|
|
* that will cause the smpboot main loop to schedule them out.
|
||
|
|
*/
|
||
|
|
cpu_hotplug_disable();
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Iterate over all (online + offline) CPUs here in case one of them
|
||
|
|
* goes offline with the should_run flag set so as to prevent its idle
|
||
|
|
* injection kthread from running when the CPU goes online again after
|
||
|
|
* the ii_dev has been freed.
|
||
|
|
*/
|
||
|
|
for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
|
||
|
|
iit = per_cpu_ptr(&idle_inject_thread, cpu);
|
||
|
|
iit->should_run = 0;
|
||
|
|
|
||
|
|
wait_task_inactive(iit->tsk, TASK_ANY);
|
||
|
|
}
|
||
|
|
|
||
|
|
cpu_hotplug_enable();
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL_NS_GPL(idle_inject_stop, IDLE_INJECT);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_setup - prepare the current task for idle injection
|
||
|
|
* @cpu: not used
|
||
|
|
*
|
||
|
|
* Called once, this function is in charge of setting the current task's
|
||
|
|
* scheduler parameters to make it an RT task.
|
||
|
|
*/
|
||
|
|
static void idle_inject_setup(unsigned int cpu)
|
||
|
|
{
|
||
|
|
sched_set_fifo(current);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_should_run - function helper for the smpboot API
|
||
|
|
* @cpu: CPU the kthread is running on
|
||
|
|
*
|
||
|
|
* Return: whether or not the thread can run.
|
||
|
|
*/
|
||
|
|
static int idle_inject_should_run(unsigned int cpu)
|
||
|
|
{
|
||
|
|
struct idle_inject_thread *iit =
|
||
|
|
per_cpu_ptr(&idle_inject_thread, cpu);
|
||
|
|
|
||
|
|
return iit->should_run;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_register_full - initialize idle injection on a set of CPUs
|
||
|
|
* @cpumask: CPUs to be affected by idle injection
|
||
|
|
* @update: This callback is called just before waking up CPUs to inject
|
||
|
|
* idle
|
||
|
|
*
|
||
|
|
* This function creates an idle injection control device structure for the
|
||
|
|
* given set of CPUs and initializes the timer associated with it. This
|
||
|
|
* function also allows to register update()callback.
|
||
|
|
* It does not start any injection cycles.
|
||
|
|
*
|
||
|
|
* Return: NULL if memory allocation fails, idle injection control device
|
||
|
|
* pointer on success.
|
||
|
|
*/
|
||
|
|
|
||
|
|
struct idle_inject_device *idle_inject_register_full(struct cpumask *cpumask,
|
||
|
|
bool (*update)(void))
|
||
|
|
{
|
||
|
|
struct idle_inject_device *ii_dev;
|
||
|
|
int cpu, cpu_rb;
|
||
|
|
|
||
|
|
ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
|
||
|
|
if (!ii_dev)
|
||
|
|
return NULL;
|
||
|
|
|
||
|
|
cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
|
||
|
|
hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
|
||
|
|
ii_dev->timer.function = idle_inject_timer_fn;
|
||
|
|
ii_dev->latency_us = UINT_MAX;
|
||
|
|
ii_dev->update = update;
|
||
|
|
|
||
|
|
for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
|
||
|
|
|
||
|
|
if (per_cpu(idle_inject_device, cpu)) {
|
||
|
|
pr_err("cpu%d is already registered\n", cpu);
|
||
|
|
goto out_rollback;
|
||
|
|
}
|
||
|
|
|
||
|
|
per_cpu(idle_inject_device, cpu) = ii_dev;
|
||
|
|
}
|
||
|
|
|
||
|
|
return ii_dev;
|
||
|
|
|
||
|
|
out_rollback:
|
||
|
|
for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
|
||
|
|
if (cpu == cpu_rb)
|
||
|
|
break;
|
||
|
|
per_cpu(idle_inject_device, cpu_rb) = NULL;
|
||
|
|
}
|
||
|
|
|
||
|
|
kfree(ii_dev);
|
||
|
|
|
||
|
|
return NULL;
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL_NS_GPL(idle_inject_register_full, IDLE_INJECT);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_register - initialize idle injection on a set of CPUs
|
||
|
|
* @cpumask: CPUs to be affected by idle injection
|
||
|
|
*
|
||
|
|
* This function creates an idle injection control device structure for the
|
||
|
|
* given set of CPUs and initializes the timer associated with it. It does not
|
||
|
|
* start any injection cycles.
|
||
|
|
*
|
||
|
|
* Return: NULL if memory allocation fails, idle injection control device
|
||
|
|
* pointer on success.
|
||
|
|
*/
|
||
|
|
struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
|
||
|
|
{
|
||
|
|
return idle_inject_register_full(cpumask, NULL);
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL_NS_GPL(idle_inject_register, IDLE_INJECT);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* idle_inject_unregister - unregister idle injection control device
|
||
|
|
* @ii_dev: idle injection control device to unregister
|
||
|
|
*
|
||
|
|
* The function stops idle injection for the given control device,
|
||
|
|
* unregisters its kthreads and frees memory allocated when that device was
|
||
|
|
* created.
|
||
|
|
*/
|
||
|
|
void idle_inject_unregister(struct idle_inject_device *ii_dev)
|
||
|
|
{
|
||
|
|
unsigned int cpu;
|
||
|
|
|
||
|
|
idle_inject_stop(ii_dev);
|
||
|
|
|
||
|
|
for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
|
||
|
|
per_cpu(idle_inject_device, cpu) = NULL;
|
||
|
|
|
||
|
|
kfree(ii_dev);
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, IDLE_INJECT);
|
||
|
|
|
||
|
|
static struct smp_hotplug_thread idle_inject_threads = {
|
||
|
|
.store = &idle_inject_thread.tsk,
|
||
|
|
.setup = idle_inject_setup,
|
||
|
|
.thread_fn = idle_inject_fn,
|
||
|
|
.thread_comm = "idle_inject/%u",
|
||
|
|
.thread_should_run = idle_inject_should_run,
|
||
|
|
};
|
||
|
|
|
||
|
|
static int __init idle_inject_init(void)
|
||
|
|
{
|
||
|
|
return smpboot_register_percpu_thread(&idle_inject_threads);
|
||
|
|
}
|
||
|
|
early_initcall(idle_inject_init);
|