* Copyright (c) 2022 Tejun Heo <tj@kernel.org>

* Copyright (c) 2022 David Vernet <dvernet@meta.com>

*/

#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void)))

enum scx_consts {

/*

* Keep built-in idle tracking even if ops.update_idle() is implemented.

*/

/*

* By default, if there are no other task to run on the CPU, ext core

* flag is specified, such tasks are passed to ops.enqueue() with

* %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info.

*/

/*

* An exiting task may schedule after PF_EXITING is set. In such cases,

* depend on pid lookups and wants to handle these tasks directly, the

* following flag can be used.

*/

/*

* If set, only tasks with policy set to SCHED_EXT are attached to

* sched_ext. If clear, SCHED_NORMAL tasks are also included.

*/

/*

* A migration disabled task can only execute on its current CPU. By

* current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr

* and thus may disagree with cpumask_weight(p->cpus_ptr).

*/

/*

* CPU cgroup support flags

SCX_OPS_ENQ_LAST |

SCX_OPS_ENQ_EXITING |

SCX_OPS_ENQ_MIGRATION_DISABLED |

SCX_OPS_SWITCH_PARTIAL |

SCX_OPS_HAS_CGROUP_WEIGHT,

};

enum scx_pick_idle_cpu_flags {

SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */

};

enum scx_kick_flags {

static struct sched_ext_ops scx_ops;

static bool scx_warned_zero_slice;

static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last);

static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting);

static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_migration_disabled);

static DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt);

static struct static_key_false scx_has_op[SCX_OPI_END] =

{ [0 ... SCX_OPI_END-1] = STATIC_KEY_FALSE_INIT };

static struct delayed_work scx_watchdog_work;

/* for %SCX_KICK_WAIT */

static unsigned long __percpu *scx_kick_cpus_pnt_seqs;

return p;

}

static enum scx_ops_enable_state scx_ops_enable_state(void)

{

return atomic_read(&scx_ops_enable_state_var);

if (!scx_rq_online(rq))

goto local;

goto global;

if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)

goto direct;

/* see %SCX_OPS_ENQ_EXITING */

if (!static_branch_unlikely(&scx_ops_enq_exiting) &&

goto local;

/* see %SCX_OPS_ENQ_MIGRATION_DISABLED */

if (!static_branch_unlikely(&scx_ops_enq_migration_disabled) &&

goto local;

if (!SCX_HAS_OP(enqueue))

goto global;

*/

touch_core_sched(rq, p);

p->scx.slice = SCX_SLICE_DFL;

local_norefill:

dispatch_enqueue(&rq->scx.local_dsq, p, enq_flags);

return;

global:

touch_core_sched(rq, p); /* see the comment in local: */

p->scx.slice = SCX_SLICE_DFL;

dispatch_enqueue(find_global_dsq(p), p, enq_flags);

}

do_enqueue_task(rq, p, enq_flags, sticky_cpu);

out:

rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;

}

static void ops_dequeue(struct task_struct *p, u64 deq_flags)

* The caller must ensure that @p and @rq are on different CPUs.

*/

static bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq,

{

int cpu = cpu_of(rq);

/*

* If @p has migration disabled, @p->cpus_ptr is updated to contain only

* easily be masked if task_allowed_on_cpu() is done first.

*/

if (unlikely(is_migration_disabled(p))) {

scx_ops_error("SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d",

p->comm, p->pid, task_cpu(p), cpu);

return false;

* picked CPU is outside the allowed mask.

*/

if (!task_allowed_on_cpu(p, cpu)) {

scx_ops_error("SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]",

cpu, p->comm, p->pid);

return false;

}

return false;

return true;

}

}

#else /* CONFIG_SMP */

static inline void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq) { WARN_ON_ONCE(1); }

static inline bool consume_remote_task(struct rq *this_rq, struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *task_rq) { return false; }

#endif /* CONFIG_SMP */

if (prev_on_rq && (!static_branch_unlikely(&scx_ops_enq_last) ||

scx_rq_bypassing(rq))) {

rq->scx.flags |= SCX_RQ_BAL_KEEP;

goto has_tasks;

}

rq->scx.flags &= ~SCX_RQ_IN_BALANCE;

*/

if (keep_prev) {

p = prev;

p->scx.slice = SCX_SLICE_DFL;

} else {

p = first_local_task(rq);

if (!p) {

scx_warned_zero_slice = true;

}

p->scx.slice = SCX_SLICE_DFL;

}

}

#ifdef CONFIG_SMP

static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags)

{

/*

* sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it

* can be a good migration opportunity with low cache and memory

if (unlikely(wake_flags & WF_EXEC))

return prev_cpu;

s32 cpu;

struct task_struct **ddsp_taskp;

cpu = SCX_CALL_OP_TASK_RET(SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU,

select_cpu, p, prev_cpu, wake_flags);

*ddsp_taskp = NULL;

if (ops_cpu_valid(cpu, "from ops.select_cpu()"))

return cpu;

else

return prev_cpu;

} else {

s32 cpu;

p->scx.slice = SCX_SLICE_DFL;

p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL;

}

return cpu;

}

}

(struct cpumask *)p->cpus_ptr);

}

static void handle_hotplug(struct rq *rq, bool online)

{

int cpu = cpu_of(rq);

atomic_long_inc(&scx_hotplug_seq);

if (scx_enabled())

if (online && SCX_HAS_OP(cpu_online))

SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_online, cpu);

rq->scx.flags &= ~SCX_RQ_ONLINE;

}

#endif /* CONFIG_SMP */

static bool check_rq_for_timeouts(struct rq *rq)

}

SCX_ATTR(ops);

static struct attribute *scx_sched_attrs[] = {

&scx_attr_ops.attr,

NULL,

};

ATTRIBUTE_GROUPS(scx_sched);

static void scx_ops_bypass(bool bypass)

{

static DEFINE_RAW_SPINLOCK(bypass_lock);

int cpu;

unsigned long flags;

WARN_ON_ONCE(scx_ops_bypass_depth <= 0);

if (scx_ops_bypass_depth != 1)

goto unlock;

} else {

scx_ops_bypass_depth--;

WARN_ON_ONCE(scx_ops_bypass_depth < 0);

if (scx_ops_bypass_depth != 0)

goto unlock;

}

atomic_inc(&scx_ops_breather_depth);

static_branch_disable(&__scx_ops_enabled);

for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)

static_branch_disable(&scx_has_op[i]);

static_branch_disable(&scx_ops_enq_last);

static_branch_disable(&scx_ops_enq_exiting);

static_branch_disable(&scx_ops_enq_migration_disabled);

static_branch_disable(&scx_ops_cpu_preempt);

synchronize_rcu();

if (ei->kind >= SCX_EXIT_ERROR) {

.at_jiffies = jiffies,

};

struct seq_buf s;

unsigned long flags;

char *buf;

int cpu;

rq_unlock(rq, &rf);

}

if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker))

memcpy(ei->dump + dump_len - sizeof(trunc_marker),

trunc_marker, sizeof(trunc_marker));

return -EINVAL;

}

return 0;

}

mutex_lock(&scx_ops_enable_mutex);

if (!scx_ops_helper) {

WRITE_ONCE(scx_ops_helper,

scx_create_rt_helper("sched_ext_ops_helper"));

static_branch_enable_cpuslocked(&scx_has_op[i]);

check_hotplug_seq(ops);

cpus_read_unlock();

ret = validate_ops(ops);

if (((void (**)(void))ops)[i])

static_branch_enable(&scx_has_op[i]);

if (ops->flags & SCX_OPS_ENQ_LAST)

static_branch_enable(&scx_ops_enq_last);

if (ops->flags & SCX_OPS_ENQ_EXITING)

static_branch_enable(&scx_ops_enq_exiting);

if (ops->flags & SCX_OPS_ENQ_MIGRATION_DISABLED)

if (scx_ops.cpu_acquire || scx_ops.cpu_release)

static_branch_enable(&scx_ops_cpu_preempt);

/*

* Lock out forks, cgroup on/offlining and moves before opening the

SCX_TG_ONLINE);

BUG_ON(rhashtable_init(&dsq_hash, &dsq_hash_params));

scx_kick_cpus_pnt_seqs =

__alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * nr_cpu_ids,

__alignof__(scx_kick_cpus_pnt_seqs[0]));

for_each_possible_cpu(cpu) {

struct rq *rq = cpu_rq(cpu);

init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL);

INIT_LIST_HEAD(&rq->scx.runnable_list);

INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals);

init_irq_work(&rq->scx.deferred_irq_work, deferred_irq_workfn);

init_irq_work(&rq->scx.kick_cpus_irq_work, kick_cpus_irq_workfn);

/********************************************************************************

* Helpers that can be called from the BPF scheduler.

*/

static bool scx_dsq_insert_preamble(struct task_struct *p, u64 enq_flags)

{

if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH))

* CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to

* the current local DSQ for running tasks and thus are not

* visible to the BPF scheduler.

*/

continue;

dispatch_dequeue(rq, p);

}

/**

* scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs

*

* All valid CPU IDs in the system are smaller than the returned value.

}

/**

* scx_bpf_task_running - Is task currently running?

* @p: task of interest

*/

return clock;

}

__bpf_kfunc_end_defs();

BTF_KFUNCS_START(scx_kfunc_ids_any)

BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap)

BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur)

BTF_ID_FLAGS(func, scx_bpf_cpuperf_set)

BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)

BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)

BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)

BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_RCU | KF_ACQUIRE)

#endif

BTF_ID_FLAGS(func, scx_bpf_now)

BTF_KFUNCS_END(scx_kfunc_ids_any)

static const struct btf_kfunc_id_set scx_kfunc_set_any = {

* check using scx_kf_allowed().

*/

if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,

&scx_kfunc_set_enqueue_dispatch)) ||

(ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,

&scx_kfunc_set_dispatch)) ||

return ret;

}

ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops);

if (ret) {

pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret);