xen-project/xen
1
0 Fork 1
mirror of https://github.com/xen-project/xen.git synced 2026-10-30 03:00:05 +01:00
Code Issues Activity

sched: rework credit2 run-queue allocation

Browse Source 
Currently the memory for each run-queue of the credit2 scheduler is
allocated at the scheduler's init function: for each cpu in the system
a struct csched2_runqueue_data is being allocated, even if the
current scheduler only handles one physical cpu or is configured to
work with a single run-queue. As each struct contains 4 cpumasks this
sums up to rather large memory sizes pretty fast.

Rework the memory allocation for run-queues to be done only when
needed, i.e. when adding a physical cpu to the scheduler requiring a
new run-queue.

In fact this fixes a bug in credit2 related to run-queue handling:
cpu_to_runqueue() will return the first free or matching run-queue,
which ever is found first. So in case a cpu is removed from credit2
this could result in e.g. run-queue 0 becoming free, so when another
cpu is added it will in any case be assigned to that free run-queue,
even if it would have found another run-queue matching later.

Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Dario Faggioli <dfaggioli@suse.com>
This commit is contained in:
Juergen Gross authored and Jan Beulich committed 2020-02-26 10:50:26 +01:00
1 parent d90bcb5f10
commit 9c84bc0046
1 file changed
+183 -188
Download Patch File Download Diff File Expand all files Collapse all files
xen/common/sched/credit2.c
+183 -188
View File
@@ -467,8 +467,12 @@ custom_param("credit2_runqueue", parse_credit2_runqueue);
struct csched2_runqueue_data { struct csched2_runqueue_data {
spinlock_t lock; /* Lock for this runqueue */ spinlock_t lock; /* Lock for this runqueue */
struct list_head rql; /* List of runqueues */
struct list_head runq; /* Ordered list of runnable vms */ struct list_head runq; /* Ordered list of runnable vms */
unsigned int refcnt; /* How many CPUs reference this runqueue */
/* (including not yet active ones) */
unsigned int nr_cpus; /* How many CPUs are sharing this runqueue */ unsigned int nr_cpus; /* How many CPUs are sharing this runqueue */
/* (only active ones) */
int id; /* ID of this runqueue (-1 if invalid) */ int id; /* ID of this runqueue (-1 if invalid) */
int load; /* Instantaneous load (num of non-idle units) */ int load; /* Instantaneous load (num of non-idle units) */
@@ -496,8 +500,8 @@ struct csched2_private {
unsigned int load_window_shift; /* Lenght of load decaying window */ unsigned int load_window_shift; /* Lenght of load decaying window */
unsigned int ratelimit_us; /* Rate limiting for this scheduler */ unsigned int ratelimit_us; /* Rate limiting for this scheduler */
cpumask_t active_queues; /* Runqueues with (maybe) active cpus */ unsigned int active_queues; /* Number of active runqueues */
struct csched2_runqueue_data *rqd; /* Data of the various runqueues */ struct list_head rql; /* List of runqueues */
cpumask_t initialized; /* CPUs part of this scheduler */ cpumask_t initialized; /* CPUs part of this scheduler */
struct list_head sdom; /* List of domains (for debug key) */ struct list_head sdom; /* List of domains (for debug key) */
@@ -508,7 +512,7 @@ struct csched2_private {
*/ */
struct csched2_pcpu { struct csched2_pcpu {
cpumask_t sibling_mask; /* Siblings in the same runqueue */ cpumask_t sibling_mask; /* Siblings in the same runqueue */
int runq_id; struct csched2_runqueue_data *rqd; /* Runqueue for this CPU */
}; };
/* /*
@@ -586,14 +590,13 @@ static inline struct csched2_dom *csched2_dom(const struct domain *d)
/* CPU to runq_id macro */ /* CPU to runq_id macro */
static inline int c2r(unsigned int cpu) static inline int c2r(unsigned int cpu)
{ {
return csched2_pcpu(cpu)->runq_id; return csched2_pcpu(cpu)->rqd->id;
} }
/* CPU to runqueue struct macro */ /* CPU to runqueue struct macro */
static inline struct csched2_runqueue_data *c2rqd(const struct scheduler *ops, static inline struct csched2_runqueue_data *c2rqd(unsigned int cpu)
unsigned int cpu)
{ {
return &csched2_priv(ops)->rqd[c2r(cpu)]; return csched2_pcpu(cpu)->rqd;
} }
/* Does the domain of this unit have a cap? */ /* Does the domain of this unit have a cap? */
@@ -804,36 +807,6 @@ static inline struct csched2_unit * runq_elem(struct list_head *elem)
return list_entry(elem, struct csched2_unit, runq_elem); return list_entry(elem, struct csched2_unit, runq_elem);
} }
static void activate_runqueue(struct csched2_private *prv, int rqi)
{
struct csched2_runqueue_data *rqd;
rqd = prv->rqd + rqi;
BUG_ON(!cpumask_empty(&rqd->active));
rqd->max_weight = 1;
rqd->id = rqi;
INIT_LIST_HEAD(&rqd->svc);
INIT_LIST_HEAD(&rqd->runq);
spin_lock_init(&rqd->lock);
__cpumask_set_cpu(rqi, &prv->active_queues);
}
static void deactivate_runqueue(struct csched2_private *prv, int rqi)
{
struct csched2_runqueue_data *rqd;
rqd = prv->rqd + rqi;
BUG_ON(!cpumask_empty(&rqd->active));
rqd->id = -1;
__cpumask_clear_cpu(rqi, &prv->active_queues);
}
static inline bool same_node(unsigned int cpua, unsigned int cpub) static inline bool same_node(unsigned int cpua, unsigned int cpub)
{ {
return cpu_to_node(cpua) == cpu_to_node(cpub); return cpu_to_node(cpua) == cpu_to_node(cpub);
@@ -850,51 +823,73 @@ static inline bool same_core(unsigned int cpua, unsigned int cpub)
cpu_to_core(cpua) == cpu_to_core(cpub); cpu_to_core(cpua) == cpu_to_core(cpub);
} }
static unsigned int static struct csched2_runqueue_data *
cpu_to_runqueue(const struct csched2_private *prv, unsigned int cpu) cpu_add_to_runqueue(struct csched2_private *prv, unsigned int cpu)
{ {
const struct csched2_runqueue_data *rqd; struct csched2_runqueue_data *rqd, *rqd_new;
unsigned int rqi; struct list_head *rqd_ins;
unsigned long flags;
int rqi = 0;
bool rqi_unused = false, rqd_valid = false;
for ( rqi = 0; rqi < nr_cpu_ids; rqi++ ) /* Prealloc in case we need it - not allowed with interrupts off. */
rqd_new = xzalloc(struct csched2_runqueue_data);
write_lock_irqsave(&prv->lock, flags);
rqd_ins = &prv->rql;
list_for_each_entry ( rqd, &prv->rql, rql )
{ {
unsigned int peer_cpu; unsigned int peer_cpu;
/* /* Remember first unused queue index. */
* As soon as we come across an uninitialized runqueue, use it. if ( !rqi_unused && rqd->id > rqi )
* In fact, either: rqi_unused = true;
* - we are initializing the first cpu, and we assign it to
* runqueue 0. This is handy, especially if we are dealing
* with the boot cpu (if credit2 is the default scheduler),
* as we would not be able to use cpu_to_socket() and similar
* helpers anyway (they're result of which is not reliable yet);
* - we have gone through all the active runqueues, and have not
* found anyone whose cpus' topology matches the one we are
* dealing with, so activating a new runqueue is what we want.
*/
if ( prv->rqd[rqi].id == -1 )
break;
rqd = prv->rqd + rqi; peer_cpu = rqd->pick_bias;
BUG_ON(cpumask_empty(&rqd->active));
peer_cpu = cpumask_first(&rqd->active);
BUG_ON(cpu_to_socket(cpu) == XEN_INVALID_SOCKET_ID || BUG_ON(cpu_to_socket(cpu) == XEN_INVALID_SOCKET_ID ||
cpu_to_socket(peer_cpu) == XEN_INVALID_SOCKET_ID); cpu_to_socket(peer_cpu) == XEN_INVALID_SOCKET_ID);
if (opt_runqueue == OPT_RUNQUEUE_CPU) /* OPT_RUNQUEUE_CPU will never find an existing runqueue. */
continue;
if ( opt_runqueue == OPT_RUNQUEUE_ALL || if ( opt_runqueue == OPT_RUNQUEUE_ALL ||
(opt_runqueue == OPT_RUNQUEUE_CORE && same_core(peer_cpu, cpu)) || (opt_runqueue == OPT_RUNQUEUE_CORE && same_core(peer_cpu, cpu)) ||
(opt_runqueue == OPT_RUNQUEUE_SOCKET && same_socket(peer_cpu, cpu)) || (opt_runqueue == OPT_RUNQUEUE_SOCKET && same_socket(peer_cpu, cpu)) ||
(opt_runqueue == OPT_RUNQUEUE_NODE && same_node(peer_cpu, cpu)) ) (opt_runqueue == OPT_RUNQUEUE_NODE && same_node(peer_cpu, cpu)) )
{
rqd_valid = true;
break; break;
}
if ( !rqi_unused )
{
rqi++;
rqd_ins = &rqd->rql;
}
} }
/* We really expect to be able to assign each cpu to a runqueue. */ if ( !rqd_valid )
BUG_ON(rqi >= nr_cpu_ids); {
if ( !rqd_new )
{
rqd = ERR_PTR(-ENOMEM);
goto out;
}
rqd = rqd_new;
rqd_new = NULL;
return rqi; list_add(&rqd->rql, rqd_ins);
rqd->pick_bias = cpu;
rqd->id = rqi;
}
rqd->refcnt++;
out:
write_unlock_irqrestore(&prv->lock, flags);
xfree(rqd_new);
return rqd;
} }
/* Find the domain with the highest weight. */ /* Find the domain with the highest weight. */
@@ -972,13 +967,13 @@ _runq_assign(struct csched2_unit *svc, struct csched2_runqueue_data *rqd)
} }
static void static void
runq_assign(const struct scheduler *ops, const struct sched_unit *unit) runq_assign(const struct sched_unit *unit)
{ {
struct csched2_unit *svc = unit->priv; struct csched2_unit *svc = unit->priv;
ASSERT(svc->rqd == NULL); ASSERT(svc->rqd == NULL);
_runq_assign(svc, c2rqd(ops, sched_unit_master(unit))); _runq_assign(svc, c2rqd(sched_unit_master(unit)));
} }
static void static void
@@ -999,11 +994,11 @@ _runq_deassign(struct csched2_unit *svc)
} }
static void static void
runq_deassign(const struct scheduler *ops, const struct sched_unit *unit) runq_deassign(const struct sched_unit *unit)
{ {
struct csched2_unit *svc = unit->priv; struct csched2_unit *svc = unit->priv;
ASSERT(svc->rqd == c2rqd(ops, sched_unit_master(unit))); ASSERT(svc->rqd == c2rqd(sched_unit_master(unit)));
_runq_deassign(svc); _runq_deassign(svc);
} }
@@ -1272,12 +1267,11 @@ update_load(const struct scheduler *ops,
update_svc_load(ops, svc, change, now); update_svc_load(ops, svc, change, now);
} }
static void static void runq_insert(struct csched2_unit *svc)
runq_insert(const struct scheduler *ops, struct csched2_unit *svc)
{ {
struct list_head *iter; struct list_head *iter;
unsigned int cpu = sched_unit_master(svc->unit); unsigned int cpu = sched_unit_master(svc->unit);
struct list_head * runq = &c2rqd(ops, cpu)->runq; struct list_head *runq = &c2rqd(cpu)->runq;
int pos = 0; int pos = 0;
ASSERT(spin_is_locked(get_sched_res(cpu)->schedule_lock)); ASSERT(spin_is_locked(get_sched_res(cpu)->schedule_lock));
@@ -1366,7 +1360,7 @@ static inline bool is_preemptable(const struct csched2_unit *svc,
static s_time_t tickle_score(const struct scheduler *ops, s_time_t now, static s_time_t tickle_score(const struct scheduler *ops, s_time_t now,
const struct csched2_unit *new, unsigned int cpu) const struct csched2_unit *new, unsigned int cpu)
{ {
struct csched2_runqueue_data *rqd = c2rqd(ops, cpu); struct csched2_runqueue_data *rqd = c2rqd(cpu);
struct csched2_unit * cur = csched2_unit(curr_on_cpu(cpu)); struct csched2_unit * cur = csched2_unit(curr_on_cpu(cpu));
const struct csched2_private *prv = csched2_priv(ops); const struct csched2_private *prv = csched2_priv(ops);
s_time_t score; s_time_t score;
@@ -1442,7 +1436,7 @@ runq_tickle(const struct scheduler *ops, struct csched2_unit *new, s_time_t now)
s_time_t max = 0; s_time_t max = 0;
struct sched_unit *unit = new->unit; struct sched_unit *unit = new->unit;
unsigned int bs, cpu = sched_unit_master(unit); unsigned int bs, cpu = sched_unit_master(unit);
struct csched2_runqueue_data *rqd = c2rqd(ops, cpu); struct csched2_runqueue_data *rqd = c2rqd(cpu);
const cpumask_t *online = cpupool_domain_master_cpumask(unit->domain); const cpumask_t *online = cpupool_domain_master_cpumask(unit->domain);
cpumask_t mask; cpumask_t mask;
@@ -1618,10 +1612,9 @@ runq_tickle(const struct scheduler *ops, struct csched2_unit *new, s_time_t now)
/* /*
* Credit-related code * Credit-related code
*/ */
static void reset_credit(const struct scheduler *ops, int cpu, s_time_t now, static void reset_credit(int cpu, s_time_t now, struct csched2_unit *snext)
struct csched2_unit *snext)
{ {
struct csched2_runqueue_data *rqd = c2rqd(ops, cpu); struct csched2_runqueue_data *rqd = c2rqd(cpu);
struct list_head *iter; struct list_head *iter;
int m; int m;
@@ -1910,7 +1903,7 @@ unpark_parked_units(const struct scheduler *ops, struct list_head *units)
* for the newly replenished budget. * for the newly replenished budget.
*/ */
ASSERT( svc->rqd != NULL ); ASSERT( svc->rqd != NULL );
ASSERT( c2rqd(ops, sched_unit_master(svc->unit)) == svc->rqd ); ASSERT( c2rqd(sched_unit_master(svc->unit)) == svc->rqd );
__set_bit(__CSFLAG_delayed_runq_add, &svc->flags); __set_bit(__CSFLAG_delayed_runq_add, &svc->flags);
} }
else if ( unit_runnable(svc->unit) ) else if ( unit_runnable(svc->unit) )
@@ -1923,7 +1916,7 @@ unpark_parked_units(const struct scheduler *ops, struct list_head *units)
*/ */
now = NOW(); now = NOW();
update_load(ops, svc->rqd, svc, 1, now); update_load(ops, svc->rqd, svc, 1, now);
runq_insert(ops, svc); runq_insert(svc);
runq_tickle(ops, svc, now); runq_tickle(ops, svc, now);
} }
list_del_init(&svc->parked_elem); list_del_init(&svc->parked_elem);
@@ -2088,7 +2081,7 @@ csched2_unit_sleep(const struct scheduler *ops, struct sched_unit *unit)
} }
else if ( unit_on_runq(svc) ) else if ( unit_on_runq(svc) )
{ {
ASSERT(svc->rqd == c2rqd(ops, sched_unit_master(unit))); ASSERT(svc->rqd == c2rqd(sched_unit_master(unit)));
update_load(ops, svc->rqd, svc, -1, NOW()); update_load(ops, svc->rqd, svc, -1, NOW());
runq_remove(svc); runq_remove(svc);
} }
@@ -2135,16 +2128,16 @@ csched2_unit_wake(const struct scheduler *ops, struct sched_unit *unit)
/* Add into the new runqueue if necessary */ /* Add into the new runqueue if necessary */
if ( svc->rqd == NULL ) if ( svc->rqd == NULL )
runq_assign(ops, unit); runq_assign(unit);
else else
ASSERT(c2rqd(ops, sched_unit_master(unit)) == svc->rqd ); ASSERT(c2rqd(sched_unit_master(unit)) == svc->rqd );
now = NOW(); now = NOW();
update_load(ops, svc->rqd, svc, 1, now); update_load(ops, svc->rqd, svc, 1, now);
/* Put the UNIT on the runq */ /* Put the UNIT on the runq */
runq_insert(ops, svc); runq_insert(svc);
runq_tickle(ops, svc, now); runq_tickle(ops, svc, now);
out: out:
@@ -2168,7 +2161,7 @@ csched2_context_saved(const struct scheduler *ops, struct sched_unit *unit)
LIST_HEAD(were_parked); LIST_HEAD(were_parked);
ASSERT(is_idle_unit(unit) || ASSERT(is_idle_unit(unit) ||
svc->rqd == c2rqd(ops, sched_unit_master(unit))); svc->rqd == c2rqd(sched_unit_master(unit)));
/* This unit is now eligible to be put on the runqueue again */ /* This unit is now eligible to be put on the runqueue again */
__clear_bit(__CSFLAG_scheduled, &svc->flags); __clear_bit(__CSFLAG_scheduled, &svc->flags);
@@ -2189,7 +2182,7 @@ csched2_context_saved(const struct scheduler *ops, struct sched_unit *unit)
{ {
ASSERT(!unit_on_runq(svc)); ASSERT(!unit_on_runq(svc));
runq_insert(ops, svc); runq_insert(svc);
runq_tickle(ops, svc, now); runq_tickle(ops, svc, now);
} }
else if ( !is_idle_unit(unit) ) else if ( !is_idle_unit(unit) )
@@ -2205,13 +2198,13 @@ static struct sched_resource *
csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit) csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
{ {
struct csched2_private *prv = csched2_priv(ops); struct csched2_private *prv = csched2_priv(ops);
int i, min_rqi = -1, min_s_rqi = -1;
unsigned int new_cpu, cpu = sched_unit_master(unit); unsigned int new_cpu, cpu = sched_unit_master(unit);
struct csched2_unit *svc = csched2_unit(unit); struct csched2_unit *svc = csched2_unit(unit);
s_time_t min_avgload = MAX_LOAD, min_s_avgload = MAX_LOAD; s_time_t min_avgload = MAX_LOAD, min_s_avgload = MAX_LOAD;
bool has_soft; bool has_soft;
struct csched2_runqueue_data *rqd, *min_rqd = NULL, *min_s_rqd = NULL;
ASSERT(!cpumask_empty(&prv->active_queues)); ASSERT(!list_empty(&prv->rql));
SCHED_STAT_CRANK(pick_resource); SCHED_STAT_CRANK(pick_resource);
@@ -2289,13 +2282,10 @@ csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
* Find both runqueues in one pass. * Find both runqueues in one pass.
*/ */
has_soft = has_soft_affinity(unit); has_soft = has_soft_affinity(unit);
for_each_cpu(i, &prv->active_queues) list_for_each_entry ( rqd, &prv->rql, rql )
{ {
struct csched2_runqueue_data *rqd;
s_time_t rqd_avgload = MAX_LOAD; s_time_t rqd_avgload = MAX_LOAD;
rqd = prv->rqd + i;
/* /*
* If none of the cpus of this runqueue is in svc's hard-affinity, * If none of the cpus of this runqueue is in svc's hard-affinity,
* skip the runqueue. * skip the runqueue.
@@ -2338,18 +2328,18 @@ csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
if ( cpumask_intersects(&mask, unit->cpu_soft_affinity) ) if ( cpumask_intersects(&mask, unit->cpu_soft_affinity) )
{ {
min_s_avgload = rqd_avgload; min_s_avgload = rqd_avgload;
min_s_rqi = i; min_s_rqd = rqd;
} }
} }
/* In any case, keep the "hard-affinity minimum" updated too. */ /* In any case, keep the "hard-affinity minimum" updated too. */
if ( rqd_avgload < min_avgload ) if ( rqd_avgload < min_avgload )
{ {
min_avgload = rqd_avgload; min_avgload = rqd_avgload;
min_rqi = i; min_rqd = rqd;
} }
} }
if ( has_soft && min_s_rqi != -1 ) if ( has_soft && min_s_rqd )
{ {
/* /*
* We have soft affinity, and we have a candidate runq, so go for it. * We have soft affinity, and we have a candidate runq, so go for it.
@@ -2369,9 +2359,9 @@ csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
cpumask_and(cpumask_scratch_cpu(cpu), cpumask_scratch_cpu(cpu), cpumask_and(cpumask_scratch_cpu(cpu), cpumask_scratch_cpu(cpu),
unit->cpu_soft_affinity); unit->cpu_soft_affinity);
cpumask_and(cpumask_scratch_cpu(cpu), cpumask_scratch_cpu(cpu), cpumask_and(cpumask_scratch_cpu(cpu), cpumask_scratch_cpu(cpu),
&prv->rqd[min_s_rqi].active); &min_s_rqd->active);
} }
else if ( min_rqi != -1 ) else if ( min_rqd )
{ {
/* /*
* Either we don't have soft-affinity, or we do, but we did not find * Either we don't have soft-affinity, or we do, but we did not find
@@ -2383,7 +2373,7 @@ csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
* with the cpus of the runq. * with the cpus of the runq.
*/ */
cpumask_and(cpumask_scratch_cpu(cpu), cpumask_scratch_cpu(cpu), cpumask_and(cpumask_scratch_cpu(cpu), cpumask_scratch_cpu(cpu),
&prv->rqd[min_rqi].active); &min_rqd->active);
} }
else else
{ {
@@ -2392,14 +2382,13 @@ csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
* contention). * contention).
*/ */
new_cpu = get_fallback_cpu(svc); new_cpu = get_fallback_cpu(svc);
min_rqi = c2r(new_cpu); min_rqd = c2rqd(new_cpu);
min_avgload = prv->rqd[min_rqi].b_avgload; min_avgload = min_rqd->b_avgload;
goto out_up; goto out_up;
} }
new_cpu = cpumask_cycle(prv->rqd[min_rqi].pick_bias, new_cpu = cpumask_cycle(min_rqd->pick_bias, cpumask_scratch_cpu(cpu));
cpumask_scratch_cpu(cpu)); min_rqd->pick_bias = new_cpu;
prv->rqd[min_rqi].pick_bias = new_cpu;
BUG_ON(new_cpu >= nr_cpu_ids); BUG_ON(new_cpu >= nr_cpu_ids);
out_up: out_up:
@@ -2414,7 +2403,7 @@ csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
} d; } d;
d.dom = unit->domain->domain_id; d.dom = unit->domain->domain_id;
d.unit = unit->unit_id; d.unit = unit->unit_id;
d.rq_id = min_rqi; d.rq_id = min_rqd->id;
d.b_avgload = min_avgload; d.b_avgload = min_avgload;
d.new_cpu = new_cpu; d.new_cpu = new_cpu;
__trace_var(TRC_CSCHED2_PICKED_CPU, 1, __trace_var(TRC_CSCHED2_PICKED_CPU, 1,
@@ -2527,7 +2516,7 @@ static void migrate(const struct scheduler *ops,
if ( on_runq ) if ( on_runq )
{ {
update_load(ops, svc->rqd, NULL, 1, now); update_load(ops, svc->rqd, NULL, 1, now);
runq_insert(ops, svc); runq_insert(svc);
runq_tickle(ops, svc, now); runq_tickle(ops, svc, now);
SCHED_STAT_CRANK(migrate_on_runq); SCHED_STAT_CRANK(migrate_on_runq);
} }
@@ -2557,9 +2546,9 @@ static bool unit_is_migrateable(const struct csched2_unit *svc,
static void balance_load(const struct scheduler *ops, int cpu, s_time_t now) static void balance_load(const struct scheduler *ops, int cpu, s_time_t now)
{ {
struct csched2_private *prv = csched2_priv(ops); struct csched2_private *prv = csched2_priv(ops);
int i, max_delta_rqi;
struct list_head *push_iter, *pull_iter; struct list_head *push_iter, *pull_iter;
bool inner_load_updated = 0; bool inner_load_updated = 0;
struct csched2_runqueue_data *rqd, *max_delta_rqd;
balance_state_t st = { .best_push_svc = NULL, .best_pull_svc = NULL }; balance_state_t st = { .best_push_svc = NULL, .best_pull_svc = NULL };
@@ -2571,22 +2560,22 @@ static void balance_load(const struct scheduler *ops, int cpu, s_time_t now)
*/ */
ASSERT(spin_is_locked(get_sched_res(cpu)->schedule_lock)); ASSERT(spin_is_locked(get_sched_res(cpu)->schedule_lock));
st.lrqd = c2rqd(ops, cpu); st.lrqd = c2rqd(cpu);
update_runq_load(ops, st.lrqd, 0, now); update_runq_load(ops, st.lrqd, 0, now);
retry: retry:
max_delta_rqi = -1; max_delta_rqd = NULL;
if ( !read_trylock(&prv->lock) ) if ( !read_trylock(&prv->lock) )
return; return;
st.load_delta = 0; st.load_delta = 0;
for_each_cpu(i, &prv->active_queues) list_for_each_entry ( rqd, &prv->rql, rql )
{ {
s_time_t delta; s_time_t delta;
st.orqd = prv->rqd + i; st.orqd = rqd;
if ( st.orqd == st.lrqd if ( st.orqd == st.lrqd
|| !spin_trylock(&st.orqd->lock) ) || !spin_trylock(&st.orqd->lock) )
@@ -2601,7 +2590,7 @@ retry:
if ( delta > st.load_delta ) if ( delta > st.load_delta )
{ {
st.load_delta = delta; st.load_delta = delta;
max_delta_rqi = i; max_delta_rqd = rqd;
} }
spin_unlock(&st.orqd->lock); spin_unlock(&st.orqd->lock);
@@ -2609,7 +2598,7 @@ retry:
/* Minimize holding the private scheduler lock. */ /* Minimize holding the private scheduler lock. */
read_unlock(&prv->lock); read_unlock(&prv->lock);
if ( max_delta_rqi == -1 ) if ( !max_delta_rqd )
goto out; goto out;
{ {
@@ -2621,10 +2610,7 @@ retry:
if ( st.orqd->b_avgload > load_max ) if ( st.orqd->b_avgload > load_max )
load_max = st.orqd->b_avgload; load_max = st.orqd->b_avgload;
cpus_max = st.lrqd->nr_cpus; cpus_max = max(st.lrqd->nr_cpus, st.orqd->nr_cpus);
i = st.orqd->nr_cpus;
if ( i > cpus_max )
cpus_max = i;
if ( unlikely(tb_init_done) ) if ( unlikely(tb_init_done) )
{ {
@@ -2660,7 +2646,7 @@ retry:
* meantime, try the process over again. This can't deadlock * meantime, try the process over again. This can't deadlock
* because if it doesn't get any other rqd locks, it will simply * because if it doesn't get any other rqd locks, it will simply
* give up and return. */ * give up and return. */
st.orqd = prv->rqd + max_delta_rqi; st.orqd = max_delta_rqd;
if ( !spin_trylock(&st.orqd->lock) ) if ( !spin_trylock(&st.orqd->lock) )
goto retry; goto retry;
@@ -2751,7 +2737,7 @@ csched2_unit_migrate(
ASSERT(cpumask_test_cpu(new_cpu, &csched2_priv(ops)->initialized)); ASSERT(cpumask_test_cpu(new_cpu, &csched2_priv(ops)->initialized));
ASSERT(cpumask_test_cpu(new_cpu, unit->cpu_hard_affinity)); ASSERT(cpumask_test_cpu(new_cpu, unit->cpu_hard_affinity));
trqd = c2rqd(ops, new_cpu); trqd = c2rqd(new_cpu);
/* /*
* Do the actual movement toward new_cpu, and update vc->processor. * Do the actual movement toward new_cpu, and update vc->processor.
@@ -2815,7 +2801,7 @@ csched2_dom_cntl(
struct csched2_unit *svc = csched2_unit(unit); struct csched2_unit *svc = csched2_unit(unit);
spinlock_t *lock = unit_schedule_lock(unit); spinlock_t *lock = unit_schedule_lock(unit);
ASSERT(svc->rqd == c2rqd(ops, sched_unit_master(unit))); ASSERT(svc->rqd == c2rqd(sched_unit_master(unit)));
svc->weight = sdom->weight; svc->weight = sdom->weight;
update_max_weight(svc->rqd, svc->weight, old_weight); update_max_weight(svc->rqd, svc->weight, old_weight);
@@ -2898,7 +2884,7 @@ csched2_dom_cntl(
if ( unit->is_running ) if ( unit->is_running )
{ {
unsigned int cpu = sched_unit_master(unit); unsigned int cpu = sched_unit_master(unit);
struct csched2_runqueue_data *rqd = c2rqd(ops, cpu); struct csched2_runqueue_data *rqd = c2rqd(cpu);
ASSERT(curr_on_cpu(cpu) == unit); ASSERT(curr_on_cpu(cpu) == unit);
@@ -3093,7 +3079,7 @@ csched2_unit_insert(const struct scheduler *ops, struct sched_unit *unit)
lock = unit_schedule_lock_irq(unit); lock = unit_schedule_lock_irq(unit);
/* Add unit to runqueue of initial processor */ /* Add unit to runqueue of initial processor */
runq_assign(ops, unit); runq_assign(unit);
unit_schedule_unlock_irq(lock, unit); unit_schedule_unlock_irq(lock, unit);
@@ -3126,7 +3112,7 @@ csched2_unit_remove(const struct scheduler *ops, struct sched_unit *unit)
/* Remove from runqueue */ /* Remove from runqueue */
lock = unit_schedule_lock_irq(unit); lock = unit_schedule_lock_irq(unit);
runq_deassign(ops, unit); runq_deassign(unit);
unit_schedule_unlock_irq(lock, unit); unit_schedule_unlock_irq(lock, unit);
@@ -3140,7 +3126,7 @@ csched2_runtime(const struct scheduler *ops, int cpu,
{ {
s_time_t time, min_time; s_time_t time, min_time;
int rt_credit; /* Proposed runtime measured in credits */ int rt_credit; /* Proposed runtime measured in credits */
struct csched2_runqueue_data *rqd = c2rqd(ops, cpu); struct csched2_runqueue_data *rqd = c2rqd(cpu);
struct list_head *runq = &rqd->runq; struct list_head *runq = &rqd->runq;
const struct csched2_private *prv = csched2_priv(ops); const struct csched2_private *prv = csched2_priv(ops);
@@ -3437,7 +3423,7 @@ static void csched2_schedule(
BUG_ON(!cpumask_test_cpu(sched_cpu, &csched2_priv(ops)->initialized)); BUG_ON(!cpumask_test_cpu(sched_cpu, &csched2_priv(ops)->initialized));
rqd = c2rqd(ops, sched_cpu); rqd = c2rqd(sched_cpu);
BUG_ON(!cpumask_test_cpu(sched_cpu, &rqd->active)); BUG_ON(!cpumask_test_cpu(sched_cpu, &rqd->active));
ASSERT(spin_is_locked(get_sched_res(sched_cpu)->schedule_lock)); ASSERT(spin_is_locked(get_sched_res(sched_cpu)->schedule_lock));
@@ -3551,7 +3537,7 @@ static void csched2_schedule(
*/ */
if ( skipped_units == 0 && snext->credit <= CSCHED2_CREDIT_RESET ) if ( skipped_units == 0 && snext->credit <= CSCHED2_CREDIT_RESET )
{ {
reset_credit(ops, sched_cpu, now, snext); reset_credit(sched_cpu, now, snext);
balance_load(ops, sched_cpu, now); balance_load(ops, sched_cpu, now);
} }
@@ -3650,7 +3636,8 @@ csched2_dump(const struct scheduler *ops)
struct list_head *iter_sdom; struct list_head *iter_sdom;
struct csched2_private *prv = csched2_priv(ops); struct csched2_private *prv = csched2_priv(ops);
unsigned long flags; unsigned long flags;
unsigned int i, j, loop; unsigned int j, loop;
struct csched2_runqueue_data *rqd;
/* /*
* We need the private scheduler lock as we access global * We need the private scheduler lock as we access global
@@ -3660,13 +3647,13 @@ csched2_dump(const struct scheduler *ops)
printk("Active queues: %d\n" printk("Active queues: %d\n"
"\tdefault-weight = %d\n", "\tdefault-weight = %d\n",
cpumask_weight(&prv->active_queues), prv->active_queues,
CSCHED2_DEFAULT_WEIGHT); CSCHED2_DEFAULT_WEIGHT);
for_each_cpu(i, &prv->active_queues) list_for_each_entry ( rqd, &prv->rql, rql )
{ {
s_time_t fraction; s_time_t fraction;
fraction = (prv->rqd[i].avgload * 100) >> prv->load_precision_shift; fraction = (rqd->avgload * 100) >> prv->load_precision_shift;
printk("Runqueue %d:\n" printk("Runqueue %d:\n"
"\tncpus = %u\n" "\tncpus = %u\n"
@@ -3675,21 +3662,21 @@ csched2_dump(const struct scheduler *ops)
"\tpick_bias = %u\n" "\tpick_bias = %u\n"
"\tinstload = %d\n" "\tinstload = %d\n"
"\taveload = %"PRI_stime" (~%"PRI_stime"%%)\n", "\taveload = %"PRI_stime" (~%"PRI_stime"%%)\n",
i, rqd->id,
prv->rqd[i].nr_cpus, rqd->nr_cpus,
CPUMASK_PR(&prv->rqd[i].active), CPUMASK_PR(&rqd->active),
prv->rqd[i].max_weight, rqd->max_weight,
prv->rqd[i].pick_bias, rqd->pick_bias,
prv->rqd[i].load, rqd->load,
prv->rqd[i].avgload, rqd->avgload,
fraction); fraction);
printk("\tidlers: %*pb\n" printk("\tidlers: %*pb\n"
"\ttickled: %*pb\n" "\ttickled: %*pb\n"
"\tfully idle cores: %*pb\n", "\tfully idle cores: %*pb\n",
CPUMASK_PR(&prv->rqd[i].idle), CPUMASK_PR(&rqd->idle),
CPUMASK_PR(&prv->rqd[i].tickled), CPUMASK_PR(&rqd->tickled),
CPUMASK_PR(&prv->rqd[i].smt_idle)); CPUMASK_PR(&rqd->smt_idle));
} }
printk("Domain info:\n"); printk("Domain info:\n");
@@ -3721,16 +3708,15 @@ csched2_dump(const struct scheduler *ops)
} }
} }
for_each_cpu(i, &prv->active_queues) list_for_each_entry ( rqd, &prv->rql, rql )
{ {
struct csched2_runqueue_data *rqd = prv->rqd + i;
struct list_head *iter, *runq = &rqd->runq; struct list_head *iter, *runq = &rqd->runq;
int loop = 0; int loop = 0;
/* We need the lock to scan the runqueue. */ /* We need the lock to scan the runqueue. */
spin_lock(&rqd->lock); spin_lock(&rqd->lock);
printk("Runqueue %d:\n", i); printk("Runqueue %d:\n", rqd->id);
for_each_cpu(j, &rqd->active) for_each_cpu(j, &rqd->active)
dump_pcpu(ops, j); dump_pcpu(ops, j);
@@ -3755,20 +3741,28 @@ csched2_dump(const struct scheduler *ops)
static void * static void *
csched2_alloc_pdata(const struct scheduler *ops, int cpu) csched2_alloc_pdata(const struct scheduler *ops, int cpu)
{ {
struct csched2_private *prv = csched2_priv(ops);
struct csched2_pcpu *spc; struct csched2_pcpu *spc;
struct csched2_runqueue_data *rqd;
spc = xzalloc(struct csched2_pcpu); spc = xzalloc(struct csched2_pcpu);
if ( spc == NULL ) if ( spc == NULL )
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
/* Not in any runqueue yet */ rqd = cpu_add_to_runqueue(prv, cpu);
spc->runq_id = -1; if ( IS_ERR(rqd) )
{
xfree(spc);
return rqd;
}
spc->rqd = rqd;
return spc; return spc;
} }
/* Returns the ID of the runqueue the cpu is assigned to. */ /* Returns the ID of the runqueue the cpu is assigned to. */
static unsigned static struct csched2_runqueue_data *
init_pdata(struct csched2_private *prv, struct csched2_pcpu *spc, init_pdata(struct csched2_private *prv, struct csched2_pcpu *spc,
unsigned int cpu) unsigned int cpu)
{ {
@@ -3778,18 +3772,23 @@ init_pdata(struct csched2_private *prv, struct csched2_pcpu *spc,
ASSERT(rw_is_write_locked(&prv->lock)); ASSERT(rw_is_write_locked(&prv->lock));
ASSERT(!cpumask_test_cpu(cpu, &prv->initialized)); ASSERT(!cpumask_test_cpu(cpu, &prv->initialized));
/* CPU data needs to be allocated, but still uninitialized. */ /* CPU data needs to be allocated, but still uninitialized. */
ASSERT(spc && spc->runq_id == -1); ASSERT(spc);
/* Figure out which runqueue to put it in */ rqd = spc->rqd;
spc->runq_id = cpu_to_runqueue(prv, cpu);
rqd = prv->rqd + spc->runq_id; ASSERT(rqd && !cpumask_test_cpu(cpu, &spc->rqd->active));
printk(XENLOG_INFO "Adding cpu %d to runqueue %d\n", cpu, spc->runq_id); printk(XENLOG_INFO "Adding cpu %d to runqueue %d\n", cpu, rqd->id);
if ( ! cpumask_test_cpu(spc->runq_id, &prv->active_queues) ) if ( !rqd->nr_cpus )
{ {
printk(XENLOG_INFO " First cpu on runqueue, activating\n"); printk(XENLOG_INFO " First cpu on runqueue, activating\n");
activate_runqueue(prv, spc->runq_id);
BUG_ON(!cpumask_empty(&rqd->active));
rqd->max_weight = 1;
INIT_LIST_HEAD(&rqd->svc);
INIT_LIST_HEAD(&rqd->runq);
spin_lock_init(&rqd->lock);
prv->active_queues++;
} }
__cpumask_set_cpu(cpu, &spc->sibling_mask); __cpumask_set_cpu(cpu, &spc->sibling_mask);
@@ -3813,7 +3812,7 @@ init_pdata(struct csched2_private *prv, struct csched2_pcpu *spc,
if ( rqd->nr_cpus == 1 ) if ( rqd->nr_cpus == 1 )
rqd->pick_bias = cpu; rqd->pick_bias = cpu;
return spc->runq_id; return rqd;
} }
/* Change the scheduler of cpu to us (Credit2). */ /* Change the scheduler of cpu to us (Credit2). */
@@ -3823,7 +3822,7 @@ csched2_switch_sched(struct scheduler *new_ops, unsigned int cpu,
{ {
struct csched2_private *prv = csched2_priv(new_ops); struct csched2_private *prv = csched2_priv(new_ops);
struct csched2_unit *svc = vdata; struct csched2_unit *svc = vdata;
unsigned rqi; struct csched2_runqueue_data *rqd;
ASSERT(pdata && svc && is_idle_unit(svc->unit)); ASSERT(pdata && svc && is_idle_unit(svc->unit));
@@ -3840,7 +3839,7 @@ csched2_switch_sched(struct scheduler *new_ops, unsigned int cpu,
sched_idle_unit(cpu)->priv = vdata; sched_idle_unit(cpu)->priv = vdata;
rqi = init_pdata(prv, pdata, cpu); rqd = init_pdata(prv, pdata, cpu);
/* /*
* Now that we know what runqueue we'll go in, double check what's said * Now that we know what runqueue we'll go in, double check what's said
@@ -3848,11 +3847,11 @@ csched2_switch_sched(struct scheduler *new_ops, unsigned int cpu,
* this scheduler, and so it's safe to have taken it /before/ our * this scheduler, and so it's safe to have taken it /before/ our
* private global lock. * private global lock.
*/ */
ASSERT(get_sched_res(cpu)->schedule_lock != &prv->rqd[rqi].lock); ASSERT(get_sched_res(cpu)->schedule_lock != &rqd->lock);
write_unlock(&prv->lock); write_unlock(&prv->lock);
return &prv->rqd[rqi].lock; return &rqd->lock;
} }
static void static void
@@ -3866,10 +3865,6 @@ csched2_deinit_pdata(const struct scheduler *ops, void *pcpu, int cpu)
write_lock_irqsave(&prv->lock, flags); write_lock_irqsave(&prv->lock, flags);
/*
* alloc_pdata is not implemented, so pcpu must be NULL. On the other
* hand, init_pdata must have been called for this pCPU.
*/
/* /*
* Scheduler specific data for this pCPU must still be there and and be * Scheduler specific data for this pCPU must still be there and and be
* valid. In fact, if we are here: * valid. In fact, if we are here:
@@ -3878,20 +3873,21 @@ csched2_deinit_pdata(const struct scheduler *ops, void *pcpu, int cpu)
* 2. init_pdata must have been called on this cpu, and deinit_pdata * 2. init_pdata must have been called on this cpu, and deinit_pdata
* (us!) must not have been called on it already. * (us!) must not have been called on it already.
*/ */
ASSERT(spc && spc->runq_id != -1); ASSERT(spc && spc->rqd);
ASSERT(cpumask_test_cpu(cpu, &prv->initialized)); ASSERT(cpumask_test_cpu(cpu, &prv->initialized));
/* Find the old runqueue and remove this cpu from it */ /* Find the old runqueue and remove this cpu from it */
rqd = prv->rqd + spc->runq_id; rqd = spc->rqd;
/* No need to save IRQs here, they're already disabled */ /* No need to save IRQs here, they're already disabled */
spin_lock(&rqd->lock); spin_lock(&rqd->lock);
printk(XENLOG_INFO "Removing cpu %d from runqueue %d\n", cpu, spc->runq_id); printk(XENLOG_INFO "Removing cpu %d from runqueue %d\n", cpu, rqd->id);
__cpumask_clear_cpu(cpu, &rqd->idle); __cpumask_clear_cpu(cpu, &rqd->idle);
__cpumask_clear_cpu(cpu, &rqd->smt_idle); __cpumask_clear_cpu(cpu, &rqd->smt_idle);
__cpumask_clear_cpu(cpu, &rqd->active); __cpumask_clear_cpu(cpu, &rqd->active);
__cpumask_clear_cpu(cpu, &rqd->tickled);
for_each_cpu ( rcpu, &rqd->active ) for_each_cpu ( rcpu, &rqd->active )
__cpumask_clear_cpu(cpu, &csched2_pcpu(rcpu)->sibling_mask); __cpumask_clear_cpu(cpu, &csched2_pcpu(rcpu)->sibling_mask);
@@ -3902,13 +3898,13 @@ csched2_deinit_pdata(const struct scheduler *ops, void *pcpu, int cpu)
if ( rqd->nr_cpus == 0 ) if ( rqd->nr_cpus == 0 )
{ {
printk(XENLOG_INFO " No cpus left on runqueue, disabling\n"); printk(XENLOG_INFO " No cpus left on runqueue, disabling\n");
deactivate_runqueue(prv, spc->runq_id);
BUG_ON(!cpumask_empty(&rqd->active));
prv->active_queues--;
} }
else if ( rqd->pick_bias == cpu ) else if ( rqd->pick_bias == cpu )
rqd->pick_bias = cpumask_first(&rqd->active); rqd->pick_bias = cpumask_first(&rqd->active);
spc->runq_id = -1;
spin_unlock(&rqd->lock); spin_unlock(&rqd->lock);
__cpumask_clear_cpu(cpu, &prv->initialized); __cpumask_clear_cpu(cpu, &prv->initialized);
@@ -3921,18 +3917,29 @@ csched2_deinit_pdata(const struct scheduler *ops, void *pcpu, int cpu)
static void static void
csched2_free_pdata(const struct scheduler *ops, void *pcpu, int cpu) csched2_free_pdata(const struct scheduler *ops, void *pcpu, int cpu)
{ {
struct csched2_private *prv = csched2_priv(ops);
struct csched2_pcpu *spc = pcpu; struct csched2_pcpu *spc = pcpu;
struct csched2_runqueue_data *rqd;
unsigned long flags;
/* if ( !spc )
* pcpu either points to a valid struct csched2_pcpu, or is NULL (if return;
* CPU bringup failed, and we're beeing called from CPU_UP_CANCELLED).
* xfree() does not really mind, but we want to be sure that either
* init_pdata has never been called, or deinit_pdata has been called
* already.
*/
ASSERT(!pcpu || spc->runq_id == -1);
ASSERT(!cpumask_test_cpu(cpu, &csched2_priv(ops)->initialized));
write_lock_irqsave(&prv->lock, flags);
rqd = spc->rqd;
ASSERT(rqd && rqd->refcnt);
ASSERT(!cpumask_test_cpu(cpu, &prv->initialized));
rqd->refcnt--;
if ( !rqd->refcnt )
list_del(&rqd->rql);
else
rqd = NULL;
write_unlock_irqrestore(&prv->lock, flags);
xfree(rqd);
xfree(pcpu); xfree(pcpu);
} }
@@ -3966,7 +3973,6 @@ csched2_global_init(void)
static int static int
csched2_init(struct scheduler *ops) csched2_init(struct scheduler *ops)
{ {
int i;
struct csched2_private *prv; struct csched2_private *prv;
printk("Initializing Credit2 scheduler\n"); printk("Initializing Credit2 scheduler\n");
@@ -3999,18 +4005,9 @@ csched2_init(struct scheduler *ops)
ops->sched_data = prv; ops->sched_data = prv;
rwlock_init(&prv->lock); rwlock_init(&prv->lock);
INIT_LIST_HEAD(&prv->rql);
INIT_LIST_HEAD(&prv->sdom); INIT_LIST_HEAD(&prv->sdom);
/* Allocate all runqueues and mark them as un-initialized */
prv->rqd = xzalloc_array(struct csched2_runqueue_data, nr_cpu_ids);
if ( !prv->rqd )
{
xfree(prv);
return -ENOMEM;
}
for ( i = 0; i < nr_cpu_ids; i++ )
prv->rqd[i].id = -1;
/* initialize ratelimit */ /* initialize ratelimit */
prv->ratelimit_us = sched_ratelimit_us; prv->ratelimit_us = sched_ratelimit_us;
@@ -4028,8 +4025,6 @@ csched2_deinit(struct scheduler *ops)
prv = csched2_priv(ops); prv = csched2_priv(ops);
ops->sched_data = NULL; ops->sched_data = NULL;
if ( prv )
xfree(prv->rqd);
xfree(prv); xfree(prv);
} }
Reference in new issue
xen-project/xen
Block a user
Blocking a user prevents them from interacting with repositories, such as opening or commenting on pull requests or issues. Learn more about blocking a user.

The note is not visible to the blocked user.