SYNC_WORKLOAD = 2
};
+struct cfqg_stats {
+#ifdef CONFIG_CFQ_GROUP_IOSCHED
+ /* total bytes transferred */
+ struct blkg_rwstat service_bytes;
+ /* total IOs serviced, post merge */
+ struct blkg_rwstat serviced;
+ /* number of ios merged */
+ struct blkg_rwstat merged;
+ /* total time spent on device in ns, may not be accurate w/ queueing */
+ struct blkg_rwstat service_time;
+ /* total time spent waiting in scheduler queue in ns */
+ struct blkg_rwstat wait_time;
+ /* number of IOs queued up */
+ struct blkg_rwstat queued;
+ /* total sectors transferred */
+ struct blkg_stat sectors;
+ /* total disk time and nr sectors dispatched by this group */
+ struct blkg_stat time;
+#ifdef CONFIG_DEBUG_BLK_CGROUP
+ /* time not charged to this cgroup */
+ struct blkg_stat unaccounted_time;
+ /* sum of number of ios queued across all samples */
+ struct blkg_stat avg_queue_size_sum;
+ /* count of samples taken for average */
+ struct blkg_stat avg_queue_size_samples;
+ /* how many times this group has been removed from service tree */
+ struct blkg_stat dequeue;
+ /* total time spent waiting for it to be assigned a timeslice. */
+ struct blkg_stat group_wait_time;
+ /* time spent idling for this blkio_group */
+ struct blkg_stat idle_time;
+ /* total time with empty current active q with other requests queued */
+ struct blkg_stat empty_time;
+ /* fields after this shouldn't be cleared on stat reset */
+ uint64_t start_group_wait_time;
+ uint64_t start_idle_time;
+ uint64_t start_empty_time;
+ uint16_t flags;
+#endif /* CONFIG_DEBUG_BLK_CGROUP */
+#endif /* CONFIG_CFQ_GROUP_IOSCHED */
+};
+
/* This is per cgroup per device grouping structure */
struct cfq_group {
/* group service_tree member */
u64 vdisktime;
unsigned int weight;
unsigned int new_weight;
- bool needs_update;
+ unsigned int dev_weight;
/* number of cfqq currently on this group */
int nr_cfqq;
/* number of requests that are on the dispatch list or inside driver */
int dispatched;
struct cfq_ttime ttime;
+ struct cfqg_stats stats;
};
struct cfq_io_cq {
#if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
-/* blkg state flags */
-enum blkg_state_flags {
- BLKG_waiting = 0,
- BLKG_idling,
- BLKG_empty,
+/* cfqg stats flags */
+enum cfqg_stats_flags {
+ CFQG_stats_waiting = 0,
+ CFQG_stats_idling,
+ CFQG_stats_empty,
};
-#define BLKG_FLAG_FNS(name) \
-static inline void blkio_mark_blkg_##name( \
- struct blkio_group_stats *stats) \
+#define CFQG_FLAG_FNS(name) \
+static inline void cfqg_stats_mark_##name(struct cfqg_stats *stats) \
{ \
- stats->flags |= (1 << BLKG_##name); \
+ stats->flags |= (1 << CFQG_stats_##name); \
} \
-static inline void blkio_clear_blkg_##name( \
- struct blkio_group_stats *stats) \
+static inline void cfqg_stats_clear_##name(struct cfqg_stats *stats) \
{ \
- stats->flags &= ~(1 << BLKG_##name); \
+ stats->flags &= ~(1 << CFQG_stats_##name); \
} \
-static inline int blkio_blkg_##name(struct blkio_group_stats *stats) \
+static inline int cfqg_stats_##name(struct cfqg_stats *stats) \
{ \
- return (stats->flags & (1 << BLKG_##name)) != 0; \
+ return (stats->flags & (1 << CFQG_stats_##name)) != 0; \
} \
-BLKG_FLAG_FNS(waiting)
-BLKG_FLAG_FNS(idling)
-BLKG_FLAG_FNS(empty)
-#undef BLKG_FLAG_FNS
+CFQG_FLAG_FNS(waiting)
+CFQG_FLAG_FNS(idling)
+CFQG_FLAG_FNS(empty)
+#undef CFQG_FLAG_FNS
/* This should be called with the queue_lock held. */
-static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
+static void cfqg_stats_update_group_wait_time(struct cfqg_stats *stats)
{
unsigned long long now;
- if (!blkio_blkg_waiting(stats))
+ if (!cfqg_stats_waiting(stats))
return;
now = sched_clock();
if (time_after64(now, stats->start_group_wait_time))
blkg_stat_add(&stats->group_wait_time,
now - stats->start_group_wait_time);
- blkio_clear_blkg_waiting(stats);
+ cfqg_stats_clear_waiting(stats);
}
/* This should be called with the queue_lock held. */
-static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
- struct blkio_policy_type *pol,
- struct blkio_group *curr_blkg)
+static void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg,
+ struct cfq_group *curr_cfqg)
{
- struct blkg_policy_data *pd = blkg->pd[pol->plid];
+ struct cfqg_stats *stats = &cfqg->stats;
- if (blkio_blkg_waiting(&pd->stats))
+ if (cfqg_stats_waiting(stats))
return;
- if (blkg == curr_blkg)
+ if (cfqg == curr_cfqg)
return;
- pd->stats.start_group_wait_time = sched_clock();
- blkio_mark_blkg_waiting(&pd->stats);
+ stats->start_group_wait_time = sched_clock();
+ cfqg_stats_mark_waiting(stats);
}
/* This should be called with the queue_lock held. */
-static void blkio_end_empty_time(struct blkio_group_stats *stats)
+static void cfqg_stats_end_empty_time(struct cfqg_stats *stats)
{
unsigned long long now;
- if (!blkio_blkg_empty(stats))
+ if (!cfqg_stats_empty(stats))
return;
now = sched_clock();
if (time_after64(now, stats->start_empty_time))
blkg_stat_add(&stats->empty_time,
now - stats->start_empty_time);
- blkio_clear_blkg_empty(stats);
+ cfqg_stats_clear_empty(stats);
}
-static void cfq_blkiocg_update_dequeue_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol,
- unsigned long dequeue)
+static void cfqg_stats_update_dequeue(struct cfq_group *cfqg)
{
- struct blkg_policy_data *pd = blkg->pd[pol->plid];
-
- lockdep_assert_held(blkg->q->queue_lock);
-
- blkg_stat_add(&pd->stats.dequeue, dequeue);
+ blkg_stat_add(&cfqg->stats.dequeue, 1);
}
-static void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg,
- struct blkio_policy_type *pol)
+static void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg)
{
- struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
-
- lockdep_assert_held(blkg->q->queue_lock);
+ struct cfqg_stats *stats = &cfqg->stats;
if (blkg_rwstat_sum(&stats->queued))
return;
* request in parent group and moved to this group while being added
* to service tree. Just ignore the event and move on.
*/
- if (blkio_blkg_empty(stats))
+ if (cfqg_stats_empty(stats))
return;
stats->start_empty_time = sched_clock();
- blkio_mark_blkg_empty(stats);
+ cfqg_stats_mark_empty(stats);
}
-static void cfq_blkiocg_update_idle_time_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol)
+static void cfqg_stats_update_idle_time(struct cfq_group *cfqg)
{
- struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+ struct cfqg_stats *stats = &cfqg->stats;
- lockdep_assert_held(blkg->q->queue_lock);
-
- if (blkio_blkg_idling(stats)) {
+ if (cfqg_stats_idling(stats)) {
unsigned long long now = sched_clock();
if (time_after64(now, stats->start_idle_time))
blkg_stat_add(&stats->idle_time,
now - stats->start_idle_time);
- blkio_clear_blkg_idling(stats);
+ cfqg_stats_clear_idling(stats);
}
}
-static void cfq_blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol)
+static void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg)
{
- struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+ struct cfqg_stats *stats = &cfqg->stats;
- lockdep_assert_held(blkg->q->queue_lock);
- BUG_ON(blkio_blkg_idling(stats));
+ BUG_ON(cfqg_stats_idling(stats));
stats->start_idle_time = sched_clock();
- blkio_mark_blkg_idling(stats);
+ cfqg_stats_mark_idling(stats);
}
-static void cfq_blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol)
+static void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg)
{
- struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
-
- lockdep_assert_held(blkg->q->queue_lock);
+ struct cfqg_stats *stats = &cfqg->stats;
blkg_stat_add(&stats->avg_queue_size_sum,
blkg_rwstat_sum(&stats->queued));
blkg_stat_add(&stats->avg_queue_size_samples, 1);
- blkio_update_group_wait_time(stats);
+ cfqg_stats_update_group_wait_time(stats);
}
#else /* CONFIG_CFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
-static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
- struct blkio_policy_type *pol,
- struct blkio_group *curr_blkg) { }
-static void blkio_end_empty_time(struct blkio_group_stats *stats) { }
-static void cfq_blkiocg_update_dequeue_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol,
- unsigned long dequeue) { }
-static void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg,
- struct blkio_policy_type *pol) { }
-static void cfq_blkiocg_update_idle_time_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol) { }
-static void cfq_blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol) { }
-static void cfq_blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol) { }
+static void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg,
+ struct cfq_group *curr_cfqg) { }
+static void cfqg_stats_end_empty_time(struct cfqg_stats *stats) { }
+static void cfqg_stats_update_dequeue(struct cfq_group *cfqg) { }
+static void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg) { }
+static void cfqg_stats_update_idle_time(struct cfq_group *cfqg) { }
+static void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg) { }
+static void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }
#endif /* CONFIG_CFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
blk_add_trace_msg((cfqd)->queue, "%s " fmt, \
blkg_path(cfqg_to_blkg((cfqg))), ##args) \
-static inline void cfq_blkiocg_update_io_add_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol,
- struct blkio_group *curr_blkg,
- bool direction, bool sync)
+static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
+ struct cfq_group *curr_cfqg, int rw)
{
- struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
- int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
-
- lockdep_assert_held(blkg->q->queue_lock);
-
- blkg_rwstat_add(&stats->queued, rw, 1);
- blkio_end_empty_time(stats);
- blkio_set_start_group_wait_time(blkg, pol, curr_blkg);
+ blkg_rwstat_add(&cfqg->stats.queued, rw, 1);
+ cfqg_stats_end_empty_time(&cfqg->stats);
+ cfqg_stats_set_start_group_wait_time(cfqg, curr_cfqg);
}
-static inline void cfq_blkiocg_update_timeslice_used(struct blkio_group *blkg,
- struct blkio_policy_type *pol, unsigned long time,
- unsigned long unaccounted_time)
+static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
+ unsigned long time, unsigned long unaccounted_time)
{
- struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
-
- lockdep_assert_held(blkg->q->queue_lock);
-
- blkg_stat_add(&stats->time, time);
+ blkg_stat_add(&cfqg->stats.time, time);
#ifdef CONFIG_DEBUG_BLK_CGROUP
- blkg_stat_add(&stats->unaccounted_time, unaccounted_time);
+ blkg_stat_add(&cfqg->stats.unaccounted_time, unaccounted_time);
#endif
}
-static inline void cfq_blkiocg_update_io_remove_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol, bool direction,
- bool sync)
+static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw)
{
- struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
- int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
-
- lockdep_assert_held(blkg->q->queue_lock);
-
- blkg_rwstat_add(&stats->queued, rw, -1);
+ blkg_rwstat_add(&cfqg->stats.queued, rw, -1);
}
-static inline void cfq_blkiocg_update_io_merged_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol, bool direction,
- bool sync)
+static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw)
{
- struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
- int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
-
- lockdep_assert_held(blkg->q->queue_lock);
-
- blkg_rwstat_add(&stats->merged, rw, 1);
+ blkg_rwstat_add(&cfqg->stats.merged, rw, 1);
}
-static inline void cfq_blkiocg_update_dispatch_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol, uint64_t bytes,
- bool direction, bool sync)
+static inline void cfqg_stats_update_dispatch(struct cfq_group *cfqg,
+ uint64_t bytes, int rw)
{
- int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
- struct blkg_policy_data *pd = blkg->pd[pol->plid];
- struct blkio_group_stats_cpu *stats_cpu;
- unsigned long flags;
-
- /* If per cpu stats are not allocated yet, don't do any accounting. */
- if (pd->stats_cpu == NULL)
- return;
-
- /*
- * Disabling interrupts to provide mutual exclusion between two
- * writes on same cpu. It probably is not needed for 64bit. Not
- * optimizing that case yet.
- */
- local_irq_save(flags);
-
- stats_cpu = this_cpu_ptr(pd->stats_cpu);
-
- blkg_stat_add(&stats_cpu->sectors, bytes >> 9);
- blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
- blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
-
- local_irq_restore(flags);
+ blkg_stat_add(&cfqg->stats.sectors, bytes >> 9);
+ blkg_rwstat_add(&cfqg->stats.serviced, rw, 1);
+ blkg_rwstat_add(&cfqg->stats.service_bytes, rw, bytes);
}
-static inline void cfq_blkiocg_update_completion_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol, uint64_t start_time,
- uint64_t io_start_time, bool direction, bool sync)
+static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
+ uint64_t start_time, uint64_t io_start_time, int rw)
{
- struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
+ struct cfqg_stats *stats = &cfqg->stats;
unsigned long long now = sched_clock();
- int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
-
- lockdep_assert_held(blkg->q->queue_lock);
if (time_after64(now, io_start_time))
blkg_rwstat_add(&stats->service_time, rw, now - io_start_time);
io_start_time - start_time);
}
+static void cfqg_stats_reset(struct blkio_group *blkg)
+{
+ struct cfq_group *cfqg = blkg_to_cfqg(blkg);
+ struct cfqg_stats *stats = &cfqg->stats;
+
+ /* queued stats shouldn't be cleared */
+ blkg_rwstat_reset(&stats->service_bytes);
+ blkg_rwstat_reset(&stats->serviced);
+ blkg_rwstat_reset(&stats->merged);
+ blkg_rwstat_reset(&stats->service_time);
+ blkg_rwstat_reset(&stats->wait_time);
+ blkg_stat_reset(&stats->time);
+#ifdef CONFIG_DEBUG_BLK_CGROUP
+ blkg_stat_reset(&stats->unaccounted_time);
+ blkg_stat_reset(&stats->avg_queue_size_sum);
+ blkg_stat_reset(&stats->avg_queue_size_samples);
+ blkg_stat_reset(&stats->dequeue);
+ blkg_stat_reset(&stats->group_wait_time);
+ blkg_stat_reset(&stats->idle_time);
+ blkg_stat_reset(&stats->empty_time);
+#endif
+}
+
#else /* CONFIG_CFQ_GROUP_IOSCHED */
static inline struct cfq_group *blkg_to_cfqg(struct blkio_group *blkg) { return NULL; }
blk_add_trace_msg((cfqd)->queue, "cfq%d " fmt, (cfqq)->pid, ##args)
#define cfq_log_cfqg(cfqd, cfqg, fmt, args...) do {} while (0)
-static inline void cfq_blkiocg_update_io_add_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol,
- struct blkio_group *curr_blkg, bool direction,
- bool sync) { }
-static inline void cfq_blkiocg_update_timeslice_used(struct blkio_group *blkg,
- struct blkio_policy_type *pol, unsigned long time,
- unsigned long unaccounted_time) { }
-static inline void cfq_blkiocg_update_io_remove_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol, bool direction,
- bool sync) { }
-static inline void cfq_blkiocg_update_io_merged_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol, bool direction,
- bool sync) { }
-static inline void cfq_blkiocg_update_dispatch_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol, uint64_t bytes,
- bool direction, bool sync) { }
-static inline void cfq_blkiocg_update_completion_stats(struct blkio_group *blkg,
- struct blkio_policy_type *pol, uint64_t start_time,
- uint64_t io_start_time, bool direction, bool sync) { }
+static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
+ struct cfq_group *curr_cfqg, int rw) { }
+static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
+ unsigned long time, unsigned long unaccounted_time) { }
+static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw) { }
+static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) { }
+static inline void cfqg_stats_update_dispatch(struct cfq_group *cfqg,
+ uint64_t bytes, int rw) { }
+static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
+ uint64_t start_time, uint64_t io_start_time, int rw) { }
#endif /* CONFIG_CFQ_GROUP_IOSCHED */
{
u64 d = delta << CFQ_SERVICE_SHIFT;
- d = d * BLKIO_WEIGHT_DEFAULT;
+ d = d * CFQ_WEIGHT_DEFAULT;
do_div(d, cfqg->weight);
return d;
}
cfq_update_group_weight(struct cfq_group *cfqg)
{
BUG_ON(!RB_EMPTY_NODE(&cfqg->rb_node));
- if (cfqg->needs_update) {
+ if (cfqg->new_weight) {
cfqg->weight = cfqg->new_weight;
- cfqg->needs_update = false;
+ cfqg->new_weight = 0;
}
}
cfq_log_cfqg(cfqd, cfqg, "del_from_rr group");
cfq_group_service_tree_del(st, cfqg);
cfqg->saved_workload_slice = 0;
- cfq_blkiocg_update_dequeue_stats(cfqg_to_blkg(cfqg),
- &blkio_policy_cfq, 1);
+ cfqg_stats_update_dequeue(cfqg);
}
static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq,
"sl_used=%u disp=%u charge=%u iops=%u sect=%lu",
used_sl, cfqq->slice_dispatch, charge,
iops_mode(cfqd), cfqq->nr_sectors);
- cfq_blkiocg_update_timeslice_used(cfqg_to_blkg(cfqg), &blkio_policy_cfq,
- used_sl, unaccounted_sl);
- cfq_blkiocg_set_start_empty_time(cfqg_to_blkg(cfqg), &blkio_policy_cfq);
+ cfqg_stats_update_timeslice_used(cfqg, used_sl, unaccounted_sl);
+ cfqg_stats_set_start_empty_time(cfqg);
}
/**
}
#ifdef CONFIG_CFQ_GROUP_IOSCHED
-static void cfq_update_blkio_group_weight(struct blkio_group *blkg,
- unsigned int weight)
-{
- struct cfq_group *cfqg = blkg_to_cfqg(blkg);
-
- cfqg->new_weight = weight;
- cfqg->needs_update = true;
-}
-
static void cfq_init_blkio_group(struct blkio_group *blkg)
{
struct cfq_group *cfqg = blkg_to_cfqg(blkg);
cfq_init_cfqg_base(cfqg);
- cfqg->weight = blkg->blkcg->weight;
+ cfqg->weight = blkg->blkcg->cfq_weight;
}
/*
cfqg_get(cfqg);
}
-static u64 blkg_prfill_weight_device(struct seq_file *sf,
- struct blkg_policy_data *pd, int off)
+static u64 cfqg_prfill_weight_device(struct seq_file *sf, void *pdata, int off)
{
- if (!pd->conf.weight)
+ struct cfq_group *cfqg = pdata;
+
+ if (!cfqg->dev_weight)
return 0;
- return __blkg_prfill_u64(sf, pd, pd->conf.weight);
+ return __blkg_prfill_u64(sf, pdata, cfqg->dev_weight);
}
-static int blkcg_print_weight_device(struct cgroup *cgrp, struct cftype *cft,
- struct seq_file *sf)
+static int cfqg_print_weight_device(struct cgroup *cgrp, struct cftype *cft,
+ struct seq_file *sf)
{
blkcg_print_blkgs(sf, cgroup_to_blkio_cgroup(cgrp),
- blkg_prfill_weight_device, BLKIO_POLICY_PROP, 0,
+ cfqg_prfill_weight_device, BLKIO_POLICY_PROP, 0,
false);
return 0;
}
-static int blkcg_print_weight(struct cgroup *cgrp, struct cftype *cft,
- struct seq_file *sf)
+static int cfq_print_weight(struct cgroup *cgrp, struct cftype *cft,
+ struct seq_file *sf)
{
- seq_printf(sf, "%u\n", cgroup_to_blkio_cgroup(cgrp)->weight);
+ seq_printf(sf, "%u\n", cgroup_to_blkio_cgroup(cgrp)->cfq_weight);
return 0;
}
-static int blkcg_set_weight_device(struct cgroup *cgrp, struct cftype *cft,
- const char *buf)
+static int cfqg_set_weight_device(struct cgroup *cgrp, struct cftype *cft,
+ const char *buf)
{
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
- struct blkg_policy_data *pd;
struct blkg_conf_ctx ctx;
+ struct cfq_group *cfqg;
int ret;
ret = blkg_conf_prep(blkcg, buf, &ctx);
return ret;
ret = -EINVAL;
- pd = ctx.blkg->pd[BLKIO_POLICY_PROP];
- if (pd && (!ctx.v || (ctx.v >= BLKIO_WEIGHT_MIN &&
- ctx.v <= BLKIO_WEIGHT_MAX))) {
- pd->conf.weight = ctx.v;
- cfq_update_blkio_group_weight(ctx.blkg, ctx.v ?: blkcg->weight);
+ cfqg = blkg_to_cfqg(ctx.blkg);
+ if (cfqg && (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN &&
+ ctx.v <= CFQ_WEIGHT_MAX))) {
+ cfqg->dev_weight = ctx.v;
+ cfqg->new_weight = cfqg->dev_weight ?: blkcg->cfq_weight;
ret = 0;
}
return ret;
}
-static int blkcg_set_weight(struct cgroup *cgrp, struct cftype *cft, u64 val)
+static int cfq_set_weight(struct cgroup *cgrp, struct cftype *cft, u64 val)
{
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
struct blkio_group *blkg;
struct hlist_node *n;
- if (val < BLKIO_WEIGHT_MIN || val > BLKIO_WEIGHT_MAX)
+ if (val < CFQ_WEIGHT_MIN || val > CFQ_WEIGHT_MAX)
return -EINVAL;
spin_lock_irq(&blkcg->lock);
- blkcg->weight = (unsigned int)val;
+ blkcg->cfq_weight = (unsigned int)val;
hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
- struct blkg_policy_data *pd = blkg->pd[BLKIO_POLICY_PROP];
+ struct cfq_group *cfqg = blkg_to_cfqg(blkg);
- if (pd && !pd->conf.weight)
- cfq_update_blkio_group_weight(blkg, blkcg->weight);
+ if (cfqg && !cfqg->dev_weight)
+ cfqg->new_weight = blkcg->cfq_weight;
}
spin_unlock_irq(&blkcg->lock);
}
#ifdef CONFIG_DEBUG_BLK_CGROUP
-static u64 blkg_prfill_avg_queue_size(struct seq_file *sf,
- struct blkg_policy_data *pd, int off)
+static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf, void *pdata, int off)
{
- u64 samples = blkg_stat_read(&pd->stats.avg_queue_size_samples);
+ struct cfq_group *cfqg = pdata;
+ u64 samples = blkg_stat_read(&cfqg->stats.avg_queue_size_samples);
u64 v = 0;
if (samples) {
- v = blkg_stat_read(&pd->stats.avg_queue_size_sum);
+ v = blkg_stat_read(&cfqg->stats.avg_queue_size_sum);
do_div(v, samples);
}
- __blkg_prfill_u64(sf, pd, v);
+ __blkg_prfill_u64(sf, pdata, v);
return 0;
}
/* print avg_queue_size */
-static int blkcg_print_avg_queue_size(struct cgroup *cgrp, struct cftype *cft,
- struct seq_file *sf)
+static int cfqg_print_avg_queue_size(struct cgroup *cgrp, struct cftype *cft,
+ struct seq_file *sf)
{
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
- blkcg_print_blkgs(sf, blkcg, blkg_prfill_avg_queue_size,
+ blkcg_print_blkgs(sf, blkcg, cfqg_prfill_avg_queue_size,
BLKIO_POLICY_PROP, 0, false);
return 0;
}
static struct cftype cfq_blkcg_files[] = {
{
.name = "weight_device",
- .read_seq_string = blkcg_print_weight_device,
- .write_string = blkcg_set_weight_device,
+ .read_seq_string = cfqg_print_weight_device,
+ .write_string = cfqg_set_weight_device,
.max_write_len = 256,
},
{
.name = "weight",
- .read_seq_string = blkcg_print_weight,
- .write_u64 = blkcg_set_weight,
+ .read_seq_string = cfq_print_weight,
+ .write_u64 = cfq_set_weight,
},
{
.name = "time",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats, time)),
+ offsetof(struct cfq_group, stats.time)),
.read_seq_string = blkcg_print_stat,
},
{
.name = "sectors",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats_cpu, sectors)),
- .read_seq_string = blkcg_print_cpu_stat,
+ offsetof(struct cfq_group, stats.sectors)),
+ .read_seq_string = blkcg_print_stat,
},
{
.name = "io_service_bytes",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats_cpu, service_bytes)),
- .read_seq_string = blkcg_print_cpu_rwstat,
+ offsetof(struct cfq_group, stats.service_bytes)),
+ .read_seq_string = blkcg_print_rwstat,
},
{
.name = "io_serviced",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats_cpu, serviced)),
- .read_seq_string = blkcg_print_cpu_rwstat,
+ offsetof(struct cfq_group, stats.serviced)),
+ .read_seq_string = blkcg_print_rwstat,
},
{
.name = "io_service_time",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats, service_time)),
+ offsetof(struct cfq_group, stats.service_time)),
.read_seq_string = blkcg_print_rwstat,
},
{
.name = "io_wait_time",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats, wait_time)),
+ offsetof(struct cfq_group, stats.wait_time)),
.read_seq_string = blkcg_print_rwstat,
},
{
.name = "io_merged",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats, merged)),
+ offsetof(struct cfq_group, stats.merged)),
.read_seq_string = blkcg_print_rwstat,
},
{
.name = "io_queued",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats, queued)),
+ offsetof(struct cfq_group, stats.queued)),
.read_seq_string = blkcg_print_rwstat,
},
#ifdef CONFIG_DEBUG_BLK_CGROUP
{
.name = "avg_queue_size",
- .read_seq_string = blkcg_print_avg_queue_size,
+ .read_seq_string = cfqg_print_avg_queue_size,
},
{
.name = "group_wait_time",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats, group_wait_time)),
+ offsetof(struct cfq_group, stats.group_wait_time)),
.read_seq_string = blkcg_print_stat,
},
{
.name = "idle_time",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats, idle_time)),
+ offsetof(struct cfq_group, stats.idle_time)),
.read_seq_string = blkcg_print_stat,
},
{
.name = "empty_time",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats, empty_time)),
+ offsetof(struct cfq_group, stats.empty_time)),
.read_seq_string = blkcg_print_stat,
},
{
.name = "dequeue",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats, dequeue)),
+ offsetof(struct cfq_group, stats.dequeue)),
.read_seq_string = blkcg_print_stat,
},
{
.name = "unaccounted_time",
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
- offsetof(struct blkio_group_stats, unaccounted_time)),
+ offsetof(struct cfq_group, stats.unaccounted_time)),
.read_seq_string = blkcg_print_stat,
},
#endif /* CONFIG_DEBUG_BLK_CGROUP */
{
elv_rb_del(&cfqq->sort_list, rq);
cfqq->queued[rq_is_sync(rq)]--;
- cfq_blkiocg_update_io_remove_stats(cfqg_to_blkg(RQ_CFQG(rq)),
- &blkio_policy_cfq, rq_data_dir(rq),
- rq_is_sync(rq));
+ cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags);
cfq_add_rq_rb(rq);
- cfq_blkiocg_update_io_add_stats(cfqg_to_blkg(RQ_CFQG(rq)),
- &blkio_policy_cfq,
- cfqg_to_blkg(cfqq->cfqd->serving_group),
- rq_data_dir(rq), rq_is_sync(rq));
+ cfqg_stats_update_io_add(RQ_CFQG(rq), cfqq->cfqd->serving_group,
+ rq->cmd_flags);
}
static struct request *
cfq_del_rq_rb(rq);
cfqq->cfqd->rq_queued--;
- cfq_blkiocg_update_io_remove_stats(cfqg_to_blkg(RQ_CFQG(rq)),
- &blkio_policy_cfq, rq_data_dir(rq),
- rq_is_sync(rq));
+ cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags);
if (rq->cmd_flags & REQ_PRIO) {
WARN_ON(!cfqq->prio_pending);
cfqq->prio_pending--;
static void cfq_bio_merged(struct request_queue *q, struct request *req,
struct bio *bio)
{
- cfq_blkiocg_update_io_merged_stats(cfqg_to_blkg(RQ_CFQG(req)),
- &blkio_policy_cfq, bio_data_dir(bio),
- cfq_bio_sync(bio));
+ cfqg_stats_update_io_merged(RQ_CFQG(req), bio->bi_rw);
}
static void
if (cfqq->next_rq == next)
cfqq->next_rq = rq;
cfq_remove_request(next);
- cfq_blkiocg_update_io_merged_stats(cfqg_to_blkg(RQ_CFQG(rq)),
- &blkio_policy_cfq, rq_data_dir(next),
- rq_is_sync(next));
+ cfqg_stats_update_io_merged(RQ_CFQG(rq), next->cmd_flags);
cfqq = RQ_CFQQ(next);
/*
static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{
del_timer(&cfqd->idle_slice_timer);
- cfq_blkiocg_update_idle_time_stats(cfqg_to_blkg(cfqq->cfqg),
- &blkio_policy_cfq);
+ cfqg_stats_update_idle_time(cfqq->cfqg);
}
static void __cfq_set_active_queue(struct cfq_data *cfqd,
if (cfqq) {
cfq_log_cfqq(cfqd, cfqq, "set_active wl_prio:%d wl_type:%d",
cfqd->serving_prio, cfqd->serving_type);
- cfq_blkiocg_update_avg_queue_size_stats(cfqg_to_blkg(cfqq->cfqg),
- &blkio_policy_cfq);
+ cfqg_stats_update_avg_queue_size(cfqq->cfqg);
cfqq->slice_start = 0;
cfqq->dispatch_start = jiffies;
cfqq->allocated_slice = 0;
sl = cfqd->cfq_slice_idle;
mod_timer(&cfqd->idle_slice_timer, jiffies + sl);
- cfq_blkiocg_update_set_idle_time_stats(cfqg_to_blkg(cfqq->cfqg),
- &blkio_policy_cfq);
+ cfqg_stats_set_start_idle_time(cfqq->cfqg);
cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu group_idle: %d", sl,
group_idle ? 1 : 0);
}
cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++;
cfqq->nr_sectors += blk_rq_sectors(rq);
- cfq_blkiocg_update_dispatch_stats(cfqg_to_blkg(cfqq->cfqg),
- &blkio_policy_cfq, blk_rq_bytes(rq),
- rq_data_dir(rq), rq_is_sync(rq));
+ cfqg_stats_update_dispatch(cfqq->cfqg, blk_rq_bytes(rq), rq->cmd_flags);
}
/*
cfq_clear_cfqq_wait_request(cfqq);
__blk_run_queue(cfqd->queue);
} else {
- cfq_blkiocg_update_idle_time_stats(
- cfqg_to_blkg(cfqq->cfqg),
- &blkio_policy_cfq);
+ cfqg_stats_update_idle_time(cfqq->cfqg);
cfq_mark_cfqq_must_dispatch(cfqq);
}
}
rq_set_fifo_time(rq, jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)]);
list_add_tail(&rq->queuelist, &cfqq->fifo);
cfq_add_rq_rb(rq);
- cfq_blkiocg_update_io_add_stats(cfqg_to_blkg(RQ_CFQG(rq)),
- &blkio_policy_cfq,
- cfqg_to_blkg(cfqd->serving_group),
- rq_data_dir(rq), rq_is_sync(rq));
+ cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group,
+ rq->cmd_flags);
cfq_rq_enqueued(cfqd, cfqq, rq);
}
cfqd->rq_in_driver--;
cfqq->dispatched--;
(RQ_CFQG(rq))->dispatched--;
- cfq_blkiocg_update_completion_stats(cfqg_to_blkg(cfqq->cfqg),
- &blkio_policy_cfq, rq_start_time_ns(rq),
- rq_io_start_time_ns(rq), rq_data_dir(rq),
- rq_is_sync(rq));
+ cfqg_stats_update_completion(cfqq->cfqg, rq_start_time_ns(rq),
+ rq_io_start_time_ns(rq), rq->cmd_flags);
cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--;
return -ENOMEM;
}
- cfqd->root_group->weight = 2*BLKIO_WEIGHT_DEFAULT;
+ cfqd->root_group->weight = 2 * CFQ_WEIGHT_DEFAULT;
/*
* Not strictly needed (since RB_ROOT just clears the node and we
static struct blkio_policy_type blkio_policy_cfq = {
.ops = {
.blkio_init_group_fn = cfq_init_blkio_group,
+ .blkio_reset_group_stats_fn = cfqg_stats_reset,
},
.plid = BLKIO_POLICY_PROP,
.pdata_size = sizeof(struct cfq_group),