blkcg: implement blkg_[rw]stat_recursive_sum() and blkg_[rw]stat_merge()

[~andy/linux] / block / blk-cgroup.c

diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 3aec4cdc89686a3fca85cdf0c24dbe86441d2a42..f9797b244eb380c37e2072b079e1a242bbb6e445 100644 (file)
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -32,6 +32,26 @@ EXPORT_SYMBOL_GPL(blkcg_root);
 
 static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
 
+static struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg,
+                                     struct request_queue *q, bool update_hint);
+
+/**
+ * blkg_for_each_descendant_pre - pre-order walk of a blkg's descendants
+ * @d_blkg: loop cursor pointing to the current descendant
+ * @pos_cgrp: used for iteration
+ * @p_blkg: target blkg to walk descendants of
+ *
+ * Walk @c_blkg through the descendants of @p_blkg.  Must be used with RCU
+ * read locked.  If called under either blkcg or queue lock, the iteration
+ * is guaranteed to include all and only online blkgs.  The caller may
+ * update @pos_cgrp by calling cgroup_rightmost_descendant() to skip
+ * subtree.
+ */
+#define blkg_for_each_descendant_pre(d_blkg, pos_cgrp, p_blkg)         \
+       cgroup_for_each_descendant_pre((pos_cgrp), (p_blkg)->blkcg->css.cgroup) \
+               if (((d_blkg) = __blkg_lookup(cgroup_to_blkcg(pos_cgrp), \
+                                             (p_blkg)->q, false)))
+
 static bool blkcg_policy_enabled(struct request_queue *q,
                                 const struct blkcg_policy *pol)
 {
@@ -127,6 +147,17 @@ err_free:
        return NULL;
 }
 
+/**
+ * __blkg_lookup - internal version of blkg_lookup()
+ * @blkcg: blkcg of interest
+ * @q: request_queue of interest
+ * @update_hint: whether to update lookup hint with the result or not
+ *
+ * This is internal version and shouldn't be used by policy
+ * implementations.  Looks up blkgs for the @blkcg - @q pair regardless of
+ * @q's bypass state.  If @update_hint is %true, the caller should be
+ * holding @q->queue_lock and lookup hint is updated on success.
+ */
 static struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg,
                                      struct request_queue *q, bool update_hint)
 {
@@ -585,6 +616,82 @@ u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 }
 EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
 
+/**
+ * blkg_stat_recursive_sum - collect hierarchical blkg_stat
+ * @pd: policy private data of interest
+ * @off: offset to the blkg_stat in @pd
+ *
+ * Collect the blkg_stat specified by @off from @pd and all its online
+ * descendants and return the sum.  The caller must be holding the queue
+ * lock for online tests.
+ */
+u64 blkg_stat_recursive_sum(struct blkg_policy_data *pd, int off)
+{
+       struct blkcg_policy *pol = blkcg_policy[pd->plid];
+       struct blkcg_gq *pos_blkg;
+       struct cgroup *pos_cgrp;
+       u64 sum;
+
+       lockdep_assert_held(pd->blkg->q->queue_lock);
+
+       sum = blkg_stat_read((void *)pd + off);
+
+       rcu_read_lock();
+       blkg_for_each_descendant_pre(pos_blkg, pos_cgrp, pd_to_blkg(pd)) {
+               struct blkg_policy_data *pos_pd = blkg_to_pd(pos_blkg, pol);
+               struct blkg_stat *stat = (void *)pos_pd + off;
+
+               if (pos_blkg->online)
+                       sum += blkg_stat_read(stat);
+       }
+       rcu_read_unlock();
+
+       return sum;
+}
+EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);
+
+/**
+ * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
+ * @pd: policy private data of interest
+ * @off: offset to the blkg_stat in @pd
+ *
+ * Collect the blkg_rwstat specified by @off from @pd and all its online
+ * descendants and return the sum.  The caller must be holding the queue
+ * lock for online tests.
+ */
+struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkg_policy_data *pd,
+                                            int off)
+{
+       struct blkcg_policy *pol = blkcg_policy[pd->plid];
+       struct blkcg_gq *pos_blkg;
+       struct cgroup *pos_cgrp;
+       struct blkg_rwstat sum;
+       int i;
+
+       lockdep_assert_held(pd->blkg->q->queue_lock);
+
+       sum = blkg_rwstat_read((void *)pd + off);
+
+       rcu_read_lock();
+       blkg_for_each_descendant_pre(pos_blkg, pos_cgrp, pd_to_blkg(pd)) {
+               struct blkg_policy_data *pos_pd = blkg_to_pd(pos_blkg, pol);
+               struct blkg_rwstat *rwstat = (void *)pos_pd + off;
+               struct blkg_rwstat tmp;
+
+               if (!pos_blkg->online)
+                       continue;
+
+               tmp = blkg_rwstat_read(rwstat);
+
+               for (i = 0; i < BLKG_RWSTAT_NR; i++)
+                       sum.cnt[i] += tmp.cnt[i];
+       }
+       rcu_read_unlock();
+
+       return sum;
+}
+EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);
+
 /**
  * blkg_conf_prep - parse and prepare for per-blkg config update
  * @blkcg: target block cgroup