4 #include <linux/kernel.h>
5 #include <linux/blkdev.h>
6 #include <linux/ktime.h>
7 #include <linux/rcupdate.h>
8 #include <linux/timer.h>
12 * 3 bits: reserved for other usage
16 #define BLK_STAT_RES_BITS 3
17 #define BLK_STAT_SIZE_BITS 12
18 #define BLK_STAT_RES_SHIFT (64 - BLK_STAT_RES_BITS)
19 #define BLK_STAT_SIZE_SHIFT (BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS)
20 #define BLK_STAT_TIME_MASK ((1ULL << BLK_STAT_SIZE_SHIFT) - 1)
21 #define BLK_STAT_SIZE_MASK \
22 (((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT)
23 #define BLK_STAT_RES_MASK (~((1ULL << BLK_STAT_RES_SHIFT) - 1))
26 * struct blk_stat_callback - Block statistics callback.
28 * A &struct blk_stat_callback is associated with a &struct request_queue. While
29 * @timer is active, that queue's request completion latencies are sorted into
30 * buckets by @bucket_fn and added to a per-cpu buffer, @cpu_stat. When the
31 * timer fires, @cpu_stat is flushed to @stat and @timer_fn is invoked.
33 struct blk_stat_callback {
35 * @list: RCU list of callbacks for a &struct request_queue.
37 struct list_head list;
40 * @timer: Timer for the next callback invocation.
42 struct timer_list timer;
45 * @cpu_stat: Per-cpu statistics buckets.
47 struct blk_rq_stat __percpu *cpu_stat;
50 * @bucket_fn: Given a request, returns which statistics bucket it
51 * should be accounted under.
53 unsigned int (*bucket_fn)(const struct request *);
56 * @buckets: Number of statistics buckets.
61 * @stat: Array of statistics buckets.
63 struct blk_rq_stat *stat;
66 * @fn: Callback function.
68 void (*timer_fn)(struct blk_stat_callback *);
71 * @data: Private pointer for the user.
78 struct blk_queue_stats *blk_alloc_queue_stats(void);
79 void blk_free_queue_stats(struct blk_queue_stats *);
81 void blk_stat_add(struct request *);
83 static inline u64 __blk_stat_time(u64 time)
85 return time & BLK_STAT_TIME_MASK;
88 static inline u64 blk_stat_time(struct blk_issue_stat *stat)
90 return __blk_stat_time(stat->stat);
93 static inline sector_t blk_capped_size(sector_t size)
95 return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1);
98 static inline sector_t blk_stat_size(struct blk_issue_stat *stat)
100 return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT;
103 static inline void blk_stat_set_issue(struct blk_issue_stat *stat,
106 stat->stat = (stat->stat & BLK_STAT_RES_MASK) |
107 (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) |
108 (((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT);
111 /* record time/size info in request but not add a callback */
112 void blk_stat_enable_accounting(struct request_queue *q);
115 * blk_stat_rq_ddir() - Bucket callback function for the request data direction.
118 * This is the same as rq_data_dir() but as a function so it can be used as
119 * @bucket_fn for blk_stat_alloc_callback().
121 * Return: Data direction of the request, either READ or WRITE.
123 unsigned int blk_stat_rq_ddir(const struct request *rq);
126 * blk_stat_alloc_callback() - Allocate a block statistics callback.
127 * @timer_fn: Timer callback function.
128 * @bucket_fn: Bucket callback function.
129 * @buckets: Number of statistics buckets.
130 * @data: Value for the @data field of the &struct blk_stat_callback.
132 * See &struct blk_stat_callback for details on the callback functions.
134 * Return: &struct blk_stat_callback on success or NULL on ENOMEM.
136 struct blk_stat_callback *
137 blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
138 unsigned int (*bucket_fn)(const struct request *),
139 unsigned int buckets, void *data);
142 * blk_stat_add_callback() - Add a block statistics callback to be run on a
144 * @q: The request queue.
147 * Note that a single &struct blk_stat_callback can only be added to a single
148 * &struct request_queue.
150 void blk_stat_add_callback(struct request_queue *q,
151 struct blk_stat_callback *cb);
154 * blk_stat_remove_callback() - Remove a block statistics callback from a
156 * @q: The request queue.
159 * When this returns, the callback is not running on any CPUs and will not be
160 * called again unless readded.
162 void blk_stat_remove_callback(struct request_queue *q,
163 struct blk_stat_callback *cb);
166 * blk_stat_free_callback() - Free a block statistics callback.
169 * @cb may be NULL, in which case this does nothing. If it is not NULL, @cb must
170 * not be associated with a request queue. I.e., if it was previously added with
171 * blk_stat_add_callback(), it must also have been removed since then with
172 * blk_stat_remove_callback().
174 void blk_stat_free_callback(struct blk_stat_callback *cb);
177 * blk_stat_is_active() - Check if a block statistics callback is currently
178 * gathering statistics.
181 static inline bool blk_stat_is_active(struct blk_stat_callback *cb)
183 return timer_pending(&cb->timer);
187 * blk_stat_activate_nsecs() - Gather block statistics during a time window in
190 * @nsecs: Number of nanoseconds to gather statistics for.
192 * The timer callback will be called when the window expires.
194 static inline void blk_stat_activate_nsecs(struct blk_stat_callback *cb,
197 mod_timer(&cb->timer, jiffies + nsecs_to_jiffies(nsecs));
201 * blk_stat_activate_msecs() - Gather block statistics during a time window in
204 * @msecs: Number of milliseconds to gather statistics for.
206 * The timer callback will be called when the window expires.
208 static inline void blk_stat_activate_msecs(struct blk_stat_callback *cb,
211 mod_timer(&cb->timer, jiffies + msecs_to_jiffies(msecs));