| 1 |
struct percpu_counter { |
| 2 |
signed long long count; |
| 3 |
}; |
| 4 |
struct blkg_rwstat { |
| 5 |
struct percpu_counter cpu_cnt[4]; |
| 6 |
}; |
| 7 |
struct cfq_group { |
| 8 |
struct blkg_rwstat service_time; |
| 9 |
}; |
| 10 |
struct cfq_queue { |
| 11 |
struct cfq_group *group; |
| 12 |
}; |
| 13 |
struct request { |
| 14 |
struct cfq_queue *active_queue; |
| 15 |
unsigned long long cmd_flags; |
| 16 |
void *priv; |
| 17 |
}; |
| 18 |
static void blkg_rwstat_add(struct blkg_rwstat *rwstat, int rw, unsigned long long val) |
| 19 |
{ |
| 20 |
struct percpu_counter *cnt; |
| 21 |
if (rw & 1) |
| 22 |
cnt = &rwstat->cpu_cnt[1]; |
| 23 |
else |
| 24 |
cnt = &rwstat->cpu_cnt[0]; |
| 25 |
cnt->count += val; |
| 26 |
if (rw & 2) |
| 27 |
cnt = &rwstat->cpu_cnt[2]; |
| 28 |
else |
| 29 |
cnt = &rwstat->cpu_cnt[3]; |
| 30 |
cnt->count += val; |
| 31 |
} |
| 32 |
extern unsigned long long rq_start_time_ns(void); |
| 33 |
extern unsigned long long rq_io_start_time_ns(void); |
| 34 |
extern int rq_is_sync(void); |
| 35 |
extern void cfq_arm_slice_timer(void); |
| 36 |
void cfq_completed_request(struct request *rq) |
| 37 |
{ |
| 38 |
struct cfq_queue *queue = rq->priv; |
| 39 |
int sync = rq_is_sync(); |
| 40 |
struct cfq_group *group = queue->group; |
| 41 |
long long start_time = rq_start_time_ns(); |
| 42 |
long long io_start_time = rq_io_start_time_ns(); |
| 43 |
int rw = rq->cmd_flags; |
| 44 |
|
| 45 |
if (io_start_time < 1) |
| 46 |
blkg_rwstat_add(&group->service_time, rw, 1 - io_start_time); |
| 47 |
blkg_rwstat_add(0, rw, io_start_time - start_time); |
| 48 |
|
| 49 |
if (rq->active_queue == queue && sync) |
| 50 |
cfq_arm_slice_timer(); |
| 51 |
} |
| 52 |
|