summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJens Axboe <jens.axboe@oracle.com>2008-05-07 10:15:46 +0200
committerJens Axboe <jens.axboe@oracle.com>2008-05-07 10:15:46 +0200
commit28f13702f03e527fcb979747a882cf366c489c50 (patch)
tree08cf04c696bf0facb083bbfbdd867e5949c5fa14
parent6d63c275572d1e6f00d4fa154f16fbb0d8c2d2bf (diff)
downloadlinux-28f13702f03e527fcb979747a882cf366c489c50.tar.gz
linux-28f13702f03e527fcb979747a882cf366c489c50.tar.bz2
linux-28f13702f03e527fcb979747a882cf366c489c50.zip
block: avoid duplicate calls to get_part() in disk stat code
get_part() is fairly expensive, as it O(N) loops over partitions to find the right one. In lots of normal IO paths we end up looking up the partition twice, to make matters even worse. Change the stat add code to accept a passed in partition instead. Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
-rw-r--r--block/blk-core.c18
-rw-r--r--drivers/block/aoe/aoecmd.c10
-rw-r--r--include/linux/genhd.h35
3 files changed, 34 insertions, 29 deletions
diff --git a/block/blk-core.c b/block/blk-core.c
index 1b7dddf94f4f..2987fe47b5ee 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -54,15 +54,16 @@ static DEFINE_PER_CPU(struct list_head, blk_cpu_done);
static void drive_stat_acct(struct request *rq, int new_io)
{
+ struct hd_struct *part;
int rw = rq_data_dir(rq);
if (!blk_fs_request(rq) || !rq->rq_disk)
return;
- if (!new_io) {
- __all_stat_inc(rq->rq_disk, merges[rw], rq->sector);
- } else {
- struct hd_struct *part = get_part(rq->rq_disk, rq->sector);
+ part = get_part(rq->rq_disk, rq->sector);
+ if (!new_io)
+ __all_stat_inc(rq->rq_disk, part, merges[rw], rq->sector);
+ else {
disk_round_stats(rq->rq_disk);
rq->rq_disk->in_flight++;
if (part) {
@@ -1538,10 +1539,11 @@ static int __end_that_request_first(struct request *req, int error,
}
if (blk_fs_request(req) && req->rq_disk) {
+ struct hd_struct *part = get_part(req->rq_disk, req->sector);
const int rw = rq_data_dir(req);
- all_stat_add(req->rq_disk, sectors[rw],
- nr_bytes >> 9, req->sector);
+ all_stat_add(req->rq_disk, part, sectors[rw],
+ nr_bytes >> 9, req->sector);
}
total_bytes = bio_nbytes = 0;
@@ -1727,8 +1729,8 @@ static void end_that_request_last(struct request *req, int error)
const int rw = rq_data_dir(req);
struct hd_struct *part = get_part(disk, req->sector);
- __all_stat_inc(disk, ios[rw], req->sector);
- __all_stat_add(disk, ticks[rw], duration, req->sector);
+ __all_stat_inc(disk, part, ios[rw], req->sector);
+ __all_stat_add(disk, part, ticks[rw], duration, req->sector);
disk_round_stats(disk);
disk->in_flight--;
if (part) {
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index 8fc429cf82b6..41f818be2f7e 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -755,11 +755,13 @@ diskstats(struct gendisk *disk, struct bio *bio, ulong duration, sector_t sector
{
unsigned long n_sect = bio->bi_size >> 9;
const int rw = bio_data_dir(bio);
+ struct hd_struct *part;
- all_stat_inc(disk, ios[rw], sector);
- all_stat_add(disk, ticks[rw], duration, sector);
- all_stat_add(disk, sectors[rw], n_sect, sector);
- all_stat_add(disk, io_ticks, duration, sector);
+ part = get_part(disk, sector);
+ all_stat_inc(disk, part, ios[rw], sector);
+ all_stat_add(disk, part, ticks[rw], duration, sector);
+ all_stat_add(disk, part, sectors[rw], n_sect, sector);
+ all_stat_add(disk, part, io_ticks, duration, sector);
}
void
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index ecd2bf63fc84..e9874e7fcdf9 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -178,17 +178,17 @@ static inline struct hd_struct *get_part(struct gendisk *gendiskp,
static inline void disk_stat_set_all(struct gendisk *gendiskp, int value) {
int i;
+
for_each_possible_cpu(i)
memset(per_cpu_ptr(gendiskp->dkstats, i), value,
- sizeof (struct disk_stats));
+ sizeof(struct disk_stats));
}
#define __part_stat_add(part, field, addnd) \
(per_cpu_ptr(part->dkstats, smp_processor_id())->field += addnd)
-#define __all_stat_add(gendiskp, field, addnd, sector) \
+#define __all_stat_add(gendiskp, part, field, addnd, sector) \
({ \
- struct hd_struct *part = get_part(gendiskp, sector); \
if (part) \
__part_stat_add(part, field, addnd); \
__disk_stat_add(gendiskp, field, addnd); \
@@ -203,11 +203,13 @@ static inline void disk_stat_set_all(struct gendisk *gendiskp, int value) {
res; \
})
-static inline void part_stat_set_all(struct hd_struct *part, int value) {
+static inline void part_stat_set_all(struct hd_struct *part, int value)
+{
int i;
+
for_each_possible_cpu(i)
memset(per_cpu_ptr(part->dkstats, i), value,
- sizeof(struct disk_stats));
+ sizeof(struct disk_stats));
}
#else /* !CONFIG_SMP */
@@ -223,9 +225,8 @@ static inline void disk_stat_set_all(struct gendisk *gendiskp, int value)
#define __part_stat_add(part, field, addnd) \
(part->dkstats.field += addnd)
-#define __all_stat_add(gendiskp, field, addnd, sector) \
+#define __all_stat_add(gendiskp, part, field, addnd, sector) \
({ \
- struct hd_struct *part = get_part(gendiskp, sector); \
if (part) \
part->dkstats.field += addnd; \
__disk_stat_add(gendiskp, field, addnd); \
@@ -276,10 +277,10 @@ static inline void part_stat_set_all(struct hd_struct *part, int value)
#define part_stat_sub(gendiskp, field, subnd) \
part_stat_add(gendiskp, field, -subnd)
-#define all_stat_add(gendiskp, field, addnd, sector) \
+#define all_stat_add(gendiskp, part, field, addnd, sector) \
do { \
preempt_disable(); \
- __all_stat_add(gendiskp, field, addnd, sector); \
+ __all_stat_add(gendiskp, part, field, addnd, sector); \
preempt_enable(); \
} while (0)
@@ -288,15 +289,15 @@ static inline void part_stat_set_all(struct hd_struct *part, int value)
#define all_stat_dec(gendiskp, field, sector) \
all_stat_add(gendiskp, field, -1, sector)
-#define __all_stat_inc(gendiskp, field, sector) \
- __all_stat_add(gendiskp, field, 1, sector)
-#define all_stat_inc(gendiskp, field, sector) \
- all_stat_add(gendiskp, field, 1, sector)
+#define __all_stat_inc(gendiskp, part, field, sector) \
+ __all_stat_add(gendiskp, part, field, 1, sector)
+#define all_stat_inc(gendiskp, part, field, sector) \
+ all_stat_add(gendiskp, part, field, 1, sector)
-#define __all_stat_sub(gendiskp, field, subnd, sector) \
- __all_stat_add(gendiskp, field, -subnd, sector)
-#define all_stat_sub(gendiskp, field, subnd, sector) \
- all_stat_add(gendiskp, field, -subnd, sector)
+#define __all_stat_sub(gendiskp, part, field, subnd, sector) \
+ __all_stat_add(gendiskp, part, field, -subnd, sector)
+#define all_stat_sub(gendiskp, part, field, subnd, sector) \
+ all_stat_add(gendiskp, part, field, -subnd, sector)
/* Inlines to alloc and free disk stats in struct gendisk */
#ifdef CONFIG_SMP