Add two new helpers to simplify I/O accounting for bio based drivers. Currently these drivers use the generic_start_io_acct and generic_end_io_acct helpers which have very cumbersome calling conventions, don't actually return the time they started accounting, and try to deal with accounting for partitions, which can't happen for bio based drivers. The new helpers will be used to subsequently replace uses of the old helpers. The main API is the bio based wrappes in blkdev.h, but for zram which wants to account rw_page based I/O lower level routines are provided as well. Signed-off-by: Christoph Hellwig <hch(a)lst.de&gt; Reviewed-by: Konstantin Khlebnikov <khlebnikov(a)yandex-team.ru&gt; --- block/blk-core.c | 34 ++++++++++++++++++++++++++++++++++ include/linux/blkdev.h | 28 ++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/block/blk-core.c b/block/blk-core.c index 77e57c2e8d602..8973104f88d90 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1432,6 +1432,40 @@ void blk_account_io_start(struct request *rq, bool new_io) part_stat_unlock(); } +unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, + unsigned int op) +{ + struct hd_struct *part = &disk->part0; + const int sgrp = op_stat_group(op); + unsigned long now = READ_ONCE(jiffies); + + part_stat_lock(); + update_io_ticks(part, now, false); + part_stat_inc(part, ios[sgrp]); + part_stat_add(part, sectors[sgrp], sectors); + part_stat_local_inc(part, in_flight[op_is_write(op)]); + part_stat_unlock(); + + return now; +} +EXPORT_SYMBOL(disk_start_io_acct); + +void disk_end_io_acct(struct gendisk *disk, unsigned int op, + unsigned long start_time) +{ + struct hd_struct *part = &disk->part0; + const int sgrp = op_stat_group(op); + unsigned long now = READ_ONCE(jiffies); + unsigned long duration = now - start_time; + + part_stat_lock(); + update_io_ticks(part, now, true); + part_stat_add(part, nsecs[sgrp], jiffies_to_nsecs(duration)); + part_stat_local_dec(part, in_flight[op_is_write(op)]); + part_stat_unlock(); +} +EXPORT_SYMBOL(disk_end_io_acct); + /* * Steal bios from a request and add them to a bio list. * The request must not have been partially completed before. diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 7d10f4e632325..6f7ff0fa8fcf8 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1892,4 +1892,32 @@ static inline void blk_wake_io_task(struct task_struct *waiter) wake_up_process(waiter); } +unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, + unsigned int op); +void disk_end_io_acct(struct gendisk *disk, unsigned int op, + unsigned long start_time); + +#ifdef CONFIG_BLOCK +/** + * bio_start_io_acct - start I/O accounting for bio based drivers + * @bio: bio to start account for + * + * Returns the start time that should be passed back to bio_end_io_acct(). + */ +static inline unsigned long bio_start_io_acct(struct bio *bio) +{ + return disk_start_io_acct(bio->bi_disk, bio_sectors(bio), bio_op(bio)); +} + +/** + * bio_end_io_acct - end I/O accounting for bio based drivers + * @bio: bio to end account for + * @start: start time returned by bio_start_io_acct() + */ +static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time) +{ + return disk_end_io_acct(bio->bi_disk, bio_op(bio), start_time); +} +#endif /* CONFIG_BLOCK */ + #endif -- 2.26.2

Switch rsxx to use the nicer bio accounting helpers. Signed-off-by: Christoph Hellwig <hch(a)lst.de&gt; Reviewed-by: Konstantin Khlebnikov <khlebnikov(a)yandex-team.ru&gt; --- drivers/block/rsxx/dev.c | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c index 8ffa8260dcafe..3ba07ab30c84f 100644 --- a/drivers/block/rsxx/dev.c +++ b/drivers/block/rsxx/dev.c @@ -96,20 +96,6 @@ static const struct block_device_operations rsxx_fops = { .ioctl = rsxx_blkdev_ioctl, }; -static void disk_stats_start(struct rsxx_cardinfo *card, struct bio *bio) -{ - generic_start_io_acct(card->queue, bio_op(bio), bio_sectors(bio), - &card->gendisk->part0); -} - -static void disk_stats_complete(struct rsxx_cardinfo *card, - struct bio *bio, - unsigned long start_time) -{ - generic_end_io_acct(card->queue, bio_op(bio), - &card->gendisk->part0, start_time); -} - static void bio_dma_done_cb(struct rsxx_cardinfo *card, void *cb_data, unsigned int error) @@ -121,7 +107,7 @@ static void bio_dma_done_cb(struct rsxx_cardinfo *card, if (atomic_dec_and_test(&meta->pending_dmas)) { if (!card->eeh_state && card->gendisk) - disk_stats_complete(card, meta->bio, meta->start_time); + bio_end_io_acct(meta->bio, meta->start_time); if (atomic_read(&meta->error)) bio_io_error(meta->bio); @@ -167,10 +153,9 @@ static blk_qc_t rsxx_make_request(struct request_queue *q, struct bio *bio) bio_meta->bio = bio; atomic_set(&bio_meta->error, 0); atomic_set(&bio_meta->pending_dmas, 0); - bio_meta->start_time = jiffies; if (!unlikely(card->halt)) - disk_stats_start(card, bio); + bio_meta->start_time = bio_start_io_acct(bio); dev_dbg(CARD_TO_DEV(card), "BIO[%c]: meta: %p addr8: x%llx size: %d\n", bio_data_dir(bio) ? 'W' : 'R', bio_meta, -- 2.26.2

Switch bcache to use the nicer bio accounting helpers, and call the routines where we also sample the start time to give coherent accounting results. Signed-off-by: Christoph Hellwig <hch(a)lst.de&gt; Reviewed-by: Konstantin Khlebnikov <khlebnikov(a)yandex-team.ru&gt; Acked-by: Coly Li <colyli(a)suse.de&gt; --- drivers/md/bcache/request.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index 77d1a26975174..22b483527176b 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -668,9 +668,7 @@ static void backing_request_endio(struct bio *bio) static void bio_complete(struct search *s) { if (s->orig_bio) { - generic_end_io_acct(s->d->disk->queue, bio_op(s->orig_bio), - &s->d->disk->part0, s->start_time); - + bio_end_io_acct(s->orig_bio, s->start_time); trace_bcache_request_end(s->d, s->orig_bio); s->orig_bio->bi_status = s->iop.status; bio_endio(s->orig_bio); @@ -730,7 +728,7 @@ static inline struct search *search_alloc(struct bio *bio, s->recoverable = 1; s->write = op_is_write(bio_op(bio)); s->read_dirty_data = 0; - s->start_time = jiffies; + s->start_time = bio_start_io_acct(bio); s->iop.c = d->c; s->iop.bio = NULL; @@ -1082,8 +1080,7 @@ static void detached_dev_end_io(struct bio *bio) bio->bi_end_io = ddip->bi_end_io; bio->bi_private = ddip->bi_private; - generic_end_io_acct(ddip->d->disk->queue, bio_op(bio), - &ddip->d->disk->part0, ddip->start_time); + bio_end_io_acct(bio, ddip->start_time); if (bio->bi_status) { struct cached_dev *dc = container_of(ddip->d, @@ -1108,7 +1105,7 @@ static void detached_dev_do_request(struct bcache_device *d, struct bio *bio) */ ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO); ddip->d = d; - ddip->start_time = jiffies; + ddip->start_time = bio_start_io_acct(bio); ddip->bi_end_io = bio->bi_end_io; ddip->bi_private = bio->bi_private; bio->bi_end_io = detached_dev_end_io; @@ -1190,11 +1187,6 @@ blk_qc_t cached_dev_make_request(struct request_queue *q, struct bio *bio) } } - generic_start_io_acct(q, - bio_op(bio), - bio_sectors(bio), - &d->disk->part0); - bio_set_dev(bio, dc->bdev); bio->bi_iter.bi_sector += dc->sb.data_offset; @@ -1311,8 +1303,6 @@ blk_qc_t flash_dev_make_request(struct request_queue *q, struct bio *bio) return BLK_QC_T_NONE; } - generic_start_io_acct(q, bio_op(bio), bio_sectors(bio), &d->disk->part0); - s = search_alloc(bio, d); cl = &s->cl; bio = &s->bio.bio; -- 2.26.2

Switch dm to use the nicer bio accounting helpers. Signed-off-by: Christoph Hellwig <hch(a)lst.de&gt; Reviewed-by: Konstantin Khlebnikov <khlebnikov(a)yandex-team.ru&gt; --- drivers/nvdimm/blk.c | 6 ++++-- drivers/nvdimm/btt.c | 6 ++++-- drivers/nvdimm/nd.h | 19 ------------------- drivers/nvdimm/pmem.c | 6 ++++-- 4 files changed, 12 insertions(+), 25 deletions(-) diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c index 43751fab9d36a..036e23aef9b04 100644 --- a/drivers/nvdimm/blk.c +++ b/drivers/nvdimm/blk.c @@ -178,7 +178,9 @@ static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio) bip = bio_integrity(bio); nsblk = q->queuedata; rw = bio_data_dir(bio); - do_acct = nd_iostat_start(bio, &start); + do_acct = blk_queue_io_stat(bio->bi_disk->queue); + if (do_acct) + start = bio_start_io_acct(bio); bio_for_each_segment(bvec, bio, iter) { unsigned int len = bvec.bv_len; @@ -195,7 +197,7 @@ static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio) } } if (do_acct) - nd_iostat_end(bio, start); + bio_end_io_acct(bio, start); bio_endio(bio); return BLK_QC_T_NONE; diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 3b09419218d6f..90c0c4bbe77b9 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -1452,7 +1452,9 @@ static blk_qc_t btt_make_request(struct request_queue *q, struct bio *bio) if (!bio_integrity_prep(bio)) return BLK_QC_T_NONE; - do_acct = nd_iostat_start(bio, &start); + do_acct = blk_queue_io_stat(bio->bi_disk->queue); + if (do_acct) + start = bio_start_io_acct(bio); bio_for_each_segment(bvec, bio, iter) { unsigned int len = bvec.bv_len; @@ -1477,7 +1479,7 @@ static blk_qc_t btt_make_request(struct request_queue *q, struct bio *bio) } } if (do_acct) - nd_iostat_end(bio, start); + bio_end_io_acct(bio, start); bio_endio(bio); return BLK_QC_T_NONE; diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index 85dbb2a322b9b..85c1ae813ea31 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -396,25 +396,6 @@ static inline int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, #endif int nd_blk_region_init(struct nd_region *nd_region); int nd_region_activate(struct nd_region *nd_region); -void __nd_iostat_start(struct bio *bio, unsigned long *start); -static inline bool nd_iostat_start(struct bio *bio, unsigned long *start) -{ - struct gendisk *disk = bio->bi_disk; - - if (!blk_queue_io_stat(disk->queue)) - return false; - - *start = jiffies; - generic_start_io_acct(disk->queue, bio_op(bio), bio_sectors(bio), - &disk->part0); - return true; -} -static inline void nd_iostat_end(struct bio *bio, unsigned long start) -{ - struct gendisk *disk = bio->bi_disk; - - generic_end_io_acct(disk->queue, bio_op(bio), &disk->part0, start); -} static inline bool is_bad_pmem(struct badblocks *bb, sector_t sector, unsigned int len) { diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 2df6994acf836..97f948f8f4e62 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -202,7 +202,9 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) if (bio->bi_opf & REQ_PREFLUSH) ret = nvdimm_flush(nd_region, bio); - do_acct = nd_iostat_start(bio, &start); + do_acct = blk_queue_io_stat(bio->bi_disk->queue); + if (do_acct) + start = bio_start_io_acct(bio); bio_for_each_segment(bvec, bio, iter) { if (op_is_write(bio_op(bio))) rc = pmem_do_write(pmem, bvec.bv_page, bvec.bv_offset, @@ -216,7 +218,7 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) } } if (do_acct) - nd_iostat_end(bio, start); + bio_end_io_acct(bio, start); if (bio->bi_opf & REQ_FUA) ret = nvdimm_flush(nd_region, bio); -- 2.26.2

Remove these now unused functions. Signed-off-by: Christoph Hellwig <hch(a)lst.de&gt; Reviewed-by: Konstantin Khlebnikov <khlebnikov(a)yandex-team.ru&gt; --- block/bio.c | 39 --------------------------------------- include/linux/bio.h | 6 ------ 2 files changed, 45 deletions(-) diff --git a/block/bio.c b/block/bio.c index 9c101a0572ca2..3e89c7b37855a 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1392,45 +1392,6 @@ void update_io_ticks(struct hd_struct *part, unsigned long now, bool end) } } -void generic_start_io_acct(struct request_queue *q, int op, - unsigned long sectors, struct hd_struct *part) -{ - const int sgrp = op_stat_group(op); - int rw = op_is_write(op); - - part_stat_lock(); - - update_io_ticks(part, jiffies, false); - part_stat_inc(part, ios[sgrp]); - part_stat_add(part, sectors[sgrp], sectors); - part_stat_local_inc(part, in_flight[rw]); - if (part->partno) - part_stat_local_inc(&part_to_disk(part)->part0, in_flight[rw]); - - part_stat_unlock(); -} -EXPORT_SYMBOL(generic_start_io_acct); - -void generic_end_io_acct(struct request_queue *q, int req_op, - struct hd_struct *part, unsigned long start_time) -{ - unsigned long now = jiffies; - unsigned long duration = now - start_time; - const int sgrp = op_stat_group(req_op); - int rw = op_is_write(req_op); - - part_stat_lock(); - - update_io_ticks(part, now, true); - part_stat_add(part, nsecs[sgrp], jiffies_to_nsecs(duration)); - part_stat_local_dec(part, in_flight[rw]); - if (part->partno) - part_stat_local_dec(&part_to_disk(part)->part0, in_flight[rw]); - - part_stat_unlock(); -} -EXPORT_SYMBOL(generic_end_io_acct); - static inline bool bio_remaining_done(struct bio *bio) { /* diff --git a/include/linux/bio.h b/include/linux/bio.h index 950c9dc44c4f2..941378ec5b39f 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -444,12 +444,6 @@ void bio_release_pages(struct bio *bio, bool mark_dirty); extern void bio_set_pages_dirty(struct bio *bio); extern void bio_check_pages_dirty(struct bio *bio); -void generic_start_io_acct(struct request_queue *q, int op, - unsigned long sectors, struct hd_struct *part); -void generic_end_io_acct(struct request_queue *q, int op, - struct hd_struct *part, - unsigned long start_time); - extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter, struct bio *src, struct bvec_iter *src_iter); extern void bio_copy_data(struct bio *dst, struct bio *src); -- 2.26.2

percpu variables have a perfectly fine working stub implementation for UP kernels, so use that. Signed-off-by: Christoph Hellwig <hch(a)lst.de&gt; Reviewed-by: Konstantin Khlebnikov <khlebnikov(a)yandex-team.ru&gt; --- block/blk.h | 2 +- block/genhd.c | 12 +++------ block/partitions/core.c | 5 ++-- include/linux/genhd.h | 13 --------- include/linux/part_stat.h | 55 ++++++++------------------------------- 5 files changed, 18 insertions(+), 69 deletions(-) diff --git a/block/blk.h b/block/blk.h index bdf5e94467aa2..0ecba2ab383d6 100644 --- a/block/blk.h +++ b/block/blk.h @@ -378,7 +378,7 @@ static inline void hd_struct_put(struct hd_struct *part) static inline void hd_free_part(struct hd_struct *part) { - free_part_stats(part); + free_percpu(part->dkstats); kfree(part->info); percpu_ref_exit(&part->ref); } diff --git a/block/genhd.c b/block/genhd.c index 094ed90964964..3e7df0a3e6bb0 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -92,7 +92,6 @@ const char *bdevname(struct block_device *bdev, char *buf) } EXPORT_SYMBOL(bdevname); -#ifdef CONFIG_SMP static void part_stat_read_all(struct hd_struct *part, struct disk_stats *stat) { int cpu; @@ -112,12 +111,6 @@ static void part_stat_read_all(struct hd_struct *part, struct disk_stats *stat) stat->io_ticks += ptr->io_ticks; } } -#else /* CONFIG_SMP */ -static void part_stat_read_all(struct hd_struct *part, struct disk_stats *stat) -{ - memcpy(stat, &part->dkstats, sizeof(struct disk_stats)); -} -#endif /* CONFIG_SMP */ static unsigned int part_in_flight(struct request_queue *q, struct hd_struct *part) @@ -1688,14 +1681,15 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id); if (disk) { - if (!init_part_stats(&disk->part0)) { + disk->part0.dkstats = alloc_percpu(struct disk_stats); + if (!disk->part0.dkstats) { kfree(disk); return NULL; } init_rwsem(&disk->lookup_sem); disk->node_id = node_id; if (disk_expand_part_tbl(disk, 0)) { - free_part_stats(&disk->part0); + free_percpu(disk->part0.dkstats); kfree(disk); return NULL; } diff --git a/block/partitions/core.c b/block/partitions/core.c index 297004fd22648..78951e33b2d7c 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -387,7 +387,8 @@ static struct hd_struct *add_partition(struct gendisk *disk, int partno, if (!p) return ERR_PTR(-EBUSY); - if (!init_part_stats(p)) { + p->dkstats = alloc_percpu(struct disk_stats); + if (!p->dkstats) { err = -ENOMEM; goto out_free; } @@ -468,7 +469,7 @@ static struct hd_struct *add_partition(struct gendisk *disk, int partno, out_free_info: kfree(p->info); out_free_stats: - free_part_stats(p); + free_percpu(p->dkstats); out_free: kfree(p); return ERR_PTR(err); diff --git a/include/linux/genhd.h b/include/linux/genhd.h index a9384449465a3..f0d6d77309a54 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -39,15 +39,6 @@ extern struct class block_class; #include <linux/fs.h> #include <linux/workqueue.h> -struct disk_stats { - u64 nsecs[NR_STAT_GROUPS]; - unsigned long sectors[NR_STAT_GROUPS]; - unsigned long ios[NR_STAT_GROUPS]; - unsigned long merges[NR_STAT_GROUPS]; - unsigned long io_ticks; - local_t in_flight[2]; -}; - #define PARTITION_META_INFO_VOLNAMELTH 64 /* * Enough for the string representation of any kind of UUID plus NULL. @@ -72,11 +63,7 @@ struct hd_struct { seqcount_t nr_sects_seq; #endif unsigned long stamp; -#ifdef CONFIG_SMP struct disk_stats __percpu *dkstats; -#else - struct disk_stats dkstats; -#endif struct percpu_ref ref; sector_t alignment_offset; diff --git a/include/linux/part_stat.h b/include/linux/part_stat.h index ece607607a864..6644197980b92 100644 --- a/include/linux/part_stat.h +++ b/include/linux/part_stat.h @@ -4,19 +4,23 @@ #include <linux/genhd.h> +struct disk_stats { + u64 nsecs[NR_STAT_GROUPS]; + unsigned long sectors[NR_STAT_GROUPS]; + unsigned long ios[NR_STAT_GROUPS]; + unsigned long merges[NR_STAT_GROUPS]; + unsigned long io_ticks; + local_t in_flight[2]; +}; + /* * Macros to operate on percpu disk statistics: * - * {disk|part|all}_stat_{add|sub|inc|dec}() modify the stat counters - * and should be called between disk_stat_lock() and - * disk_stat_unlock(). + * {disk|part|all}_stat_{add|sub|inc|dec}() modify the stat counters and should + * be called between disk_stat_lock() and disk_stat_unlock(). * * part_stat_read() can be called at any time. - * - * part_stat_{add|set_all}() and {init|free}_part_stats are for - * internal use only. */ -#ifdef CONFIG_SMP #define part_stat_lock() ({ rcu_read_lock(); get_cpu(); }) #define part_stat_unlock() do { put_cpu(); rcu_read_unlock(); } while (0) @@ -44,43 +48,6 @@ static inline void part_stat_set_all(struct hd_struct *part, int value) sizeof(struct disk_stats)); } -static inline int init_part_stats(struct hd_struct *part) -{ - part->dkstats = alloc_percpu(struct disk_stats); - if (!part->dkstats) - return 0; - return 1; -} - -static inline void free_part_stats(struct hd_struct *part) -{ - free_percpu(part->dkstats); -} - -#else /* !CONFIG_SMP */ -#define part_stat_lock() ({ rcu_read_lock(); 0; }) -#define part_stat_unlock() rcu_read_unlock() - -#define part_stat_get(part, field) ((part)->dkstats.field) -#define part_stat_get_cpu(part, field, cpu) part_stat_get(part, field) -#define part_stat_read(part, field) part_stat_get(part, field) - -static inline void part_stat_set_all(struct hd_struct *part, int value) -{ - memset(&part->dkstats, value, sizeof(struct disk_stats)); -} - -static inline int init_part_stats(struct hd_struct *part) -{ - return 1; -} - -static inline void free_part_stats(struct hd_struct *part) -{ -} - -#endif /* CONFIG_SMP */ - #define part_stat_read_accum(part, field) \ (part_stat_read(part, field[STAT_READ]) + \ part_stat_read(part, field[STAT_WRITE]) + \ -- 2.26.2

From: Konstantin Khlebnikov <khlebnikov(a)yandex-team.ru&gt; Move the non-"new_io" branch of blk_account_io_start() into separate function. Fix merge accounting for discards (they were counted as write merges). The new blk_account_io_merge_bio() doesn't call update_io_ticks() unlike blk_account_io_start(), as there is no reason for that. Signed-off-by: Konstantin Khlebnikov <khlebnikov(a)yandex-team.ru&gt; [hch: rebased] Signed-off-by: Christoph Hellwig <hch(a)lst.de&gt; --- block/blk-core.c | 25 ++++++++++++++++--------- block/blk-exec.c | 2 +- block/blk-mq.c | 2 +- block/blk.h | 2 +- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/block/blk-core.c b/block/blk-core.c index c1675d43c2da0..bf2f7d4bc0c1c 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -636,6 +636,16 @@ void blk_put_request(struct request *req) } EXPORT_SYMBOL(blk_put_request); +static void blk_account_io_merge_bio(struct request *req) +{ + if (!blk_do_io_stat(req)) + return; + + part_stat_lock(); + part_stat_inc(req->part, merges[op_stat_group(req_op(req))]); + part_stat_unlock(); +} + bool bio_attempt_back_merge(struct request *req, struct bio *bio, unsigned int nr_segs) { @@ -656,7 +666,7 @@ bool bio_attempt_back_merge(struct request *req, struct bio *bio, bio_crypt_free_ctx(bio); - blk_account_io_start(req, false); + blk_account_io_merge_bio(req); return true; } @@ -682,7 +692,7 @@ bool bio_attempt_front_merge(struct request *req, struct bio *bio, bio_crypt_do_front_merge(req, bio); - blk_account_io_start(req, false); + blk_account_io_merge_bio(req); return true; } @@ -704,7 +714,7 @@ bool bio_attempt_discard_merge(struct request_queue *q, struct request *req, req->__data_len += bio->bi_iter.bi_size; req->nr_phys_segments = segments + 1; - blk_account_io_start(req, false); + blk_account_io_merge_bio(req); return true; no_merge: req_set_nomerge(q, req); @@ -1329,7 +1339,7 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request * return BLK_STS_IOERR; if (blk_queue_io_stat(q)) - blk_account_io_start(rq, true); + blk_account_io_start(rq); /* * Since we have a scheduler attached on the top device, @@ -1433,16 +1443,13 @@ void blk_account_io_done(struct request *req, u64 now) } } -void blk_account_io_start(struct request *rq, bool new_io) +void blk_account_io_start(struct request *rq) { if (!blk_do_io_stat(rq)) return; part_stat_lock(); - if (!new_io) - part_stat_inc(rq->part, merges[rq_data_dir(rq)]); - else - rq->part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); + rq->part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); update_io_ticks(rq->part, jiffies, false); part_stat_unlock(); } diff --git a/block/blk-exec.c b/block/blk-exec.c index e20a852ae432d..85324d53d072f 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c @@ -55,7 +55,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk, rq->rq_disk = bd_disk; rq->end_io = done; - blk_account_io_start(rq, true); + blk_account_io_start(rq); /* * don't check dying flag for MQ because the request won't diff --git a/block/blk-mq.c b/block/blk-mq.c index cac11945f6023..c606c74463ccd 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1822,7 +1822,7 @@ static void blk_mq_bio_to_request(struct request *rq, struct bio *bio, blk_rq_bio_prep(rq, bio, nr_segs); blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO); - blk_account_io_start(rq, true); + blk_account_io_start(rq); } static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, diff --git a/block/blk.h b/block/blk.h index 0ecba2ab383d6..428f7e5d70a86 100644 --- a/block/blk.h +++ b/block/blk.h @@ -185,7 +185,7 @@ bool bio_attempt_discard_merge(struct request_queue *q, struct request *req, bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs, struct request **same_queue_rq); -void blk_account_io_start(struct request *req, bool new_io); +void blk_account_io_start(struct request *req); void blk_account_io_done(struct request *req, u64 now); /* -- 2.26.2

From: Konstantin Khlebnikov <khlebnikov(a)yandex-team.ru&gt; Most architectures have fast path to access percpu for current cpu. The required preempt_disable() is provided by part_stat_lock(). Signed-off-by: Konstantin Khlebnikov <khlebnikov(a)yandex-team.ru&gt; [hch: rebased] Signed-off-by: Christoph Hellwig <hch(a)lst.de&gt; --- include/linux/part_stat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/part_stat.h b/include/linux/part_stat.h index a6b0938ce82e9..24125778ef3ec 100644 --- a/include/linux/part_stat.h +++ b/include/linux/part_stat.h @@ -54,7 +54,7 @@ static inline void part_stat_set_all(struct hd_struct *part, int value) part_stat_read(part, field[STAT_DISCARD])) #define __part_stat_add(part, field, addnd) \ - (part_stat_get(part, field) += (addnd)) + __this_cpu_add((part)->dkstats->field, addnd) #define part_stat_add(part, field, addnd) do { \ __part_stat_add((part), field, addnd); \ -- 2.26.2