summaryrefslogtreecommitdiffstats
path: root/drivers/block
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/block')
-rw-r--r--drivers/block/as-iosched.c8
-rw-r--r--drivers/block/deadline-iosched.c8
-rw-r--r--drivers/block/genhd.c25
-rw-r--r--drivers/block/ioctl.c74
-rw-r--r--drivers/block/ll_rw_blk.c111
-rw-r--r--drivers/block/loop.c81
-rw-r--r--drivers/block/pktcdvd.c36
7 files changed, 199 insertions, 144 deletions
diff --git a/drivers/block/as-iosched.c b/drivers/block/as-iosched.c
index 638db06de2be..3410b4d294b9 100644
--- a/drivers/block/as-iosched.c
+++ b/drivers/block/as-iosched.c
@@ -1871,20 +1871,22 @@ static int as_init_queue(request_queue_t *q, elevator_t *e)
if (!arq_pool)
return -ENOMEM;
- ad = kmalloc(sizeof(*ad), GFP_KERNEL);
+ ad = kmalloc_node(sizeof(*ad), GFP_KERNEL, q->node);
if (!ad)
return -ENOMEM;
memset(ad, 0, sizeof(*ad));
ad->q = q; /* Identify what queue the data belongs to */
- ad->hash = kmalloc(sizeof(struct list_head)*AS_HASH_ENTRIES,GFP_KERNEL);
+ ad->hash = kmalloc_node(sizeof(struct list_head)*AS_HASH_ENTRIES,
+ GFP_KERNEL, q->node);
if (!ad->hash) {
kfree(ad);
return -ENOMEM;
}
- ad->arq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, arq_pool);
+ ad->arq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+ mempool_free_slab, arq_pool, q->node);
if (!ad->arq_pool) {
kfree(ad->hash);
kfree(ad);
diff --git a/drivers/block/deadline-iosched.c b/drivers/block/deadline-iosched.c
index 7f79f3dd0165..4bc2fea73273 100644
--- a/drivers/block/deadline-iosched.c
+++ b/drivers/block/deadline-iosched.c
@@ -711,18 +711,20 @@ static int deadline_init_queue(request_queue_t *q, elevator_t *e)
if (!drq_pool)
return -ENOMEM;
- dd = kmalloc(sizeof(*dd), GFP_KERNEL);
+ dd = kmalloc_node(sizeof(*dd), GFP_KERNEL, q->node);
if (!dd)
return -ENOMEM;
memset(dd, 0, sizeof(*dd));
- dd->hash = kmalloc(sizeof(struct list_head)*DL_HASH_ENTRIES,GFP_KERNEL);
+ dd->hash = kmalloc_node(sizeof(struct list_head)*DL_HASH_ENTRIES,
+ GFP_KERNEL, q->node);
if (!dd->hash) {
kfree(dd);
return -ENOMEM;
}
- dd->drq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, drq_pool);
+ dd->drq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+ mempool_free_slab, drq_pool, q->node);
if (!dd->drq_pool) {
kfree(dd->hash);
kfree(dd);
diff --git a/drivers/block/genhd.c b/drivers/block/genhd.c
index 53f7d846b747..47fd3659a061 100644
--- a/drivers/block/genhd.c
+++ b/drivers/block/genhd.c
@@ -40,7 +40,7 @@ static inline int major_to_index(int major)
#ifdef CONFIG_PROC_FS
/* get block device names in somewhat random order */
-int get_blkdev_list(char *p)
+int get_blkdev_list(char *p, int used)
{
struct blk_major_name *n;
int i, len;
@@ -49,10 +49,18 @@ int get_blkdev_list(char *p)
down(&block_subsys_sem);
for (i = 0; i < ARRAY_SIZE(major_names); i++) {
- for (n = major_names[i]; n; n = n->next)
+ for (n = major_names[i]; n; n = n->next) {
+ /*
+ * If the curent string plus the 5 extra characters
+ * in the line would run us off the page, then we're done
+ */
+ if ((len + used + strlen(n->name) + 5) >= PAGE_SIZE)
+ goto page_full;
len += sprintf(p+len, "%3d %s\n",
n->major, n->name);
+ }
}
+page_full:
up(&block_subsys_sem);
return len;
@@ -582,10 +590,16 @@ struct seq_operations diskstats_op = {
.show = diskstats_show
};
-
struct gendisk *alloc_disk(int minors)
{
- struct gendisk *disk = kmalloc(sizeof(struct gendisk), GFP_KERNEL);
+ return alloc_disk_node(minors, -1);
+}
+
+struct gendisk *alloc_disk_node(int minors, int node_id)
+{
+ struct gendisk *disk;
+
+ disk = kmalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
if (disk) {
memset(disk, 0, sizeof(struct gendisk));
if (!init_disk_stats(disk)) {
@@ -594,7 +608,7 @@ struct gendisk *alloc_disk(int minors)
}
if (minors > 1) {
int size = (minors - 1) * sizeof(struct hd_struct *);
- disk->part = kmalloc(size, GFP_KERNEL);
+ disk->part = kmalloc_node(size, GFP_KERNEL, node_id);
if (!disk->part) {
kfree(disk);
return NULL;
@@ -610,6 +624,7 @@ struct gendisk *alloc_disk(int minors)
}
EXPORT_SYMBOL(alloc_disk);
+EXPORT_SYMBOL(alloc_disk_node);
struct kobject *get_disk(struct gendisk *disk)
{
diff --git a/drivers/block/ioctl.c b/drivers/block/ioctl.c
index 6d7bcc9da9e7..6e278474f9a8 100644
--- a/drivers/block/ioctl.c
+++ b/drivers/block/ioctl.c
@@ -133,11 +133,9 @@ static int put_u64(unsigned long arg, u64 val)
return put_user(val, (u64 __user *)arg);
}
-int blkdev_ioctl(struct inode *inode, struct file *file, unsigned cmd,
- unsigned long arg)
+static int blkdev_locked_ioctl(struct file *file, struct block_device *bdev,
+ unsigned cmd, unsigned long arg)
{
- struct block_device *bdev = inode->i_bdev;
- struct gendisk *disk = bdev->bd_disk;
struct backing_dev_info *bdi;
int ret, n;
@@ -190,36 +188,72 @@ int blkdev_ioctl(struct inode *inode, struct file *file, unsigned cmd,
return put_ulong(arg, bdev->bd_inode->i_size >> 9);
case BLKGETSIZE64:
return put_u64(arg, bdev->bd_inode->i_size);
+ }
+ return -ENOIOCTLCMD;
+}
+
+static int blkdev_driver_ioctl(struct inode *inode, struct file *file,
+ struct gendisk *disk, unsigned cmd, unsigned long arg)
+{
+ int ret;
+ if (disk->fops->unlocked_ioctl)
+ return disk->fops->unlocked_ioctl(file, cmd, arg);
+
+ if (disk->fops->ioctl) {
+ lock_kernel();
+ ret = disk->fops->ioctl(inode, file, cmd, arg);
+ unlock_kernel();
+ return ret;
+ }
+
+ return -ENOTTY;
+}
+
+int blkdev_ioctl(struct inode *inode, struct file *file, unsigned cmd,
+ unsigned long arg)
+{
+ struct block_device *bdev = inode->i_bdev;
+ struct gendisk *disk = bdev->bd_disk;
+ int ret, n;
+
+ switch(cmd) {
case BLKFLSBUF:
if (!capable(CAP_SYS_ADMIN))
return -EACCES;
- if (disk->fops->ioctl) {
- ret = disk->fops->ioctl(inode, file, cmd, arg);
- /* -EINVAL to handle old uncorrected drivers */
- if (ret != -EINVAL && ret != -ENOTTY)
- return ret;
- }
+
+ ret = blkdev_driver_ioctl(inode, file, disk, cmd, arg);
+ /* -EINVAL to handle old uncorrected drivers */
+ if (ret != -EINVAL && ret != -ENOTTY)
+ return ret;
+
+ lock_kernel();
fsync_bdev(bdev);
invalidate_bdev(bdev, 0);
+ unlock_kernel();
return 0;
+
case BLKROSET:
- if (disk->fops->ioctl) {
- ret = disk->fops->ioctl(inode, file, cmd, arg);
- /* -EINVAL to handle old uncorrected drivers */
- if (ret != -EINVAL && ret != -ENOTTY)
- return ret;
- }
+ ret = blkdev_driver_ioctl(inode, file, disk, cmd, arg);
+ /* -EINVAL to handle old uncorrected drivers */
+ if (ret != -EINVAL && ret != -ENOTTY)
+ return ret;
if (!capable(CAP_SYS_ADMIN))
return -EACCES;
if (get_user(n, (int __user *)(arg)))
return -EFAULT;
+ lock_kernel();
set_device_ro(bdev, n);
+ unlock_kernel();
return 0;
- default:
- if (disk->fops->ioctl)
- return disk->fops->ioctl(inode, file, cmd, arg);
}
- return -ENOTTY;
+
+ lock_kernel();
+ ret = blkdev_locked_ioctl(file, bdev, cmd, arg);
+ unlock_kernel();
+ if (ret != -ENOIOCTLCMD)
+ return ret;
+
+ return blkdev_driver_ioctl(inode, file, disk, cmd, arg);
}
/* Most of the generic ioctls are handled in the normal fallback path.
diff --git a/drivers/block/ll_rw_blk.c b/drivers/block/ll_rw_blk.c
index 81fe3a0c1fe7..fd94ea27d594 100644
--- a/drivers/block/ll_rw_blk.c
+++ b/drivers/block/ll_rw_blk.c
@@ -28,6 +28,7 @@
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/writeback.h>
+#include <linux/blkdev.h>
/*
* for max sense size
@@ -716,7 +717,7 @@ struct request *blk_queue_find_tag(request_queue_t *q, int tag)
{
struct blk_queue_tag *bqt = q->queue_tags;
- if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
+ if (unlikely(bqt == NULL || tag >= bqt->max_depth))
return NULL;
return bqt->tag_index[tag];
@@ -774,9 +775,9 @@ EXPORT_SYMBOL(blk_queue_free_tags);
static int
init_tag_map(request_queue_t *q, struct blk_queue_tag *tags, int depth)
{
- int bits, i;
struct request **tag_index;
unsigned long *tag_map;
+ int nr_ulongs;
if (depth > q->nr_requests * 2) {
depth = q->nr_requests * 2;
@@ -788,24 +789,17 @@ init_tag_map(request_queue_t *q, struct blk_queue_tag *tags, int depth)
if (!tag_index)
goto fail;
- bits = (depth / BLK_TAGS_PER_LONG) + 1;
- tag_map = kmalloc(bits * sizeof(unsigned long), GFP_ATOMIC);
+ nr_ulongs = ALIGN(depth, BITS_PER_LONG) / BITS_PER_LONG;
+ tag_map = kmalloc(nr_ulongs * sizeof(unsigned long), GFP_ATOMIC);
if (!tag_map)
goto fail;
memset(tag_index, 0, depth * sizeof(struct request *));
- memset(tag_map, 0, bits * sizeof(unsigned long));
+ memset(tag_map, 0, nr_ulongs * sizeof(unsigned long));
tags->max_depth = depth;
- tags->real_max_depth = bits * BITS_PER_LONG;
tags->tag_index = tag_index;
tags->tag_map = tag_map;
- /*
- * set the upper bits if the depth isn't a multiple of the word size
- */
- for (i = depth; i < bits * BLK_TAGS_PER_LONG; i++)
- __set_bit(i, tag_map);
-
return 0;
fail:
kfree(tag_index);
@@ -870,32 +864,24 @@ int blk_queue_resize_tags(request_queue_t *q, int new_depth)
struct blk_queue_tag *bqt = q->queue_tags;
struct request **tag_index;
unsigned long *tag_map;
- int bits, max_depth;
+ int max_depth, nr_ulongs;
if (!bqt)
return -ENXIO;
/*
- * don't bother sizing down
- */
- if (new_depth <= bqt->real_max_depth) {
- bqt->max_depth = new_depth;
- return 0;
- }
-
- /*
* save the old state info, so we can copy it back
*/
tag_index = bqt->tag_index;
tag_map = bqt->tag_map;
- max_depth = bqt->real_max_depth;
+ max_depth = bqt->max_depth;
if (init_tag_map(q, bqt, new_depth))
return -ENOMEM;
memcpy(bqt->tag_index, tag_index, max_depth * sizeof(struct request *));
- bits = max_depth / BLK_TAGS_PER_LONG;
- memcpy(bqt->tag_map, tag_map, bits * sizeof(unsigned long));
+ nr_ulongs = ALIGN(max_depth, BITS_PER_LONG) / BITS_PER_LONG;
+ memcpy(bqt->tag_map, tag_map, nr_ulongs * sizeof(unsigned long));
kfree(tag_index);
kfree(tag_map);
@@ -925,11 +911,16 @@ void blk_queue_end_tag(request_queue_t *q, struct request *rq)
BUG_ON(tag == -1);
- if (unlikely(tag >= bqt->real_max_depth))
+ if (unlikely(tag >= bqt->max_depth))
+ /*
+ * This can happen after tag depth has been reduced.
+ * FIXME: how about a warning or info message here?
+ */
return;
if (unlikely(!__test_and_clear_bit(tag, bqt->tag_map))) {
- printk("attempt to clear non-busy tag (%d)\n", tag);
+ printk(KERN_ERR "%s: attempt to clear non-busy tag (%d)\n",
+ __FUNCTION__, tag);
return;
}
@@ -938,7 +929,8 @@ void blk_queue_end_tag(request_queue_t *q, struct request *rq)
rq->tag = -1;
if (unlikely(bqt->tag_index[tag] == NULL))
- printk("tag %d is missing\n", tag);
+ printk(KERN_ERR "%s: tag %d is missing\n",
+ __FUNCTION__, tag);
bqt->tag_index[tag] = NULL;
bqt->busy--;
@@ -967,24 +959,20 @@ EXPORT_SYMBOL(blk_queue_end_tag);
int blk_queue_start_tag(request_queue_t *q, struct request *rq)
{
struct blk_queue_tag *bqt = q->queue_tags;
- unsigned long *map = bqt->tag_map;
- int tag = 0;
+ int tag;
if (unlikely((rq->flags & REQ_QUEUED))) {
printk(KERN_ERR
- "request %p for device [%s] already tagged %d",
- rq, rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->tag);
+ "%s: request %p for device [%s] already tagged %d",
+ __FUNCTION__, rq,
+ rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->tag);
BUG();
}
- for (map = bqt->tag_map; *map == -1UL; map++) {
- tag += BLK_TAGS_PER_LONG;
-
- if (tag >= bqt->max_depth)
- return 1;
- }
+ tag = find_first_zero_bit(bqt->tag_map, bqt->max_depth);
+ if (tag >= bqt->max_depth)
+ return 1;
- tag += ffz(*map);
__set_bit(tag, bqt->tag_map);
rq->flags |= REQ_QUEUED;
@@ -1020,7 +1008,8 @@ void blk_queue_invalidate_tags(request_queue_t *q)
rq = list_entry_rq(tmp);
if (rq->tag == -1) {
- printk("bad tag found on list\n");
+ printk(KERN_ERR
+ "%s: bad tag found on list\n", __FUNCTION__);
list_del_init(&rq->queuelist);
rq->flags &= ~REQ_QUEUED;
} else
@@ -1450,7 +1439,7 @@ EXPORT_SYMBOL(blk_remove_plug);
*/
void __generic_unplug_device(request_queue_t *q)
{
- if (test_bit(QUEUE_FLAG_STOPPED, &q->queue_flags))
+ if (unlikely(test_bit(QUEUE_FLAG_STOPPED, &q->queue_flags)))
return;
if (!blk_remove_plug(q))
@@ -1645,7 +1634,8 @@ static int blk_init_free_list(request_queue_t *q)
init_waitqueue_head(&rl->wait[WRITE]);
init_waitqueue_head(&rl->drain);
- rl->rq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, request_cachep);
+ rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+ mempool_free_slab, request_cachep, q->node);
if (!rl->rq_pool)
return -ENOMEM;
@@ -1657,8 +1647,15 @@ static int __make_request(request_queue_t *, struct bio *);
request_queue_t *blk_alloc_queue(int gfp_mask)
{
- request_queue_t *q = kmem_cache_alloc(requestq_cachep, gfp_mask);
+ return blk_alloc_queue_node(gfp_mask, -1);
+}
+EXPORT_SYMBOL(blk_alloc_queue);
+
+request_queue_t *blk_alloc_queue_node(int gfp_mask, int node_id)
+{
+ request_queue_t *q;
+ q = kmem_cache_alloc_node(requestq_cachep, gfp_mask, node_id);
if (!q)
return NULL;
@@ -1671,8 +1668,7 @@ request_queue_t *blk_alloc_queue(int gfp_mask)
return q;
}
-
-EXPORT_SYMBOL(blk_alloc_queue);
+EXPORT_SYMBOL(blk_alloc_queue_node);
/**
* blk_init_queue - prepare a request queue for use with a block device
@@ -1705,13 +1701,22 @@ EXPORT_SYMBOL(blk_alloc_queue);
* blk_init_queue() must be paired with a blk_cleanup_queue() call
* when the block device is deactivated (such as at module unload).
**/
+
request_queue_t *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
{
- request_queue_t *q = blk_alloc_queue(GFP_KERNEL);
+ return blk_init_queue_node(rfn, lock, -1);
+}
+EXPORT_SYMBOL(blk_init_queue);
+
+request_queue_t *
+blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
+{
+ request_queue_t *q = blk_alloc_queue_node(GFP_KERNEL, node_id);
if (!q)
return NULL;
+ q->node = node_id;
if (blk_init_free_list(q))
goto out_init;
@@ -1754,12 +1759,11 @@ out_init:
kmem_cache_free(requestq_cachep, q);
return NULL;
}
-
-EXPORT_SYMBOL(blk_init_queue);
+EXPORT_SYMBOL(blk_init_queue_node);
int blk_get_queue(request_queue_t *q)
{
- if (!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)) {
+ if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
atomic_inc(&q->refcnt);
return 0;
}
@@ -1838,7 +1842,6 @@ static void __freed_request(request_queue_t *q, int rw)
clear_queue_congested(q, rw);
if (rl->count[rw] + 1 <= q->nr_requests) {
- smp_mb();
if (waitqueue_active(&rl->wait[rw]))
wake_up(&rl->wait[rw]);
@@ -1966,7 +1969,6 @@ static struct request *get_request_wait(request_queue_t *q, int rw)
DEFINE_WAIT(wait);
struct request *rq;
- generic_unplug_device(q);
do {
struct request_list *rl = &q->rq;
@@ -1978,6 +1980,7 @@ static struct request *get_request_wait(request_queue_t *q, int rw)
if (!rq) {
struct io_context *ioc;
+ generic_unplug_device(q);
io_schedule();
/*
@@ -2581,7 +2584,7 @@ static int __make_request(request_queue_t *q, struct bio *bio)
spin_lock_prefetch(q->queue_lock);
barrier = bio_barrier(bio);
- if (barrier && (q->ordered == QUEUE_ORDERED_NONE)) {
+ if (unlikely(barrier) && (q->ordered == QUEUE_ORDERED_NONE)) {
err = -EOPNOTSUPP;
goto end_io;
}
@@ -2682,7 +2685,7 @@ get_rq:
/*
* REQ_BARRIER implies no merging, but lets make it explicit
*/
- if (barrier)
+ if (unlikely(barrier))
req->flags |= (REQ_HARDBARRIER | REQ_NOMERGE);
req->errors = 0;
@@ -2806,7 +2809,7 @@ static inline void block_wait_queue_running(request_queue_t *q)
{
DEFINE_WAIT(wait);
- while (test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags)) {
+ while (unlikely(test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags))) {
struct request_list *rl = &q->rq;
prepare_to_wait_exclusive(&rl->drain, &wait,
@@ -2915,7 +2918,7 @@ end_io:
goto end_io;
}
- if (test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))
+ if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
goto end_io;
block_wait_queue_running(q);
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index 6f011d0d8e97..b35e08876dd4 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -472,17 +472,11 @@ static int do_bio_filebacked(struct loop_device *lo, struct bio *bio)
*/
static void loop_add_bio(struct loop_device *lo, struct bio *bio)
{
- unsigned long flags;
-
- spin_lock_irqsave(&lo->lo_lock, flags);
if (lo->lo_biotail) {
lo->lo_biotail->bi_next = bio;
lo->lo_biotail = bio;
} else
lo->lo_bio = lo->lo_biotail = bio;
- spin_unlock_irqrestore(&lo->lo_lock, flags);
-
- up(&lo->lo_bh_mutex);
}
/*
@@ -492,14 +486,12 @@ static struct bio *loop_get_bio(struct loop_device *lo)
{
struct bio *bio;
- spin_lock_irq(&lo->lo_lock);
if ((bio = lo->lo_bio)) {
if (bio == lo->lo_biotail)
lo->lo_biotail = NULL;
lo->lo_bio = bio->bi_next;
bio->bi_next = NULL;
}
- spin_unlock_irq(&lo->lo_lock);
return bio;
}
@@ -509,35 +501,28 @@ static int loop_make_request(request_queue_t *q, struct bio *old_bio)
struct loop_device *lo = q->queuedata;
int rw = bio_rw(old_bio);
- if (!lo)
- goto out;
+ if (rw == READA)
+ rw = READ;
+
+ BUG_ON(!lo || (rw != READ && rw != WRITE));
spin_lock_irq(&lo->lo_lock);
if (lo->lo_state != Lo_bound)
- goto inactive;
- atomic_inc(&lo->lo_pending);
- spin_unlock_irq(&lo->lo_lock);
-
- if (rw == WRITE) {
- if (lo->lo_flags & LO_FLAGS_READ_ONLY)
- goto err;
- } else if (rw == READA) {
- rw = READ;
- } else if (rw != READ) {
- printk(KERN_ERR "loop: unknown command (%x)\n", rw);
- goto err;
- }
+ goto out;
+ if (unlikely(rw == WRITE && (lo->lo_flags & LO_FLAGS_READ_ONLY)))
+ goto out;
+ lo->lo_pending++;
loop_add_bio(lo, old_bio);
+ spin_unlock_irq(&lo->lo_lock);
+ up(&lo->lo_bh_mutex);
return 0;
-err:
- if (atomic_dec_and_test(&lo->lo_pending))
- up(&lo->lo_bh_mutex);
+
out:
+ if (lo->lo_pending == 0)
+ up(&lo->lo_bh_mutex);
+ spin_unlock_irq(&lo->lo_lock);
bio_io_error(old_bio, old_bio->bi_size);
return 0;
-inactive:
- spin_unlock_irq(&lo->lo_lock);
- goto out;
}
/*
@@ -560,13 +545,11 @@ static void do_loop_switch(struct loop_device *, struct switch_request *);
static inline void loop_handle_bio(struct loop_device *lo, struct bio *bio)
{
- int ret;
-
if (unlikely(!bio->bi_bdev)) {
do_loop_switch(lo, bio->bi_private);
bio_put(bio);
} else {
- ret = do_bio_filebacked(lo, bio);
+ int ret = do_bio_filebacked(lo, bio);
bio_endio(bio, bio->bi_size, ret);
}
}
@@ -594,7 +577,7 @@ static int loop_thread(void *data)
set_user_nice(current, -20);
lo->lo_state = Lo_bound;
- atomic_inc(&lo->lo_pending);
+ lo->lo_pending = 1;
/*
* up sem, we are running
@@ -602,26 +585,37 @@ static int loop_thread(void *data)
up(&lo->lo_sem);
for (;;) {
- down_interruptible(&lo->lo_bh_mutex);
+ int pending;
+
/*
- * could be upped because of tear-down, not because of
- * pending work
+ * interruptible just to not contribute to load avg
*/
- if (!atomic_read(&lo->lo_pending))
+ if (down_interruptible(&lo->lo_bh_mutex))
+ continue;
+
+ spin_lock_irq(&lo->lo_lock);
+
+ /*
+ * could be upped because of tear-down, not pending work
+ */
+ if (unlikely(!lo->lo_pending)) {
+ spin_unlock_irq(&lo->lo_lock);
break;
+ }
bio = loop_get_bio(lo);
- if (!bio) {
- printk("loop: missing bio\n");
- continue;
- }
+ lo->lo_pending--;
+ pending = lo->lo_pending;
+ spin_unlock_irq(&lo->lo_lock);
+
+ BUG_ON(!bio);
loop_handle_bio(lo, bio);
/*
* upped both for pending work and tear-down, lo_pending
* will hit zero then
*/
- if (atomic_dec_and_test(&lo->lo_pending))
+ if (unlikely(!pending))
break;
}
@@ -900,7 +894,8 @@ static int loop_clr_fd(struct loop_device *lo, struct block_device *bdev)
spin_lock_irq(&lo->lo_lock);
lo->lo_state = Lo_rundown;
- if (atomic_dec_and_test(&lo->lo_pending))
+ lo->lo_pending--;
+ if (!lo->lo_pending)
up(&lo->lo_bh_mutex);
spin_unlock_irq(&lo->lo_lock);
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index bc56770bcc90..7f3d78de265c 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -467,14 +467,12 @@ static int pkt_set_speed(struct pktcdvd_device *pd, unsigned write_speed, unsign
* Queue a bio for processing by the low-level CD device. Must be called
* from process context.
*/
-static void pkt_queue_bio(struct pktcdvd_device *pd, struct bio *bio, int high_prio_read)
+static void pkt_queue_bio(struct pktcdvd_device *pd, struct bio *bio)
{
spin_lock(&pd->iosched.lock);
if (bio_data_dir(bio) == READ) {
pkt_add_list_last(bio, &pd->iosched.read_queue,
&pd->iosched.read_queue_tail);
- if (high_prio_read)
- pd->iosched.high_prio_read = 1;
} else {
pkt_add_list_last(bio, &pd->iosched.write_queue,
&pd->iosched.write_queue_tail);
@@ -490,15 +488,16 @@ static void pkt_queue_bio(struct pktcdvd_device *pd, struct bio *bio, int high_p
* requirements for CDRW drives:
* - A cache flush command must be inserted before a read request if the
* previous request was a write.
- * - Switching between reading and writing is slow, so don't it more often
+ * - Switching between reading and writing is slow, so don't do it more often
* than necessary.
+ * - Optimize for throughput at the expense of latency. This means that streaming
+ * writes will never be interrupted by a read, but if the drive has to seek
+ * before the next write, switch to reading instead if there are any pending
+ * read requests.
* - Set the read speed according to current usage pattern. When only reading
* from the device, it's best to use the highest possible read speed, but
* when switching often between reading and writing, it's better to have the
* same read and write speeds.
- * - Reads originating from user space should have higher priority than reads
- * originating from pkt_gather_data, because some process is usually waiting
- * on reads of the first kind.
*/
static void pkt_iosched_process_queue(struct pktcdvd_device *pd)
{
@@ -512,21 +511,24 @@ static void pkt_iosched_process_queue(struct pktcdvd_device *pd)
for (;;) {
struct bio *bio;
- int reads_queued, writes_queued, high_prio_read;
+ int reads_queued, writes_queued;
spin_lock(&pd->iosched.lock);
reads_queued = (pd->iosched.read_queue != NULL);
writes_queued = (pd->iosched.write_queue != NULL);
- if (!reads_queued)
- pd->iosched.high_prio_read = 0;
- high_prio_read = pd->iosched.high_prio_read;
spin_unlock(&pd->iosched.lock);
if (!reads_queued && !writes_queued)
break;
if (pd->iosched.writing) {
- if (high_prio_read || (!writes_queued && reads_queued)) {
+ int need_write_seek = 1;
+ spin_lock(&pd->iosched.lock);
+ bio = pd->iosched.write_queue;
+ spin_unlock(&pd->iosched.lock);
+ if (bio && (bio->bi_sector == pd->iosched.last_write))
+ need_write_seek = 0;
+ if (need_write_seek && reads_queued) {
if (atomic_read(&pd->cdrw.pending_bios) > 0) {
VPRINTK("pktcdvd: write, waiting\n");
break;
@@ -559,8 +561,10 @@ static void pkt_iosched_process_queue(struct pktcdvd_device *pd)
if (bio_data_dir(bio) == READ)
pd->iosched.successive_reads += bio->bi_size >> 10;
- else
+ else {
pd->iosched.successive_reads = 0;
+ pd->iosched.last_write = bio->bi_sector + bio_sectors(bio);
+ }
if (pd->iosched.successive_reads >= HI_SPEED_SWITCH) {
if (pd->read_speed == pd->write_speed) {
pd->read_speed = MAX_SPEED;
@@ -765,7 +769,7 @@ static void pkt_gather_data(struct pktcdvd_device *pd, struct packet_data *pkt)
atomic_inc(&pkt->io_wait);
bio->bi_rw = READ;
- pkt_queue_bio(pd, bio, 0);
+ pkt_queue_bio(pd, bio);
frames_read++;
}
@@ -1062,7 +1066,7 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt)
atomic_set(&pkt->io_wait, 1);
pkt->w_bio->bi_rw = WRITE;
- pkt_queue_bio(pd, pkt->w_bio, 0);
+ pkt_queue_bio(pd, pkt->w_bio);
}
static void pkt_finish_packet(struct packet_data *pkt, int uptodate)
@@ -2120,7 +2124,7 @@ static int pkt_make_request(request_queue_t *q, struct bio *bio)
cloned_bio->bi_private = psd;
cloned_bio->bi_end_io = pkt_end_io_read_cloned;
pd->stats.secs_r += bio->bi_size >> 9;
- pkt_queue_bio(pd, cloned_bio, 1);
+ pkt_queue_bio(pd, cloned_bio);
return 0;
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-03-13 14:37:23 +0000