diff options
author | Tiwei Bie <tiwei.bie@intel.com> | 2018-11-21 18:03:28 +0800 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2018-11-26 22:17:39 -0800 |
commit | f51f982682e2a612fa642dfee47cc8da63677148 (patch) | |
tree | d7fee1b20490d8d2dff82d2f7ae254a200a6fdc0 /drivers/virtio | |
parent | 1ce9e6055fa0a9043405c5604cf19169ec5379ff (diff) | |
download | linux-f51f982682e2a612fa642dfee47cc8da63677148.tar.gz linux-f51f982682e2a612fa642dfee47cc8da63677148.tar.bz2 linux-f51f982682e2a612fa642dfee47cc8da63677148.zip |
virtio_ring: leverage event idx in packed ring
Leverage the EVENT_IDX feature in packed ring to suppress
events when it's available.
Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers/virtio')
-rw-r--r-- | drivers/virtio/virtio_ring.c | 77 |
1 files changed, 71 insertions, 6 deletions
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index b63eee2034e7..40e4d3798d16 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1222,7 +1222,7 @@ unmap_release: static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); - u16 flags; + u16 new, old, off_wrap, flags, wrap_counter, event_idx; bool needs_kick; union { struct { @@ -1240,6 +1240,8 @@ static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq) */ virtio_mb(vq->weak_barriers); + old = vq->packed.next_avail_idx - vq->num_added; + new = vq->packed.next_avail_idx; vq->num_added = 0; snapshot.u32 = *(u32 *)vq->packed.vring.device; @@ -1248,7 +1250,20 @@ static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq) LAST_ADD_TIME_CHECK(vq); LAST_ADD_TIME_INVALID(vq); - needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE); + if (flags != VRING_PACKED_EVENT_FLAG_DESC) { + needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE); + goto out; + } + + off_wrap = le16_to_cpu(snapshot.off_wrap); + + wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR; + event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR); + if (wrap_counter != vq->packed.avail_wrap_counter) + event_idx -= vq->packed.vring.num; + + needs_kick = vring_need_event(event_idx, new, old); +out: END_USE(vq); return needs_kick; } @@ -1365,6 +1380,18 @@ static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq, vq->packed.used_wrap_counter ^= 1; } + /* + * If we expect an interrupt for the next entry, tell host + * by writing event index and flush out the write before + * the read in the next get_buf call. + */ + if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC) + virtio_store_mb(vq->weak_barriers, + &vq->packed.vring.driver->off_wrap, + cpu_to_le16(vq->last_used_idx | + (vq->packed.used_wrap_counter << + VRING_PACKED_EVENT_F_WRAP_CTR))); + LAST_ADD_TIME_INVALID(vq); END_USE(vq); @@ -1393,8 +1420,22 @@ static unsigned virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq) * more to do. */ + if (vq->event) { + vq->packed.vring.driver->off_wrap = + cpu_to_le16(vq->last_used_idx | + (vq->packed.used_wrap_counter << + VRING_PACKED_EVENT_F_WRAP_CTR)); + /* + * We need to update event offset and event wrap + * counter first before updating event flags. + */ + virtio_wmb(vq->weak_barriers); + } + if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { - vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_ENABLE; + vq->packed.event_flags_shadow = vq->event ? + VRING_PACKED_EVENT_FLAG_DESC : + VRING_PACKED_EVENT_FLAG_ENABLE; vq->packed.vring.driver->flags = cpu_to_le16(vq->packed.event_flags_shadow); } @@ -1420,6 +1461,7 @@ static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); u16 used_idx, wrap_counter; + u16 bufs; START_USE(vq); @@ -1428,11 +1470,34 @@ static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq) * more to do. */ - used_idx = vq->last_used_idx; - wrap_counter = vq->packed.used_wrap_counter; + if (vq->event) { + /* TODO: tune this threshold */ + bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4; + wrap_counter = vq->packed.used_wrap_counter; + + used_idx = vq->last_used_idx + bufs; + if (used_idx >= vq->packed.vring.num) { + used_idx -= vq->packed.vring.num; + wrap_counter ^= 1; + } + + vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx | + (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR)); + + /* + * We need to update event offset and event wrap + * counter first before updating event flags. + */ + virtio_wmb(vq->weak_barriers); + } else { + used_idx = vq->last_used_idx; + wrap_counter = vq->packed.used_wrap_counter; + } if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { - vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_ENABLE; + vq->packed.event_flags_shadow = vq->event ? + VRING_PACKED_EVENT_FLAG_DESC : + VRING_PACKED_EVENT_FLAG_ENABLE; vq->packed.vring.driver->flags = cpu_to_le16(vq->packed.event_flags_shadow); } |