|
@@ -11,7 +11,9 @@
|
|
|
#include "qemu/osdep.h"
|
|
|
#include "qapi/error.h"
|
|
|
#include "hw/virtio/vhost.h"
|
|
|
+#include "hw/virtio/vhost-user.h"
|
|
|
#include "hw/virtio/vhost-backend.h"
|
|
|
+#include "hw/virtio/virtio.h"
|
|
|
#include "hw/virtio/virtio-net.h"
|
|
|
#include "chardev/char-fe.h"
|
|
|
#include "sysemu/kvm.h"
|
|
@@ -30,6 +32,7 @@
|
|
|
|
|
|
#define VHOST_MEMORY_MAX_NREGIONS 8
|
|
|
#define VHOST_USER_F_PROTOCOL_FEATURES 30
|
|
|
+#define VHOST_USER_SLAVE_MAX_FDS 8
|
|
|
|
|
|
/*
|
|
|
* Maximum size of virtio device config space
|
|
@@ -47,6 +50,8 @@ enum VhostUserProtocolFeature {
|
|
|
VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
|
|
|
VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
|
|
|
VHOST_USER_PROTOCOL_F_CONFIG = 9,
|
|
|
+ VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
|
|
|
+ VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
|
|
|
VHOST_USER_PROTOCOL_F_MAX
|
|
|
};
|
|
|
|
|
@@ -91,6 +96,7 @@ typedef enum VhostUserSlaveRequest {
|
|
|
VHOST_USER_SLAVE_NONE = 0,
|
|
|
VHOST_USER_SLAVE_IOTLB_MSG = 1,
|
|
|
VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
|
|
|
+ VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
|
|
|
VHOST_USER_SLAVE_MAX
|
|
|
} VhostUserSlaveRequest;
|
|
|
|
|
@@ -135,6 +141,12 @@ static VhostUserConfig c __attribute__ ((unused));
|
|
|
+ sizeof(c.size) \
|
|
|
+ sizeof(c.flags))
|
|
|
|
|
|
+typedef struct VhostUserVringArea {
|
|
|
+ uint64_t u64;
|
|
|
+ uint64_t size;
|
|
|
+ uint64_t offset;
|
|
|
+} VhostUserVringArea;
|
|
|
+
|
|
|
typedef struct {
|
|
|
VhostUserRequest request;
|
|
|
|
|
@@ -156,6 +168,7 @@ typedef union {
|
|
|
struct vhost_iotlb_msg iotlb;
|
|
|
VhostUserConfig config;
|
|
|
VhostUserCryptoSession session;
|
|
|
+ VhostUserVringArea area;
|
|
|
} VhostUserPayload;
|
|
|
|
|
|
typedef struct VhostUserMsg {
|
|
@@ -173,7 +186,8 @@ static VhostUserMsg m __attribute__ ((unused));
|
|
|
|
|
|
struct vhost_user {
|
|
|
struct vhost_dev *dev;
|
|
|
- CharBackend *chr;
|
|
|
+ /* Shared between vhost devs of the same virtio device */
|
|
|
+ VhostUserState *user;
|
|
|
int slave_fd;
|
|
|
NotifierWithReturn postcopy_notifier;
|
|
|
struct PostCopyFD postcopy_fd;
|
|
@@ -199,7 +213,7 @@ static bool ioeventfd_enabled(void)
|
|
|
static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
|
|
|
{
|
|
|
struct vhost_user *u = dev->opaque;
|
|
|
- CharBackend *chr = u->chr;
|
|
|
+ CharBackend *chr = u->user->chr;
|
|
|
uint8_t *p = (uint8_t *) msg;
|
|
|
int r, size = VHOST_USER_HDR_SIZE;
|
|
|
|
|
@@ -285,7 +299,7 @@ static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
|
|
|
int *fds, int fd_num)
|
|
|
{
|
|
|
struct vhost_user *u = dev->opaque;
|
|
|
- CharBackend *chr = u->chr;
|
|
|
+ CharBackend *chr = u->user->chr;
|
|
|
int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
|
|
|
|
|
|
/*
|
|
@@ -636,9 +650,37 @@ static int vhost_user_set_vring_num(struct vhost_dev *dev,
|
|
|
return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
|
|
|
}
|
|
|
|
|
|
+static void vhost_user_host_notifier_restore(struct vhost_dev *dev,
|
|
|
+ int queue_idx)
|
|
|
+{
|
|
|
+ struct vhost_user *u = dev->opaque;
|
|
|
+ VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
|
|
|
+ VirtIODevice *vdev = dev->vdev;
|
|
|
+
|
|
|
+ if (n->addr && !n->set) {
|
|
|
+ virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true);
|
|
|
+ n->set = true;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static void vhost_user_host_notifier_remove(struct vhost_dev *dev,
|
|
|
+ int queue_idx)
|
|
|
+{
|
|
|
+ struct vhost_user *u = dev->opaque;
|
|
|
+ VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
|
|
|
+ VirtIODevice *vdev = dev->vdev;
|
|
|
+
|
|
|
+ if (n->addr && n->set) {
|
|
|
+ virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
|
|
|
+ n->set = false;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
static int vhost_user_set_vring_base(struct vhost_dev *dev,
|
|
|
struct vhost_vring_state *ring)
|
|
|
{
|
|
|
+ vhost_user_host_notifier_restore(dev, ring->index);
|
|
|
+
|
|
|
return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
|
|
|
}
|
|
|
|
|
@@ -672,6 +714,8 @@ static int vhost_user_get_vring_base(struct vhost_dev *dev,
|
|
|
.hdr.size = sizeof(msg.payload.state),
|
|
|
};
|
|
|
|
|
|
+ vhost_user_host_notifier_remove(dev, ring->index);
|
|
|
+
|
|
|
if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
|
|
|
return -1;
|
|
|
}
|
|
@@ -845,6 +889,66 @@ static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
+static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
|
|
|
+ VhostUserVringArea *area,
|
|
|
+ int fd)
|
|
|
+{
|
|
|
+ int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
|
|
|
+ size_t page_size = qemu_real_host_page_size;
|
|
|
+ struct vhost_user *u = dev->opaque;
|
|
|
+ VhostUserState *user = u->user;
|
|
|
+ VirtIODevice *vdev = dev->vdev;
|
|
|
+ VhostUserHostNotifier *n;
|
|
|
+ void *addr;
|
|
|
+ char *name;
|
|
|
+
|
|
|
+ if (!virtio_has_feature(dev->protocol_features,
|
|
|
+ VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
|
|
|
+ vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
|
|
|
+ return -1;
|
|
|
+ }
|
|
|
+
|
|
|
+ n = &user->notifier[queue_idx];
|
|
|
+
|
|
|
+ if (n->addr) {
|
|
|
+ virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
|
|
|
+ object_unparent(OBJECT(&n->mr));
|
|
|
+ munmap(n->addr, page_size);
|
|
|
+ n->addr = NULL;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Sanity check. */
|
|
|
+ if (area->size != page_size) {
|
|
|
+ return -1;
|
|
|
+ }
|
|
|
+
|
|
|
+ addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
|
|
|
+ fd, area->offset);
|
|
|
+ if (addr == MAP_FAILED) {
|
|
|
+ return -1;
|
|
|
+ }
|
|
|
+
|
|
|
+ name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
|
|
|
+ user, queue_idx);
|
|
|
+ memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
|
|
|
+ page_size, addr);
|
|
|
+ g_free(name);
|
|
|
+
|
|
|
+ if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
|
|
|
+ munmap(addr, page_size);
|
|
|
+ return -1;
|
|
|
+ }
|
|
|
+
|
|
|
+ n->addr = addr;
|
|
|
+ n->set = true;
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
static void slave_read(void *opaque)
|
|
|
{
|
|
|
struct vhost_dev *dev = opaque;
|
|
@@ -854,10 +958,10 @@ static void slave_read(void *opaque)
|
|
|
int size, ret = 0;
|
|
|
struct iovec iov;
|
|
|
struct msghdr msgh;
|
|
|
- int fd = -1;
|
|
|
+ int fd[VHOST_USER_SLAVE_MAX_FDS];
|
|
|
char control[CMSG_SPACE(sizeof(fd))];
|
|
|
struct cmsghdr *cmsg;
|
|
|
- size_t fdsize;
|
|
|
+ int i, fdsize = 0;
|
|
|
|
|
|
memset(&msgh, 0, sizeof(msgh));
|
|
|
msgh.msg_iov = &iov;
|
|
@@ -865,6 +969,8 @@ static void slave_read(void *opaque)
|
|
|
msgh.msg_control = control;
|
|
|
msgh.msg_controllen = sizeof(control);
|
|
|
|
|
|
+ memset(fd, -1, sizeof(fd));
|
|
|
+
|
|
|
/* Read header */
|
|
|
iov.iov_base = &hdr;
|
|
|
iov.iov_len = VHOST_USER_HDR_SIZE;
|
|
@@ -885,7 +991,7 @@ static void slave_read(void *opaque)
|
|
|
if (cmsg->cmsg_level == SOL_SOCKET &&
|
|
|
cmsg->cmsg_type == SCM_RIGHTS) {
|
|
|
fdsize = cmsg->cmsg_len - CMSG_LEN(0);
|
|
|
- memcpy(&fd, CMSG_DATA(cmsg), fdsize);
|
|
|
+ memcpy(fd, CMSG_DATA(cmsg), fdsize);
|
|
|
break;
|
|
|
}
|
|
|
}
|
|
@@ -911,16 +1017,21 @@ static void slave_read(void *opaque)
|
|
|
case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
|
|
|
ret = vhost_user_slave_handle_config_change(dev);
|
|
|
break;
|
|
|
+ case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
|
|
|
+ ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
|
|
|
+ fd[0]);
|
|
|
+ break;
|
|
|
default:
|
|
|
error_report("Received unexpected msg type.");
|
|
|
- if (fd != -1) {
|
|
|
- close(fd);
|
|
|
- }
|
|
|
ret = -EINVAL;
|
|
|
}
|
|
|
|
|
|
- /* Message handlers need to make sure that fd will be consumed. */
|
|
|
- fd = -1;
|
|
|
+ /* Close the remaining file descriptors. */
|
|
|
+ for (i = 0; i < fdsize; i++) {
|
|
|
+ if (fd[i] != -1) {
|
|
|
+ close(fd[i]);
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
/*
|
|
|
* REPLY_ACK feature handling. Other reply types has to be managed
|
|
@@ -954,8 +1065,10 @@ err:
|
|
|
qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
|
|
|
close(u->slave_fd);
|
|
|
u->slave_fd = -1;
|
|
|
- if (fd != -1) {
|
|
|
- close(fd);
|
|
|
+ for (i = 0; i < fdsize; i++) {
|
|
|
+ if (fd[i] != -1) {
|
|
|
+ close(fd[i]);
|
|
|
+ }
|
|
|
}
|
|
|
return;
|
|
|
}
|
|
@@ -1083,7 +1196,7 @@ static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
|
|
|
static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
|
|
|
{
|
|
|
struct vhost_user *u = dev->opaque;
|
|
|
- CharBackend *chr = u->chr;
|
|
|
+ CharBackend *chr = u->user->chr;
|
|
|
int ufd;
|
|
|
VhostUserMsg msg = {
|
|
|
.hdr.request = VHOST_USER_POSTCOPY_ADVISE,
|
|
@@ -1221,7 +1334,7 @@ static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-static int vhost_user_init(struct vhost_dev *dev, void *opaque)
|
|
|
+static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque)
|
|
|
{
|
|
|
uint64_t features, protocol_features;
|
|
|
struct vhost_user *u;
|
|
@@ -1230,7 +1343,7 @@ static int vhost_user_init(struct vhost_dev *dev, void *opaque)
|
|
|
assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
|
|
|
|
|
|
u = g_new0(struct vhost_user, 1);
|
|
|
- u->chr = opaque;
|
|
|
+ u->user = opaque;
|
|
|
u->slave_fd = -1;
|
|
|
u->dev = dev;
|
|
|
dev->opaque = u;
|
|
@@ -1306,7 +1419,7 @@ static int vhost_user_init(struct vhost_dev *dev, void *opaque)
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-static int vhost_user_cleanup(struct vhost_dev *dev)
|
|
|
+static int vhost_user_backend_cleanup(struct vhost_dev *dev)
|
|
|
{
|
|
|
struct vhost_user *u;
|
|
|
|
|
@@ -1620,10 +1733,40 @@ vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
+static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
|
|
|
+ MemoryRegionSection *section)
|
|
|
+{
|
|
|
+ bool result;
|
|
|
+
|
|
|
+ result = memory_region_get_fd(section->mr) >= 0;
|
|
|
+
|
|
|
+ return result;
|
|
|
+}
|
|
|
+
|
|
|
+VhostUserState *vhost_user_init(void)
|
|
|
+{
|
|
|
+ VhostUserState *user = g_new0(struct VhostUserState, 1);
|
|
|
+
|
|
|
+ return user;
|
|
|
+}
|
|
|
+
|
|
|
+void vhost_user_cleanup(VhostUserState *user)
|
|
|
+{
|
|
|
+ int i;
|
|
|
+
|
|
|
+ for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
|
|
|
+ if (user->notifier[i].addr) {
|
|
|
+ object_unparent(OBJECT(&user->notifier[i].mr));
|
|
|
+ munmap(user->notifier[i].addr, qemu_real_host_page_size);
|
|
|
+ user->notifier[i].addr = NULL;
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
const VhostOps user_ops = {
|
|
|
.backend_type = VHOST_BACKEND_TYPE_USER,
|
|
|
- .vhost_backend_init = vhost_user_init,
|
|
|
- .vhost_backend_cleanup = vhost_user_cleanup,
|
|
|
+ .vhost_backend_init = vhost_user_backend_init,
|
|
|
+ .vhost_backend_cleanup = vhost_user_backend_cleanup,
|
|
|
.vhost_backend_memslots_limit = vhost_user_memslots_limit,
|
|
|
.vhost_set_log_base = vhost_user_set_log_base,
|
|
|
.vhost_set_mem_table = vhost_user_set_mem_table,
|
|
@@ -1650,4 +1793,5 @@ const VhostOps user_ops = {
|
|
|
.vhost_set_config = vhost_user_set_config,
|
|
|
.vhost_crypto_create_session = vhost_user_crypto_create_session,
|
|
|
.vhost_crypto_close_session = vhost_user_crypto_close_session,
|
|
|
+ .vhost_backend_mem_section_filter = vhost_user_mem_section_filter,
|
|
|
};
|