[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: [Qemu-devel] [RFC PATCH RDMA support v3: 05/10] RDMA connection esta
From: |
Paolo Bonzini |
Subject: |
Re: [Qemu-devel] [RFC PATCH RDMA support v3: 05/10] RDMA connection establishment (migration-rdma.c). |
Date: |
Mon, 11 Mar 2013 14:41:21 +0100 |
User-agent: |
Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130219 Thunderbird/17.0.3 |
Il 11/03/2013 05:33, address@hidden ha scritto:
> From: "Michael R. Hines" <address@hidden>
>
> Use 'migrate rdma:host:port' to begin the migration.
>
> The TCP control channel has finally been eliminated
> when RDMA is used. Documentation of the use of SEND message
> for transferring device state is covered in docs/rdma.txt
>
> Signed-off-by: Michael R. Hines <address@hidden>
> ---
> migration-rdma.c | 198
> ++++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 198 insertions(+)
> create mode 100644 migration-rdma.c
>
> diff --git a/migration-rdma.c b/migration-rdma.c
> new file mode 100644
> index 0000000..822b17a
> --- /dev/null
> +++ b/migration-rdma.c
> @@ -0,0 +1,198 @@
> +/*
> + * Copyright (C) 2013 Michael R. Hines <address@hidden>
> + * Copyright (C) 2013 Jiuxing Liu <address@hidden>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; under version 2 of the License.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program; if not, see <http://www.gnu.org/licenses/>.
> + */
> +#include "migration/rdma.h"
> +#include "qemu-common.h"
> +#include "migration/migration.h"
> +#include "migration/qemu-file.h"
> +#include <stdio.h>
> +#include <sys/types.h>
> +#include <sys/socket.h>
> +#include <netdb.h>
> +#include <arpa/inet.h>
> +#include <string.h>
> +
> +//#define DEBUG_MIGRATION_RDMA
> +
> +#ifdef DEBUG_MIGRATION_RDMA
> +#define DPRINTF(fmt, ...) \
> + do { printf("migration-rdma: " fmt, ## __VA_ARGS__); } while (0)
> +#else
> +#define DPRINTF(fmt, ...) \
> + do { } while (0)
> +#endif
> +
> +static int rdma_accept_incoming_migration(RDMAData *rdma, Error **errp)
> +{
> + int ret;
> +
> + ret = qemu_rdma_migrate_listen(rdma, rdma->host, rdma->port);
> + if (ret) {
> + qemu_rdma_print("rdma migration: error listening!");
> + goto err_rdma_server_wait;
> + }
> +
> + ret = qemu_rdma_alloc_qp(&rdma->rdma_ctx);
> + if (ret) {
> + qemu_rdma_print("rdma migration: error allocating qp!");
> + goto err_rdma_server_wait;
> + }
> +
> + ret = qemu_rdma_migrate_accept(&rdma->rdma_ctx, NULL, NULL, NULL, 0);
> + if (ret) {
> + qemu_rdma_print("rdma migration: error accepting connection!");
> + goto err_rdma_server_wait;
> + }
> +
> + ret = qemu_rdma_post_recv_qemu_file(rdma);
> + if (ret) {
> + qemu_rdma_print("rdma migration: error posting second qemu file
> recv!");
> + goto err_rdma_server_wait;
> + }
> +
> + ret = qemu_rdma_post_send_remote_info(rdma);
> + if (ret) {
> + qemu_rdma_print("rdma migration: error sending remote info!");
> + goto err_rdma_server_wait;
> + }
> +
> + ret = qemu_rdma_wait_for_wrid(rdma, RDMA_WRID_SEND_REMOTE_INFO);
> + if (ret < 0) {
> + qemu_rdma_print("rdma migration: polling remote info error!");
> + goto err_rdma_server_wait;
> + }
> +
> + rdma->total_bytes = 0;
> + rdma->enabled = 1;
> + qemu_rdma_dump_gid("server_connect", rdma->rdma_ctx.cm_id);
> + return 0;
> +
> +err_rdma_server_wait:
> + qemu_rdma_cleanup(rdma);
> + return -1;
> +
> +}
> +
> +int rdma_start_incoming_migration(const char * host_port, Error **errp)
> +{
> + QEMUFile *f;
> + int ret;
> + RDMAData rdma;
> +
> + if ((ret = qemu_rdma_data_init(&rdma, host_port, errp)) < 0)
> + return ret;
> +
> + ret = qemu_rdma_server_init(&rdma, NULL);
> +
> + DPRINTF("Starting RDMA-based incoming migration\n");
> +
> + if (!ret) {
> + DPRINTF("qemu_rdma_server_init success\n");
> + ret = qemu_rdma_server_prepare(&rdma, NULL);
> +
> + if (!ret) {
> + DPRINTF("qemu_rdma_server_prepare success\n");
> +
> + ret = rdma_accept_incoming_migration(&rdma, NULL);
> + if(!ret)
> + DPRINTF("qemu_rdma_accept_incoming_migration success\n");
> + f = qemu_fopen_rdma(&rdma);
> + if (f == NULL) {
> + fprintf(stderr, "could not qemu_fopen RDMA\n");
> + ret = -EIO;
> + }
> +
> + process_incoming_migration(f);
> + }
> + }
> +
> + return ret;
> +}
> +
> +/*
> + * Not sure what this is for yet...
> + */
> +static int qemu_rdma_errno(MigrationState *s)
> +{
> + return 0;
> +}
> +
> +/*
> + * SEND messages for device state only.
> + * pc.ram is handled elsewhere...
> + */
> +static int qemu_rdma_send(MigrationState *s, const void * buf, size_t size)
> +{
> + size_t len, remaining = size;
> + uint8_t * data = (void *) buf;
> +
> + if (qemu_rdma_write_flush(&s->rdma) < 0) {
> + qemu_file_set_error(s->file, -EIO);
> + return -EIO;
> + }
> +
> + while(remaining) {
> + len = MIN(remaining, RDMA_SEND_INCREMENT);
> + remaining -= len;
> + DPRINTF("iter Sending %" PRId64 "-byte SEND\n", len);
> +
> + if(qemu_rdma_exchange_send(&s->rdma, data, len) < 0)
> + return -EINVAL;
> +
> + data += len;
> + }
> +
> + return size;
> +}
> +
> +static int qemu_rdma_close(MigrationState *s)
> +{
> + DPRINTF("qemu_rdma_close\n");
> + qemu_rdma_cleanup(&s->rdma);
> + return 0;
> +}
> +
> +void rdma_start_outgoing_migration(MigrationState *s, const char *host_port,
> Error **errp)
> +{
> + int ret;
> +
> +#ifndef CONFIG_RDMA
> + error_set(errp, QERR_FEATURE_DISABLED, "rdma migration");
> + return;
> +#endif
> +
> + if (qemu_rdma_data_init(&s->rdma, host_port, errp) < 0)
> + return;
> +
> + ret = qemu_rdma_client_init(&s->rdma, NULL);
> + if(!ret) {
> + DPRINTF("qemu_rdma_client_init success\n");
> + ret = qemu_rdma_client_connect(&s->rdma, NULL);
> +
> + if(!ret) {
> + s->get_error = qemu_rdma_errno;
> + s->write = qemu_rdma_send;
> + s->close = qemu_rdma_close;
> + s->fd = -2;
> + DPRINTF("qemu_rdma_client_connect success\n");
> + migrate_fd_connect(s);
> + return;
> + }
> + }
> +
> + s->fd = -1;
> + migrate_fd_error(s);
> +}
>
Note that as soon as the pending migration patches hit upstream, almost
all of this code will move to QEMUFileRDMA (in particular qemu_rdma_send
and qemu_rdma_close). It should become simpler.
Paolo
- Re: [Qemu-devel] [RFC PATCH RDMA support v3: 05/10] RDMA connection establishment (migration-rdma.c).,
Paolo Bonzini <=