Re: [Qemu-devel] [PATCH v3 25/27] migration: add support for encrypting data with TLS

[Dr. David Alan Gilbert]

* Daniel P. Berrange (address@hidden) wrote:
> This extends the migration_set_incoming_channel and
> migration_set_outgoing_channel methods so that they
> will automatically wrap the QIOChannel in a
> QIOChannelTLS instance if TLS credentials are configured
> in the migration parameters.

Reviewed-by: Dr. David Alan Gilbert <address@hidden>

You might like to check how it behaves with a migrate_cancel
after connect but before the handshake has finished.

I think it will probably work - the cancel does a shutdown()
call so it'll probably cause it to nuke the underlying conenction
that will then cause the handshake callback.

Dave

> This allows TLS to work for tcp, unix, fd and exec
> migration protocols. It does not (currently) work for
> RDMA since it does not use these APIs, but it is
> unlikely that TLS would be desired with RDMA anyway
> since it would degrade the performance to that seen
> with TCP defeating the purpose of using RDMA.
> 
> On the target host, QEMU would be launched with a set
> of TLS credentials for a server endpoint
> 
>  $ qemu-system-x86_64 -monitor stdio -incoming defer \
>     -object 
> tls-creds-x509,dir=/home/berrange/security/qemutls,endpoint=server,id=tls0 \
>     ...other args...
> 
> To enable incoming TLS migration 2 monitor commands are
> then used
> 
>   (qemu) migrate_set_str_parameter tls-creds tls0
>   (qemu) migrate_incoming tcp:myhostname:9000
> 
> On the source host, QEMU is launched in a similar
> manner but using client endpoint credentials
> 
>  $ qemu-system-x86_64 -monitor stdio \
>     -object 
> tls-creds-x509,dir=/home/berrange/security/qemutls,endpoint=client,id=tls0 \
>     ...other args...
> 
> To enable outgoing TLS migration 2 monitor commands are
> then used
> 
>   (qemu) migrate_set_str_parameter tls-creds tls0
>   (qemu) migrate tcp:otherhostname:9000
> 
> Thanks to earlier improvements to error reporting,
> TLS errors can be seen 'info migrate' when doing a
> detached migration. For example:
> 
>   (qemu) info migrate
>   capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: 
> off compress: off events: off x-postcopy-ram: off
>   Migration status: failed
>   total time: 0 milliseconds
>   error description: TLS handshake failed: The TLS connection was 
> non-properly terminated.
> 
> Or
> 
>   (qemu) info migrate
>   capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: 
> off compress: off events: off x-postcopy-ram: off
>   Migration status: failed
>   total time: 0 milliseconds
>   error description: Certificate does not match the hostname localhost
> 
> Signed-off-by: Daniel P. Berrange <address@hidden>
> ---
>  include/migration/migration.h |  12 +++-
>  migration/Makefile.objs       |   1 +
>  migration/exec.c              |   2 +-
>  migration/fd.c                |   2 +-
>  migration/migration.c         |  40 +++++++++--
>  migration/socket.c            |  34 +++++++--
>  migration/tls.c               | 160 
> ++++++++++++++++++++++++++++++++++++++++++
>  trace-events                  |  12 +++-
>  8 files changed, 246 insertions(+), 17 deletions(-)
>  create mode 100644 migration/tls.c
> 
> diff --git a/include/migration/migration.h b/include/migration/migration.h
> index 999a5ee..6310ff4 100644
> --- a/include/migration/migration.h
> +++ b/include/migration/migration.h
> @@ -186,8 +186,18 @@ void qemu_start_incoming_migration(const char *uri, 
> Error **errp);
>  void migration_set_incoming_channel(MigrationState *s,
>                                      QIOChannel *ioc);
>  
> +void migration_tls_set_incoming_channel(MigrationState *s,
> +                                        QIOChannel *ioc,
> +                                        Error **errp);
> +
>  void migration_set_outgoing_channel(MigrationState *s,
> -                                    QIOChannel *ioc);
> +                                    QIOChannel *ioc,
> +                                    const char *hostname);
> +
> +void migration_tls_set_outgoing_channel(MigrationState *s,
> +                                        QIOChannel *ioc,
> +                                        const char *hostname,
> +                                        Error **errp);
>  
>  uint64_t migrate_max_downtime(void);
>  
> diff --git a/migration/Makefile.objs b/migration/Makefile.objs
> index 7b9051c..e68b54d 100644
> --- a/migration/Makefile.objs
> +++ b/migration/Makefile.objs
> @@ -1,4 +1,5 @@
>  common-obj-y += migration.o socket.o fd.o exec.o
> +common-obj-y += tls.o
>  common-obj-y += vmstate.o
>  common-obj-y += qemu-file.o
>  common-obj-y += qemu-file-channel.o
> diff --git a/migration/exec.c b/migration/exec.c
> index 4f439b4..a5debc6 100644
> --- a/migration/exec.c
> +++ b/migration/exec.c
> @@ -36,7 +36,7 @@ void exec_start_outgoing_migration(MigrationState *s, const 
> char *command, Error
>          return;
>      }
>  
> -    migration_set_outgoing_channel(s, ioc);
> +    migration_set_outgoing_channel(s, ioc, NULL);
>      object_unref(OBJECT(ioc));
>  }
>  
> diff --git a/migration/fd.c b/migration/fd.c
> index 1a7fd43..e089bf4 100644
> --- a/migration/fd.c
> +++ b/migration/fd.c
> @@ -36,7 +36,7 @@ void fd_start_outgoing_migration(MigrationState *s, const 
> char *fdname, Error **
>          return;
>      }
>  
> -    migration_set_outgoing_channel(s, ioc);
> +    migration_set_outgoing_channel(s, ioc, NULL);
>      object_unref(OBJECT(ioc));
>  }
>  
> diff --git a/migration/migration.c b/migration/migration.c
> index c7bc1c7..84393c0 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -35,6 +35,7 @@
>  #include "exec/memory.h"
>  #include "exec/address-spaces.h"
>  #include "io/channel-buffer.h"
> +#include "io/channel-tls.h"
>  
>  #define MAX_THROTTLE  (32 << 20)      /* Migration transfer speed throttling 
> */
>  
> @@ -420,20 +421,47 @@ void process_incoming_migration(QEMUFile *f)
>  void migration_set_incoming_channel(MigrationState *s,
>                                      QIOChannel *ioc)
>  {
> -    QEMUFile *f = qemu_fopen_channel_input(ioc);
> +    trace_migration_set_incoming_channel(
> +        ioc, object_get_typename(OBJECT(ioc)));
>  
> -    process_incoming_migration(f);
> +    if (s->parameters.tls_creds &&
> +        !object_dynamic_cast(OBJECT(ioc),
> +                             TYPE_QIO_CHANNEL_TLS)) {
> +        Error *local_err = NULL;
> +        migration_tls_set_incoming_channel(s, ioc, &local_err);
> +        if (local_err) {
> +            error_report_err(local_err);
> +        }
> +    } else {
> +        QEMUFile *f = qemu_fopen_channel_input(ioc);
> +        process_incoming_migration(f);
> +    }
>  }
>  
>  
>  void migration_set_outgoing_channel(MigrationState *s,
> -                                    QIOChannel *ioc)
> +                                    QIOChannel *ioc,
> +                                    const char *hostname)
>  {
> -    QEMUFile *f = qemu_fopen_channel_output(ioc);
> +    trace_migration_set_outgoing_channel(
> +        ioc, object_get_typename(OBJECT(ioc)), hostname);
>  
> -    s->to_dst_file = f;
> +    if (s->parameters.tls_creds &&
> +        !object_dynamic_cast(OBJECT(ioc),
> +                             TYPE_QIO_CHANNEL_TLS)) {
> +        Error *local_err = NULL;
> +        migration_tls_set_outgoing_channel(s, ioc, hostname, &local_err);
> +        if (local_err) {
> +            migrate_fd_error(s, local_err);
> +            error_free(local_err);
> +        }
> +    } else {
> +        QEMUFile *f = qemu_fopen_channel_output(ioc);
>  
> -    migrate_fd_connect(s);
> +        s->to_dst_file = f;
> +
> +        migrate_fd_connect(s);
> +    }
>  }
>  
>  
> diff --git a/migration/socket.c b/migration/socket.c
> index 7e0d9ee..5670f70 100644
> --- a/migration/socket.c
> +++ b/migration/socket.c
> @@ -54,20 +54,35 @@ static SocketAddress *unix_build_address(const char *path)
>  }
>  
>  
> +struct SocketConnectData {
> +    MigrationState *s;
> +    char *hostname;
> +};
> +
> +static void socket_connect_data_free(void *opaque)
> +{
> +    struct SocketConnectData *data = opaque;
> +    if (!data) {
> +        return;
> +    }
> +    g_free(data->hostname);
> +    g_free(data);
> +}
> +
>  static void socket_outgoing_migration(Object *src,
>                                        Error *err,
>                                        gpointer opaque)
>  {
> -    MigrationState *s = opaque;
> +    struct SocketConnectData *data = opaque;
>      QIOChannel *sioc = QIO_CHANNEL(src);
>  
>      if (err) {
>          trace_migration_socket_outgoing_error(error_get_pretty(err));
> -        s->to_dst_file = NULL;
> -        migrate_fd_error(s, err);
> +        data->s->to_dst_file = NULL;
> +        migrate_fd_error(data->s, err);
>      } else {
> -        trace_migration_socket_outgoing_connected();
> -        migration_set_outgoing_channel(s, sioc);
> +        trace_migration_socket_outgoing_connected(data->hostname);
> +        migration_set_outgoing_channel(data->s, sioc, data->hostname);
>      }
>      object_unref(src);
>  }
> @@ -77,11 +92,16 @@ static void 
> socket_start_outgoing_migration(MigrationState *s,
>                                              Error **errp)
>  {
>      QIOChannelSocket *sioc = qio_channel_socket_new();
> +    struct SocketConnectData *data = g_new0(struct SocketConnectData, 1);
> +    data->s = s;
> +    if (saddr->type == SOCKET_ADDRESS_KIND_INET) {
> +        data->hostname = g_strdup(saddr->u.inet->host);
> +    }
>      qio_channel_socket_connect_async(sioc,
>                                       saddr,
>                                       socket_outgoing_migration,
> -                                     s,
> -                                     NULL);
> +                                     data,
> +                                     socket_connect_data_free);
>      qapi_free_SocketAddress(saddr);
>  }
>  
> diff --git a/migration/tls.c b/migration/tls.c
> new file mode 100644
> index 0000000..12a20d6
> --- /dev/null
> +++ b/migration/tls.c
> @@ -0,0 +1,160 @@
> +/*
> + * QEMU migration TLS support
> + *
> + * Copyright (c) 2015 Red Hat, Inc.
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library; if not, see 
> <http://www.gnu.org/licenses/>.
> + *
> + */
> +
> +#include "qemu/osdep.h"
> +#include "migration/migration.h"
> +#include "io/channel-tls.h"
> +#include "crypto/tlscreds.h"
> +#include "qemu/error-report.h"
> +#include "trace.h"
> +
> +static QCryptoTLSCreds *
> +migration_tls_get_creds(MigrationState *s,
> +                        QCryptoTLSCredsEndpoint endpoint,
> +                        Error **errp)
> +{
> +    Object *creds;
> +    QCryptoTLSCreds *ret;
> +
> +    creds = object_resolve_path_component(
> +        object_get_objects_root(), s->parameters.tls_creds);
> +    if (!creds) {
> +        error_setg(errp, "No TLS credentials with id '%s'",
> +                   s->parameters.tls_creds);
> +        return NULL;
> +    }
> +    ret = (QCryptoTLSCreds *)object_dynamic_cast(
> +        creds, TYPE_QCRYPTO_TLS_CREDS);
> +    if (!ret) {
> +        error_setg(errp, "Object with id '%s' is not TLS credentials",
> +                   s->parameters.tls_creds);
> +        return NULL;
> +    }
> +    if (ret->endpoint != endpoint) {
> +        error_setg(errp,
> +                   "Expected TLS credentials for a %s endpoint",
> +                   endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT ?
> +                   "client" : "server");
> +        return NULL;
> +    }
> +
> +    object_ref(OBJECT(ret));
> +    return ret;
> +}
> +
> +
> +static void migration_tls_incoming_handshake(Object *src,
> +                                             Error *err,
> +                                             gpointer opaque)
> +{
> +    QIOChannel *ioc = QIO_CHANNEL(src);
> +
> +    if (err) {
> +        trace_migration_tls_incoming_handshake_error(error_get_pretty(err));
> +        error_report("%s", error_get_pretty(err));
> +    } else {
> +        trace_migration_tls_incoming_handshake_complete();
> +        migration_set_incoming_channel(migrate_get_current(), ioc);
> +    }
> +    object_unref(OBJECT(ioc));
> +}
> +
> +void migration_tls_set_incoming_channel(MigrationState *s,
> +                                        QIOChannel *ioc,
> +                                        Error **errp)
> +{
> +    QCryptoTLSCreds *creds;
> +    QIOChannelTLS *tioc;
> +
> +    creds = migration_tls_get_creds(
> +        s, QCRYPTO_TLS_CREDS_ENDPOINT_SERVER, errp);
> +    if (!creds) {
> +        return;
> +    }
> +
> +    tioc = qio_channel_tls_new_server(
> +        ioc, creds,
> +        NULL, /* XXX pass ACL name */
> +        errp);
> +    if (!tioc) {
> +        return;
> +    }
> +
> +    trace_migration_tls_incoming_handshake_start();
> +    qio_channel_tls_handshake(tioc,
> +                              migration_tls_incoming_handshake,
> +                              NULL,
> +                              NULL);
> +}
> +
> +
> +static void migration_tls_outgoing_handshake(Object *src,
> +                                             Error *err,
> +                                             gpointer opaque)
> +{
> +    MigrationState *s = opaque;
> +    QIOChannel *ioc = QIO_CHANNEL(src);
> +
> +    if (err) {
> +        trace_migration_tls_outgoing_handshake_error(error_get_pretty(err));
> +        s->to_dst_file = NULL;
> +        migrate_fd_error(s, err);
> +    } else {
> +        trace_migration_tls_outgoing_handshake_complete();
> +        migration_set_outgoing_channel(s, ioc, NULL);
> +    }
> +    object_unref(OBJECT(ioc));
> +}
> +
> +
> +void migration_tls_set_outgoing_channel(MigrationState *s,
> +                                        QIOChannel *ioc,
> +                                        const char *hostname,
> +                                        Error **errp)
> +{
> +    QCryptoTLSCreds *creds;
> +    QIOChannelTLS *tioc;
> +
> +    creds = migration_tls_get_creds(
> +        s, QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT, errp);
> +    if (!creds) {
> +        return;
> +    }
> +
> +    if (s->parameters.tls_hostname) {
> +        hostname = s->parameters.tls_hostname;
> +    }
> +    if (!hostname) {
> +        error_setg(errp, "No hostname available for TLS");
> +        return;
> +    }
> +
> +    tioc = qio_channel_tls_new_client(
> +        ioc, creds, hostname, errp);
> +    if (!tioc) {
> +        return;
> +    }
> +
> +    trace_migration_tls_outgoing_handshake_start(hostname);
> +    qio_channel_tls_handshake(tioc,
> +                              migration_tls_outgoing_handshake,
> +                              s,
> +                              NULL);
> +}
> diff --git a/trace-events b/trace-events
> index 5b8da12..1dd2066 100644
> --- a/trace-events
> +++ b/trace-events
> @@ -1516,6 +1516,8 @@ migrate_state_too_big(void) ""
>  migrate_transferred(uint64_t tranferred, uint64_t time_spent, double 
> bandwidth, uint64_t size) "transferred %" PRIu64 " time_spent %" PRIu64 " 
> bandwidth %g max_size %" PRId64
>  process_incoming_migration_co_end(int ret, int ps) "ret=%d postcopy-state=%d"
>  process_incoming_migration_co_postcopy_end_main(void) ""
> +migration_set_incoming_channel(void *ioc, const char *ioctype) "ioc=%p 
> ioctype=%s"
> +migration_set_outgoing_channel(void *ioc, const char *ioctype, const char 
> *hostname)  "ioc=%p ioctype=%s hostname=%s"
>  
>  # migration/rdma.c
>  qemu_rdma_accept_incoming_migration(void) ""
> @@ -1610,9 +1612,17 @@ migration_fd_incoming(int fd) "fd=%d"
>  
>  # migration/socket.c
>  migration_socket_incoming_accepted(void) ""
> -migration_socket_outgoing_connected(void) ""
> +migration_socket_outgoing_connected(const char *hostname) "hostname=%s"
>  migration_socket_outgoing_error(const char *err) "error=%s"
>  
> +# migration/tls.c
> +migration_tls_outgoing_handshake_start(const char *hostname) "hostname=%s"
> +migration_tls_outgoing_handshake_error(const char *err) "err=%s"
> +migration_tls_outgoing_handshake_complete(void) ""
> +migration_tls_incoming_handshake_start(void) ""
> +migration_tls_incoming_handshake_error(const char *err) "err=%s"
> +migration_tls_incoming_handshake_complete(void) ""
> +
>  # kvm-all.c
>  kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
>  kvm_vm_ioctl(int type, void *arg) "type 0x%x, arg %p"
> -- 
> 2.5.0
> 
--
Dr. David Alan Gilbert / address@hidden / Manchester, UK