Re: [Qemu-devel] [PATCH v2 1/4] coroutine: introduce coroutines

[Blue Swirl]

On Thu, May 12, 2011 at 12:54 PM, Stefan Hajnoczi
<address@hidden> wrote:
> From: Kevin Wolf <address@hidden>
>
> Asynchronous code is becoming very complex.  At the same time
> synchronous code is growing because it is convenient to write.
> Sometimes duplicate code paths are even added, one synchronous and the
> other asynchronous.  This patch introduces coroutines which allow code
> that looks synchronous but is asynchronous under the covers.
>
> A coroutine has its own stack and is therefore able to preserve state
> across blocking operations, which traditionally require callback
> functions and manual marshalling of parameters.
>
> Creating and starting a coroutine is easy:
>
>  coroutine = qemu_coroutine_create(my_coroutine);
>  qemu_coroutine_enter(coroutine, my_data);
>
> The coroutine then executes until it returns or yields:
>
>  void coroutine_fn my_coroutine(void *opaque) {
>      MyData *my_data = opaque;
>
>      /* do some work */
>
>      qemu_coroutine_yield();
>
>      /* do some more work */
>  }
>
> Yielding switches control back to the caller of qemu_coroutine_enter().
> This is typically used to switch back to the main thread's event loop
> after issuing an asynchronous I/O request.  The request callback will
> then invoke qemu_coroutine_enter() once more to switch back to the
> coroutine.
>
> Note that coroutines never execute concurrently and should only be used
> from threads which hold the global mutex.  This restriction makes
> programming with coroutines easier than with threads.  Race conditions
> cannot occur since only one coroutine may be active at any time.  Other
> coroutines can only run across yield.
>
> This coroutines implementation is based on the gtk-vnc implementation
> written by Anthony Liguori <address@hidden> but it has been
> significantly rewritten by Kevin Wolf <address@hidden> to use
> setjmp()/longjmp() instead of the more expensive swapcontext().
>
> Signed-off-by: Kevin Wolf <address@hidden>
> Signed-off-by: Stefan Hajnoczi <address@hidden>
> ---
>  Makefile.objs        |    7 +++
>  coroutine-ucontext.c |   73 +++++++++++++++++++++++++++++
>  coroutine-win32.c    |   57 +++++++++++++++++++++++
>  qemu-coroutine-int.h |   53 +++++++++++++++++++++
>  qemu-coroutine.c     |  125 
> ++++++++++++++++++++++++++++++++++++++++++++++++++
>  qemu-coroutine.h     |   82 +++++++++++++++++++++++++++++++++
>  trace-events         |    5 ++
>  7 files changed, 402 insertions(+), 0 deletions(-)
>  create mode 100644 coroutine-ucontext.c
>  create mode 100644 coroutine-win32.c
>  create mode 100644 qemu-coroutine-int.h
>  create mode 100644 qemu-coroutine.c
>  create mode 100644 qemu-coroutine.h
>
> diff --git a/Makefile.objs b/Makefile.objs
> index 9d8851e..cba6c2b 100644
> --- a/Makefile.objs
> +++ b/Makefile.objs
> @@ -11,6 +11,12 @@ oslib-obj-$(CONFIG_WIN32) += oslib-win32.o
>  oslib-obj-$(CONFIG_POSIX) += oslib-posix.o
>
>  #######################################################################
> +# coroutines
> +coroutine-obj-y = qemu-coroutine.o
> +coroutine-obj-$(CONFIG_POSIX) += coroutine-ucontext.o
> +coroutine-obj-$(CONFIG_WIN32) += coroutine-win32.o
> +
> +#######################################################################
>  # block-obj-y is code used by both qemu system emulation and qemu-img
>
>  block-obj-y = cutils.o cache-utils.o qemu-malloc.o qemu-option.o module.o 
> async.o
> @@ -67,6 +73,7 @@ common-obj-y += readline.o console.o cursor.o qemu-error.o
>  common-obj-y += $(oslib-obj-y)
>  common-obj-$(CONFIG_WIN32) += os-win32.o
>  common-obj-$(CONFIG_POSIX) += os-posix.o
> +common-obj-y += $(coroutine-obj-y)
>
>  common-obj-y += tcg-runtime.o host-utils.o
>  common-obj-y += irq.o ioport.o input.o
> diff --git a/coroutine-ucontext.c b/coroutine-ucontext.c
> new file mode 100644
> index 0000000..3b14ebf
> --- /dev/null
> +++ b/coroutine-ucontext.c
> @@ -0,0 +1,73 @@
> +/*
> + * ucontext coroutine initialization code
> + *
> + * Copyright (C) 2006  Anthony Liguori <address@hidden>
> + * Copyright (C) 2011  Kevin Wolf <address@hidden>
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.0 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 
> USA

Please use the web link version.

> + */
> +
> +/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */

What is the problem?

> +#ifdef _FORTIFY_SOURCE
> +#undef _FORTIFY_SOURCE
> +#endif
> +#include <setjmp.h>
> +#include <stdint.h>
> +#include <ucontext.h>
> +#include "qemu-coroutine-int.h"
> +
> +static Coroutine *new_coroutine;
> +
> +static void continuation_trampoline(void)
> +{
> +    Coroutine *co = new_coroutine;
> +
> +    /* Initialize longjmp environment and switch back to
> +     * qemu_coroutine_init_env() in the old ucontext. */
> +    if (!setjmp(co->env)) {
> +        return;
> +    }
> +
> +    while (true) {
> +        co->entry(co->data);
> +        if (!setjmp(co->env)) {
> +            longjmp(co->caller->env, COROUTINE_TERMINATE);
> +        }
> +    }
> +}
> +
> +int qemu_coroutine_init_env(Coroutine *co, size_t stack_size)
> +{
> +    ucontext_t old_uc, uc;
> +
> +    /* Create a new ucontext for switching to the coroutine stack and setting
> +     * up a longjmp environment. */
> +    if (getcontext(&uc) == -1) {
> +        return -errno;
> +    }
> +
> +    uc.uc_link = &old_uc;
> +    uc.uc_stack.ss_sp = co->stack;
> +    uc.uc_stack.ss_size = stack_size;
> +    uc.uc_stack.ss_flags = 0;
> +
> +    new_coroutine = co;
> +    makecontext(&uc, (void *)continuation_trampoline, 0);
> +
> +    /* Initialize the longjmp environment */
> +    swapcontext(&old_uc, &uc);
> +
> +    return 0;
> +}
> diff --git a/coroutine-win32.c b/coroutine-win32.c
> new file mode 100644
> index 0000000..99141dd
> --- /dev/null
> +++ b/coroutine-win32.c
> @@ -0,0 +1,57 @@
> +/*
> + * Win32 coroutine initialization code
> + *
> + * Copyright (c) 2011 Kevin Wolf <address@hidden>
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a 
> copy
> + * of this software and associated documentation files (the "Software"), to 
> deal
> + * in the Software without restriction, including without limitation the 
> rights
> + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
> + * copies of the Software, and to permit persons to whom the Software is
> + * furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
> FROM,
> + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
> + * THE SOFTWARE.
> + */
> +
> +#include "qemu-coroutine-int.h"
> +
> +static void __attribute__((used)) trampoline(Coroutine *co)
> +{
> +    if (!setjmp(co->env)) {
> +        return;
> +    }
> +
> +    while (true) {
> +        co->entry(co->data);
> +        if (!setjmp(co->env)) {
> +            longjmp(co->caller->env, COROUTINE_TERMINATE);
> +        }
> +    }
> +}
> +
> +int qemu_coroutine_init_env(Coroutine *co, size_t stack_size)
> +{
> +#ifdef __i386__
> +    asm volatile(
> +        "mov %%esp, %%ebx;"
> +        "mov %0, %%esp;"
> +        "pushl %1;"
> +        "call _trampoline;"
> +        "mov %%ebx, %%esp;"
> +        : : "r" (co->stack + stack_size), "r" (co) : "ebx"
> +    );
> +#else
> +    #error This host architecture is not supported for win32
> +#endif
> +
> +    return 0;
> +}
> diff --git a/qemu-coroutine-int.h b/qemu-coroutine-int.h
> new file mode 100644
> index 0000000..71c6ee9
> --- /dev/null
> +++ b/qemu-coroutine-int.h
> @@ -0,0 +1,53 @@
> +/*
> + * Coroutine internals
> + *
> + * Copyright (c) 2011 Kevin Wolf <address@hidden>
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a 
> copy
> + * of this software and associated documentation files (the "Software"), to 
> deal
> + * in the Software without restriction, including without limitation the 
> rights
> + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
> + * copies of the Software, and to permit persons to whom the Software is
> + * furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
> FROM,
> + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
> + * THE SOFTWARE.
> + */
> +
> +#ifndef QEMU_COROUTINE_INT_H
> +#define QEMU_COROUTINE_INT_H
> +
> +#include <setjmp.h>
> +#include "qemu-common.h"
> +#include "qemu-queue.h"
> +#include "qemu-coroutine.h"
> +
> +enum {
> +    /* setjmp() return values */
> +    COROUTINE_YIELD = 1,
> +    COROUTINE_TERMINATE = 2,
> +};
> +
> +struct Coroutine {
> +    struct Coroutine *caller;
> +
> +    char *stack;
> +
> +    /* Used to pass arguments/return values for coroutines */
> +    void *data;
> +    CoroutineEntry *entry;
> +
> +    jmp_buf env;
> +};
> +
> +int qemu_coroutine_init_env(Coroutine *co, size_t stack_size);
> +
> +#endif
> diff --git a/qemu-coroutine.c b/qemu-coroutine.c
> new file mode 100644
> index 0000000..80bed14
> --- /dev/null
> +++ b/qemu-coroutine.c
> @@ -0,0 +1,125 @@
> +/*
> + * QEMU coroutines
> + *
> + * Copyright IBM, Corp. 2011
> + *
> + * Authors:
> + *  Stefan Hajnoczi    <address@hidden>
> + *  Kevin Wolf         <address@hidden>
> + *
> + * This work is licensed under the terms of the GNU LGPL, version 2 or later.
> + * See the COPYING.LIB file in the top-level directory.
> + *
> + */
> +
> +/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */
> +#ifdef _FORTIFY_SOURCE
> +#undef _FORTIFY_SOURCE
> +#endif
> +#include <setjmp.h>
> +
> +#include "trace.h"
> +#include "qemu-queue.h"
> +#include "qemu-common.h"
> +#include "qemu-coroutine.h"
> +#include "qemu-coroutine-int.h"
> +
> +static __thread Coroutine leader;

$ cat thread.c
static __thread int sigusr1_wfd;
$ gcc thread.c -c -pthread
thread.c:1: error: thread-local storage not supported for this target
$ gcc -v
Reading specs from
/usr/bin/../lib/gcc-lib/sparc64-unknown-openbsd4.9/4.2.1/specs
Target: sparc64-unknown-openbsd4.9
Configured with: OpenBSD/sparc64 system compiler
Thread model: posix
gcc version 4.2.1 20070719

> +static __thread Coroutine *current;
> +
> +static void coroutine_terminate(Coroutine *co)
> +{
> +    trace_qemu_coroutine_terminate(co);
> +    qemu_free(co->stack);
> +    qemu_free(co);
> +}
> +
> +static Coroutine *coroutine_new(void)
> +{
> +    const size_t stack_size = 4 << 20;
> +    Coroutine *co;
> +
> +    co = qemu_mallocz(sizeof(*co));
> +    co->stack = qemu_malloc(stack_size);
> +    qemu_coroutine_init_env(co, stack_size);
> +    return co;
> +}
> +
> +Coroutine *qemu_coroutine_create(CoroutineEntry *entry)
> +{
> +    Coroutine *co;
> +
> +    co = coroutine_new();
> +    co->entry = entry;
> +
> +    return co;
> +}
> +
> +Coroutine *coroutine_fn qemu_coroutine_self(void)
> +{
> +    if (current == NULL) {
> +        current = &leader;
> +    }
> +
> +    return current;
> +}
> +
> +bool qemu_in_coroutine(void)
> +{
> +    return qemu_coroutine_self() != &leader;
> +}
> +
> +static void *coroutine_swap(Coroutine *from, Coroutine *to, void *opaque)
> +{
> +    int ret;
> +    void *to_data;
> +
> +    to->data = opaque;
> +
> +    ret = setjmp(from->env);
> +    switch (ret) {
> +    case COROUTINE_YIELD:
> +        return from->data;
> +    case COROUTINE_TERMINATE:
> +        current = to->caller;
> +        to_data = to->data;
> +        coroutine_terminate(to);
> +        return to_data;
> +    default:
> +        /* Switch to called coroutine */
> +        current = to;
> +        longjmp(to->env, COROUTINE_YIELD);
> +        return NULL;
> +    }
> +}
> +
> +void qemu_coroutine_enter(Coroutine *co, void *opaque)
> +{
> +    Coroutine *self = qemu_coroutine_self();
> +
> +    trace_qemu_coroutine_enter(self, co, opaque);
> +
> +    if (co->caller) {
> +        fprintf(stderr, "Co-routine re-entered recursively\n");
> +        abort();
> +    }
> +
> +    co->caller = self;
> +    coroutine_swap(self, co, opaque);
> +}
> +
> +void *coroutine_fn qemu_coroutine_yield(void)
> +{
> +    Coroutine *self = qemu_coroutine_self();
> +    Coroutine *to = self->caller;
> +
> +    trace_qemu_coroutine_yield(self, self->caller);
> +
> +    if (!to) {
> +        fprintf(stderr, "Co-routine is yielding to no one\n");
> +        abort();
> +    }
> +
> +    self->caller = NULL;
> +    return coroutine_swap(self, to, NULL);
> +}
> diff --git a/qemu-coroutine.h b/qemu-coroutine.h
> new file mode 100644
> index 0000000..b79b4bf
> --- /dev/null
> +++ b/qemu-coroutine.h
> @@ -0,0 +1,82 @@
> +/*
> + * QEMU coroutine implementation
> + *
> + * Copyright IBM, Corp. 2011
> + *
> + * Authors:
> + *  Stefan Hajnoczi    <address@hidden>
> + *
> + * This work is licensed under the terms of the GNU LGPL, version 2 or later.
> + * See the COPYING.LIB file in the top-level directory.
> + *
> + */
> +
> +#ifndef QEMU_COROUTINE_H
> +#define QEMU_COROUTINE_H
> +
> +#include <stdbool.h>
> +
> +/**
> + * Mark a function that executes in coroutine context
> + *
> + * Functions that execute in coroutine context cannot be called directly from
> + * normal functions.  In the future it would be nice to enable compiler or
> + * static checker support for catching such errors.  This annotation might 
> make
> + * it possible and in the meantime it serves as documentation.
> + *
> + * For example:
> + *
> + *   static void coroutine_fn foo(void) {
> + *       ....
> + *   }
> + */
> +#define coroutine_fn
> +
> +typedef struct Coroutine Coroutine;
> +
> +/**
> + * Coroutine entry point
> + *
> + * When the coroutine is entered for the first time, opaque is passed in as 
> an
> + * argument.
> + *
> + * When this function returns, the coroutine is destroyed automatically and
> + * execution continues in the caller who last entered the coroutine.
> + */
> +typedef void coroutine_fn CoroutineEntry(void *opaque);
> +
> +/**
> + * Create a new coroutine
> + *
> + * Use qemu_coroutine_enter() to actually transfer control to the coroutine.
> + */
> +Coroutine *qemu_coroutine_create(CoroutineEntry *entry);
> +
> +/**
> + * Transfer control to a coroutine
> + *
> + * The opaque argument is made available to the coroutine either as the entry
> + * function argument if this is the first time a new coroutine is entered, or
> + * as the return value from qemu_coroutine_yield().
> + */
> +void qemu_coroutine_enter(Coroutine *coroutine, void *opaque);
> +
> +/**
> + * Transfer control back to a coroutine's caller
> + *
> + * The return value is the argument passed back in from the next
> + * qemu_coroutine_enter().
> + */
> +void *coroutine_fn qemu_coroutine_yield(void);
> +
> +/**
> + * Get the currently executing coroutine
> + */
> +Coroutine *coroutine_fn qemu_coroutine_self(void);
> +
> +/**
> + * Return whether or not currently inside a coroutine
> + */
> +bool qemu_in_coroutine(void);
> +
> +#endif /* QEMU_COROUTINE_H */
> diff --git a/trace-events b/trace-events
> index 4f965e2..2d4db05 100644
> --- a/trace-events
> +++ b/trace-events
> @@ -361,3 +361,8 @@ disable milkymist_uart_pulse_irq_tx(void) "Pulse IRQ TX"
>  # hw/milkymist-vgafb.c
>  disable milkymist_vgafb_memory_read(uint32_t addr, uint32_t value) "addr 
> %08x value %08x"
>  disable milkymist_vgafb_memory_write(uint32_t addr, uint32_t value) "addr 
> %08x value %08x"
> +
> +# qemu-coroutine.c
> +qemu_coroutine_enter(void *from, void *to, void *opaque) "from %p to %p 
> opaque %p"
> +qemu_coroutine_yield(void *from, void *to) "from %p to %p"
> +qemu_coroutine_terminate(void *co) "self %p"

Not disabled?