[Qemu-devel] [RFC 33/38] cpu: introduce cpu_tcg_sched_work to run work while other CPUs sleep

[Emilio G. Cota]

This is similar in intent to the async_safe_work mechanism. The main
differences are:

- Work is run on a single CPU thread *after* all others are put to sleep

- Sleeping threads are woken up by the worker thread upon completing its job

- A flag as been added to tcg_ctx so that only one thread can schedule
  work at a time. The flag is checked every time tb_lock is acquired.

- Handles the possibility of CPU threads being created after the existing
  CPUs are put to sleep. This is easily triggered with many threads on
  a many-core host in usermode.

- Works for both softmmu and usermode

Signed-off-by: Emilio G. Cota <address@hidden>
---
 cpu-exec.c              | 89 +++++++++++++++++++++++++++++++++++++++++++++++++
 exec.c                  |  4 +++
 include/exec/exec-all.h |  5 +++
 include/qom/cpu.h       | 20 +++++++++++
 tcg/tcg.h               |  1 +
 translate-all.c         | 23 ++++++++++++-
 6 files changed, 141 insertions(+), 1 deletion(-)

diff --git a/cpu-exec.c b/cpu-exec.c
index ff08da8..378ce52 100644
--- a/cpu-exec.c
+++ b/cpu-exec.c
@@ -393,6 +393,57 @@ static inline void cpu_exit_loop_lock_reset(CPUState *cpu)
 { }
 #endif
 
+static inline void cpu_sleep_other(CPUState *cpu, CPUState *curr)
+{
+    assert(cpu->tcg_sleep_owner == NULL);
+    qemu_mutex_lock(cpu->tcg_work_lock);
+    cpu->tcg_sleep_requests++;
+    cpu->tcg_sleep_owner = curr;
+    qemu_mutex_unlock(cpu->tcg_work_lock);
+#ifdef CONFIG_SOFTMMU
+    cpu_exit(cpu);
+#else
+    /* cannot call cpu_exit(); cpu->exit_request is not for usermode */
+    smp_wmb();
+    cpu->tcg_exit_req = 1;
+#endif
+}
+
+/* call with no locks held */
+static inline void cpu_sleep_others(CPUState *curr)
+{
+    CPUState *cpu;
+
+    CPU_FOREACH(cpu) {
+        if (cpu == curr) {
+            continue;
+        }
+        cpu_sleep_other(cpu, curr);
+    }
+    /* wait until all other threads are out of the execution loop */
+    synchronize_rcu();
+}
+
+static inline void cpu_wake_others(CPUState *curr)
+{
+    CPUState *cpu;
+
+    CPU_FOREACH(cpu) {
+        if (cpu == curr) {
+            continue;
+        }
+        if (cpu->tcg_sleep_owner != curr) {
+            assert(!cpu->inited);
+            continue;
+        }
+        qemu_mutex_lock(cpu->tcg_work_lock);
+        cpu->tcg_sleep_requests--;
+        cpu->tcg_sleep_owner = NULL;
+        qemu_cond_signal(cpu->tcg_work_cond);
+        qemu_mutex_unlock(cpu->tcg_work_lock);
+    }
+}
+
 /* main execution loop */
 
 int cpu_exec(CPUState *cpu)
@@ -410,6 +461,44 @@ int cpu_exec(CPUState *cpu)
 
     current_cpu = cpu;
 
+    /*
+     * Prevent threads that were created during a TCG work critical section
+     * (and that therefore didn't have cpu->tcg_work_owner set) from executing.
+     * What we do is then to not let them run by sending them out of the CPU
+     * loop until the tcg_work_pending flag goes down.
+     */
+    if (unlikely(!cpu->inited)) {
+        tb_lock();
+        tb_unlock();
+        cpu->inited = true;
+    }
+
+    if (cpu->tcg_work_func) {
+        cpu_sleep_others(cpu);
+        /*
+         * At this point all existing threads are sleeping.
+         * With the check above we make sure that threads that might be
+         * concurrently added at this point won't execute until the end of the
+         * work window, so we can safely call the work function.
+         */
+        cpu->tcg_work_func(cpu->tcg_work_arg);
+        cpu->tcg_work_func = NULL;
+        cpu->tcg_work_arg = NULL;
+
+        /* mark the end of the TCG work critical section */
+        tb_lock_nocheck();
+        tcg_ctx.tb_ctx.work_pending = false;
+        tb_unlock();
+        cpu_wake_others(cpu);
+    }
+
+    qemu_mutex_lock(cpu->tcg_work_lock);
+    assert(cpu->tcg_sleep_requests >= 0);
+    while (unlikely(cpu->tcg_sleep_requests)) {
+        qemu_cond_wait(cpu->tcg_work_cond, cpu->tcg_work_lock);
+    }
+    qemu_mutex_unlock(cpu->tcg_work_lock);
+
 #ifndef CONFIG_USER_ONLY
     /* FIXME: user-mode emulation probably needs a similar mechanism as well,
      * for example for tb_flush.
diff --git a/exec.c b/exec.c
index 58cd096..45a9761 100644
--- a/exec.c
+++ b/exec.c
@@ -579,6 +579,10 @@ void cpu_exec_init(CPUState *cpu, Error **errp)
 
     qemu_mutex_init(&cpu->tb_jmp_cache_lock);
     seqlock_init(&cpu->tb_jmp_cache_sequence, &cpu->tb_jmp_cache_lock);
+    cpu->tcg_work_cond = g_malloc(sizeof(*cpu->tcg_work_cond));
+    qemu_cond_init(cpu->tcg_work_cond);
+    cpu->tcg_work_lock = g_malloc(sizeof(*cpu->tcg_work_lock));
+    qemu_mutex_init(cpu->tcg_work_lock);
 #ifndef CONFIG_USER_ONLY
     cpu->as = &address_space_memory;
     cpu->thread_id = qemu_get_thread_id();
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 050e820..be8315c 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -198,6 +198,11 @@ struct TBContext {
     int nb_tbs;
     /* any access to the tbs or the page table must use this lock */
     QemuMutex tb_lock;
+    /*
+     * This ensures that only one thread can perform safe work at a time.
+     * Protected by tb_lock; check the flag right after acquiring the lock.
+     */
+    bool work_pending;
 
     /* statistics */
     int tb_flush_count;
diff --git a/include/qom/cpu.h b/include/qom/cpu.h
index ab484be..aba7edb 100644
--- a/include/qom/cpu.h
+++ b/include/qom/cpu.h
@@ -273,6 +273,15 @@ struct CPUState {
     bool stop;
     bool stopped;
     bool cpu_loop_exit_locked;
+    bool inited;
+    /* tcg_work_* protected by tcg_work_lock */
+    QemuCond *tcg_work_cond;
+    QemuMutex *tcg_work_lock;
+    void (*tcg_work_func)(void *arg);
+    void *tcg_work_arg;
+    CPUState *tcg_sleep_owner;
+    int tcg_sleep_requests;
+
     volatile sig_atomic_t exit_request;
     uint32_t interrupt_request;
     int singlestep_enabled;
@@ -582,6 +591,17 @@ void async_run_safe_work_on_cpu(CPUState *cpu, void 
(*func)(void *data),
 bool async_safe_work_pending(void);
 
 /**
+ * cpu_tcg_sched_work:
+ * @cpu: CPU thread to schedule the work on
+ * @func: function to be called when all other CPU threads are asleep
+ * @arg: argument to be passed to @func
+ *
+ * Schedule work to be done while all other CPU threads are put to sleep.
+ * Call with tb_lock held.
+ */
+void cpu_tcg_sched_work(CPUState *cpu, void (*func)(void *arg), void *arg);
+
+/**
  * qemu_get_cpu:
  * @index: The address@hidden value of the CPU to obtain.
  *
diff --git a/tcg/tcg.h b/tcg/tcg.h
index 9a873ac..1229f7e 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -596,6 +596,7 @@ void tcg_pool_reset(TCGContext *s);
 void tcg_pool_delete(TCGContext *s);
 
 void tb_lock(void);
+void tb_lock_nocheck(void);
 void tb_unlock(void);
 bool tb_lock_recursive(void);
 void tb_lock_reset(void);
diff --git a/translate-all.c b/translate-all.c
index 8f8c402..f3f7fb2 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -133,13 +133,24 @@ TCGContext tcg_ctx;
 /* translation block context */
 __thread int have_tb_lock;
 
-void tb_lock(void)
+/* acquire tb_lock without checking for pending work */
+void tb_lock_nocheck(void)
 {
     assert(!have_tb_lock);
     qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
     have_tb_lock++;
 }
 
+void tb_lock(void)
+{
+    tb_lock_nocheck();
+    if (unlikely(tcg_ctx.tb_ctx.work_pending)) {
+        assert(current_cpu);
+        current_cpu->exception_index = EXCP_INTERRUPT;
+        cpu_loop_exit(current_cpu);
+    }
+}
+
 void tb_unlock(void)
 {
     assert(have_tb_lock);
@@ -961,6 +972,16 @@ static void tb_page_check(void)
 
 #endif
 
+void cpu_tcg_sched_work(CPUState *cpu, void (*func)(void *arg), void *arg)
+{
+    assert(have_tb_lock);
+    tcg_ctx.tb_ctx.work_pending = true;
+    cpu->tcg_work_func = func;
+    cpu->tcg_work_arg = arg;
+    cpu->exception_index = EXCP_INTERRUPT;
+    cpu_loop_exit(cpu);
+}
+
 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
 {
     TranslationBlock *tb1;
-- 
1.9.1