[Qemu-devel] [RFC][PATCH v2] add realtime option

[Satoru Moriya]

Changelog
v1 -> v2
 - add RFC tag again
 - change semantics as follows
   - set event threads' priority to maxprio
   - set vcpu threads' priority to maxprio - 1
 - isolate all the posix stuff and put them into os_prioritize() in
   os-posix.c/qemu-os-win32.h to avoid breaking win32 build
 - introduce qemu_init_realtime(), qemu_realtime_is_enable and
   qemu_realtime_get_parameters() and struct QemuRealtimeInfo to
   keep realtime option and remove related global variables in vl.c
 - add other benchmark(qpid-latency-test) result

We have some plans to migrate old enterprise/control systems which
require low latency (msec order) to kvm virtualized environment. In
order to satify the requirements, this patch adds realtime option to qemu:

-realtime maxprio=<prio>,policy=<pol>

This option change the scheduling policy and priority to realtime
one (event threads: maxprio, vcpu threads: maxprio - 1) and mlock all
qemu and guest memory.

Of course, we need more improvements to keep latency low in qemu
virtualized environment and this is a first step. OTOH, we can
meet the requirement of our first migration project with this patch.

[ Note ]
This version doesn't support vhost, vpnc and linux-aio.

These are some basic performance results:

Host : 4 core, 4GB
Guest: 1 core, 512MB

Benchmark: qpid-latency-test
http://qpid.apache.org/
https://access.redhat.com/knowledge/docs/en-US/Red_Hat_Enterprise_MRG/2/html/Messaging_Installation_and_Configuration_Guide/qpid_latency_test.html

 Command:
 - qemu
   $ qemu -smp 1 -m 512 -enable-kvm -netdev tap,id=hostnet1 -device
     virtio-net-pci,netdev=hostnet1 -drive file=vm.img,if=virtio
     (-realtime maxprio=99,policy=fifo)

 - benchmark
   $ chrt -f 99 qpid-latency-test --tcp-nodelay --rate 10000 -b <server>

 Results: worst latency (msec) from 100 run
  - no load
    1. normal qemu  : 17.468400
    2. chrt qemu(*) : 10.019900
    3. realtime qemu:  8.048370

  - load (iperf, server:vm, client:other physical sercer)
    4. normal qemu  : 26.711100
    5. chrt qemu    :  8.485140
    6. realtime qemu: 10.176700

 (*) $ chrt -f -p 99 <event_thread_tid>
     $ chrt -f -p 98 <vcpu_thread_tid>

Any comments are welcome.

Regards,
Satoru

Signed-off-by: Satoru Moriya <address@hidden>
---
 cpus.c                    | 17 +++++++++++++++++
 include/qemu/thread.h     |  4 ++++
 include/sysemu/os-posix.h |  1 +
 include/sysemu/os-win32.h |  1 +
 os-posix.c                | 48 +++++++++++++++++++++++++++++++++++++++++++++++
 qemu-config.c             | 16 ++++++++++++++++
 qemu-options.hx           |  9 +++++++++
 qemu-thread-posix.c       | 27 ++++++++++++++++++++++++++
 qemu-thread-win32.c       | 13 +++++++++++++
 vl.c                      | 33 ++++++++++++++++++++++++++++++++
 10 files changed, 169 insertions(+)

diff --git a/cpus.c b/cpus.c
index 4a7782a..a049970 100644
--- a/cpus.c
+++ b/cpus.c
@@ -734,6 +734,9 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
     CPUArchState *env = arg;
     CPUState *cpu = ENV_GET_CPU(env);
     int r;
+    int rt_policy, rt_priority;
+    struct sched_param sp;
+
 
     qemu_mutex_lock(&qemu_global_mutex);
     qemu_thread_get_self(cpu->thread);
@@ -746,6 +749,20 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
         exit(1);
     }
 
+    if (qemu_realtime_is_enabled()) {
+        qemu_realtime_get_parameters(&rt_policy, &rt_priority);
+        /*
+         * vcpu threads' priority must be set to event thread priority -1
+         * to avoid starvation.
+         */
+        sp.sched_priority = rt_priority - 1;
+        r = sched_setscheduler(0, rt_policy, &sp);
+        if (r < 0) {
+            perror("Setting realtime policy failed");
+            exit(1);
+        }
+    }
+
     qemu_kvm_init_cpu_signals(env);
 
     /* signal CPU creation */
diff --git a/include/qemu/thread.h b/include/qemu/thread.h
index c02404b..3d8b3d2 100644
--- a/include/qemu/thread.h
+++ b/include/qemu/thread.h
@@ -53,4 +53,8 @@ void qemu_thread_get_self(QemuThread *thread);
 bool qemu_thread_is_self(QemuThread *thread);
 void qemu_thread_exit(void *retval);
 
+void qemu_init_realtime(int, int);
+bool qemu_realtime_is_enabled(void);
+void qemu_realtime_get_parameters(int *, int *);
+
 #endif
diff --git a/include/sysemu/os-posix.h b/include/sysemu/os-posix.h
index 7f198e4..e5995b0 100644
--- a/include/sysemu/os-posix.h
+++ b/include/sysemu/os-posix.h
@@ -31,6 +31,7 @@ void os_set_proc_name(const char *s);
 void os_setup_signal_handling(void);
 void os_daemonize(void);
 void os_setup_post(void);
+void os_prioritize(const char *, int);
 
 typedef struct timeval qemu_timeval;
 #define qemu_gettimeofday(tp) gettimeofday(tp, NULL)
diff --git a/include/sysemu/os-win32.h b/include/sysemu/os-win32.h
index d0e9234..946b566 100644
--- a/include/sysemu/os-win32.h
+++ b/include/sysemu/os-win32.h
@@ -78,6 +78,7 @@ static inline void os_daemonize(void) {}
 static inline void os_setup_post(void) {}
 void os_set_line_buffering(void);
 static inline void os_set_proc_name(const char *dummy) {}
+static inline void os_prioritize(const char *pol, int prio) {}
 
 #if !defined(EPROTONOSUPPORT)
 # define EPROTONOSUPPORT EINVAL
diff --git a/os-posix.c b/os-posix.c
index 5c64518..8fe0fa2 100644
--- a/os-posix.c
+++ b/os-posix.c
@@ -33,12 +33,14 @@
 #include <pwd.h>
 #include <grp.h>
 #include <libgen.h>
+#include <sched.h>
 
 /* Needed early for CONFIG_BSD etc. */
 #include "config-host.h"
 #include "sysemu/sysemu.h"
 #include "net/slirp.h"
 #include "qemu-options.h"
+#include "qemu-thread.h"
 
 #ifdef CONFIG_LINUX
 #include <sys/prctl.h>
@@ -363,3 +365,49 @@ bool is_daemonized(void)
 {
     return daemonize;
 }
+
+void os_prioritize(const char *rt_sched_policy, int max_sched_priority)
+{
+    int rt_pol, sys_min_prio, sys_max_prio;
+
+    if (rt_sched_policy) {
+        if (!strcmp(rt_sched_policy, "rr")) {
+            rt_pol = SCHED_RR;
+        } else if (!strcmp(rt_sched_policy, "fifo")) {
+            rt_pol = SCHED_FIFO;
+        } else {
+            fprintf(stderr, "qemu: invalid option value '%s'\n",
+                    rt_sched_policy);
+            exit(1);
+        }
+    } else {
+        rt_pol = SCHED_RR;
+    }
+
+    sys_min_prio = sched_get_priority_min(rt_pol);
+    sys_max_prio = sched_get_priority_max(rt_pol);
+
+    if (max_sched_priority < sys_min_prio + 1) {
+        /*
+         * We set event threads' priority to max_sched_priorty and
+         * vcpu threads' to max_sched_priority - 1 in order to avoid
+         * starvation. So, it must be > sys_min_prio + 1.
+         */
+        fprintf(stderr, "qemu: invalid option maxprio=%d. It must be >= %d\n",
+                max_sched_priority, sys_min_prio + 1);
+        exit(1);
+    }
+
+    if (sys_max_prio < max_sched_priority) {
+        fprintf(stderr, "qemu: invalid option maxprio=%d. It must be <= %d\n",
+                max_sched_priority, sys_max_prio);
+        exit(1);
+    }
+
+    qemu_init_realtime(rt_pol, max_sched_priority);
+
+    if (mlockall(MCL_CURRENT | MCL_FUTURE)) {
+        perror("mlockall");
+        exit(1);
+    }
+}
diff --git a/qemu-config.c b/qemu-config.c
index 2188c3e..b945d07 100644
--- a/qemu-config.c
+++ b/qemu-config.c
@@ -647,6 +647,21 @@ static QemuOptsList qemu_object_opts = {
     },
 };
 
+static QemuOptsList qemu_realtime_opts = {
+    .name = "realtime",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_realtime_opts.head),
+    .desc = {
+        {
+            .name = "maxprio",
+            .type = QEMU_OPT_NUMBER,
+        }, {
+            .name = "policy",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
 static QemuOptsList *vm_config_groups[32] = {
     &qemu_drive_opts,
     &qemu_chardev_opts,
@@ -664,6 +679,7 @@ static QemuOptsList *vm_config_groups[32] = {
     &qemu_sandbox_opts,
     &qemu_add_fd_opts,
     &qemu_object_opts,
+    &qemu_realtime_opts,
     NULL,
 };
 
diff --git a/qemu-options.hx b/qemu-options.hx
index 9df0cde..968a20a 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -2447,6 +2447,15 @@ STEXI
 Do not start CPU at startup (you must type 'c' in the monitor).
 ETEXI
 
+DEF("realtime", HAS_ARG, QEMU_OPTION_realtime,
+    "-realtime maxprio=prio[,policy=pol]\n",
+    QEMU_ARCH_ALL)
+STEXI
address@hidden -realtime address@hidden,address@hidden
address@hidden -realtime
+run qemu as realtime process with priority @var{prio} and policy @var{pol}.
+ETEXI
+
 DEF("gdb", HAS_ARG, QEMU_OPTION_gdb, \
     "-gdb dev        wait for gdb connection on 'dev'\n", QEMU_ARCH_ALL)
 STEXI
diff --git a/qemu-thread-posix.c b/qemu-thread-posix.c
index 7be292e..10a97cc 100644
--- a/qemu-thread-posix.c
+++ b/qemu-thread-posix.c
@@ -22,6 +22,15 @@
 #include <sys/time.h>
 #include "qemu/thread.h"
 
+struct QemuRealtimeInfo {
+    bool is_realtime;
+    int policy;
+    int max_priority;
+};
+typedef struct QemuRealtimeInfo QemuRealtimeInfo;
+
+static QemuRealtimeInfo rt_info;
+
 static void error_exit(int err, const char *msg)
 {
     fprintf(stderr, "qemu: %s: %s\n", msg, strerror(err));
@@ -324,3 +333,21 @@ void *qemu_thread_join(QemuThread *thread)
     }
     return ret;
 }
+
+void qemu_init_realtime(int rt_sched_policy, int max_sched_priority)
+{
+    rt_info.is_realtime = true;
+    rt_info.policy = rt_sched_policy;
+    rt_info.max_priority = max_sched_priority;
+}
+
+bool qemu_realtime_is_enabled(void)
+{
+    return rt_info.is_realtime;
+}
+
+void qemu_realtime_get_parameters(int *policy, int *max_priority)
+{
+    *policy = rt_info.policy;
+    *max_priority = rt_info.max_priority;
+}
diff --git a/qemu-thread-win32.c b/qemu-thread-win32.c
index 8037b39..3beebcf 100644
--- a/qemu-thread-win32.c
+++ b/qemu-thread-win32.c
@@ -369,3 +369,16 @@ bool qemu_thread_is_self(QemuThread *thread)
 {
     return GetCurrentThreadId() == thread->tid;
 }
+
+void qemu_init_realtime(int rt_sched_policy, int max_sched_priority)
+{
+}
+
+bool qemu_realtime_is_enabled(void)
+{
+    return false;
+}
+
+void qemu_realtime_get_parameters(int *policy, int *max_priority)
+{
+}
diff --git a/vl.c b/vl.c
index e6a8d89..c310587 100644
--- a/vl.c
+++ b/vl.c
@@ -29,6 +29,7 @@
 #include <sys/time.h>
 #include <zlib.h>
 #include "qemu/bitmap.h"
+#include "qemu-thread.h"
 
 /* Needed early for CONFIG_BSD etc. */
 #include "config-host.h"
@@ -1148,6 +1149,17 @@ static void smp_parse(const char *optarg)
         max_cpus = smp_cpus;
 }
 
+static void configure_realtime(QemuOpts *opts)
+{
+    const char *pol;
+    int prio;
+
+    pol = qemu_opt_get(opts, "policy");
+    prio = qemu_opt_get_number(opts, "maxprio", 1);
+
+    os_prioritize(pol, prio);
+}
+
 /***********************************************************/
 /* USB devices */
 
@@ -1754,9 +1766,22 @@ static void main_loop(void)
 {
     bool nonblocking;
     int last_io = 0;
+    int rt_policy, rt_priority;
+    struct sched_param sp;
 #ifdef CONFIG_PROFILER
     int64_t ti;
 #endif
+
+    if (qemu_realtime_is_enabled()) {
+        qemu_realtime_get_parameters(&rt_policy, &rt_priority);
+
+        sp.sched_priority = rt_priority;;
+        if (sched_setscheduler(0, rt_policy, &sp) < 0) {
+            perror("Setting realtime policy failed");
+            exit(1);
+        }
+    }
+
     do {
         nonblocking = !kvm_enabled() && last_io > 0;
 #ifdef CONFIG_PROFILER
@@ -2758,6 +2783,14 @@ int main(int argc, char **argv, char **envp)
                 }
                 numa_add(optarg);
                 break;
+            case QEMU_OPTION_realtime:
+                opts = qemu_opts_parse(qemu_find_opts("realtime"), optarg, 0);
+                if (!opts) {
+                    fprintf(stderr, "parse error: %s\n", optarg);
+                    exit(1);
+                }
+                configure_realtime(opts);
+                break;
             case QEMU_OPTION_display:
                 display_type = select_display(optarg);
                 break;
-- 
1.7.11.7