[Qemu-devel] [PATCH] Add realtime option

[Satoru Moriya]

We have some plans to migrate old enterprise/control systems which
require low latency (msec order) to kvm virtualized environment.
In order to satisfy the requirements, this patch adds realtime option
to qemu:

 -realtime maxprio=<prio>,policy=<pol>

This option change the scheduling policy and priority to realtime one
(only vcpu thread) as specified with argument and mlock all qemu and
guest memory.

Of course, we need much more improvements to keep latency low in qemu
virtualized environment and this is a first step. OTOH, we can meet the
requirement of our first migration project with this patch.

These are basic performance test results:

Host : 4 core, 4GB, 3.7.0-rc3
Guest: 1 core, 512MB, 3.6.3-1.fc17

Benchmark: cyclictest
https://rt.wiki.kernel.org/index.php/Cyclictest

Command:
 $ cyclictest -p 99 -n -m -q -l 100000

Results:
 - no load (1:normal qemu, 2:realtime qemu)
   1. T: 0 ( 544) P:99 I:1000 C:100000 Min: 11 Act: 32 Avg: 157 Max: 10029
   2. T: 0 ( 449) P:99 I:1000 C:100000 Min: 16 Act: 30 Avg:  29 Max:   540

 - load (heavy network traffic) (3:normal qemu, 4: realtime qemu)
   3. T: 0 (3455) P:99 I:1000 C:100000 Min: 10 Act: 38 Avg: 364 Max: 18394
   4. T: 0 ( 493) P:99 I:1000 C:100000 Min: 12 Act: 21 Avg:  76 Max: 10796

Signed-off-by: Satoru Moriya <address@hidden>
---
 cpus.c          | 10 ++++++++++
 cpus.h          |  3 +++
 qemu-config.c   | 16 ++++++++++++++++
 qemu-options.hx |  9 +++++++++
 vl.c            | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 89 insertions(+)

diff --git a/cpus.c b/cpus.c
index d9c332f..456e6ea 100644
--- a/cpus.c
+++ b/cpus.c
@@ -734,6 +734,7 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
     CPUArchState *env = arg;
     CPUState *cpu = ENV_GET_CPU(env);
     int r;
+    struct sched_param sp;
 
     qemu_mutex_lock(&qemu_global_mutex);
     qemu_thread_get_self(cpu->thread);
@@ -746,6 +747,15 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
         exit(1);
     }
 
+    if (realtime) {
+        sp.sched_priority = realtime_prio;
+        r = sched_setscheduler(0, realtime_pol, &sp);
+        if (r < 0) {
+            perror("Setting realtime policy failed");
+            exit(1);
+        }
+    }
+
     qemu_kvm_init_cpu_signals(env);
 
     /* signal CPU creation */
diff --git a/cpus.h b/cpus.h
index 81bd817..a6b2688 100644
--- a/cpus.h
+++ b/cpus.h
@@ -16,6 +16,9 @@ void qtest_clock_warp(int64_t dest);
 /* vl.c */
 extern int smp_cores;
 extern int smp_threads;
+extern int realtime;
+extern int realtime_prio;
+extern int realtime_pol;
 void set_numa_modes(void);
 void set_cpu_log(const char *optarg);
 void set_cpu_log_filename(const char *optarg);
diff --git a/qemu-config.c b/qemu-config.c
index 3154cac..13290c6 100644
--- a/qemu-config.c
+++ b/qemu-config.c
@@ -658,6 +658,21 @@ QemuOptsList qemu_boot_opts = {
             .type = QEMU_OPT_STRING,
         },
         { /*End of list */ }
+    },
+};
+
+QemuOptsList qemu_realtime_opts = {
+    .name = "realtime",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_realtime_opts.head),
+    .desc = {
+        {
+            .name = "maxprio",
+            .type = QEMU_OPT_NUMBER,
+        }, {
+            .name = "policy",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* End of List */ }
     },
 };
 
@@ -699,6 +714,7 @@ static QemuOptsList *vm_config_groups[32] = {
     &qemu_iscsi_opts,
     &qemu_sandbox_opts,
     &qemu_add_fd_opts,
+    &qemu_realtime_opts,
     NULL,
 };
 
diff --git a/qemu-options.hx b/qemu-options.hx
index fe8f15c..eb8ba05 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -2405,6 +2405,15 @@ STEXI
 Do not start CPU at startup (you must type 'c' in the monitor).
 ETEXI
 
+DEF("realtime", HAS_ARG, QEMU_OPTION_realtime,
+    "-realtime maxprio=prio[,policy=pol]\n",
+    QEMU_ARCH_ALL)
+STEXI
address@hidden -realtime address@hidden,address@hidden
address@hidden -realtime
+run qemu as a realtime process with priority @var{prio} and policy @var{pol}.
+ETEXI
+
 DEF("gdb", HAS_ARG, QEMU_OPTION_gdb, \
     "-gdb dev        wait for gdb connection on 'dev'\n", QEMU_ARCH_ALL)
 STEXI
diff --git a/vl.c b/vl.c
index 0f5b07b..a08fe79 100644
--- a/vl.c
+++ b/vl.c
@@ -248,6 +248,10 @@ int nb_numa_nodes;
 uint64_t node_mem[MAX_NODES];
 unsigned long *node_cpumask[MAX_NODES];
 
+int realtime;
+int realtime_prio;
+int realtime_pol;
+
 uint8_t qemu_uuid[16];
 
 static QEMUBootSetHandler *boot_set_handler;
@@ -1151,6 +1155,45 @@ static void smp_parse(const char *optarg)
         max_cpus = smp_cpus;
 }
 
+static void configure_realtime(QemuOpts *opts) {
+    int prio, max_prio, min_prio;
+    const char *pol;
+
+    pol = qemu_opt_get(opts, "policy");
+    if (pol) {
+        if (!strcmp(pol, "rr")) {
+            realtime_pol = SCHED_RR;
+        } else if (!strcmp(pol, "fifo")) {
+            realtime_pol = SCHED_FIFO;
+        } else {
+            fprintf(stderr, "qemu: invalid option value '%s'\n", pol);
+            exit(1);
+        }
+    } else {
+        realtime_pol = SCHED_RR;
+    }
+    prio = qemu_opt_get_number(opts, "maxprio", 1);
+
+    min_prio = sched_get_priority_min(realtime_pol);
+    max_prio = sched_get_priority_max(realtime_pol);
+
+    if (prio < min_prio) {
+        realtime_prio = min_prio;
+    } else if (max_prio < prio) {
+        realtime_prio = max_prio;
+    } else {
+        realtime_prio = prio;
+    }
+
+    if (mlockall(MCL_CURRENT | MCL_FUTURE)) {
+        perror("mlock");
+        exit(1);
+    }
+
+    realtime = 1;
+}
+
 /***********************************************************/
 /* USB devices */
 
@@ -2712,6 +2755,14 @@ int main(int argc, char **argv, char **envp)
                 }
                 numa_add(optarg);
                 break;
+            case QEMU_OPTION_realtime:
+                opts = qemu_opts_parse(qemu_find_opts("realtime"), optarg, 0);
+                if (!opts) {
+                    fprintf(stderr, "parse error: %s\n", optarg);
+                    exit(1);
+                }
+                configure_realtime(opts);
+                break;
             case QEMU_OPTION_display:
                 display_type = select_display(optarg);
                 break;
--
1.7.11.7