[Qemu-devel] [PATCH 01/15] qapi: add code generator for qmp-types (v2)

[Anthony Liguori]

Only generate qmp-types.[ch].  These files contain the type definitions for
QMP along with the alloc/free functions for these types.  Functions to convert
enum values to integers and vice versa are also included.

qmp-types is used both within QEMU and within libqmp

Special alloc/free functions are provided to ensure that all structures are
padded when allocated.  This makes sure that libqmp can provide a forward
compatible interface since all additions to a structure will have a boolean
enable flag.

The free function is convenient since individual structures may have pointers
that also require freeing.

Signed-off-by: Anthony Liguori <address@hidden>
---
v1 -> v2
 - modify code generator to use multiline strings instead of prints
 - support proxy commands
 - support async commands

diff --git a/Makefile b/Makefile
index 6b1d716..6b9fd69 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,7 @@ GENERATED_HEADERS = config-host.h trace.h qemu-options.def
 ifeq ($(TRACE_BACKEND),dtrace)
 GENERATED_HEADERS += trace-dtrace.h
 endif
+GENERATED_HEADERS += qmp-types.h
 
 ifneq ($(wildcard config-host.mak),)
 # Put the all: rule here so that config-host.mak can contain dependencies.
@@ -146,6 +147,14 @@ trace-dtrace.o: trace-dtrace.dtrace $(GENERATED_HEADERS)
 
 simpletrace.o: simpletrace.c $(GENERATED_HEADERS)
 
+qmp-types.c: $(SRC_PATH)/qmp-schema.json $(SRC_PATH)/qmp-gen.py
+       $(call quiet-command,python $(SRC_PATH)/qmp-gen.py --types-body < $< > 
$@, "  GEN   $@")
+
+qmp-types.h: $(SRC_PATH)/qmp-schema.json $(SRC_PATH)/qmp-gen.py
+       $(call quiet-command,python $(SRC_PATH)/qmp-gen.py --types-header < $< 
> $@, "  GEN   $@")
+
+qmp-types.o: qmp-types.c qmp-types.h
+
 version.o: $(SRC_PATH)/version.rc config-host.mak
        $(call quiet-command,$(WINDRES) -I. -o $@ $<,"  RC    $(TARGET_DIR)$@")
 
diff --git a/Makefile.objs b/Makefile.objs
index 69f0383..710d99f 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -15,7 +15,7 @@ oslib-obj-$(CONFIG_POSIX) += oslib-posix.o
 
 block-obj-y = cutils.o cache-utils.o qemu-malloc.o qemu-option.o module.o
 block-obj-y += nbd.o block.o aio.o aes.o qemu-config.o
-block-obj-y += error.o
+block-obj-y += error.o qmp-types.o
 block-obj-$(CONFIG_POSIX) += posix-aio-compat.o
 block-obj-$(CONFIG_LINUX_AIO) += linux-aio.o
 
diff --git a/ordereddict.py b/ordereddict.py
new file mode 100644
index 0000000..e17269f
--- /dev/null
+++ b/ordereddict.py
@@ -0,0 +1,128 @@
+# Copyright (c) 2009 Raymond Hettinger
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+#     The above copyright notice and this permission notice shall be
+#     included in all copies or substantial portions of the Software.
+#
+#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+#     OTHER DEALINGS IN THE SOFTWARE.
+
+from UserDict import DictMixin
+
+class OrderedDict(dict, DictMixin):
+
+    def __init__(self, *args, **kwds):
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        try:
+            self.__end
+        except AttributeError:
+            self.clear()
+        self.update(*args, **kwds)
+
+    def clear(self):
+        self.__end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.__map = {}                 # key --> [key, prev, next]
+        dict.clear(self)
+
+    def __setitem__(self, key, value):
+        if key not in self:
+            end = self.__end
+            curr = end[1]
+            curr[2] = end[1] = self.__map[key] = [key, curr, end]
+        dict.__setitem__(self, key, value)
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+        key, prev, next = self.__map.pop(key)
+        prev[2] = next
+        next[1] = prev
+
+    def __iter__(self):
+        end = self.__end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.__end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def popitem(self, last=True):
+        if not self:
+            raise KeyError('dictionary is empty')
+        if last:
+            key = reversed(self).next()
+        else:
+            key = iter(self).next()
+        value = self.pop(key)
+        return key, value
+
+    def __reduce__(self):
+        items = [[k, self[k]] for k in self]
+        tmp = self.__map, self.__end
+        del self.__map, self.__end
+        inst_dict = vars(self).copy()
+        self.__map, self.__end = tmp
+        if inst_dict:
+            return (self.__class__, (items,), inst_dict)
+        return self.__class__, (items,)
+
+    def keys(self):
+        return list(self)
+
+    setdefault = DictMixin.setdefault
+    update = DictMixin.update
+    pop = DictMixin.pop
+    values = DictMixin.values
+    items = DictMixin.items
+    iterkeys = DictMixin.iterkeys
+    itervalues = DictMixin.itervalues
+    iteritems = DictMixin.iteritems
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, self.items())
+
+    def copy(self):
+        return self.__class__(self)
+
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        d = cls()
+        for key in iterable:
+            d[key] = value
+        return d
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedDict):
+            if len(self) != len(other):
+                return False
+            for p, q in  zip(self.items(), other.items()):
+                if p != q:
+                    return False
+            return True
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        return not self == other
+
diff --git a/qmp-gen.py b/qmp-gen.py
new file mode 100644
index 0000000..cded2f6
--- /dev/null
+++ b/qmp-gen.py
@@ -0,0 +1,516 @@
+##
+# QAPI Code Generator
+#
+# Copyright IBM, Corp. 2011
+#
+# Authors:
+#  Anthony Liguori   <address@hidden>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+##
+import sys
+from ordereddict import OrderedDict
+
+enum_types = []
+event_types = {}
+
+def qmp_is_proxy_cmd(name):
+    return name.startswith('guest-')
+
+def qmp_is_async_cmd(name):
+    return name.startswith('guest-')
+
+def qmp_is_stateful_cmd(name):
+    return name in ['qmp_capabilities', 'put-event', 'getfd', 'closefd']
+
+def c_var(name):
+    return '_'.join(name.split('-'))
+
+def genindent(count):
+    ret = ""
+    for i in range(count):
+        ret += " "
+    return ret
+
+indent_level = 0
+
+def push_indent():
+    global indent_level
+    indent_level += 4
+
+def pop_indent():
+    global indent_level
+    indent_level -= 4
+
+def cgen(code, **kwds):
+    indent = genindent(indent_level)
+    lines = code.split('\n')
+    lines = map(lambda x: indent + x, lines)
+    return '\n'.join(lines) % kwds + '\n'
+
+def mcgen(code, **kwds):
+    return cgen('\n'.join(code.split('\n')[1:-1]), **kwds)
+
+def is_dict(obj):
+    if type(obj) in [dict, OrderedDict]:
+        return True
+    return False
+
+def qmp_array_type_to_c(typename):
+    if type(typename) == list or is_dict(typename):
+        return qmp_type_to_c(typename)
+    elif typename == 'int':
+        return 'IntArray *'
+    elif typename == 'str':
+        return 'StringArray *'
+    elif typename == 'bool':
+        return 'BoolArray *'
+    elif typename == 'number':
+        return 'DoubleArray *'
+    else:
+        return qmp_type_to_c(typename)
+
+def qmp_type_should_free(typename):
+    if (type(typename) == list or
+        typename == 'str' or
+        (typename not in ['int', 'bool', 'number'] and
+         typename not in enum_types and not typename.isupper())):
+        return True
+    return False
+
+def qmp_free_func(typename):
+    if type(typename) == list:
+        return qmp_free_func(typename[0])
+    elif typename == 'str':
+        return 'qemu_free'
+    else:
+        return 'qmp_free_%s' % (de_camel_case(typename))
+
+def qmp_type_is_event(typename):
+    if type(typename) == str and typename.isupper():
+        return True
+    return False
+
+def qmp_type_to_c(typename, retval=False, indent=0):
+    if type(typename) == list:
+        return qmp_array_type_to_c(typename[0])
+    elif is_dict(typename):
+        string = 'struct {\n'
+        for argname, argtype, optional in parse_args(typename):
+            if optional:
+                string += "%sbool has_%s;\n" % (genindent(indent + 4), 
c_var(argname))
+            string += "%s%s %s;\n" % (genindent(indent + 4),
+                                      qmp_type_to_c(argtype, True,
+                                                    indent=(indent + 4)),
+                                      c_var(argname))
+        string += "%s}" % genindent(indent)
+        return string
+    elif typename == 'int':
+        return 'int64_t'
+    elif not retval and typename == 'str':
+        return 'const char *'
+    elif retval and typename == 'str':
+        return 'char *'
+    elif typename == 'bool':
+        return 'bool'
+    elif typename == 'number':
+        return 'double'
+    elif typename == 'none':
+        return 'void'
+    elif typename in enum_types:
+        return typename
+    elif qmp_type_is_event(typename):
+        return 'struct %s *' % qmp_event_to_c(typename)
+    else:
+        return 'struct %s *' % typename
+
+def qmp_type_to_qobj(typename):
+    return 'qmp_marshal_type_%s' % typename
+
+def qmp_type_from_qobj(typename):
+    return 'qmp_unmarshal_type_%s' % typename
+
+def parse_args(typeinfo):
+    for member in typeinfo:
+        argname = member
+        argtype = typeinfo[member]
+        optional = False
+        if member.startswith('*'):
+            argname = member[1:]
+            optional = True
+        yield (argname, argtype, optional)
+
+def de_camel_case(name):
+    new_name = ''
+    for ch in name:
+        if ch.isupper() and new_name:
+            new_name += '_'
+        new_name += ch.lower()
+    return new_name
+
+def camel_case(name):
+    new_name = ''
+    first = True
+    for ch in name:
+        if ch in ['_', '-']:
+            first = True
+        elif first:
+            new_name += ch.upper()
+            first = False
+        else:
+            new_name += ch.lower()
+    return new_name
+
+def qmp_event_to_c(name):
+    return '%sEvent' % camel_case(name)
+
+def qmp_event_func_to_c(name):
+    return '%sFunc' % camel_case(name)
+
+def enum_abbreviation(name):
+    ab = ''
+    for ch in name:
+        if ch.isupper():
+            ab += ch
+    return ab
+
+def gen_type_declaration(name, typeinfo):
+    ret = ''
+    if type(typeinfo) == str:
+        ret += mcgen('''
+
+typedef %(type)s %(name)s;
+''',
+                     type=qmp_type_to_c(typeinfo),
+                     name=name)
+    elif is_dict(typeinfo) and not name.isupper():
+        ret += mcgen('''
+
+typedef struct %(name)s %(name)s;
+struct %(name)s {
+''', name=name)
+        for argname, argtype, optional in parse_args(typeinfo):
+            if optional:
+                ret += cgen('    bool has_%(c_name)s;',
+                            c_name=c_var(argname))
+            ret += cgen('    %(type)s %(c_name)s;',
+                        type=qmp_type_to_c(argtype, True, indent=4),
+                        c_name=c_var(argname))
+        ret += mcgen('''
+    %(c_name)s *next;
+};
+
+%(name)s *qmp_alloc_%(dcc_name)s(void);
+void qmp_free_%(dcc_name)s(%(name)s *obj);
+''',
+                     c_name=c_var(name), name=name,
+                     dcc_name=de_camel_case(name))
+    elif is_dict(typeinfo) and name.isupper():
+        arglist = ['void *opaque']
+        for argname, argtype, optional in parse_args(typeinfo):
+            arglist.append('%s %s' % (qmp_type_to_c(argtype), argname))
+        ret += mcgen('''
+
+typedef void (%(event_func)s)(%(args)s);
+
+typedef struct %(c_event)s {
+    QmpSignal *signal;
+    %(event_func)s *func;
+} %(c_event)s;
+''',
+                     event_func=qmp_event_func_to_c(name),
+                     args=', '.join(arglist),
+                     c_event=qmp_event_to_c(name))
+    return ret
+
+def gen_metatype_free(typeinfo, prefix):
+    ret = ''
+
+    for argname, argtype, optional in parse_args(typeinfo):
+        if type(argtype) == list:
+            argtype = argtype[0]
+
+        if is_dict(argtype):
+            if optional:
+                ret += cgen('    if (%(prefix)shas_%(c_name)s) {',
+                            prefix=prefix, c_name=c_var(argname))
+                push_indent()
+            ret += gen_metatype_free(argtype, '%s%s.' % (prefix, argname))
+            if optional:
+                pop_indent()
+                ret += cgen('    }')
+        elif qmp_type_should_free(argtype):
+            if optional:
+                ret += mcgen('''
+    if (%(prefix)shas_%(c_name)s) {
+        %(free)s(%(prefix)s%(c_name)s);
+    }
+''',
+                             prefix=prefix, c_name=c_var(argname),
+                             free=qmp_free_func(argtype))
+            else:
+                ret += mcgen('''
+    %(free)s(%(prefix)s%(c_name)s);
+''',
+                             prefix=prefix, c_name=c_var(argname),
+                             free=qmp_free_func(argtype))
+
+    return ret
+
+def gen_type_definition(name, typeinfo):
+    return mcgen('''
+
+void qmp_free_%(dcc_name)s(%(name)s *obj)
+{
+    if (!obj) {
+        return;
+    }
+%(type_free)s
+
+    %(free)s(obj->next);
+    qemu_free(obj);
+}
+
+%(name)s *qmp_alloc_%(dcc_name)s(void)
+{
+    BUILD_ASSERT(sizeof(%(name)s) < 512);
+    return qemu_mallocz(512);
+}
+''',
+                dcc_name=de_camel_case(name), name=name,
+                free=qmp_free_func(name),
+                type_free=gen_metatype_free(typeinfo, 'obj->'))
+
+def gen_enum_declaration(name, entries):
+    ret = mcgen('''
+
+typedef enum %(name)s {
+''', name=name)
+    i = 0
+    for entry in entries:
+        ret += cgen('    %(abrev)s_%(name)s = %(value)d,',
+                    abrev=enum_abbreviation(name),
+                    name=entry.upper(), value=i)
+        i += 1
+    ret += mcgen('''
+} %(name)s;
+
+%(name)s qmp_type_%(dcc_name)s_from_str(const char *str, Error **errp);
+const char *qmp_type_%(dcc_name)s_to_str(%(name)s value, Error **errp);
+''',
+                 name=name, dcc_name=de_camel_case(name))
+    return ret
+
+def gen_enum_definition(name, entries):
+    ret = mcgen('''
+
+%(name)s qmp_type_%(dcc_name)s_from_str(const char *str, Error **errp)
+{
+''',
+                name=name,
+                dcc_name=de_camel_case(name))
+    first = True
+    for entry in entries:
+        prefix = '} else '
+        if first:
+            prefix = ''
+            first = False
+        ret += mcgen('''
+    %(prefix)sif (strcmp(str, "%(entry)s") == 0) {
+        return %(abrev)s_%(value)s;
+''',
+                     prefix=prefix, entry=entry,
+                     abrev=enum_abbreviation(name), value=entry.upper())
+
+    ret += mcgen('''
+    } else {
+        error_set(errp, QERR_ENUM_VALUE_INVALID, "%(name)s", str);
+        return %(abrev)s_%(value)s;
+    }
+}
+
+const char *qmp_type_%(dcc_name)s_to_str(%(name)s value, Error **errp)
+{
+''',
+                 name=name, abrev=enum_abbreviation(name),
+                 value=entries[0].upper(), dcc_name=de_camel_case(name))
+
+    first = True
+    for entry in entries:
+        enum = '%s_%s' % (enum_abbreviation(name), entry.upper())
+        prefix = '} else '
+        if first:
+            prefix = ''
+            first = False
+        ret += mcgen('''
+    %(prefix)sif (value == %(enum)s) {
+        return "%(entry)s";
+''',
+                     entry=entry, prefix=prefix, enum=enum)
+    ret += mcgen('''
+    } else {
+        char buf[32];
+        snprintf(buf, sizeof(buf), "%%d", value);
+        error_set(errp, QERR_ENUM_VALUE_INVALID, "%(name)s", buf);
+        return NULL;
+    }
+}
+''',
+                 name=name)
+    return ret
+
+def tokenize(data):
+    while len(data):
+        if data[0] in ['{', '}', ':', ',', '[', ']']:
+            yield data[0]
+            data = data[1:]
+        elif data[0] in ' \n':
+            data = data[1:]
+        elif data[0] == "'":
+            data = data[1:]
+            string = ''
+            while data[0] != "'":
+                string += data[0]
+                data = data[1:]
+            data = data[1:]
+            yield string
+
+def parse_value(tokens):
+    if tokens[0] == '{':
+        ret = OrderedDict()
+        tokens = tokens[1:]
+        while tokens[0] != '}':
+            key = tokens[0]
+            tokens = tokens[1:] 
+
+            tokens = tokens[1:] # :
+
+            value, tokens = parse_value(tokens)
+
+            if tokens[0] == ',':
+                tokens = tokens[1:]
+
+            ret[key] = value
+        tokens = tokens[1:]
+        return ret, tokens
+    elif tokens[0] == '[':
+        ret = []
+        tokens = tokens[1:]
+        while tokens[0] != ']':
+            value, tokens = parse_value(tokens)
+            if tokens[0] == ',':
+                tokens = tokens[1:]
+            ret.append(value)
+        tokens = tokens[1:]
+        return ret, tokens
+    else:
+        return tokens[0], tokens[1:]
+
+def ordered_eval(string):
+    return parse_value(map(lambda x: x, tokenize(string)))[0]
+#    return eval(string)
+
+def generate(kind):
+    global enum_types
+    global event_types
+    global indent_level
+
+    enum_types = []
+    event_types = {}
+    indent_level = 0
+
+    ret = mcgen('''
+/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT */
+''')
+
+    if kind == 'types-header':
+        ret += mcgen('''
+#ifndef QMP_TYPES_H
+#define QMP_TYPES_H
+
+#include "qmp-types-core.h"
+''')
+    elif kind == 'types-body':
+        ret += mcgen('''
+#include "qmp-types.h"
+#include "qmp-marshal-types.h"
+''')
+    
+    exprs = []
+    expr = ''
+    
+    for line in sys.stdin:
+        if line.startswith('#') or line == '\n':
+            continue
+    
+        if line.startswith(' '):
+            expr += line
+        elif expr:
+            s = ordered_eval(expr)
+            exprs.append(s)
+            expr = line
+        else:
+            expr += line
+    
+    if expr:
+        s = ordered_eval(expr)
+        exprs.append(s)
+    
+    for s in exprs:
+       if s.has_key('type'):
+           name = s['type']
+           data = s['data']
+
+           if kind == 'types-body':
+               ret += gen_type_definition(name, data)
+           elif kind == 'types-header':
+               ret += gen_type_declaration(name, data)
+       elif s.has_key('enum'):
+           name = s['enum']
+           data = s['data']
+
+           enum_types.append(s['enum'])
+           if kind == 'types-header':
+               ret += gen_enum_declaration(name, data)
+           elif kind == 'types-body':
+               ret += gen_enum_definition(name, data)
+       elif s.has_key('event'):
+           name = s['event']
+           data = {}
+           if s.has_key('data'):
+               data = s['data']
+
+           event_types[name] = data
+           if kind == 'types-header':
+               ret += gen_type_declaration(name, data)
+       elif s.has_key('command'):
+            name = s['command']
+            options = {}
+            if s.has_key('data'):
+                options = s['data']
+            retval = 'none'
+            if s.has_key('returns'):
+                retval = s['returns']
+
+    if kind.endswith('header'):
+        ret += cgen('#endif')
+
+    return ret
+
+def main(args):
+    if len(args) != 1:
+        return 1
+    if not args[0].startswith('--'):
+        return 1
+
+    kind = args[0][2:]
+
+    ret = generate(kind)
+
+    sys.stdout.write(ret)
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
diff --git a/qmp-schema.json b/qmp-schema.json
new file mode 100644
index 0000000..e69de29
diff --git a/qmp-types-core.h b/qmp-types-core.h
new file mode 100644
index 0000000..a018ba7
--- /dev/null
+++ b/qmp-types-core.h
@@ -0,0 +1,29 @@
+/*
+ * QAPI
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Anthony Liguori   <address@hidden>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.  See
+ * the COPYING.LIB file in the top-level directory.
+ */
+#ifndef QMP_TYPES_CORE_H
+#define QMP_TYPES_CORE_H
+
+#include <stdint.h>
+#include <stdbool.h>
+#include "error.h"
+
+typedef struct QmpSignal QmpSignal;
+typedef struct QmpCommandState QmpCommandState;
+typedef struct QmpState QmpState;
+
+#define BUILD_ASSERT(cond) do {     \
+    (void)sizeof(int[-1+!!(cond)]); \
+} while (0)
+
+#define BUILD_BUG() BUILD_ASSERT(0)
+
+#endif
-- 
1.7.0.4