Re: [PATCH] Add msgpack

[Lukas Gradl]

Leo Famulari <address@hidden> writes:

> On Sat, Jun 11, 2016 at 06:56:38PM -0500, Lukas Gradl wrote:
>> It seems that zlib is only required for tests.  In README and README.md
>> zlib is never  mentioned among the dependencies, which also supports
>> that it is only needed for tests.  I was not aware of that before (Shame
>> on me).
>
> Is zlib really only required for tests? I see this:
>
> $ grep -rIi zlib /gnu/store/xn8z7k6j7zm4qz14bm29fgk0kwwvz3c4-msgpack-1.4.1
> /gnu/store/xn8z7k6j7zm4qz14bm29fgk0kwwvz3c4-msgpack-1.4.1/include/msgpack/zbuffer.h:#include
>  <zlib.h>
> /gnu/store/xn8z7k6j7zm4qz14bm29fgk0kwwvz3c4-msgpack-1.4.1/include/msgpack/zbuffer.hpp:#include
>  <zlib.h>
>
> So, there are headers in the msgpack output that require zlib.

Oh, that is true.  I searched around, but I could not find anything
about zbuffer.h[pp].  I am not sure what it is used for.  FWIW, it is
only referenced in src/Makefile, src/Makefile.in, src/Makefile.am
CMakeLists.txt and in the tests:


$ grep -ri zbuffer
src/Makefile:   ../include/msgpack/zbuffer.hpp \
src/Makefile:   ../include/msgpack/vrefbuffer.h ../include/msgpack/zbuffer.h \
src/Makefile:   ../include/msgpack/zbuffer.hpp ../include/msgpack/zone.hpp \
src/Makefile:   ../include/msgpack/vrefbuffer.h ../include/msgpack/zbuffer.h \
src/Makefile.in:@ENABLE_CXX_TRUE@       ../include/msgpack/zbuffer.hpp \
src/Makefile.in:        ../include/msgpack/vrefbuffer.h 
../include/msgpack/zbuffer.h \
src/Makefile.in:        ../include/msgpack/zbuffer.hpp 
../include/msgpack/zone.hpp \
src/Makefile.in:        ../include/msgpack/vrefbuffer.h 
../include/msgpack/zbuffer.h \
src/Makefile.am:                ../include/msgpack/zbuffer.h \
src/Makefile.am:                ../include/msgpack/zbuffer.hpp \
ChangeLog:  * Fix zbuffer with empty string problem (#303)
include/msgpack/zbuffer.hpp:#ifndef MSGPACK_ZBUFFER_HPP
include/msgpack/zbuffer.hpp:#define MSGPACK_ZBUFFER_HPP
include/msgpack/zbuffer.hpp:#ifndef MSGPACK_ZBUFFER_RESERVE_SIZE
include/msgpack/zbuffer.hpp:#define MSGPACK_ZBUFFER_RESERVE_SIZE 512
include/msgpack/zbuffer.hpp:#ifndef MSGPACK_ZBUFFER_INIT_SIZE
include/msgpack/zbuffer.hpp:#define MSGPACK_ZBUFFER_INIT_SIZE 8192
include/msgpack/zbuffer.hpp:class zbuffer {
include/msgpack/zbuffer.hpp:    zbuffer(int level = Z_DEFAULT_COMPRESSION,
include/msgpack/zbuffer.hpp:            size_t init_size = 
MSGPACK_ZBUFFER_INIT_SIZE)
include/msgpack/zbuffer.hpp:    ~zbuffer()
include/msgpack/zbuffer.hpp:            if(m_stream.avail_out < 
MSGPACK_ZBUFFER_RESERVE_SIZE) {
include/msgpack/zbuffer.hpp:    zbuffer(const zbuffer&);
include/msgpack/zbuffer.hpp:    zbuffer& operator=(const zbuffer&);
include/msgpack/zbuffer.hpp:    zbuffer(const zbuffer&) = delete;
include/msgpack/zbuffer.hpp:    zbuffer& operator=(const zbuffer&) = delete;
include/msgpack/zbuffer.hpp:#endif /* msgpack/zbuffer.hpp */
include/msgpack/zbuffer.h:#ifndef MSGPACK_ZBUFFER_H
include/msgpack/zbuffer.h:#define MSGPACK_ZBUFFER_H
include/msgpack/zbuffer.h: * @defgroup msgpack_zbuffer Compressed buffer
include/msgpack/zbuffer.h:typedef struct msgpack_zbuffer {
include/msgpack/zbuffer.h:} msgpack_zbuffer;
include/msgpack/zbuffer.h:#ifndef MSGPACK_ZBUFFER_INIT_SIZE
include/msgpack/zbuffer.h:#define MSGPACK_ZBUFFER_INIT_SIZE 8192
include/msgpack/zbuffer.h:static inline bool msgpack_zbuffer_init(
include/msgpack/zbuffer.h:    msgpack_zbuffer* zbuf, int level, size_t 
init_size);
include/msgpack/zbuffer.h:static inline void 
msgpack_zbuffer_destroy(msgpack_zbuffer* zbuf);
include/msgpack/zbuffer.h:static inline msgpack_zbuffer* 
msgpack_zbuffer_new(int level, size_t init_size);
include/msgpack/zbuffer.h:static inline void 
msgpack_zbuffer_free(msgpack_zbuffer* zbuf);
include/msgpack/zbuffer.h:static inline char* 
msgpack_zbuffer_flush(msgpack_zbuffer* zbuf);
include/msgpack/zbuffer.h:static inline const char* msgpack_zbuffer_data(const 
msgpack_zbuffer* zbuf);
include/msgpack/zbuffer.h:static inline size_t msgpack_zbuffer_size(const 
msgpack_zbuffer* zbuf);
include/msgpack/zbuffer.h:static inline bool 
msgpack_zbuffer_reset(msgpack_zbuffer* zbuf);
include/msgpack/zbuffer.h:static inline void 
msgpack_zbuffer_reset_buffer(msgpack_zbuffer* zbuf);
include/msgpack/zbuffer.h:static inline char* 
msgpack_zbuffer_release_buffer(msgpack_zbuffer* zbuf);
include/msgpack/zbuffer.h:#ifndef MSGPACK_ZBUFFER_RESERVE_SIZE
include/msgpack/zbuffer.h:#define MSGPACK_ZBUFFER_RESERVE_SIZE 512
include/msgpack/zbuffer.h:static inline int msgpack_zbuffer_write(void* data, 
const char* buf, size_t len);
include/msgpack/zbuffer.h:static inline bool 
msgpack_zbuffer_expand(msgpack_zbuffer* zbuf);
include/msgpack/zbuffer.h:static inline bool 
msgpack_zbuffer_init(msgpack_zbuffer* zbuf,
include/msgpack/zbuffer.h:    memset(zbuf, 0, sizeof(msgpack_zbuffer));
include/msgpack/zbuffer.h:static inline void 
msgpack_zbuffer_destroy(msgpack_zbuffer* zbuf)
include/msgpack/zbuffer.h:static inline msgpack_zbuffer* 
msgpack_zbuffer_new(int level, size_t init_size)
include/msgpack/zbuffer.h:    msgpack_zbuffer* zbuf = 
(msgpack_zbuffer*)malloc(sizeof(msgpack_zbuffer));
include/msgpack/zbuffer.h:    if(!msgpack_zbuffer_init(zbuf, level, init_size)) 
{
include/msgpack/zbuffer.h:static inline void 
msgpack_zbuffer_free(msgpack_zbuffer* zbuf)
include/msgpack/zbuffer.h:    msgpack_zbuffer_destroy(zbuf);
include/msgpack/zbuffer.h:static inline bool 
msgpack_zbuffer_expand(msgpack_zbuffer* zbuf)
include/msgpack/zbuffer.h:static inline int msgpack_zbuffer_write(void* data, 
const char* buf, size_t len)
include/msgpack/zbuffer.h:    msgpack_zbuffer* zbuf = (msgpack_zbuffer*)data;
include/msgpack/zbuffer.h:        if(zbuf->stream.avail_out < 
MSGPACK_ZBUFFER_RESERVE_SIZE) {
include/msgpack/zbuffer.h:            if(!msgpack_zbuffer_expand(zbuf)) {
include/msgpack/zbuffer.h:static inline char* 
msgpack_zbuffer_flush(msgpack_zbuffer* zbuf)
include/msgpack/zbuffer.h:            if(!msgpack_zbuffer_expand(zbuf)) {
include/msgpack/zbuffer.h:static inline const char* msgpack_zbuffer_data(const 
msgpack_zbuffer* zbuf)
include/msgpack/zbuffer.h:static inline size_t msgpack_zbuffer_size(const 
msgpack_zbuffer* zbuf)
include/msgpack/zbuffer.h:static inline void 
msgpack_zbuffer_reset_buffer(msgpack_zbuffer* zbuf)
include/msgpack/zbuffer.h:static inline bool 
msgpack_zbuffer_reset(msgpack_zbuffer* zbuf)
include/msgpack/zbuffer.h:    msgpack_zbuffer_reset_buffer(zbuf);
include/msgpack/zbuffer.h:static inline char* 
msgpack_zbuffer_release_buffer(msgpack_zbuffer* zbuf)
include/msgpack/zbuffer.h:#endif /* msgpack/zbuffer.h */
CMakeLists.txt:    include/msgpack/zbuffer.h
CMakeLists.txt:        include/msgpack/zbuffer.hpp
test/buffer.cpp:#include <msgpack/zbuffer.hpp>
test/buffer.cpp:#include <msgpack/zbuffer.h>
test/buffer.cpp:TEST(buffer, zbuffer)
test/buffer.cpp:    msgpack::zbuffer zbuf;
test/buffer.cpp:TEST(buffer, zbuffer_c)
test/buffer.cpp:    msgpack_zbuffer zbuf;
test/buffer.cpp:    EXPECT_TRUE(msgpack_zbuffer_init(&zbuf, 1, 
MSGPACK_ZBUFFER_INIT_SIZE));
test/buffer.cpp:    EXPECT_EQ(0, msgpack_zbuffer_write(&zbuf, "a", 1));
test/buffer.cpp:    EXPECT_EQ(0, msgpack_zbuffer_write(&zbuf, "a", 1));
test/buffer.cpp:    EXPECT_EQ(0, msgpack_zbuffer_write(&zbuf, "a", 1));
test/buffer.cpp:    EXPECT_EQ(0, msgpack_zbuffer_write(&zbuf, "", 0));
test/buffer.cpp:    EXPECT_TRUE(msgpack_zbuffer_flush(&zbuf) != NULL);
test/buffer.cpp:    msgpack_zbuffer_destroy(&zbuf);


Judging from this, I am inclined to think that msgpack does not use
these two headers for anything but tests.  But I am not sure about
this.

> If you are at the stage where you are using Ring based on these
> packages, I wonder if it is not using the part of msgpack that uses
> zlib? Or, if it's finding zlib in the environment (if you are on a
> foreign distro)?

I am on GuixSD, but I do not have any packages besides opendht that
would depend on msgpack, yet.  I do not have a working definition for
Ring.  At this point, I do not see any way I could really test if
msgpack works properly with/without zlib.

FWIW, I looked at what other distributions do:

The Arch PKGBUILD does not refer to Zlib at all:
https://git.archlinux.org/svntogit/community.git/tree/trunk/PKGBUILD?h=packages/msgpack-c

Gentoo has Zlib for tests:
https://gitweb.gentoo.org/repo/gentoo.git/tree/dev-libs/msgpack/msgpack-1.1.0.ebuild

Debian seems to not refer to Zlib:
https://packages.debian.org/jessie/libmsgpackc2
https://packages.debian.org/jessie/libmsgpack3
https://packages.debian.org/jessie/libmsgpack-dev

This Nix expression does not mention zlib:
https://github.com/NixOS/nixpkgs/blob/f199be0fafcff0713dd4340b4373e575294aad58/pkgs/development/libraries/libmsgpack/generic.nix#L41


However, all of these seem to be different versions of msgpack.


On one hand, since these distributions seem to not include zlib as a
dependency, maybe zbuffer.h and zbuffer.hpp are not meant to be used or are
seldom used by other projects.  On the other hand msgpack installs these
headers, so they should be useful.

>
> Or have I misunderstood?

No, I missed that.  My mistake.  But I am not sure what to do about
this.

I am working on packaging pjproject now.  After that I think I will have
most of the dependencies of libring.  At that point I could probably
test if msgpack works the way it is now.  I am perfectly fine if we wait
with the inclusion of msgpack until then. What do you think?

Best,
Lukas