[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[bug#29995] [PATCH 3/5] daemon: Add gzip log compression.
|
From: |
Ludovic Courtès |
|
Subject: |
[bug#29995] [PATCH 3/5] daemon: Add gzip log compression. |
|
Date: |
Fri, 5 Jan 2018 18:02:56 +0100 |
* nix/nix-daemon/guix-daemon.cc (GUIX_OPT_LOG_COMPRESSION): New macro.
(options): Mark "disable-log-compression" as hidden and add
"log-compression".
(parse_opt): Handle GUIX_OPT_LOG_COMPRESSION.
* nix/libstore/build.cc (DerivationGoal): Add 'gzLogFile'.
(openLogFile): Initialize it when 'logCompression' is COMPRESSION_GZIP.
(closeLogFile, handleChildOutput): Honor 'gzLogFile'.
* nix/libstore/globals.hh (Settings)[compressLog]: Remove.
[logCompression]: New field.
(CompressionType): New enum.
* nix/libstore/globals.cc (Settings::Settings): Initialize it.
(update): Remove '_get' call for 'compressLog'.
* nix/local.mk (guix_daemon_LDADD, guix_register_LDADD): Add -lz.
* guix/store.scm (log-file): Handle '.gz' log files.
* tests/guix-daemon.sh: Add test with '--log-compression=gzip'.
* doc/guix.texi (Invoking guix-daemon): Adjust accordingly.
* config-daemon.ac: Check for libz and zlib.h.
---
config-daemon.ac | 6 ++++++
doc/guix.texi | 9 +++++----
guix/store.scm | 6 ++++--
nix/libstore/build.cc | 45 +++++++++++++++++++++++++++++++++++++++----
nix/libstore/globals.cc | 4 ++--
nix/libstore/globals.hh | 8 +++++++-
nix/local.mk | 6 +++---
nix/nix-daemon/guix-daemon.cc | 25 +++++++++++++++++++++---
tests/guix-daemon.sh | 38 +++++++++++++++++++++++++++++++++++-
9 files changed, 127 insertions(+), 20 deletions(-)
diff --git a/config-daemon.ac b/config-daemon.ac
index 42b59819d..59f6f2713 100644
--- a/config-daemon.ac
+++ b/config-daemon.ac
@@ -18,6 +18,12 @@ if test "x$guix_build_daemon" = "xyes"; then
dnl Use 64-bit file system calls so that we can support files > 2 GiB.
AC_SYS_LARGEFILE
+ dnl Look for zlib, a required dependency.
+ AC_CHECK_LIB([z], [gzdopen], [true],
+ [AC_MSG_ERROR([Guix requires zlib. See http://www.zlib.net/.])])
+ AC_CHECK_HEADERS([zlib.h], [true],
+ [AC_MSG_ERROR([Guix requires zlib. See http://www.zlib.net/.])])
+
dnl Look for libbz2, a required dependency.
AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
[AC_MSG_ERROR([Guix requires libbz2, which is part of bzip2. See
http://www.bzip.org/.])])
diff --git a/doc/guix.texi b/doc/guix.texi
index f6ed3ef87..bd9bcb73a 100644
--- a/doc/guix.texi
+++ b/doc/guix.texi
@@ -13,7 +13,7 @@
@set OPENPGP-SIGNING-KEY-ID 3CE464558A84FDC69DB40CFB090B11993D9AEBB5
@copying
-Copyright @copyright{} 2012, 2013, 2014, 2015, 2016, 2017 Ludovic
address@hidden
+Copyright @copyright{} 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic
address@hidden
Copyright @copyright{} 2013, 2014, 2016 Andreas address@hidden
Copyright @copyright{} 2013 Nikita address@hidden
Copyright @copyright{} 2014, 2015, 2016 Alex address@hidden
@@ -1234,12 +1234,13 @@ processes to gain access to undeclared dependencies.
It is necessary,
though, when @command{guix-daemon} is running under an unprivileged user
account.
address@hidden --disable-log-compression
-Disable compression of the build logs.
address@hidden address@hidden
+Compress build logs according to @var{type}, one of @code{gzip},
address@hidden, or @code{none}.
Unless @code{--lose-logs} is used, all the build logs are kept in the
@var{localstatedir}. To save space, the daemon automatically compresses
-them with bzip2 by default. This option disables that.
+them with bzip2 by default.
@item --disable-deduplication
@cindex deduplication
diff --git a/guix/store.scm b/guix/store.scm
index e6e45ba89..89db46b8e 100644
--- a/guix/store.scm
+++ b/guix/store.scm
@@ -1,5 +1,5 @@
;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès
<address@hidden>
+;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès
<address@hidden>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -1567,8 +1567,10 @@ must be an absolute store file name, or a derivation
file name."
"/log/guix/drvs/"
(string-take base 2) "/"
(string-drop base 2)))
+ (log.gz (string-append log ".gz"))
(log.bz2 (string-append log ".bz2")))
- (cond ((file-exists? log.bz2) log.bz2)
+ (cond ((file-exists? log.gz) log.gz)
+ ((file-exists? log.bz2) log.bz2)
((file-exists? log) log)
(else #f))))
(else
diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc
index d68e8b2bc..5bf3e3aac 100644
--- a/nix/libstore/build.cc
+++ b/nix/libstore/build.cc
@@ -31,6 +31,7 @@
#include <pwd.h>
#include <grp.h>
+#include <zlib.h>
#include <bzlib.h>
/* Includes required for chroot support. */
@@ -744,6 +745,7 @@ private:
/* File descriptor for the log file. */
FILE * fLogFile;
+ gzFile gzLogFile;
BZFILE * bzLogFile;
AutoCloseFD fdLogFile;
@@ -892,6 +894,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const
StringSet & wantedOut
, needRestart(false)
, retrySubstitution(false)
, fLogFile(0)
+ , gzLogFile(0)
, bzLogFile(0)
, useChroot(false)
, buildMode(buildMode)
@@ -2599,8 +2602,25 @@ Path DerivationGoal::openLogFile()
Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir %
string(baseName, 0, 2)).str();
createDirs(dir);
- if (settings.compressLog) {
+ switch (settings.logCompression)
+ {
+ case COMPRESSION_GZIP: {
+ Path logFileName = (format("%1%/%2%.gz") % dir % string(baseName,
2)).str();
+ AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY |
O_TRUNC, 0666);
+ if (fd == -1) throw SysError(format("creating log file `%1%'") %
logFileName);
+ closeOnExec(fd);
+ /* Note: FD will be closed by 'gzclose'. */
+ if (!(gzLogFile = gzdopen(fd.borrow(), "w")))
+ throw Error(format("cannot open compressed log file `%1%'") %
logFileName);
+
+ gzbuffer(gzLogFile, 32768);
+ gzsetparams(gzLogFile, Z_BEST_COMPRESSION, Z_DEFAULT_STRATEGY);
+
+ return logFileName;
+ }
+
+ case COMPRESSION_BZIP2: {
Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName,
2)).str();
AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY |
O_TRUNC, 0666);
if (fd == -1) throw SysError(format("creating log file `%1%'") %
logFileName);
@@ -2614,20 +2634,30 @@ Path DerivationGoal::openLogFile()
throw Error(format("cannot open compressed log file `%1%'") %
logFileName);
return logFileName;
+ }
- } else {
+ case COMPRESSION_NONE: {
Path logFileName = (format("%1%/%2%") % dir % string(baseName,
2)).str();
fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC,
0666);
if (fdLogFile == -1) throw SysError(format("creating log file `%1%'")
% logFileName);
closeOnExec(fdLogFile);
return logFileName;
+ }
}
+
+ abort();
}
void DerivationGoal::closeLogFile()
{
- if (bzLogFile) {
+ if (gzLogFile) {
+ int err;
+ err = gzclose(gzLogFile);
+ gzLogFile = NULL;
+ if (err != Z_OK) throw Error(format("cannot close compressed log file
(gzip error = %1%)") % err);
+ }
+ else if (bzLogFile) {
int err;
BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0);
bzLogFile = 0;
@@ -2695,7 +2725,14 @@ void DerivationGoal::handleChildOutput(int fd, const
string & data)
}
if (verbosity >= settings.buildVerbosity)
writeToStderr(data);
- if (bzLogFile) {
+
+ if (gzLogFile) {
+ if (data.size() > 0) {
+ int count, err;
+ count = gzwrite(gzLogFile, data.data(), data.size());
+ if (count == 0) throw Error(format("cannot write to compressed
log file (gzip error = %1%)") % gzerror(gzLogFile, &err));
+ }
+ } else if (bzLogFile) {
int err;
BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(),
data.size());
if (err != BZ_OK) throw Error(format("cannot write to compressed
log file (BZip2 error = %1%)") % err);
diff --git a/nix/libstore/globals.cc b/nix/libstore/globals.cc
index 65dad24d9..82d528dc9 100644
--- a/nix/libstore/globals.cc
+++ b/nix/libstore/globals.cc
@@ -45,7 +45,7 @@ Settings::Settings()
useSshSubstituter = false;
impersonateLinux26 = false;
keepLog = true;
- compressLog = true;
+ logCompression = COMPRESSION_BZIP2;
maxLogSize = 0;
cacheFailure = false;
pollInterval = 5;
@@ -162,7 +162,7 @@ void Settings::update()
_get(useChroot, "build-use-chroot");
_get(impersonateLinux26, "build-impersonate-linux-26");
_get(keepLog, "build-keep-log");
- _get(compressLog, "build-compress-log");
+ // _get(logCompression, "build-log-compression");
_get(maxLogSize, "build-max-log-size");
_get(cacheFailure, "build-cache-failure");
_get(pollInterval, "build-poll-interval");
diff --git a/nix/libstore/globals.hh b/nix/libstore/globals.hh
index 7beb1a55c..81cf2f52d 100644
--- a/nix/libstore/globals.hh
+++ b/nix/libstore/globals.hh
@@ -8,6 +8,12 @@
namespace nix {
+enum CompressionType
+{
+ COMPRESSION_NONE = 0,
+ COMPRESSION_GZIP = 1,
+ COMPRESSION_BZIP2 = 2
+};
struct Settings {
@@ -169,7 +175,7 @@ struct Settings {
bool keepLog;
/* Whether to compress logs. */
- bool compressLog;
+ enum CompressionType logCompression;
/* Maximum number of bytes a builder can write to stdout/stderr
before being killed (0 means no limit). */
diff --git a/nix/local.mk b/nix/local.mk
index 9e0c457be..d802da617 100644
--- a/nix/local.mk
+++ b/nix/local.mk
@@ -1,5 +1,5 @@
# GNU Guix --- Functional package management for GNU
-# Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <address@hidden>
+# Copyright © 2012, 2013, 2014, 2015, 2016, 2018 Ludovic Courtès
<address@hidden>
# Copyright © 2016 Mathieu Lirzin <address@hidden>
#
# This file is part of GNU Guix.
@@ -132,7 +132,7 @@ guix_daemon_CPPFLAGS = \
-I$(top_srcdir)/%D%/libstore
guix_daemon_LDADD = \
- libstore.a libutil.a libformat.a -lbz2 \
+ libstore.a libutil.a libformat.a -lz -lbz2 \
$(SQLITE3_LIBS) $(LIBGCRYPT_LIBS)
guix_daemon_headers = \
@@ -149,7 +149,7 @@ guix_register_CPPFLAGS = \
# XXX: Should we start using shared libs?
guix_register_LDADD = \
- libstore.a libutil.a libformat.a -lbz2 \
+ libstore.a libutil.a libformat.a -lz -lbz2 \
$(SQLITE3_LIBS) $(LIBGCRYPT_LIBS)
diff --git a/nix/nix-daemon/guix-daemon.cc b/nix/nix-daemon/guix-daemon.cc
index 796335820..a1ef90dfd 100644
--- a/nix/nix-daemon/guix-daemon.cc
+++ b/nix/nix-daemon/guix-daemon.cc
@@ -1,5 +1,5 @@
/* GNU Guix --- Functional package management for GNU
- Copyright (C) 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès
<address@hidden>
+ Copyright (C) 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès
<address@hidden>
Copyright (C) 2006, 2010, 2012, 2014 Eelco Dolstra <address@hidden>
This file is part of GNU Guix.
@@ -88,6 +88,7 @@ builds derivations on behalf of its clients.");
#define GUIX_OPT_BUILD_ROUNDS 17
#define GUIX_OPT_TIMEOUT 18
#define GUIX_OPT_MAX_SILENT_TIME 19
+#define GUIX_OPT_LOG_COMPRESSION 20
static const struct argp_option options[] =
{
@@ -120,8 +121,11 @@ static const struct argp_option options[] =
n_("build each derivation N times in a row") },
{ "lose-logs", GUIX_OPT_LOSE_LOGS, 0, 0,
n_("do not keep build logs") },
- { "disable-log-compression", GUIX_OPT_DISABLE_LOG_COMPRESSION, 0, 0,
+ { "disable-log-compression", GUIX_OPT_DISABLE_LOG_COMPRESSION, 0,
+ OPTION_HIDDEN, // deprecated
n_("disable compression of the build logs") },
+ { "log-compression", GUIX_OPT_LOG_COMPRESSION, "TYPE", 0,
+ n_("use the specified compression type for build logs") },
/* '--disable-deduplication' was known as '--disable-store-optimization'
up to Guix 0.7 included, so keep the alias around. */
@@ -197,8 +201,21 @@ parse_opt (int key, char *arg, struct argp_state *state)
settings.set("build-extra-chroot-dirs", chroot_dirs);
break;
}
+ case GUIX_OPT_LOG_COMPRESSION:
+ if (strcmp (arg, "none") == 0)
+ settings.logCompression = COMPRESSION_NONE;
+ else if (strcmp (arg, "gzip") == 0)
+ settings.logCompression = COMPRESSION_GZIP;
+ else if (strcmp (arg, "bzip2") == 0)
+ settings.logCompression = COMPRESSION_BZIP2;
+ else
+ {
+ fprintf (stderr, _("error: %s: unknown compression type\n"), arg);
+ exit (EXIT_FAILURE);
+ }
+ break;
case GUIX_OPT_DISABLE_LOG_COMPRESSION:
- settings.compressLog = false;
+ settings.logCompression = COMPRESSION_NONE;
break;
case GUIX_OPT_BUILD_USERS_GROUP:
settings.buildUsersGroup = arg;
@@ -487,6 +504,8 @@ main (int argc, char *argv[])
/* Effect all the changes made via 'settings.set'. */
settings.update ();
+ printMsg(lvlDebug,
+ format ("build log compression: %1%") % settings.logCompression);
if (settings.useSubstitutes)
{
diff --git a/tests/guix-daemon.sh b/tests/guix-daemon.sh
index 7212e3eb6..6f91eb58b 100644
--- a/tests/guix-daemon.sh
+++ b/tests/guix-daemon.sh
@@ -1,5 +1,5 @@
# GNU Guix --- Functional package management for GNU
-# Copyright © 2012, 2014, 2015, 2016, 2017 Ludovic Courtès <address@hidden>
+# Copyright © 2012, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès
<address@hidden>
#
# This file is part of GNU Guix.
#
@@ -193,3 +193,39 @@ do
GUIX_DAEMON_SOCKET="$socket" guile -c "$client_code"
kill "$daemon_pid"
done
+
+# Log compression.
+
+guix-daemon --listen="$socket" --disable-chroot --debug --log-compression=gzip
&
+daemon_pid=$!
+
+stamp="compressed-build-log-test-$$-`date +%H%M%S`"
+client_code="
+ (use-modules (guix) (gnu packages bootstrap))
+
+ (with-store store
+ (run-with-store store
+ (mlet %store-monad ((drv (lower-object
+ (computed-file \"compressed-log-test\"
+ #~(begin
+ (display \"$stamp\")
+ (newline)
+ (mkdir #\$output))
+ #:guile %bootstrap-guile))))
+ (display (derivation-file-name drv))
+ (newline)
+ (return #t))))
+"
+
+GUIX_DAEMON_SOCKET="$socket"
+export GUIX_DAEMON_SOCKET
+
+drv=`guile -c "$client_code"`
+guix build "$drv"
+
+log=`guix build "$drv" --log-file`
+test -f "$log"
+case "$log" in
+ *.gz) test "`gunzip -c < "$log"`" = "$stamp" ;;
+ *) false ;;
+esac
--
2.15.1
- [bug#29995] [PATCH 0/5] Publish build logs, compress as gzip, Ludovic Courtès, 2018/01/05
- [bug#29995] [PATCH 1/5] publish: Use 'x-raw-file' internal response header., Ludovic Courtès, 2018/01/05
- [bug#29995] [PATCH 0/5] Publish build logs, compress as gzip, Ludovic Courtès, 2018/01/05
- bug#29995: [PATCH 0/5] Publish build logs, compress as gzip, Ludovic Courtès, 2018/01/07
- [bug#29995] [PATCH 0/5] Publish build logs, compress as gzip, Ludovic Courtès, 2018/01/07