02/02: daemon: Add "builtin:download" derivation builder.

[Ludovic Courtès]

civodul pushed a commit to branch wip-oob-download
in repository guix.

commit 9decba80efe66af52f6f60dd2d0f9cd2fa02a6d1
Author: Eelco Dolstra <address@hidden>
Date:   Mon Jul 20 04:30:16 2015 +0200

    daemon: Add "builtin:download" derivation builder.
    
    This ensures that 1) the derivation doesn't change when Guix changes;
    2) the derivation closure doesn't contain Guix and its dependencies; 3)
    we don't have to rely on ugly chroot hacks.
    
    Adapted from Nix commit 0a2bee307b20411f5b0dda0c662b1f9bb9e0e131.
    
    * nix/libstore/build.cc (isBuiltin): New function.
    (DerivationGoal::runChild): Add special case for 'isBuiltin(drv)'.
    * nix/libstore/builtins.cc, nix/libstore/builtins.hh: New files.
    * nix/local.mk (libstore_a_SOURCES): Add builtins.cc.
    (libstore_headers): Add builtins.hh.
    * tests/derivations.scm ("unknown built-in builder")
    ("'download' built-in builder")
    ("'download' built-in builder, invalid hash")
    ("'download' built-in builder, not fixed-output"): New test.
    
    Co-authored-by: Ludovic Courtès <address@hidden>
---
 nix/libstore/build.cc    |   33 +++++++++++++++++++++++++++------
 nix/libstore/builtins.cc |   39 +++++++++++++++++++++++++++++++++++++++
 nix/libstore/builtins.hh |    9 +++++++++
 nix/local.mk             |    2 ++
 tests/derivations.scm    |   44 ++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 121 insertions(+), 6 deletions(-)

diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc
index ae78e65..3402ddc 100644
--- a/nix/libstore/build.cc
+++ b/nix/libstore/build.cc
@@ -8,6 +8,7 @@
 #include "util.hh"
 #include "archive.hh"
 #include "affinity.hh"
+#include "builtins.hh"
 
 #include <map>
 #include <sstream>
@@ -1256,6 +1257,12 @@ bool substitutesAllowed(const Derivation & drv)
 }
 
 
+static bool isBuiltin(const Derivation & drv)
+{
+    return string(drv.builder, 0, 8) == "builtin:";
+}
+
+
 void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
@@ -2255,6 +2262,26 @@ void DerivationGoal::runChild()
                 throw SysError("setuid failed");
         }
 
+        restoreSIGPIPE();
+
+        /* Indicate that we managed to set up the build environment. */
+        writeFull(STDERR_FILENO, "\n");
+
+        /* Execute the program.  This should not return. */
+        if (isBuiltin(drv)) {
+            try {
+                logType = ltFlat;
+                if (drv.builder == "builtin:download")
+                    builtinDownload(drv);
+                else
+                    throw Error(format("unsupported builtin function '%1%'") % 
string(drv.builder, 8));
+                _exit(0);
+            } catch (std::exception & e) {
+                writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n");
+                _exit(1);
+            }
+        }
+
         /* Fill in the arguments. */
         Strings args;
         string builderBasename = baseNameOf(drv.builder);
@@ -2262,12 +2289,6 @@ void DerivationGoal::runChild()
         foreach (Strings::iterator, i, drv.args)
             args.push_back(rewriteHashes(*i, rewritesToTmp));
 
-        restoreSIGPIPE();
-
-        /* Indicate that we managed to set up the build environment. */
-        writeFull(STDERR_FILENO, "\n");
-
-        /* Execute the program.  This should not return. */
         execve(drv.builder.c_str(), stringsToCharPtrs(args).data(), 
stringsToCharPtrs(envStrs).data());
 
         throw SysError(format("executing `%1%'") % drv.builder);
diff --git a/nix/libstore/builtins.cc b/nix/libstore/builtins.cc
new file mode 100644
index 0000000..84f09e8
--- /dev/null
+++ b/nix/libstore/builtins.cc
@@ -0,0 +1,39 @@
+#include "builtins.hh"
+#include "util.hh"
+
+namespace nix {
+
+void builtinDownload(const Derivation & drv)
+{
+    // Make sure we get a fixed-output derivation.  If we did not check that,
+    // then the result of our download would not be checked.
+    if (!isFixedOutputDrv(drv)) {
+       throw Error("'download' built-in function passed a derivation \
+that is not fixed-output");
+    }
+
+    auto getAttr = [&](const string & name) {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % 
name);
+        return i->second;
+    };
+
+    auto url = getAttr("url");
+    auto out = getAttr("out");
+
+    // Invoke 'guix download'.
+    Strings args;
+    args.push_back("download");
+    args.push_back("-o");
+    args.push_back(out);
+    args.push_back(url);                         // TODO: Pass a mirror file.
+    runProgram("guix", true, args);
+
+    auto executable = drv.env.find("executable");
+    if (executable != drv.env.end() && executable->second == "1") {
+        if (chmod(out.c_str(), 0755) == -1)
+            throw SysError(format("making '%1%' executable") % out);
+    }
+}
+
+}
diff --git a/nix/libstore/builtins.hh b/nix/libstore/builtins.hh
new file mode 100644
index 0000000..fefc6d3
--- /dev/null
+++ b/nix/libstore/builtins.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "derivations.hh"
+
+namespace nix {
+
+void builtinDownload(const Derivation & drv);
+
+}
diff --git a/nix/local.mk b/nix/local.mk
index c666edd..c08e47b 100644
--- a/nix/local.mk
+++ b/nix/local.mk
@@ -87,6 +87,7 @@ libstore_a_SOURCES =                          \
   %D%/libstore/build.cc                                \
   %D%/libstore/pathlocks.cc                    \
   %D%/libstore/derivations.cc                  \
+  %D%/libstore/builtins.cc                     \
   %D%/libstore/sqlite.cc
 
 libstore_headers =                             \
@@ -98,6 +99,7 @@ libstore_headers =                            \
   %D%/libstore/misc.hh                         \
   %D%/libstore/local-store.hh                  \
   %D%/libstore/sqlite.hh                       \
+  %D%/libstore/builtins.hh                     \
   %D%/libstore/store-api.hh
 
 libstore_a_CPPFLAGS =                          \
diff --git a/tests/derivations.scm b/tests/derivations.scm
index d8553b2..d13014f 100644
--- a/tests/derivations.scm
+++ b/tests/derivations.scm
@@ -205,6 +205,50 @@
                 (= (stat:ino (lstat file1))
                    (stat:ino (lstat file2))))))))
 
+(test-assert "unknown built-in builder"
+  (let ((drv (derivation %store "ohoh" "builtin:does-not-exist" '())))
+    (guard (c ((nix-protocol-error? c)
+               (string-contains (nix-protocol-error-message c) "failed")))
+      (build-derivations %store (list drv))
+      #f)))
+
+(test-assert "'download' built-in builder"
+  (let* ((text   (random-text))
+         (source (add-text-to-store %store "hello" text))
+         (url    (string-append "file://" source))
+         (drv    (derivation %store "world"
+                             "builtin:download" '()
+                             #:env-vars `(("url" . ,url))
+                             #:hash-algo 'sha256
+                             #:hash (sha256 (string->utf8 text)))))
+    (and (build-derivations %store (list drv))
+         (= (stat:ino (stat (derivation->output-path drv)))
+            (stat:ino (stat source))))))
+
+(test-assert "'download' built-in builder, invalid hash"
+  (let* ((source (add-text-to-store %store "hello" "hi!"))
+         (url    (string-append "file://" source))
+         (drv    (derivation %store "world"
+                             "builtin:download" '()
+                             #:env-vars `(("url" . ,url))
+                             #:hash-algo 'sha256
+                             #:hash (sha256 #vu8()))))
+    (guard (c ((nix-protocol-error? c)
+               (string-contains (nix-protocol-error-message c) "failed")))
+      (build-derivations %store (list drv))
+      #f)))
+
+(test-assert "'download' built-in builder, not fixed-output"
+  (let* ((source (add-text-to-store %store "hello" "hi!"))
+         (url    (string-append "file://" source))
+         (drv    (derivation %store "world"
+                             "builtin:download" '()
+                             #:env-vars `(("url" . ,url)))))
+    (guard (c ((nix-protocol-error? c)
+               (string-contains (nix-protocol-error-message c) "failed")))
+      (build-derivations %store (list drv))
+      #f)))
+
 (test-equal "derivation-name"
   "foo-0.0"
   (let ((drv (derivation %store "foo-0.0" %bash '())))