master 84e0749: EMBA container build improvements for Emacs build testing.

[Teodor Zlatanov]

branch: master
commit 84e0749b8b180bb94a5c32ebda11b5f22942dc22
Author: Ted Zlatanov <tzz@lifelogs.com>
Commit: Ted Zlatanov <tzz@lifelogs.com>

    EMBA container build improvements for Emacs build testing.
    
    * test/infra/gitlab-ci.yml: Moved from .gitlab-ci.yml. Use the
    EMBA container registry with a different login token storage file
    for each commit. Split test stages into prep, build, fast tests,
    normal tests, platform tests, and slow (everything) and use
    templates where possible.
    
    * .gitlab-ci.yml: Include test/infra/gitlab-ci.yml and move all
    content there.
---
 .gitlab-ci.yml                             | 142 +----------------------------
 test/infra/Dockerfile.emba                 |   2 +-
 .gitlab-ci.yml => test/infra/gitlab-ci.yml | 112 ++++++++++++++++-------
 3 files changed, 82 insertions(+), 174 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eb88476..3138f41 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2021 Free Software Foundation, Inc.
+# Copyright (C) 2021 Free Software Foundation, Inc.
 #
 #  This file is part of GNU Emacs.
 #
@@ -24,141 +24,5 @@
 # Maintainer: Ted Zlatanov <tzz@lifelogs.com>
 # URL: https://emba.gnu.org/emacs/emacs
 
-# Never run merge request pipelines, they usually duplicate push pipelines
-# see 
https://docs.gitlab.com/ee/ci/yaml/README.html#common-if-clauses-for-rules
-workflow:
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-      when: never
-    - when: always
-
-variables:
-  GIT_STRATEGY: fetch
-  EMACS_EMBA_CI: 1
-
-default:
-  image: docker:19.03.12
-  timeout: 3 hours
-  before_script:
-    - docker info
-
-.job-template:
-  # these will be cached across builds
-  cache:
-    key: ${CI_COMMIT_REF_SLUG}
-    paths: []
-    policy: pull-push
-  # these will be saved for followup builds
-  artifacts:
-    expire_in: 24 hrs
-    paths: []
-      # - "test/**/*.log"
-      # - "**/*.log"
-
-.test-template:
-  rules:
-    - changes:
-        - "**/Makefile.in"
-        - .gitlab-ci.yml
-        - aclocal.m4
-        - autogen.sh
-        - configure.ac
-        - lib/*.{h,c}
-        - lisp/**/*.el
-        - src/*.{h,c}
-        - test/infra/*
-        - test/lisp/**/*.el
-        - test/src/*.el
-    - changes:
-        # gfilemonitor, kqueue
-        - src/gfilenotify.c
-        - src/kqueue.c
-        # MS Windows
-        - "**/w32*"
-        # GNUstep
-        - lisp/term/ns-win.el
-        - src/ns*.{h,m}
-        - src/macfont.{h,m}
-      when: never
-
-  # using the variables for each job
-  script:
-    - docker build --target ${target} -t ${target}:${CI_COMMIT_REF_SLUG} -t 
${target}:${CI_COMMIT_SHA} -f test/infra/Dockerfile.emba .
-    # TODO: with make -j4 several of the tests were failing, for example 
shadowfile-tests, but passed without it
-    - docker run -i --rm -e EMACS_EMBA_CI=${EMACS_EMBA_CI} 
${target}:${CI_COMMIT_SHA} make ${make_params}
-
-stages:
-  - fast
-  - normal
-  - slow
-
-test-fast:
-  stage: fast
-  extends: [.job-template, .test-template]
-  variables:
-    target: emacs-inotify
-    make_params: "-C test check"
-
-test-lisp:
-  stage: normal
-  extends: [.job-template, .test-template]
-  variables:
-    target: emacs-inotify
-    make_params: "-C test check-lisp"
-
-test-net:
-  stage: normal
-  extends: [.job-template, .test-template]
-  variables:
-    target: emacs-inotify
-    make_params: "-C test check-net"
-
-test-filenotify-gio:
-  # This tests file monitor libraries gfilemonitor and gio.
-  stage: normal
-  extends: [.job-template, .test-template]
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-      changes:
-        - "**/Makefile.in"
-        - .gitlab-ci.yml
-        - lisp/autorevert.el
-        - lisp/filenotify.el
-        - lisp/net/tramp-sh.el
-        - src/gfilenotify.c
-        - test/infra/*
-        - test/lisp/autorevert-tests.el
-        - test/lisp/filenotify-tests.el
-  variables:
-    target: emacs-filenotify-gio
-    make_params: "-k -C test autorevert-tests filenotify-tests"
-
-test-gnustep:
-  # This tests the GNUstep build process
-  stage: normal
-  extends: [.job-template, .test-template]
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-      changes:
-        - "**/Makefile.in"
-        - .gitlab-ci.yml
-        - configure.ac
-        - src/ns*.{h,m}
-        - src/macfont.{h,m}
-        - lisp/term/ns-win.el
-        - nextstep/**/*
-        - test/infra/*
-  variables:
-    target: emacs-gnustep
-    make_params: install
-
-test-all:
-  # This tests also file monitor libraries inotify and inotifywatch.
-  stage: slow
-  extends: [.job-template, .test-template]
-  rules:
-    # note there's no "changes" section, so this always runs on a schedule
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-  variables:
-    target: emacs-inotify
-    make_params: check-expensive
+# Just load from test/infra, to keep build automation files there.
+include: '/test/infra/gitlab-ci.yml'
diff --git a/test/infra/Dockerfile.emba b/test/infra/Dockerfile.emba
index dd41982..421264d 100644
--- a/test/infra/Dockerfile.emba
+++ b/test/infra/Dockerfile.emba
@@ -41,7 +41,7 @@ COPY . /checkout
 WORKDIR /checkout
 RUN ./autogen.sh autoconf
 RUN ./configure --without-makeinfo
-RUN make bootstrap
+RUN make -j4 bootstrap
 RUN make -j4
 
 FROM emacs-base as emacs-filenotify-gio
diff --git a/.gitlab-ci.yml b/test/infra/gitlab-ci.yml
similarity index 69%
copy from .gitlab-ci.yml
copy to test/infra/gitlab-ci.yml
index eb88476..d893455 100644
--- a/.gitlab-ci.yml
+++ b/test/infra/gitlab-ci.yml
@@ -35,17 +35,24 @@ workflow:
 variables:
   GIT_STRATEGY: fetch
   EMACS_EMBA_CI: 1
+  # # Use TLS 
https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled
+  # DOCKER_HOST: tcp://docker:2376
+  # DOCKER_TLS_CERTDIR: "/certs"
+  # Put the configuration for each run in a separate directory to avoid 
conflicts
+  DOCKER_CONFIG: "/.docker-config-${CI_COMMIT_SHA}"
 
 default:
   image: docker:19.03.12
   timeout: 3 hours
   before_script:
     - docker info
+    - echo "docker registry is ${CI_REGISTRY}"
+    - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} 
${CI_REGISTRY}
 
 .job-template:
   # these will be cached across builds
   cache:
-    key: ${CI_COMMIT_REF_SLUG}
+    key: ${CI_COMMIT_SHA}
     paths: []
     policy: pull-push
   # these will be saved for followup builds
@@ -55,6 +62,38 @@ default:
       # - "test/**/*.log"
       # - "**/*.log"
 
+.build-template:
+  script:
+    - docker build --pull --target ${target} -t 
${CI_REGISTRY_IMAGE}:${target}-${CI_COMMIT_SHA} -f test/infra/Dockerfile.emba .
+    - docker push ${CI_REGISTRY_IMAGE}:${target}-${CI_COMMIT_SHA}
+
+.gnustep-template:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      changes:
+        - "**/Makefile.in"
+        - .gitlab-ci.yml
+        - configure.ac
+        - src/ns*.{h,m}
+        - src/macfont.{h,m}
+        - lisp/term/ns-win.el
+        - nextstep/**/*
+        - test/infra/*
+
+.filenotify-gio-template:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      changes:
+        - "**/Makefile.in"
+        - .gitlab-ci.yml
+        - lisp/autorevert.el
+        - lisp/filenotify.el
+        - lisp/net/tramp-sh.el
+        - src/gfilenotify.c
+        - test/infra/*
+        - test/lisp/autorevert-tests.el
+        - test/lisp/filenotify-tests.el
+
 .test-template:
   rules:
     - changes:
@@ -83,30 +122,58 @@ default:
 
   # using the variables for each job
   script:
-    - docker build --target ${target} -t ${target}:${CI_COMMIT_REF_SLUG} -t 
${target}:${CI_COMMIT_SHA} -f test/infra/Dockerfile.emba .
+    - docker pull ${CI_REGISTRY_IMAGE}:${target}-${CI_COMMIT_SHA}
     # TODO: with make -j4 several of the tests were failing, for example 
shadowfile-tests, but passed without it
-    - docker run -i --rm -e EMACS_EMBA_CI=${EMACS_EMBA_CI} 
${target}:${CI_COMMIT_SHA} make ${make_params}
+    - docker run -i --rm -e EMACS_EMBA_CI=${EMACS_EMBA_CI} 
${CI_REGISTRY_IMAGE}:${target}-${CI_COMMIT_SHA} make ${make_params}
 
 stages:
+  - prep-images
+  - build-images
   - fast
   - normal
+  - platform-images
+  - platforms
   - slow
 
-test-fast:
+prep-image-base:
+  stage: prep-images
+  extends: [.job-template, .build-template]
+  variables:
+    target: emacs-base
+
+build-image-inotify:
+  stage: build-images
+  extends: [.job-template, .build-template]
+  variables:
+    target: emacs-inotify
+
+test-fast-inotify:
   stage: fast
   extends: [.job-template, .test-template]
   variables:
     target: emacs-inotify
     make_params: "-C test check"
 
-test-lisp:
+build-image-filenotify-gio:
+  stage: platform-images
+  extends: [.job-template, .build-template, .filenotify-gio-template]
+  variables:
+    target: emacs-filenotify-gio
+
+build-image-gnustep:
+  stage: platform-images
+  extends: [.job-template, .build-template, .gnustep-template]
+  variables:
+    target: emacs-gnustep
+
+test-lisp-inotify:
   stage: normal
   extends: [.job-template, .test-template]
   variables:
     target: emacs-inotify
     make_params: "-C test check-lisp"
 
-test-net:
+test-net-inotify:
   stage: normal
   extends: [.job-template, .test-template]
   variables:
@@ -115,44 +182,21 @@ test-net:
 
 test-filenotify-gio:
   # This tests file monitor libraries gfilemonitor and gio.
-  stage: normal
-  extends: [.job-template, .test-template]
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-      changes:
-        - "**/Makefile.in"
-        - .gitlab-ci.yml
-        - lisp/autorevert.el
-        - lisp/filenotify.el
-        - lisp/net/tramp-sh.el
-        - src/gfilenotify.c
-        - test/infra/*
-        - test/lisp/autorevert-tests.el
-        - test/lisp/filenotify-tests.el
+  stage: platforms
+  extends: [.job-template, .test-template, .filenotify-gio-template]
   variables:
     target: emacs-filenotify-gio
     make_params: "-k -C test autorevert-tests filenotify-tests"
 
 test-gnustep:
   # This tests the GNUstep build process
-  stage: normal
-  extends: [.job-template, .test-template]
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-      changes:
-        - "**/Makefile.in"
-        - .gitlab-ci.yml
-        - configure.ac
-        - src/ns*.{h,m}
-        - src/macfont.{h,m}
-        - lisp/term/ns-win.el
-        - nextstep/**/*
-        - test/infra/*
+  stage: platforms
+  extends: [.job-template, .test-template, .gnustep-template]
   variables:
     target: emacs-gnustep
     make_params: install
 
-test-all:
+test-all-inotify:
   # This tests also file monitor libraries inotify and inotifywatch.
   stage: slow
   extends: [.job-template, .test-template]