ci: clean up .travis.yaml

Clean up Travis CI config, extract multi-step initializations into scripts.
Also, move kernel-building tests to happen last to not block lightweight
Debian and Ubuntu tests.

Signed-off-by: Andrii Nakryiko <andriin@fb.com>

.travis.yml

@@ -8,16 +8,9 @@ env:
     global:
         - PROJECT_NAME='libbpf'
         - AUTHOR_EMAIL="$(git log -1 --pretty=\"%aE\")"
-        - CI_MANAGERS="$TRAVIS_BUILD_DIR/travis-ci/managers"
-        - VMTEST_ROOT="$TRAVIS_BUILD_DIR/travis-ci/vmtest"
         - REPO_ROOT="$TRAVIS_BUILD_DIR"
-        - VMTEST_SETUPCMD="PROJECT_NAME=${PROJECT_NAME} ./${PROJECT_NAME}/travis-ci/vmtest/run_selftests.sh"
+        - CI_ROOT="$REPO_ROOT/travis-ci"
-    jobs:
+        - VMTEST_ROOT="$CI_ROOT/vmtest"
-        # Setup command override.
-        # 5.5.0-rc6 is built from bpf-next; TODO(hex@): remove when pahole v1.16 is available
-        - KERNEL=5.5.0-rc6
-        - KERNEL=5.5.0
-        - KERNEL=LATEST
 
 addons:
     apt:
@@ -28,25 +21,6 @@ addons:
             - elfutils
             - libcap-dev
             - libelf-dev
-install: sudo adduser "${USER}" kvm
-before_script:
-    - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-    - echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" | sudo tee -a /etc/apt/sources.list
-    - echo "deb http://archive.ubuntu.com/ubuntu eoan main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
-    - sudo apt-get -qq update
-    - sudo apt-get -y install dwarves=1.15-1
-    - sudo apt-get -qq -y install clang-10 lld-10 llvm-10
-    - KERNEL="${KERNEL}" ${VMTEST_ROOT}/prepare_selftests.sh travis-ci/vmtest/bpf-next
-    # Escape whitespace characters.
-    - setup_cmd=$(sed 's/\([[:space:]]\)/\\\1/g' <<< "${VMTEST_SETUPCMD}")
-    - if [[ "${KERNEL}" = 'LATEST' ]]; then
-        sudo -E sudo -E -u "${USER}" "${VMTEST_ROOT}/run.sh" -b travis-ci/vmtest/bpf-next -o -d ~ -s "${setup_cmd}" ~/root.img;
-      else
-        sudo -E sudo -E -u "${USER}" "${VMTEST_ROOT}/run.sh" -k "${KERNEL}*" -o -d ~ -s "${setup_cmd}" ~/root.img;
-      fi; exitstatus=$?
-    - test $exitstatus -le 1
-script:
-    - test $exitstatus -eq 0
 
 stages:
     # Run Coverity periodically instead of for each PR for following reasons:
@@ -60,130 +34,79 @@ stages:
 
 jobs:
     include:
-        - stage: Build & test
+        - stage: Build
-          name: Debian Testing
+          name: Debian Build
           language: bash
-          env:
+          install:        $CI_ROOT/managers/debian.sh SETUP
-              - DEBIAN_RELEASE="testing"
+          script:         $CI_ROOT/managers/debian.sh RUN || travis_terminate
-              - CONT_NAME="libbpf-debian-$DEBIAN_RELEASE"
+          after_script:   $CI_ROOT/managers/debian.sh CLEANUP
-          before_install:
-              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-              - docker --version
-          install:
-              - $CI_MANAGERS/debian.sh SETUP
-          # Override before_script: so VMTEST before_install commands are not executed.
-          before_script: true
-          script:
-              - $CI_MANAGERS/debian.sh RUN || travis_terminate
-          after_script:
-              - $CI_MANAGERS/debian.sh CLEANUP
 
-        - name: Debian Testing (ASan+UBSan)
+        - name: Debian Build (ASan+UBSan)
           language: bash
-          env:
+          install:        $CI_ROOT/managers/debian.sh SETUP
-              - DEBIAN_RELEASE="testing"
+          script:         $CI_ROOT/managers/debian.sh RUN_ASAN || travis_terminate
-              - CONT_NAME="libbpf-debian-$DEBIAN_RELEASE"
+          after_script:   $CI_ROOT/managers/debian.sh CLEANUP
-          before_install:
-              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-              - docker --version
-          install:
-              - $CI_MANAGERS/debian.sh SETUP
-          before_script: true
-          script:
-              - $CI_MANAGERS/debian.sh RUN_ASAN || travis_terminate
-          after_script:
-              - $CI_MANAGERS/debian.sh CLEANUP
 
-        - name: Debian Testing (clang)
+        - name: Debian Build (clang)
           language: bash
-          env:
+          install:        $CI_ROOT/managers/debian.sh SETUP
-              - DEBIAN_RELEASE="testing"
+          script:         $CI_ROOT/managers/debian.sh RUN_CLANG || travis_terminate
-              - CONT_NAME="libbpf-debian-$DEBIAN_RELEASE"
+          after_script:   $CI_ROOT/managers/debian.sh CLEANUP
-          before_install:
-              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-              - docker --version
-          install:
-              - $CI_MANAGERS/debian.sh SETUP
-          before_script: true
-          script:
-              - $CI_MANAGERS/debian.sh RUN_CLANG || travis_terminate
-          after_script:
-              - $CI_MANAGERS/debian.sh CLEANUP
 
-        - name: Debian Testing (clang ASan+UBSan)
+        - name: Debian Build (clang ASan+UBSan)
           language: bash
-          env:
+          install:        $CI_ROOT/managers/debian.sh SETUP
-              - DEBIAN_RELEASE="testing"
+          script:         $CI_ROOT/managers/debian.sh RUN_CLANG_ASAN || travis_terminate
-              - CONT_NAME="libbpf-debian-$DEBIAN_RELEASE"
+          after_script:   $CI_ROOT/managers/debian.sh CLEANUP
-          before_install:
-              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-              - docker --version
-          install:
-              - $CI_MANAGERS/debian.sh SETUP
-          before_script: true
-          script:
-              - $CI_MANAGERS/debian.sh RUN_CLANG_ASAN || travis_terminate
-          after_script:
-              - $CI_MANAGERS/debian.sh CLEANUP
 
-        - name: Debian Testing (gcc-8)
+        - name: Debian Build (gcc-8)
           language: bash
-          env:
+          install:        $CI_ROOT/managers/debian.sh SETUP
-              - DEBIAN_RELEASE="testing"
+          script:         $CI_ROOT/managers/debian.sh RUN_GCC8 || travis_terminate
-              - CONT_NAME="libbpf-debian-$DEBIAN_RELEASE"
+          after_script:   $CI_ROOT/managers/debian.sh CLEANUP
-          before_install:
-              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-              - docker --version
-          install:
-              - $CI_MANAGERS/debian.sh SETUP
-          before_script: true
-          script:
-              - $CI_MANAGERS/debian.sh RUN_GCC8 || travis_terminate
-          after_script:
-              - $CI_MANAGERS/debian.sh CLEANUP
 
-        - name: Debian Testing (gcc-8 ASan+UBSan)
+        - name: Debian Build (gcc-8 ASan+UBSan)
           language: bash
-          env:
+          install:        $CI_ROOT/managers/debian.sh SETUP
-              - DEBIAN_RELEASE="testing"
+          script:         $CI_ROOT/managers/debian.sh RUN_GCC8_ASAN || travis_terminate
-              - CONT_NAME="libbpf-debian-$DEBIAN_RELEASE"
+          after_script:   $CI_ROOT/managers/debian.sh CLEANUP
-          before_install:
-              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-              - docker --version
-          install:
-              - $CI_MANAGERS/debian.sh SETUP
-          before_script: true
-          script:
-              - $CI_MANAGERS/debian.sh RUN_GCC8_ASAN || travis_terminate
-          after_script:
-              - $CI_MANAGERS/debian.sh CLEANUP
 
-        - name: Ubuntu Bionic
+        - name: Ubuntu Bionic Build
           language: bash
-          before_script: true
+          script: sudo $CI_ROOT/managers/ubuntu.sh || travis_terminate
-          script:
-              - sudo $CI_MANAGERS/ubuntu.sh || travis_terminate
 
-        - name: Ubuntu Bionic (arm)
+        - name: Ubuntu Bionic Build (arm)
           arch: arm64
           language: bash
-          before_script: true
+          script: sudo $CI_ROOT/managers/ubuntu.sh || travis_terminate
-          script:
-              - sudo $CI_MANAGERS/ubuntu.sh || travis_terminate
 
-        - name: Ubuntu Bionic (s390x)
+        - name: Ubuntu Bionic Build (s390x)
           arch: s390x
           language: bash
-          before_script: true
+          script: sudo $CI_ROOT/managers/ubuntu.sh || travis_terminate
-          script:
-              - sudo $CI_MANAGERS/ubuntu.sh || travis_terminate
 
-        - name: Ubuntu Bionic (ppc64le)
+        - name: Ubuntu Bionic Build (ppc64le)
           arch: ppc64le
           language: bash
-          before_script: true
+          script: sudo $CI_ROOT/managers/ubuntu.sh || travis_terminate
-          script:
+
-              - sudo $CI_MANAGERS/ubuntu.sh || travis_terminate
+        - stage: Build & Test
+          name: Kernel 5.5.0 + selftests
+          language: bash
+          env: KERNEL=5.5.0
+          script:  $CI_ROOT/vmtest/run_vmtest.sh || travis_terminate
+
+        # 5.5.0-rc6 is built from bpf-next; TODO(hex@): remove when pahole v1.16 is available
+        - name: Kernel 5.5.0-rc6 (pahole 1.16) + selftests
+          language: bash
+          env: KERNEL=5.5.0-rc6
+          install: sudo adduser "${USER}" kvm
+          script:  $CI_ROOT/vmtest/run_vmtest.sh || travis_terminate
+
+        - name: Kernel LATEST + selftests
+          language: bash
+          env: KERNEL=LATEST
+          script:  $CI_ROOT/vmtest/run_vmtest.sh || travis_terminate
 
         - stage: Coverity
           language: bash
@@ -203,7 +126,5 @@ jobs:
               - sudo apt-get update
               - sudo apt-get -y build-dep libelf-dev
               - sudo apt-get install -y libelf-dev pkg-config
-          # Override before_script: so VMTEST before_script commands are not executed.
-          before_script: true
           script:
               - scripts/coverity.sh || travis_terminate

travis-ci/managers/debian.sh

@@ -2,7 +2,7 @@
 
 PHASES=(${@:-SETUP RUN RUN_ASAN CLEANUP})
 DEBIAN_RELEASE="${DEBIAN_RELEASE:-testing}"
-CONT_NAME="${CONT_NAME:-debian-$DEBIAN_RELEASE-$RANDOM}"
+CONT_NAME="${CONT_NAME:-libbpf-debian-$DEBIAN_RELEASE}"
 ENV_VARS="${ENV_VARS:-}"
 DOCKER_RUN="${DOCKER_RUN:-docker run}"
 REPO_ROOT="${REPO_ROOT:-$PWD}"
@@ -30,6 +30,10 @@ for phase in "${PHASES[@]}"; do
         SETUP)
             info "Setup phase"
             info "Using Debian $DEBIAN_RELEASE"
+
+            sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+            docker --version
+
             docker pull debian:$DEBIAN_RELEASE
             info "Starting container $CONT_NAME"
             $DOCKER_RUN -v $REPO_ROOT:/build:rw \
@@ -57,7 +61,7 @@ for phase in "${PHASES[@]}"; do
             docker_exec mkdir build install
             docker_exec ${CC:-cc} --version
             info "build"
-            docker_exec make CFLAGS="${CFLAGS}" -C ./src -B OBJDIR=../build
+	    docker_exec make -j$((4*$(nproc))) CFLAGS="${CFLAGS}" -C ./src -B OBJDIR=../build
             info "ldd build/libbpf.so:"
             docker_exec ldd build/libbpf.so
             if ! docker_exec ldd build/libbpf.so | grep -q libelf; then
@@ -65,7 +69,7 @@ for phase in "${PHASES[@]}"; do
                 exit 1
             fi
             info "install"
-            docker_exec make -C src OBJDIR=../build DESTDIR=../install install
+            docker_exec make -j$((4*$(nproc))) -C src OBJDIR=../build DESTDIR=../install install
             docker_exec rm -rf build install
             ;;
         CLEANUP)

travis-ci/managers/ubuntu.sh

@@ -17,11 +17,11 @@ cd $REPO_ROOT
 CFLAGS="-g -O2 -Werror -Wall -fsanitize=address,undefined"
 mkdir build install
 cc --version
-make CFLAGS="${CFLAGS}" -C ./src -B OBJDIR=../build
+make -j$((4*$(nproc))) CFLAGS="${CFLAGS}" -C ./src -B OBJDIR=../build
 ldd build/libbpf.so
 if ! ldd build/libbpf.so | grep -q libelf; then
     echo "FAIL: No reference to libelf.so in libbpf.so!"
     exit 1
 fi
-make -C src OBJDIR=../build DESTDIR=../install install
+make -j$((4*$(nproc))) -C src OBJDIR=../build DESTDIR=../install install
 rm -rf build install

travis-ci/vmtest/run_vmtest.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -eux
+
+VMTEST_SETUPCMD="PROJECT_NAME=${PROJECT_NAME} ./${PROJECT_NAME}/travis-ci/vmtest/run_selftests.sh"
+
+echo "KERNEL: $KERNEL"
+
+# Install required packages
+wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" | sudo tee -a /etc/apt/sources.list
+echo "deb http://archive.ubuntu.com/ubuntu eoan main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
+sudo apt-get -qq update
+sudo apt-get -y install dwarves=1.15-1
+sudo apt-get -qq -y install clang-10 lld-10 llvm-10
+
+# Build selftests (and latest kernel, if necessary)
+KERNEL="${KERNEL}" ${VMTEST_ROOT}/prepare_selftests.sh travis-ci/vmtest/bpf-next
+
+# Escape whitespace characters.
+setup_cmd=$(sed 's/\([[:space:]]\)/\\\1/g' <<< "${VMTEST_SETUPCMD}")
+
+if [[ "${KERNEL}" = 'LATEST' ]]; then
+  sudo -E sudo -E -u "${USER}" "${VMTEST_ROOT}/run.sh" -b travis-ci/vmtest/bpf-next -o -d ~ -s "${setup_cmd}" ~/root.img;
+else
+  sudo -E sudo -E -u "${USER}" "${VMTEST_ROOT}/run.sh" -k "${KERNEL}*" -o -d ~ -s "${setup_cmd}" ~/root.img;
+fi