Rename travis-ci/ directory to ci/

We are no longer using Travis. As such, we should move away from a lot of CI functionality located in a folder called travis-ci/. This change renames the travis-ci/ directory to the more generic ci/. To preserve backwards compatibility until all "consumers" have transitioned, we add a symbolic link called travis-ci back. It will be removed in the near term future. Signed-off-by: Daniel Müller <deso@posteo.net>
2026-03-31 05:39:06 +08:00 · 2022-08-17 14:23:41 -07:00
parent d700dcf162
commit bfdf7653e0
23 changed files with 2 additions and 1 deletions
--- a/ci/rootfs/mkrootfs_arch.sh
+++ b/ci/rootfs/mkrootfs_arch.sh
@@ -0,0 +1,107 @@
+#!/bin/bash
+
+# This script is based on drgn script for generating Arch Linux bootstrap
+# images.
+# https://github.com/osandov/drgn/blob/master/scripts/vmtest/mkrootfs.sh
+
+set -euo pipefail
+
+usage () {
+	USAGE_STRING="usage: $0 [NAME]
+       $0 -h
+
+Build an Arch Linux root filesystem image for testing libbpf in a virtual
+machine.
+
+The image is generated as a zstd-compressed tarball.
+
+This must be run as root, as most of the installation is done in a chroot.
+
+Arguments:
+  NAME   name of generated image file (default:
+         libbpf-vmtest-rootfs-\$DATE.tar.zst)
+
+Options:
+  -h     display this help message and exit"
+
+	case "$1" in
+		out)
+			echo "$USAGE_STRING"
+			exit 0
+			;;
+		err)
+			echo "$USAGE_STRING" >&2
+			exit 1
+			;;
+	esac
+}
+
+while getopts "h" OPT; do
+	case "$OPT" in
+		h)
+			usage out
+			;;
+		*)
+			usage err
+			;;
+	esac
+done
+if [[ $OPTIND -eq $# ]]; then
+	NAME="${!OPTIND}"
+elif [[ $OPTIND -gt $# ]]; then
+	NAME="libbpf-vmtest-rootfs-$(date +%Y.%m.%d).tar.zst"
+else
+	usage err
+fi
+
+pacman_conf=
+root=
+trap 'rm -rf "$pacman_conf" "$root"' EXIT
+pacman_conf="$(mktemp -p "$PWD")"
+cat > "$pacman_conf" << "EOF"
+[options]
+Architecture = x86_64
+CheckSpace
+SigLevel = Required DatabaseOptional
+[core]
+Include = /etc/pacman.d/mirrorlist
+[extra]
+Include = /etc/pacman.d/mirrorlist
+[community]
+Include = /etc/pacman.d/mirrorlist
+EOF
+root="$(mktemp -d -p "$PWD")"
+
+packages=(
+	busybox
+	# libbpf dependencies.
+	libelf
+	zlib
+	# selftests test_progs dependencies.
+	binutils
+	elfutils
+	ethtool
+	glibc
+	iproute2
+	# selftests test_verifier dependencies.
+	libcap
+)
+
+pacstrap -C "$pacman_conf" -cGM "$root" "${packages[@]}"
+
+# Remove unnecessary files from the chroot.
+
+# We don't need the pacman databases anymore.
+rm -rf "$root/var/lib/pacman/sync/"
+# We don't need D, Fortran, or Go.
+ rm -f "$root/usr/lib/libgdruntime."* \
+	"$root/usr/lib/libgphobos."* \
+	"$root/usr/lib/libgfortran."* \
+	"$root/usr/lib/libgo."*
+# We don't need any documentation.
+rm -rf "$root/usr/share/{doc,help,man,texinfo}"
+
+"$(dirname "$0")"/mkrootfs_tweak.sh "$root"
+
+tar -C "$root" -c . | zstd -T0 -19 -o "$NAME"
+chmod 644 "$NAME"
--- a/ci/rootfs/mkrootfs_debian.sh
+++ b/ci/rootfs/mkrootfs_debian.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+# This script builds a Debian root filesystem image for testing libbpf in a
+# virtual machine. Requires debootstrap >= 1.0.95 and zstd.
+
+# Use e.g. ./mkrootfs_debian.sh --arch=s390x to generate a rootfs for a
+# foreign architecture. Requires configured binfmt_misc, e.g. using
+# Debian/Ubuntu's qemu-user-binfmt package or
+# https://github.com/multiarch/qemu-user-static.
+
+set -e -u -x -o pipefail
+
+# Check whether we are root now in order to avoid confusing errors later.
+if [ "$(id -u)" != 0 ]; then
+	echo "$0 must run as root" >&2
+	exit 1
+fi
+
+# Create a working directory and schedule its deletion.
+root=$(mktemp -d -p "$PWD")
+trap 'rm -r "$root"' EXIT
+
+# Install packages.
+packages=(
+	binutils
+	busybox
+	elfutils
+	ethtool
+	iproute2
+	iptables
+	libcap2
+	libelf1
+	strace
+	zlib1g
+)
+packages=$(IFS=, && echo "${packages[*]}")
+debootstrap --include="$packages" --variant=minbase "$@" bookworm "$root"
+
+# Remove the init scripts (tests use their own). Also remove various
+# unnecessary files in order to save space.
+rm -rf \
+	"$root"/etc/rcS.d \
+	"$root"/usr/share/{doc,info,locale,man,zoneinfo} \
+	"$root"/var/cache/apt/archives/* \
+	"$root"/var/lib/apt/lists/*
+
+# Apply common tweaks.
+"$(dirname "$0")"/mkrootfs_tweak.sh "$root"
+
+# Save the result.
+name="libbpf-vmtest-rootfs-$(date +%Y.%m.%d).tar.zst"
+rm -f "$name"
+tar -C "$root" -c . | zstd -T0 -19 -o "$name"
--- a/ci/rootfs/mkrootfs_tweak.sh
+++ b/ci/rootfs/mkrootfs_tweak.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+# This script prepares a mounted root filesystem for testing libbpf in a virtual
+# machine.
+set -e -u -x -o pipefail
+root=$1
+shift
+
+chroot "${root}" /bin/busybox --install
+
+cat > "$root/etc/inittab" << "EOF"
+::sysinit:/etc/init.d/rcS
+::ctrlaltdel:/sbin/reboot
+::shutdown:/sbin/swapoff -a
+::shutdown:/bin/umount -a -r
+::restart:/sbin/init
+EOF
+chmod 644 "$root/etc/inittab"
+
+mkdir -m 755 -p "$root/etc/init.d" "$root/etc/rcS.d"
+cat > "$root/etc/rcS.d/S10-mount" << "EOF"
+#!/bin/sh
+
+set -eux
+
+/bin/mount proc /proc -t proc
+
+# Mount devtmpfs if not mounted
+if [[ -z $(/bin/mount -t devtmpfs) ]]; then
+	/bin/mount devtmpfs /dev -t devtmpfs
+fi
+
+/bin/mount sysfs /sys -t sysfs
+/bin/mount bpffs /sys/fs/bpf -t bpf
+/bin/mount debugfs /sys/kernel/debug -t debugfs
+
+echo 'Listing currently mounted file systems'
+/bin/mount
+EOF
+chmod 755 "$root/etc/rcS.d/S10-mount"
+
+cat > "$root/etc/rcS.d/S40-network" << "EOF"
+#!/bin/sh
+
+set -eux
+
+ip link set lo up
+EOF
+chmod 755 "$root/etc/rcS.d/S40-network"
+
+cat > "$root/etc/init.d/rcS" << "EOF"
+#!/bin/sh
+
+set -eux
+
+for path in /etc/rcS.d/S*; do
+	[ -x "$path" ] && "$path"
+done
+EOF
+chmod 755 "$root/etc/init.d/rcS"
+
+chmod 755 "$root"
--- a/ci/rootfs/s390x-self-hosted-builder/README.md
+++ b/ci/rootfs/s390x-self-hosted-builder/README.md
@@ -0,0 +1,107 @@
+# IBM Z self-hosted builder
+
+libbpf CI uses an IBM-provided z15 self-hosted builder. There are no IBM Z
+builds of GitHub (GH) Actions runner, and stable qemu-user has problems with .NET
+apps, so the builder runs the x86_64 runner version with qemu-user built from
+the master branch.
+
+We are currently supporting runners for the following repositories:
+* libbpf/libbpf
+* kernel-patches/bpf
+* kernel-patches/vmtest
+
+Below instructions are directly applicable to libbpf, and require minor
+modifications for kernel-patches repos. Currently, qemu-user-static Docker
+image is shared between all GitHub runners, but separate actions-runner-\*
+service / Docker image is created for each runner type.
+
+## Configuring the builder.
+
+### Install prerequisites.
+
+```
+$ sudo apt install -y docker.io  # Ubuntu
+```
+
+### Add services.
+
+```
+$ sudo cp *.service /etc/systemd/system/
+$ sudo systemctl daemon-reload
+```
+
+### Create a config file.
+
+```
+$ sudo tee /etc/actions-runner-libbpf
+repo=<owner>/<name>
+access_token=<ghp_***>
+```
+
+Access token should have the repo scope, consult
+https://docs.github.com/en/rest/reference/actions#create-a-registration-token-for-a-repository
+for details.
+
+### Autostart the x86_64 emulation support.
+
+This step is important, you would not be able to build docker container
+without having this service running. If container build fails, make sure
+service is running properly.
+
+```
+$ sudo systemctl enable --now qemu-user-static
+```
+
+### Autostart the runner.
+
+```
+$ sudo systemctl enable --now actions-runner-libbpf
+```
+
+## Rebuilding the image
+
+In order to update the `iiilinuxibmcom/actions-runner-libbpf` image, e.g. to
+get the latest OS security fixes, use the following commands:
+
+```
+$ sudo docker build \
+      --pull \
+      -f actions-runner-libbpf.Dockerfile \
+      -t iiilinuxibmcom/actions-runner-libbpf \
+      .
+$ sudo systemctl restart actions-runner-libbpf
+```
+
+## Removing persistent data
+
+The `actions-runner-libbpf` service stores various temporary data, such as
+runner registration information, work directories and logs, in the
+`actions-runner-libbpf` volume. In order to remove it and start from scratch,
+e.g. when upgrading the runner or switching it to a different repository, use
+the following commands:
+
+```
+$ sudo systemctl stop actions-runner-libbpf
+$ sudo docker rm -f actions-runner-libbpf
+$ sudo docker volume rm actions-runner-libbpf
+```
+
+## Troubleshooting
+
+In order to check if service is running, use the following command:
+
+```
+$ sudo systemctl status <service name>
+```
+
+In order to get logs for service:
+
+```
+$ journalctl -u <service name>
+```
+
+In order to check which containers are currently active:
+
+```
+$ sudo docker ps
+```
--- a/ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile
+++ b/ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile
@@ -0,0 +1,50 @@
+# Self-Hosted IBM Z Github Actions Runner.
+
+# Temporary image: amd64 dependencies.
+FROM amd64/ubuntu:20.04 as ld-prefix
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get -y install ca-certificates libicu66 libssl1.1
+
+# Main image.
+FROM s390x/ubuntu:20.04
+
+# Packages for libbpf testing that are not installed by .github/actions/setup.
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get -y install \
+        bc \
+        bison \
+        cmake \
+        cpu-checker \
+        curl \
+        flex \
+        git \
+        jq \
+        linux-image-generic \
+        qemu-system-s390x \
+        rsync \
+        software-properties-common \
+        sudo \
+        tree
+
+# amd64 dependencies.
+COPY --from=ld-prefix / /usr/x86_64-linux-gnu/
+RUN ln -fs ../lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 /usr/x86_64-linux-gnu/lib64/
+RUN ln -fs /etc/resolv.conf /usr/x86_64-linux-gnu/etc/
+ENV QEMU_LD_PREFIX=/usr/x86_64-linux-gnu
+
+# amd64 Github Actions Runner.
+ARG version=2.285.0
+RUN useradd -m actions-runner
+RUN echo "actions-runner ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
+RUN echo "Defaults env_keep += \"DEBIAN_FRONTEND\"" >>/etc/sudoers
+RUN usermod -a -G kvm actions-runner
+USER actions-runner
+ENV USER=actions-runner
+WORKDIR /home/actions-runner
+RUN curl -L https://github.com/actions/runner/releases/download/v${version}/actions-runner-linux-x64-${version}.tar.gz | tar -xz
+VOLUME /home/actions-runner
+
+# Scripts.
+COPY fs/ /
+ENTRYPOINT ["/usr/bin/entrypoint"]
+CMD ["/usr/bin/actions-runner"]
--- a/ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.service
+++ b/ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.service
@@ -0,0 +1,24 @@
+[Unit]
+Description=Self-Hosted IBM Z Github Actions Runner
+Wants=qemu-user-static
+After=qemu-user-static
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+Restart=always
+ExecStart=/usr/bin/docker run \
+              --device=/dev/kvm \
+              --env-file=/etc/actions-runner-libbpf \
+              --init \
+              --interactive \
+              --name=actions-runner-libbpf \
+              --rm \
+              --volume=actions-runner-libbpf:/home/actions-runner \
+              iiilinuxibmcom/actions-runner-libbpf
+ExecStop=/bin/sh -c "docker exec actions-runner-libbpf kill -INT -- -1"
+ExecStop=/bin/sh -c "docker wait actions-runner-libbpf"
+ExecStop=/bin/sh -c "docker rm actions-runner-libbpf"
+
+[Install]
+WantedBy=multi-user.target
--- a/ci/rootfs/s390x-self-hosted-builder/fs/usr/bin/actions-runner
+++ b/ci/rootfs/s390x-self-hosted-builder/fs/usr/bin/actions-runner
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+#
+# Ephemeral runner startup script.
+#
+# Expects the following environment variables:
+#
+# - repo=<owner>/<name>
+# - access_token=<ghp_***>
+#
+
+set -e -u
+
+# Check the cached registration token.
+token_file=registration-token.json
+set +e
+expires_at=$(jq --raw-output .expires_at "$token_file" 2>/dev/null)
+status=$?
+set -e
+if [[ $status -ne 0 || $(date +%s) -ge $(date -d "$expires_at" +%s) ]]; then
+    # Refresh the cached registration token.
+    curl \
+        -X POST \
+        -H "Accept: application/vnd.github.v3+json" \
+        -H "Authorization: token $access_token" \
+        "https://api.github.com/repos/$repo/actions/runners/registration-token" \
+        -o "$token_file"
+fi
+
+# (Re-)register the runner.
+registration_token=$(jq --raw-output .token "$token_file")
+./config.sh remove --token "$registration_token" || true
+./config.sh \
+    --url "https://github.com/$repo" \
+    --token "$registration_token" \
+    --labels z15 \
+    --ephemeral
+
+# Run one job.
+./run.sh
--- a/ci/rootfs/s390x-self-hosted-builder/fs/usr/bin/entrypoint
+++ b/ci/rootfs/s390x-self-hosted-builder/fs/usr/bin/entrypoint
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+#
+# Container entrypoint that waits for all spawned processes.
+#
+
+set -e -u
+
+# /dev/kvm has host permissions, fix it.
+if [ -e /dev/kvm ]; then
+    sudo chown root:kvm /dev/kvm
+fi
+
+# Create a FIFO and start reading from its read end.
+tempdir=$(mktemp -d "/tmp/done.XXXXXXXXXX")
+trap 'rm -r "$tempdir"' EXIT
+done="$tempdir/pipe"
+mkfifo "$done"
+cat "$done" & waiter=$!
+
+# Start the workload. Its descendants will inherit the FIFO's write end.
+status=0
+if [ "$#" -eq 0 ]; then
+    bash 9>"$done" || status=$?
+else
+    "$@" 9>"$done" || status=$?
+fi
+
+# When the workload and all of its descendants exit, the FIFO's write end will
+# be closed and `cat "$done"` will exit. Wait until it happens. This is needed
+# in order to handle SelfUpdater, which the workload may start in background
+# before exiting.
+wait "$waiter"
+
+exit "$status"
--- a/ci/rootfs/s390x-self-hosted-builder/qemu-user-static.service
+++ b/ci/rootfs/s390x-self-hosted-builder/qemu-user-static.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Support for transparent execution of non-native binaries with QEMU user emulation
+
+[Service]
+Type=oneshot
+# The source code for iiilinuxibmcom/qemu-user-static is at https://github.com/iii-i/qemu-user-static/tree/v6.1.0-1
+# TODO: replace it with multiarch/qemu-user-static once version >6.1 is available
+ExecStart=/usr/bin/docker run --rm --interactive --privileged iiilinuxibmcom/qemu-user-static --reset -p yes
+
+[Install]
+WantedBy=multi-user.target