From 1987a34fc92ab9518ecbf0d3f1c8f8827c3b74ac Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Tue, 30 Mar 2021 00:36:50 +0200 Subject: [PATCH] vmtest: use libguestfs for disk image manipulations Running vmtest inside a container removes the ability to use certain root powers, among other things - mounting arbitrary images. Use libguestfs in order to avoid having to mount anything. Signed-off-by: Ilya Leoshkevich --- .github/actions/setup/action.yml | 2 +- travis-ci/vmtest/run.sh | 142 ++++++++++++++++++++----------- 2 files changed, 91 insertions(+), 53 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index e85a434..63bbff5 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -12,7 +12,7 @@ runs: echo 'echo ::group::Env setup' > /tmp/ci_setup echo export DEBIAN_FRONTEND=noninteractive >> /tmp/ci_setup echo sudo apt-get update >> /tmp/ci_setup - echo sudo apt-get install -y aptitude qemu-kvm zstd binutils-dev elfutils libcap-dev libelf-dev libdw-dev >> /tmp/ci_setup + echo sudo apt-get install -y aptitude qemu-kvm zstd binutils-dev elfutils libcap-dev libelf-dev libdw-dev libguestfs-tools >> /tmp/ci_setup echo export PROJECT_NAME='libbpf' >> /tmp/ci_setup echo export AUTHOR_EMAIL="$(git log -1 --pretty=\"%aE\")" >> /tmp/ci_setup echo export REPO_ROOT=$GITHUB_WORKSPACE >> /tmp/ci_setup diff --git a/travis-ci/vmtest/run.sh b/travis-ci/vmtest/run.sh index 9819d6c..c6021f2 100755 --- a/travis-ci/vmtest/run.sh +++ b/travis-ci/vmtest/run.sh @@ -15,7 +15,7 @@ Run "${PROJECT_NAME}" tests in a virtual machine. This exits with status 0 on success, 1 if the virtual machine ran successfully but tests failed, and 2 if we encountered a fatal error. -This script uses sudo to mount and modify the disk image. +This script uses sudo to work around a libguestfs bug. Arguments: IMG path of virtual machine disk image to create @@ -177,6 +177,11 @@ else fi IMG="${!OPTIND}" fi +if [[ "${SOURCE_FULLCOPY}" == "1" ]]; then + img_size=2G +else + img_size=8G +fi unset URLS cache_urls() { @@ -242,15 +247,29 @@ cp_img() { create_rootfs_img() { local path="$1" set_nocow "$path" - truncate -s 2G "$path" + truncate -s "$img_size" "$path" mkfs.ext4 -q "$path" } download_rootfs() { local rootfsversion="$1" - local dir="$2" download "${ARCH}/${PROJECT_NAME}-vmtest-rootfs-$rootfsversion.tar.zst" | - zstd -d | sudo tar -C "$dir" -x + zstd -d +} + +tar_in() { + local dst_path="$1" + # guestfish --remote does not forward file descriptors, which prevents + # us from using `tar-in -` or bash process substitution. We don't want + # to copy all the data into a temporary file, so use a FIFO. + tmp=$(mktemp -d) + mkfifo "$tmp/fifo" + cat >"$tmp/fifo" & + local cat_pid=$! + guestfish --remote tar-in "$tmp/fifo" "$dst_path" + wait "$cat_pid" + rm -r "$tmp" + tmp= } if (( LIST )); then @@ -297,18 +316,12 @@ echo "Disk image: $IMG" >&2 tmp= ARCH_DIR="$DIR/$ARCH" mkdir -p "$ARCH_DIR" -mnt="$(mktemp -d -p "$DIR" mnt.XXXXXXXXXX)" cleanup() { if [[ -n $tmp ]]; then - rm -f "$tmp" || true - fi - if mountpoint -q "$mnt"; then - sudo umount "$mnt" || true - fi - if [[ -d "$mnt" ]]; then - rmdir "$mnt" || true + rm -rf "$tmp" || true fi + guestfish --remote exit 2>/dev/null || true } trap cleanup EXIT @@ -324,12 +337,19 @@ else fi fi -# Mount and set up the rootfs image. +# Mount and set up the rootfs image. Use a persistent guestfish session in +# order to avoid the startup overhead. +# Work around https://bugs.launchpad.net/fuel/+bug/1467579. +sudo chmod +r /boot/vmlinuz* +eval "$(guestfish --listen)" if (( ONESHOT )); then rm -f "$IMG" create_rootfs_img "$IMG" - sudo mount -o loop "$IMG" "$mnt" - download_rootfs "$ROOTFSVERSION" "$mnt" + guestfish --remote \ + add "$IMG" label:img : \ + launch : \ + mount /dev/disk/guestfs/img / + download_rootfs "$ROOTFSVERSION" | tar_in / else if (( ! SKIPIMG )); then rootfs_img="${ARCH_DIR}/${PROJECT_NAME}-vmtest-rootfs-${ROOTFSVERSION}.img" @@ -337,13 +357,15 @@ else if [[ ! -e $rootfs_img ]]; then tmp="$(mktemp "$rootfs_img.XXX.part")" set_nocow "$tmp" - truncate -s 2G "$tmp" + truncate -s "$img_size" "$tmp" mkfs.ext4 -q "$tmp" - sudo mount -o loop "$tmp" "$mnt" - download_rootfs "$ROOTFSVERSION" "$mnt" + # libguestfs supports hotplugging only with a libvirt + # backend, which we are not using here, so handle the + # temporary image in a separate session. + download_rootfs "$ROOTFSVERSION" | + guestfish -a "$tmp" tar-in - / - sudo umount "$mnt" mv "$tmp" "$rootfs_img" tmp= fi @@ -351,11 +373,14 @@ else rm -f "$IMG" cp_img "$rootfs_img" "$IMG" fi - sudo mount -o loop "$IMG" "$mnt" + guestfish --remote \ + add "$IMG" label:img : \ + launch : \ + mount /dev/disk/guestfs/img / fi # Install vmlinux. -vmlinux="$mnt/boot/vmlinux-${KERNELRELEASE}" +vmlinux="/boot/vmlinux-${KERNELRELEASE}" if [[ -v BUILDDIR || $ONESHOT -eq 0 ]]; then if [[ -v BUILDDIR ]]; then source_vmlinux="${BUILDDIR}/vmlinux" @@ -368,15 +393,14 @@ if [[ -v BUILDDIR || $ONESHOT -eq 0 ]]; then tmp= fi fi - echo "Copying vmlinux..." >&2 - sudo rsync -cp --chmod 0644 "$source_vmlinux" "$vmlinux" else - # We could use "sudo zstd -o", but let's not run zstd as root with - # input from the internet. - download "${ARCH}/vmlinux-${KERNELRELEASE}.zst" | - zstd -d | sudo tee "$vmlinux" > /dev/null - sudo chmod 644 "$vmlinux" + source_vmlinux="${ARCH_DIR}/vmlinux-${KERNELRELEASE}" + download "${ARCH}/vmlinux-${KERNELRELEASE}.zst" | zstd -d >"$source_vmlinux" fi +echo "Copying vmlinux..." >&2 +guestfish --remote \ + upload "$source_vmlinux" "$vmlinux" : \ + chmod 644 "$vmlinux" travis_fold end vmlinux_setup @@ -384,7 +408,7 @@ REPO_PATH="${SELFTEST_REPO_PATH:-travis-ci/vmtest/bpf-next}" LIBBPF_PATH="${REPO_ROOT}" \ VMTEST_ROOT="${VMTEST_ROOT}" \ REPO_PATH="${REPO_PATH}" \ - VMLINUX_BTF=${vmlinux} ${VMTEST_ROOT}/build_selftests.sh + VMLINUX_BTF=$(realpath ${source_vmlinux}) ${VMTEST_ROOT}/build_selftests.sh travis_fold start bpftool_checks "Running bpftool checks..." if [[ "${KERNEL}" = 'LATEST' ]]; then @@ -402,23 +426,31 @@ if (( SKIPSOURCE )); then else echo "Copying source files..." >&2 # Copy the source files in. - sudo mkdir -p -m 0755 "$mnt/${PROJECT_NAME}" + guestfish --remote \ + mkdir-p "/${PROJECT_NAME}" : \ + chmod 0755 "/${PROJECT_NAME}" if [[ "${SOURCE_FULLCOPY}" == "1" ]]; then - git ls-files -z | sudo rsync --files-from=- -0cpt . "$mnt/${PROJECT_NAME}" + git ls-files -z | tar --null --files-from=- -c | tar_in "/${PROJECT_NAME}" else - sudo mkdir -p -m 0755 ${mnt}/${PROJECT_NAME}/{selftests,travis-ci} + guestfish --remote \ + mkdir-p "/${PROJECT_NAME}/selftests" : \ + chmod 0755 "/${PROJECT_NAME}/selftests" : \ + mkdir-p "/${PROJECT_NAME}/travis-ci" : \ + chmod 0755 "/${PROJECT_NAME}/travis-ci" tree --du -shaC "${REPO_ROOT}/selftests/bpf" - sudo rsync -avm "${REPO_ROOT}/selftests/bpf" "$mnt/${PROJECT_NAME}/selftests/" - sudo rsync -avm "${REPO_ROOT}/travis-ci/vmtest" "$mnt/${PROJECT_NAME}/travis-ci/" - fi - + tar -C "${REPO_ROOT}/selftests" -c bpf | tar_in "/${PROJECT_NAME}/selftests" + tar -C "${REPO_ROOT}/travis-ci" -c vmtest | tar_in "/${PROJECT_NAME}/travis-ci" + fi fi -setup_script="#!/bin/sh +tmp=$(mktemp) +cat <"$tmp" +"#!/bin/sh echo 'Skipping setup commands' echo 0 > /exitstatus -chmod 644 /exitstatus" +chmod 644 /exitstatus +HERE # Create the init scripts. if [[ ! -z SETUPCMD ]]; then @@ -427,30 +459,38 @@ if [[ ! -z SETUPCMD ]]; then kernel="${KERNELRELEASE}" if [[ -v BUILDDIR ]]; then kernel='latest'; fi setup_envvars="export KERNEL=${kernel}" - setup_script=$(printf "#!/bin/sh + cat <"$tmp" +#!/bin/sh set -eux echo 'Running setup commands' -%s -set +e; %s; exitstatus=\$?; set -e +${setup_envvars} +set +e; ${setup_cmd}; exitstatus=\$?; set -e echo \$exitstatus > /exitstatus -chmod 644 /exitstatus" "${setup_envvars}" "${setup_cmd}") +chmod 644 /exitstatus +HERE fi -echo "${setup_script}" | sudo tee "$mnt/etc/rcS.d/S50-run-tests" > /dev/null -sudo chmod 755 "$mnt/etc/rcS.d/S50-run-tests" +guestfish --remote \ + upload "$tmp" /etc/rcS.d/S50-run-tests : \ + chmod 755 /etc/rcS.d/S50-run-tests fold_shutdown="$(travis_fold start shutdown)" -poweroff_script="#!/bin/sh +cat <"$tmp" +#!/bin/sh echo ${fold_shutdown} echo -e '\033[1;33mShutdown\033[0m\n' -poweroff" -echo "${poweroff_script}" | sudo tee "$mnt/etc/rcS.d/S99-poweroff" > /dev/null -sudo chmod 755 "$mnt/etc/rcS.d/S99-poweroff" +poweroff +HERE +guestfish --remote \ + upload "$tmp" /etc/rcS.d/S99-poweroff : \ + chmod 755 /etc/rcS.d/S99-poweroff +rm "$tmp" +tmp= -sudo umount "$mnt" +guestfish --remote exit echo "Starting VM with $(nproc) CPUs..." @@ -480,14 +520,12 @@ esac -drive file="$IMG",format=raw,index=1,media=disk,if=virtio,cache=none \ -kernel "$vmlinuz" -append "root=/dev/vda rw console=$console kernel.panic=-1 $APPEND" -sudo mount -o loop "$IMG" "$mnt" -if exitstatus="$(cat "$mnt/exitstatus" 2>/dev/null)"; then +if exitstatus="$(guestfish --ro -a "$IMG" -i cat /exitstatus 2>/dev/null)"; then printf '\nTests exit status: %s\n' "$exitstatus" >&2 else printf '\nCould not read tests exit status\n' >&2 exitstatus=1 fi -sudo umount "$mnt" travis_fold end shutdown