mirror/libbpf
1
0
Fork 0
mirror of https://github.com/netdata/libbpf.git synced 2026-03-20 16:29:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

sync_kernel.sh: revamp sync script to work in the presence of merges (#15)

Browse Source 
Previous version of script relied on squashing baseline commit and
rebasing rest of commits on top of it. This doesn't work well with git
histories containing merges. This patch changes approach by cherry-picking
commits that have libbpf-related changes and then rewriting history since
last checkpoint.

This still might fail if there were manually resolved merge conflicts for
libbpf, but it's the best we can get as far as I can tell.

Script now also verifies that state of libbpf in Linux's repository exactly
matches the state of libbpf in github repo.

If everything goes smoothing (including verification step), we clean up after
ourselves and only leave libbpf-sync-XXX branch in github's libbpf repo to
push to remote.

Signed-off-by: Andrii Nakryiko <andriin@fb.com>
This commit is contained in:
Andrii Nakryiko
2019-03-02 21:19:17 -08:00
committed by yonghong-song
parent d5fa4150f0
commit 947e40e1fd
1 changed files with 94 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
scripts/sync-kernel.sh
View File

@@ -22,80 +22,129 @@ set -eu
WORKDIR=$(pwd)
trap "cd ${WORKDIR}; exit" INT TERM EXIT
echo "WORKDIR: ${WORKDIR}"
echo "LINUX REPO: ${LINUX_REPO}"
echo "LIBBPF REPO: ${LIBBPF_REPO}"
echo "WORKDIR: ${WORKDIR}"
echo "LINUX REPO: ${LINUX_REPO}"
echo "LIBBPF REPO: ${LIBBPF_REPO}"
SUFFIX=$(date --utc +%Y-%m-%dT%H-%M-%S.%3NZ)
BASELINE_COMMIT=${3-$(cat ${LIBBPF_REPO}/CHECKPOINT-COMMIT)}
# Use current kernel repo HEAD as a source of patches
cd ${LINUX_REPO}
CHECKPOINT_COMMIT=$(git rev-parse HEAD)
git branch libbpf-${SUFFIX} HEAD
echo "SUFFIX: ${SUFFIX}"
echo "BASELINE COMMIT: ${BASELINE_COMMIT}"
echo "CHECKPOINT COMMIT: ${CHECKPOINT_COMMIT}"
TIP_SYM_REF=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD)
TIP_COMMIT=$(git rev-parse HEAD)
BASELINE_TAG=libbpf-baseline-${SUFFIX}
TIP_TAG=libbpf-tip-${SUFFIX}
VIEW_TAG=libbpf-view-${SUFFIX}
LIBBPF_SYNC_TAG=libbpf-sync-${SUFFIX}
# Squash state of kernel repo at baseline into single commit
SQUASHED_BASELINE_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE ${BASELINE_COMMIT}")
echo "SQUASHED BASELINE COMMIT: ${SQUASHED_BASELINE_COMMIT}"
SQUASH_BASE_TAG=libbpf-squash-base-${SUFFIX}
SQUASH_TIP_TAG=libbpf-squash-tip-${SUFFIX}
SQUASH_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE SQUASH ${BASELINE_COMMIT}")
# Re-apply the rest of commits
git rebase --onto ${SQUASHED_BASELINE_COMMIT} ${BASELINE_COMMIT} libbpf-${SUFFIX}
echo "SUFFIX: ${SUFFIX}"
echo "BASELINE COMMIT: $(git log --pretty=oneline --no-walk ${BASELINE_COMMIT})"
echo "TIP COMMIT: $(git log --pretty=oneline --no-walk ${TIP_COMMIT})"
echo "SQUASH COMMIT: ${SQUASH_COMMIT}"
echo "BASELINE TAG: ${BASELINE_TAG}"
echo "TIP TAG: ${TIP_TAG}"
echo "SQUASH BASE TAG: ${SQUASH_BASE_TAG}"
echo "SQUASH TIP TAG: ${SQUASH_TIP_TAG}"
echo "VIEW TAG: ${VIEW_TAG}"
echo "LIBBPF SYNC TAG: ${LIBBPF_SYNC_TAG}"
# Move all libbpf files into libbpf-projection directory
git filter-branch --prune-empty -f --tree-filter ' \
mkdir -p libbpf-projection/include/uapi/linux libbpf-projection/include/tools && \
git mv -kf tools/lib/bpf libbpf-projection/src && \
git mv -kf tools/include/uapi/linux/{bpf_common.h,bpf.h,btf.h,if_link.h,netlink.h} libbpf-projection/include/uapi/linux && \
git mv -kf tools/include/tools/libc_compat.h libbpf-projection/include/tools && \
git rm --ignore-unmatch -f libbpf-projection/src/{Makefile,Build,test_libbpf.cpp,.gitignore} \
' libbpf-${SUFFIX}
TMP_DIR=$(mktemp -d)
echo "TEMP DIR: ${TMP_DIR}"
echo "PATCHES+COVER: ${TMP_DIR}/patches"
echo "PATCHSET: ${TMP_DIR}/patchset.patch"
# Make libbpf-projection a new root directory
git filter-branch --prune-empty -f --subdirectory-filter libbpf-projection libbpf-${SUFFIX}
git branch ${BASELINE_TAG} ${BASELINE_COMMIT}
git branch ${TIP_TAG} ${TIP_COMMIT}
git branch ${SQUASH_BASE_TAG} ${SQUASH_COMMIT}
git checkout -b ${SQUASH_TIP_TAG} ${SQUASH_COMMIT}
# We don't want to include SQUASHED_BASELINE_COMMIT into patch set,
# but git doesn't allow to exclude rev A in A..B rev range, so instead
# we calculate number of patches to include
COMMIT_CNT=$(($(git rev-list --count ${SQUASHED_BASELINE_COMMIT}..libbpf-${SUFFIX}) - 1))
# Cherry-pick new commits onto squashed baseline commit
LIBBPF_PATHS=(tools/lib/bpf tools/include/uapi/linux/{bpf_common.h,bpf.h,btf.h,if_link.h,netlink.h} tools/include/tools/libc_compat.h)
LIBBPF_NEW_COMMITS=$(git rev-list --topo-order --reverse ${BASELINE_TAG}..${TIP_TAG} ${LIBBPF_PATHS[@]})
for LIBBPF_NEW_COMMIT in ${LIBBPF_NEW_COMMITS}; do
git cherry-pick ${LIBBPF_NEW_COMMIT}
done
# If baseline is the only commit with libbpf-related changes, bail out
LIBBPF_TREE_FILTER=' \
mkdir -p __libbpf/include/uapi/linux __libbpf/include/tools && \
git mv -kf tools/lib/bpf __libbpf/src && \
git mv -kf tools/include/uapi/linux/{bpf_common.h,bpf.h,btf.h,if_link.h,netlink.h} __libbpf/include/uapi/linux && \
git mv -kf tools/include/tools/libc_compat.h __libbpf/include/tools && \
git rm --ignore-unmatch -f __libbpf/src/{Makefile,Build,test_libbpf.cpp,.gitignore} \
'
# Move all libbpf files into __libbpf directory.
git filter-branch --prune-empty -f --tree-filter "${LIBBPF_TREE_FILTER}" ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG}
# Make __libbpf a new root directory
git filter-branch --prune-empty -f --subdirectory-filter __libbpf ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG}
# If there are no new commits with libbpf-related changes, bail out
COMMIT_CNT=$(git rev-list --count ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG})
if ((${COMMIT_CNT} <= 0)); then
echo "No new changes to apply, we are done!"
exit 2
fi
# Exclude baseline commit and generate nice cover letter with summary
git format-patch -${COMMIT_CNT} --cover-letter -o /tmp/libbpf-${SUFFIX}.patches
git format-patch ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --cover-letter -o ${TMP_DIR}/patches
# Now generate single-file patchset w/o cover to apply on top of libbpf repo
git format-patch ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --stdout > ${TMP_DIR}/patchset.patch
# Now generate single-file patchset to apply on top of libbpf repo,
# this time including "sync commit" as well
git format-patch -${COMMIT_CNT} --stdout > /tmp/libbpf-${SUFFIX}.patchset
echo "PATCHSET: /tmp/libbpf-${SUFFIX}.patchset"
cd ${WORKDIR}
cd ${LIBBPF_REPO}
git checkout -b libbpf-sync-${SUFFIX}
git am /tmp/libbpf-${SUFFIX}.patchset
# Now is time to re-apply libbpf-related linux patches to libbpf repo
cd ${WORKDIR} && cd ${LIBBPF_REPO}
git checkout -b ${LIBBPF_SYNC_TAG}
git am --committer-date-is-author-date ${TMP_DIR}/patchset.patch
# Use generated cover-letter as a template for "sync commit" with
# baseline and checkpoint commits from kernel repo (and leave summary
# from cover letter intact, of course)
echo ${CHECKPOINT_COMMIT} > CHECKPOINT-COMMIT && \
git add CHECKPOINT-COMMIT && \
awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' /tmp/libbpf-${SUFFIX}.patches/0000-cover-letter.patch | \
echo ${TIP_COMMIT} > CHECKPOINT-COMMIT && \
git add CHECKPOINT-COMMIT && \
awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' ${TMP_DIR}/patches/0000-cover-letter.patch | \
sed "s/\*\*\* BLURB HERE \*\*\*/\
sync: latest libbpf changes from kernel\n\
\n\
Syncing latest libbpf commits from kernel repository.\n\
Baseline commit: ${BASELINE_COMMIT}\n\
Checkpoint commit: ${CHECKPOINT_COMMIT}/" | \
Checkpoint commit: ${TIP_COMMIT}/" | \
git commit --file=-
# Clean up a bit after ourselvest
rm -r /tmp/libbpf-${SUFFIX}.patches /tmp/libbpf-${SUFFIX}.patchset
echo "SUCCESS! ${COMMIT_CNT} commits synced."
echo "Verifying Linux's and Github's libbpf state"
LIBBPF_VIEW_PATHS=(src include/uapi/linux/{bpf_common.h,bpf.h,btf.h,if_link.h,netlink.h} include/tools/libc_compat.h)
LIBBPF_VIEW_EXCLUDE_REGEX='^src/(Makefile|Build|test_libbpf.cpp|\.gitignore)$'
cd ${WORKDIR} && cd ${LINUX_REPO}
LINUX_ABS_DIR=$(pwd)
git checkout -b ${VIEW_TAG} ${TIP_COMMIT}
git filter-branch -f --tree-filter "${LIBBPF_TREE_FILTER}" ${VIEW_TAG}^..${VIEW_TAG}
git filter-branch -f --subdirectory-filter __libbpf ${VIEW_TAG}^..${VIEW_TAG}
git ls-files -- ${LIBBPF_VIEW_PATHS[@]} > ${TMP_DIR}/linux-view.ls
cd ${WORKDIR} && cd ${LIBBPF_REPO}
GITHUB_ABS_DIR=$(pwd)
git ls-files -- ${LIBBPF_VIEW_PATHS[@]} | grep -v -E "${LIBBPF_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/github-view.ls
echo "Comparing list of files..."
diff ${TMP_DIR}/linux-view.ls ${TMP_DIR}/github-view.ls
echo "Comparing file contents..."
for F in $(cat ${TMP_DIR}/linux-view.ls); do
diff "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"
done
echo "Contents appear identical!"
echo "Cleaning up..."
rm -r ${TMP_DIR}
cd ${WORKDIR} && cd ${LINUX_REPO}
git checkout ${TIP_SYM_REF}
git branch -D ${BASELINE_TAG} ${TIP_TAG} ${SQUASH_BASE_TAG} ${SQUASH_TIP_TAG} ${VIEW_TAG}
cd ${WORKDIR}
echo "DONE."
echo "Success! ${COMMIT_CNT} commits applied."