mirror/libbpf
1
0
Fork 0
mirror of https://github.com/netdata/libbpf.git synced 2026-04-04 07:39:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

sync_kernel.sh: revamp sync script to work in the presence of merges (#15)

Browse Source 
Previous version of script relied on squashing baseline commit and
rebasing rest of commits on top of it. This doesn't work well with git
histories containing merges. This patch changes approach by cherry-picking
commits that have libbpf-related changes and then rewriting history since
last checkpoint.

This still might fail if there were manually resolved merge conflicts for
libbpf, but it's the best we can get as far as I can tell.

Script now also verifies that state of libbpf in Linux's repository exactly
matches the state of libbpf in github repo.

If everything goes smoothing (including verification step), we clean up after
ourselves and only leave libbpf-sync-XXX branch in github's libbpf repo to
push to remote.

Signed-off-by: Andrii Nakryiko <andriin@fb.com>
This commit is contained in:
Andrii Nakryiko
2019-03-02 21:19:17 -08:00
committed by yonghong-song
parent d5fa4150f0
commit 947e40e1fd
1 changed files with 94 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
scripts/sync-kernel.sh
View File

@@ -22,80 +22,129 @@ set -eu
WORKDIR=$(pwd) WORKDIR=$(pwd)
trap "cd ${WORKDIR}; exit" INT TERM EXIT trap "cd ${WORKDIR}; exit" INT TERM EXIT
echo "WORKDIR: ${WORKDIR}" echo "WORKDIR: ${WORKDIR}"
echo "LINUX REPO: ${LINUX_REPO}" echo "LINUX REPO: ${LINUX_REPO}"
echo "LIBBPF REPO: ${LIBBPF_REPO}" echo "LIBBPF REPO: ${LIBBPF_REPO}"
SUFFIX=$(date --utc +%Y-%m-%dT%H-%M-%S.%3NZ) SUFFIX=$(date --utc +%Y-%m-%dT%H-%M-%S.%3NZ)
BASELINE_COMMIT=${3-$(cat ${LIBBPF_REPO}/CHECKPOINT-COMMIT)} BASELINE_COMMIT=${3-$(cat ${LIBBPF_REPO}/CHECKPOINT-COMMIT)}
# Use current kernel repo HEAD as a source of patches # Use current kernel repo HEAD as a source of patches
cd ${LINUX_REPO} cd ${LINUX_REPO}
CHECKPOINT_COMMIT=$(git rev-parse HEAD) TIP_SYM_REF=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD)
git branch libbpf-${SUFFIX} HEAD TIP_COMMIT=$(git rev-parse HEAD)
BASELINE_TAG=libbpf-baseline-${SUFFIX}
echo "SUFFIX: ${SUFFIX}" TIP_TAG=libbpf-tip-${SUFFIX}
echo "BASELINE COMMIT: ${BASELINE_COMMIT}" VIEW_TAG=libbpf-view-${SUFFIX}
echo "CHECKPOINT COMMIT: ${CHECKPOINT_COMMIT}" LIBBPF_SYNC_TAG=libbpf-sync-${SUFFIX}
# Squash state of kernel repo at baseline into single commit # Squash state of kernel repo at baseline into single commit
SQUASHED_BASELINE_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE ${BASELINE_COMMIT}") SQUASH_BASE_TAG=libbpf-squash-base-${SUFFIX}
echo "SQUASHED BASELINE COMMIT: ${SQUASHED_BASELINE_COMMIT}" SQUASH_TIP_TAG=libbpf-squash-tip-${SUFFIX}
SQUASH_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE SQUASH ${BASELINE_COMMIT}")
# Re-apply the rest of commits echo "SUFFIX: ${SUFFIX}"
git rebase --onto ${SQUASHED_BASELINE_COMMIT} ${BASELINE_COMMIT} libbpf-${SUFFIX} echo "BASELINE COMMIT: $(git log --pretty=oneline --no-walk ${BASELINE_COMMIT})"
echo "TIP COMMIT: $(git log --pretty=oneline --no-walk ${TIP_COMMIT})"
echo "SQUASH COMMIT: ${SQUASH_COMMIT}"
echo "BASELINE TAG: ${BASELINE_TAG}"
echo "TIP TAG: ${TIP_TAG}"
echo "SQUASH BASE TAG: ${SQUASH_BASE_TAG}"
echo "SQUASH TIP TAG: ${SQUASH_TIP_TAG}"
echo "VIEW TAG: ${VIEW_TAG}"
echo "LIBBPF SYNC TAG: ${LIBBPF_SYNC_TAG}"
# Move all libbpf files into libbpf-projection directory TMP_DIR=$(mktemp -d)
git filter-branch --prune-empty -f --tree-filter ' \ echo "TEMP DIR: ${TMP_DIR}"
mkdir -p libbpf-projection/include/uapi/linux libbpf-projection/include/tools && \ echo "PATCHES+COVER: ${TMP_DIR}/patches"
git mv -kf tools/lib/bpf libbpf-projection/src && \ echo "PATCHSET: ${TMP_DIR}/patchset.patch"
git mv -kf tools/include/uapi/linux/{bpf_common.h,bpf.h,btf.h,if_link.h,netlink.h} libbpf-projection/include/uapi/linux && \
git mv -kf tools/include/tools/libc_compat.h libbpf-projection/include/tools && \
git rm --ignore-unmatch -f libbpf-projection/src/{Makefile,Build,test_libbpf.cpp,.gitignore} \
' libbpf-${SUFFIX}
# Make libbpf-projection a new root directory git branch ${BASELINE_TAG} ${BASELINE_COMMIT}
git filter-branch --prune-empty -f --subdirectory-filter libbpf-projection libbpf-${SUFFIX} git branch ${TIP_TAG} ${TIP_COMMIT}
git branch ${SQUASH_BASE_TAG} ${SQUASH_COMMIT}
git checkout -b ${SQUASH_TIP_TAG} ${SQUASH_COMMIT}
# We don't want to include SQUASHED_BASELINE_COMMIT into patch set, # Cherry-pick new commits onto squashed baseline commit
# but git doesn't allow to exclude rev A in A..B rev range, so instead LIBBPF_PATHS=(tools/lib/bpf tools/include/uapi/linux/{bpf_common.h,bpf.h,btf.h,if_link.h,netlink.h} tools/include/tools/libc_compat.h)
# we calculate number of patches to include LIBBPF_NEW_COMMITS=$(git rev-list --topo-order --reverse ${BASELINE_TAG}..${TIP_TAG} ${LIBBPF_PATHS[@]})
COMMIT_CNT=$(($(git rev-list --count ${SQUASHED_BASELINE_COMMIT}..libbpf-${SUFFIX}) - 1)) for LIBBPF_NEW_COMMIT in ${LIBBPF_NEW_COMMITS}; do
git cherry-pick ${LIBBPF_NEW_COMMIT}
done
# If baseline is the only commit with libbpf-related changes, bail out LIBBPF_TREE_FILTER=' \
mkdir -p __libbpf/include/uapi/linux __libbpf/include/tools && \
git mv -kf tools/lib/bpf __libbpf/src && \
git mv -kf tools/include/uapi/linux/{bpf_common.h,bpf.h,btf.h,if_link.h,netlink.h} __libbpf/include/uapi/linux && \
git mv -kf tools/include/tools/libc_compat.h __libbpf/include/tools && \
git rm --ignore-unmatch -f __libbpf/src/{Makefile,Build,test_libbpf.cpp,.gitignore} \
'
# Move all libbpf files into __libbpf directory.
git filter-branch --prune-empty -f --tree-filter "${LIBBPF_TREE_FILTER}" ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG}
# Make __libbpf a new root directory
git filter-branch --prune-empty -f --subdirectory-filter __libbpf ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG}
# If there are no new commits with libbpf-related changes, bail out
COMMIT_CNT=$(git rev-list --count ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG})
if ((${COMMIT_CNT} <= 0)); then if ((${COMMIT_CNT} <= 0)); then
echo "No new changes to apply, we are done!" echo "No new changes to apply, we are done!"
exit 2 exit 2
fi fi
# Exclude baseline commit and generate nice cover letter with summary # Exclude baseline commit and generate nice cover letter with summary
git format-patch -${COMMIT_CNT} --cover-letter -o /tmp/libbpf-${SUFFIX}.patches git format-patch ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --cover-letter -o ${TMP_DIR}/patches
# Now generate single-file patchset w/o cover to apply on top of libbpf repo
git format-patch ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --stdout > ${TMP_DIR}/patchset.patch
# Now generate single-file patchset to apply on top of libbpf repo, # Now is time to re-apply libbpf-related linux patches to libbpf repo
# this time including "sync commit" as well cd ${WORKDIR} && cd ${LIBBPF_REPO}
git format-patch -${COMMIT_CNT} --stdout > /tmp/libbpf-${SUFFIX}.patchset git checkout -b ${LIBBPF_SYNC_TAG}
echo "PATCHSET: /tmp/libbpf-${SUFFIX}.patchset" git am --committer-date-is-author-date ${TMP_DIR}/patchset.patch
cd ${WORKDIR}
cd ${LIBBPF_REPO}
git checkout -b libbpf-sync-${SUFFIX}
git am /tmp/libbpf-${SUFFIX}.patchset
# Use generated cover-letter as a template for "sync commit" with # Use generated cover-letter as a template for "sync commit" with
# baseline and checkpoint commits from kernel repo (and leave summary # baseline and checkpoint commits from kernel repo (and leave summary
# from cover letter intact, of course) # from cover letter intact, of course)
echo ${CHECKPOINT_COMMIT} > CHECKPOINT-COMMIT && \ echo ${TIP_COMMIT} > CHECKPOINT-COMMIT && \
git add CHECKPOINT-COMMIT && \ git add CHECKPOINT-COMMIT && \
awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' /tmp/libbpf-${SUFFIX}.patches/0000-cover-letter.patch | \ awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' ${TMP_DIR}/patches/0000-cover-letter.patch | \
sed "s/\*\*\* BLURB HERE \*\*\*/\ sed "s/\*\*\* BLURB HERE \*\*\*/\
sync: latest libbpf changes from kernel\n\ sync: latest libbpf changes from kernel\n\
\n\ \n\
Syncing latest libbpf commits from kernel repository.\n\ Syncing latest libbpf commits from kernel repository.\n\
Baseline commit: ${BASELINE_COMMIT}\n\ Baseline commit: ${BASELINE_COMMIT}\n\
Checkpoint commit: ${CHECKPOINT_COMMIT}/" | \ Checkpoint commit: ${TIP_COMMIT}/" | \
git commit --file=- git commit --file=-
# Clean up a bit after ourselvest echo "SUCCESS! ${COMMIT_CNT} commits synced."
rm -r /tmp/libbpf-${SUFFIX}.patches /tmp/libbpf-${SUFFIX}.patchset
echo "Verifying Linux's and Github's libbpf state"
LIBBPF_VIEW_PATHS=(src include/uapi/linux/{bpf_common.h,bpf.h,btf.h,if_link.h,netlink.h} include/tools/libc_compat.h)
LIBBPF_VIEW_EXCLUDE_REGEX='^src/(Makefile|Build|test_libbpf.cpp|\.gitignore)$'
cd ${WORKDIR} && cd ${LINUX_REPO}
LINUX_ABS_DIR=$(pwd)
git checkout -b ${VIEW_TAG} ${TIP_COMMIT}
git filter-branch -f --tree-filter "${LIBBPF_TREE_FILTER}" ${VIEW_TAG}^..${VIEW_TAG}
git filter-branch -f --subdirectory-filter __libbpf ${VIEW_TAG}^..${VIEW_TAG}
git ls-files -- ${LIBBPF_VIEW_PATHS[@]} > ${TMP_DIR}/linux-view.ls
cd ${WORKDIR} && cd ${LIBBPF_REPO}
GITHUB_ABS_DIR=$(pwd)
git ls-files -- ${LIBBPF_VIEW_PATHS[@]} | grep -v -E "${LIBBPF_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/github-view.ls
echo "Comparing list of files..."
diff ${TMP_DIR}/linux-view.ls ${TMP_DIR}/github-view.ls
echo "Comparing file contents..."
for F in $(cat ${TMP_DIR}/linux-view.ls); do
diff "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"
done
echo "Contents appear identical!"
echo "Cleaning up..."
rm -r ${TMP_DIR}
cd ${WORKDIR} && cd ${LINUX_REPO}
git checkout ${TIP_SYM_REF}
git branch -D ${BASELINE_TAG} ${TIP_TAG} ${SQUASH_BASE_TAG} ${SQUASH_TIP_TAG} ${VIEW_TAG}
cd ${WORKDIR}
echo "DONE."
echo "Success! ${COMMIT_CNT} commits applied."