From 7e89be4022f639061dbeae9e8d3a42a20836ec05 Mon Sep 17 00:00:00 2001 From: Yucong Sun Date: Thu, 11 Nov 2021 15:17:43 -0800 Subject: [PATCH 001/131] Migrate vmtest to modular actions in libbpf/ci --- .github/actions/build-selftests/action.yml | 30 + .../build-selftests}/build_selftests.sh | 16 +- .github/actions/build-selftests/helpers.sh | 44 ++ .../prepare_selftests-4.9.0.sh | 0 .../prepare_selftests-5.5.0.sh | 0 .../actions/build-selftests}/vmlinux.h | 0 .github/actions/vmtest/action.yml | 87 ++- .github/workflows/build.yml | 81 +++ .github/workflows/test.yml | 77 +-- travis-ci/{vmtest => rootfs}/mkrootfs_arch.sh | 0 .../{vmtest => rootfs}/mkrootfs_debian.sh | 0 .../{vmtest => rootfs}/mkrootfs_tweak.sh | 0 .../s390x-self-hosted-builder/README.md | 0 .../actions-runner-libbpf.Dockerfile | 0 .../actions-runner-libbpf.service | 0 .../fs/usr/bin/actions-runner | 0 .../fs/usr/bin/entrypoint | 0 .../qemu-user-static.service | 0 travis-ci/vmtest/build_pahole.sh | 34 -- travis-ci/vmtest/checkout_latest_kernel.sh | 54 -- travis-ci/vmtest/configs/INDEX | 9 - travis-ci/vmtest/helpers.sh | 2 - travis-ci/vmtest/prepare_selftests.sh | 21 - travis-ci/vmtest/run.sh | 569 ------------------ travis-ci/vmtest/run_selftests.sh | 6 +- travis-ci/vmtest/run_vmtest.sh | 53 -- 26 files changed, 241 insertions(+), 842 deletions(-) create mode 100644 .github/actions/build-selftests/action.yml rename {travis-ci/vmtest => .github/actions/build-selftests}/build_selftests.sh (69%) create mode 100644 .github/actions/build-selftests/helpers.sh rename {travis-ci/vmtest => .github/actions/build-selftests}/prepare_selftests-4.9.0.sh (100%) rename {travis-ci/vmtest => .github/actions/build-selftests}/prepare_selftests-5.5.0.sh (100%) rename {travis-ci/vmtest => .github/actions/build-selftests}/vmlinux.h (100%) create mode 100644 .github/workflows/build.yml rename travis-ci/{vmtest => rootfs}/mkrootfs_arch.sh (100%) rename travis-ci/{vmtest => rootfs}/mkrootfs_debian.sh (100%) rename travis-ci/{vmtest => rootfs}/mkrootfs_tweak.sh (100%) rename travis-ci/{vmtest => rootfs}/s390x-self-hosted-builder/README.md (100%) rename travis-ci/{vmtest => rootfs}/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile (100%) rename travis-ci/{vmtest => rootfs}/s390x-self-hosted-builder/actions-runner-libbpf.service (100%) rename travis-ci/{vmtest => rootfs}/s390x-self-hosted-builder/fs/usr/bin/actions-runner (100%) rename travis-ci/{vmtest => rootfs}/s390x-self-hosted-builder/fs/usr/bin/entrypoint (100%) rename travis-ci/{vmtest => rootfs}/s390x-self-hosted-builder/qemu-user-static.service (100%) delete mode 100755 travis-ci/vmtest/build_pahole.sh delete mode 100755 travis-ci/vmtest/checkout_latest_kernel.sh delete mode 100644 travis-ci/vmtest/configs/INDEX mode change 100644 => 100755 travis-ci/vmtest/helpers.sh delete mode 100755 travis-ci/vmtest/prepare_selftests.sh delete mode 100755 travis-ci/vmtest/run.sh delete mode 100755 travis-ci/vmtest/run_vmtest.sh diff --git a/.github/actions/build-selftests/action.yml b/.github/actions/build-selftests/action.yml new file mode 100644 index 0000000..af7981b --- /dev/null +++ b/.github/actions/build-selftests/action.yml @@ -0,0 +1,30 @@ +name: 'build-selftests' +description: 'Build BPF selftests' +inputs: + repo-path: + description: 'where is the source code' + required: true + kernel: + description: 'kernel version or LATEST' + required: true + default: 'LATEST' + vmlinux: + description: 'where is vmlinux file' + required: true + default: '${{ github.workspace }}/vmlinux' + +runs: + using: "composite" + steps: + - shell: bash + run: | + echo "::group::Setup Env" + sudo apt-get install -y qemu-kvm zstd binutils-dev elfutils libcap-dev libelf-dev libdw-dev python3-docutils + echo "::endgroup::" + - shell: bash + run: | + export KERNEL=${{ inputs.kernel }} + export REPO_ROOT="${{ github.workspace }}" + export REPO_PATH="${{ inputs.repo-path }}" + export VMLINUX_BTF="${{ inputs.vmlinux }}" + ${{ github.action_path }}/build_selftests.sh diff --git a/travis-ci/vmtest/build_selftests.sh b/.github/actions/build-selftests/build_selftests.sh similarity index 69% rename from travis-ci/vmtest/build_selftests.sh rename to .github/actions/build-selftests/build_selftests.sh index 18dc613..05e3d39 100755 --- a/travis-ci/vmtest/build_selftests.sh +++ b/.github/actions/build-selftests/build_selftests.sh @@ -2,16 +2,16 @@ set -euo pipefail -source $(cd $(dirname $0) && pwd)/helpers.sh +THISDIR="$(cd $(dirname $0) && pwd)" + +source ${THISDIR}/helpers.sh travis_fold start prepare_selftests "Building selftests" -sudo apt-get -y install python3-docutils # for rst2man - LLVM_VER=14 LIBBPF_PATH="${REPO_ROOT}" -PREPARE_SELFTESTS_SCRIPT=${VMTEST_ROOT}/prepare_selftests-${KERNEL}.sh +PREPARE_SELFTESTS_SCRIPT=${THISDIR}/prepare_selftests-${KERNEL}.sh if [ -f "${PREPARE_SELFTESTS_SCRIPT}" ]; then (cd "${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf" && ${PREPARE_SELFTESTS_SCRIPT}) fi @@ -19,9 +19,10 @@ fi if [[ "${KERNEL}" = 'LATEST' ]]; then VMLINUX_H= else - VMLINUX_H=${VMTEST_ROOT}/vmlinux.h + VMLINUX_H=${THISDIR}/vmlinux.h fi +cd ${REPO_ROOT}/${REPO_PATH} make \ CLANG=clang-${LLVM_VER} \ LLC=llc-${LLVM_VER} \ @@ -29,7 +30,8 @@ make \ VMLINUX_BTF="${VMLINUX_BTF}" \ VMLINUX_H=${VMLINUX_H} \ -C "${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf" \ - -j $((4*$(nproc))) >/dev/null + -j $((4*$(nproc))) > /dev/null +cd - mkdir ${LIBBPF_PATH}/selftests cp -R "${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf" \ ${LIBBPF_PATH}/selftests @@ -37,6 +39,4 @@ cd ${LIBBPF_PATH} rm selftests/bpf/.gitignore git add selftests -git add "${VMTEST_ROOT}"/configs/blacklist/BLACKLIST-* "${VMTEST_ROOT}"/configs/whitelist/WHITELIST-* - travis_fold end prepare_selftests diff --git a/.github/actions/build-selftests/helpers.sh b/.github/actions/build-selftests/helpers.sh new file mode 100644 index 0000000..856660a --- /dev/null +++ b/.github/actions/build-selftests/helpers.sh @@ -0,0 +1,44 @@ +# $1 - start or end +# $2 - fold identifier, no spaces +# $3 - fold section description +travis_fold() { + local YELLOW='\033[1;33m' + local NOCOLOR='\033[0m' + if [ -z ${GITHUB_WORKFLOW+x} ]; then + echo travis_fold:$1:$2 + if [ ! -z "${3:-}" ]; then + echo -e "${YELLOW}$3${NOCOLOR}" + fi + echo + else + if [ $1 = "start" ]; then + line="::group::$2" + if [ ! -z "${3:-}" ]; then + line="$line - ${YELLOW}$3${NOCOLOR}" + fi + else + line="::endgroup::" + fi + echo -e "$line" + fi +} + +__print() { + local TITLE="" + if [[ -n $2 ]]; then + TITLE=" title=$2" + fi + echo "::$1${TITLE}::$3" +} + +# $1 - title +# $2 - message +print_error() { + __print error $1 $2 +} + +# $1 - title +# $2 - message +print_notice() { + __print notice $1 $2 +} diff --git a/travis-ci/vmtest/prepare_selftests-4.9.0.sh b/.github/actions/build-selftests/prepare_selftests-4.9.0.sh similarity index 100% rename from travis-ci/vmtest/prepare_selftests-4.9.0.sh rename to .github/actions/build-selftests/prepare_selftests-4.9.0.sh diff --git a/travis-ci/vmtest/prepare_selftests-5.5.0.sh b/.github/actions/build-selftests/prepare_selftests-5.5.0.sh similarity index 100% rename from travis-ci/vmtest/prepare_selftests-5.5.0.sh rename to .github/actions/build-selftests/prepare_selftests-5.5.0.sh diff --git a/travis-ci/vmtest/vmlinux.h b/.github/actions/build-selftests/vmlinux.h similarity index 100% rename from travis-ci/vmtest/vmlinux.h rename to .github/actions/build-selftests/vmlinux.h diff --git a/.github/actions/vmtest/action.yml b/.github/actions/vmtest/action.yml index b0deb51..2f86607 100644 --- a/.github/actions/vmtest/action.yml +++ b/.github/actions/vmtest/action.yml @@ -5,31 +5,78 @@ inputs: description: 'kernel version or LATEST' required: true default: 'LATEST' - kernel-rev: - description: 'CHECKPOINT or rev/tag/branch' + arch: + description: 'what arch to test' required: true - default: 'CHECKPOINT' - kernel-origin: - description: 'kernel repo' - required: true - default: 'https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git' + default: 'x86_64' pahole: - description: 'pahole rev/tag/branch' + description: 'pahole rev or master' required: true default: 'master' - pahole-origin: - description: 'pahole repo' - required: true - default: 'https://git.kernel.org/pub/scm/devel/pahole/pahole.git' runs: using: "composite" steps: - - run: | - source /tmp/ci_setup - export KERNEL=${{ inputs.kernel }} - export KERNEL_BRANCH=${{ inputs.kernel-rev }} - export KERNEL_ORIGIN=${{ inputs.kernel-origin }} - export PAHOLE_BRANCH=${{ inputs.pahole }} - export PAHOLE_ORIGIN=${{ inputs.pahole-origin }} - $CI_ROOT/vmtest/run_vmtest.sh + # setup envinronment + - name: Setup environment + uses: libbpf/ci/setup-build-env@master + with: + pahole: ${{ inputs.pahole }} + # 1. download CHECKPOINT kernel source + - name: Get checkpoint commit shell: bash + run: | + cat CHECKPOINT-COMMIT + echo "CHECKPOINT=$(cat CHECKPOINT-COMMIT)" >> $GITHUB_ENV + - name: Get kernel source at checkpoint + uses: libbpf/ci/get-linux-source@master + with: + repo: 'https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git' + rev: ${{ env.CHECKPOINT }} + dest: '${{ github.workspace }}/.kernel' + - name: Prepare to build BPF selftests + shell: bash + run: | + echo "::group::Prepare buidling selftest" + cd .kernel + cp ${{ github.workspace }}/travis-ci/vmtest/configs/config-latest.${{ inputs.arch }} .config + make olddefconfig && make prepare + cd - + echo "::endgroup::" + # 2. if kernel == LATEST, build kernel image from tree + - name: Build kernel image + if: ${{ inputs.kernel == 'LATEST' }} + shell: bash + run: | + echo "::group::Build Kernel Image" + cd .kernel + make -j $((4*$(nproc))) all > /dev/null + cp vmlinux ${{ github.workspace }} + cd - + echo "::endgroup::" + # else, just download prebuilt kernel image + - name: Download prebuilt kernel + if: ${{ inputs.kernel != 'LATEST' }} + uses: libbpf/ci/download-vmlinux@master + with: + kernel: ${{ inputs.kernel }} + arch: ${{ inputs.arch }} + # 3. build selftests + - name: Build BPF selftests + uses: ./.github/actions/build-selftests + with: + repo-path: '.kernel' + kernel: ${{ inputs.kernel }} + # 4. prepare rootfs + - name: prepare rootfs + uses: libbpf/ci/prepare-rootfs@master + with: + kernel: ${{ inputs.kernel }} + project-name: 'libbpf' + arch: ${{ inputs.arch }} + # 5. run selftest in QEMU + - name: Run selftests + uses: libbpf/ci/run-qemu@master + with: + img: '/tmp/root.img' + vmlinuz: 'vmlinuz' + arch: ${{ inputs.arch }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..25768d6 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,81 @@ +name: libbpf-build + +on: + pull_request: + push: + schedule: + - cron: '0 18 * * *' + +concurrency: + group: ci-build-${{ github.head_ref }} + cancel-in-progress: true + +jobs: + + debian: + runs-on: ubuntu-latest + name: Debian Build (${{ matrix.name }}) + strategy: + fail-fast: false + matrix: + include: + - name: default + target: RUN + - name: ASan+UBSan + target: RUN_ASAN + - name: clang + target: RUN_CLANG + - name: clang ASan+UBSan + target: RUN_CLANG_ASAN + - name: gcc-10 + target: RUN_GCC10 + - name: gcc-10 ASan+UBSan + target: RUN_GCC10_ASAN + steps: + - uses: actions/checkout@v2 + name: Checkout + - uses: ./.github/actions/setup + name: Setup + - uses: ./.github/actions/debian + name: Build + with: + target: ${{ matrix.target }} + + ubuntu: + runs-on: ubuntu-latest + name: Ubuntu Focal Build (${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - arch: aarch64 + - arch: ppc64le + - arch: s390x + - arch: x86 + steps: + - uses: actions/checkout@v2 + name: Checkout + - uses: ./.github/actions/setup + name: Pre-Setup + - run: source /tmp/ci_setup && sudo -E $CI_ROOT/managers/ubuntu.sh + if: matrix.arch == 'x86' + name: Setup + - uses: uraimo/run-on-arch-action@v2.0.5 + name: Build in docker + if: matrix.arch != 'x86' + with: + distro: + ubuntu20.04 + arch: + ${{ matrix.arch }} + setup: + cp /tmp/ci_setup $GITHUB_WORKSPACE + dockerRunArgs: | + --volume "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}" + shell: /bin/bash + install: | + export DEBIAN_FRONTEND=noninteractive + export TZ="America/Los_Angeles" + apt-get update -y + apt-get install -y tzdata build-essential sudo + run: source ${GITHUB_WORKSPACE}/ci_setup && $CI_ROOT/managers/ubuntu.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64dbbcb..5904843 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,12 +20,16 @@ jobs: include: - kernel: 'LATEST' runs_on: ubuntu-latest - - kernel: 'LATEST' - runs_on: z15 + arch: 'x86_64' - kernel: '5.5.0' runs_on: ubuntu-latest + arch: 'x86_64' - kernel: '4.9.0' runs_on: ubuntu-latest + arch: 'x86_64' + - kernel: 'LATEST' + runs_on: z15 + arch: 's390x' steps: - uses: actions/checkout@v2 name: Checkout @@ -35,71 +39,4 @@ jobs: name: vmtest with: kernel: ${{ matrix.kernel }} - - debian: - runs-on: ubuntu-latest - name: Debian Build (${{ matrix.name }}) - strategy: - fail-fast: false - matrix: - include: - - name: default - target: RUN - - name: ASan+UBSan - target: RUN_ASAN - - name: clang - target: RUN_CLANG - - name: clang ASan+UBSan - target: RUN_CLANG_ASAN - - name: gcc-10 - target: RUN_GCC10 - - name: gcc-10 ASan+UBSan - target: RUN_GCC10_ASAN - steps: - - uses: actions/checkout@v2 - name: Checkout - - uses: ./.github/actions/setup - name: Setup - - uses: ./.github/actions/debian - name: Build - with: - target: ${{ matrix.target }} - - ubuntu: - runs-on: ubuntu-latest - name: Ubuntu Focal Build (${{ matrix.arch }}) - strategy: - fail-fast: false - matrix: - include: - - arch: aarch64 - - arch: ppc64le - - arch: s390x - - arch: x86 - steps: - - uses: actions/checkout@v2 - name: Checkout - - uses: ./.github/actions/setup - name: Pre-Setup - - run: source /tmp/ci_setup && sudo -E $CI_ROOT/managers/ubuntu.sh - if: matrix.arch == 'x86' - name: Setup - - uses: uraimo/run-on-arch-action@v2.0.5 - name: Build in docker - if: matrix.arch != 'x86' - with: - distro: - ubuntu20.04 - arch: - ${{ matrix.arch }} - setup: - cp /tmp/ci_setup $GITHUB_WORKSPACE - dockerRunArgs: | - --volume "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}" - shell: /bin/bash - install: | - export DEBIAN_FRONTEND=noninteractive - export TZ="America/Los_Angeles" - apt-get update -y - apt-get install -y tzdata build-essential sudo - run: source ${GITHUB_WORKSPACE}/ci_setup && $CI_ROOT/managers/ubuntu.sh + arch: ${{ matrix.arch }} diff --git a/travis-ci/vmtest/mkrootfs_arch.sh b/travis-ci/rootfs/mkrootfs_arch.sh similarity index 100% rename from travis-ci/vmtest/mkrootfs_arch.sh rename to travis-ci/rootfs/mkrootfs_arch.sh diff --git a/travis-ci/vmtest/mkrootfs_debian.sh b/travis-ci/rootfs/mkrootfs_debian.sh similarity index 100% rename from travis-ci/vmtest/mkrootfs_debian.sh rename to travis-ci/rootfs/mkrootfs_debian.sh diff --git a/travis-ci/vmtest/mkrootfs_tweak.sh b/travis-ci/rootfs/mkrootfs_tweak.sh similarity index 100% rename from travis-ci/vmtest/mkrootfs_tweak.sh rename to travis-ci/rootfs/mkrootfs_tweak.sh diff --git a/travis-ci/vmtest/s390x-self-hosted-builder/README.md b/travis-ci/rootfs/s390x-self-hosted-builder/README.md similarity index 100% rename from travis-ci/vmtest/s390x-self-hosted-builder/README.md rename to travis-ci/rootfs/s390x-self-hosted-builder/README.md diff --git a/travis-ci/vmtest/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile b/travis-ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile similarity index 100% rename from travis-ci/vmtest/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile rename to travis-ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile diff --git a/travis-ci/vmtest/s390x-self-hosted-builder/actions-runner-libbpf.service b/travis-ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.service similarity index 100% rename from travis-ci/vmtest/s390x-self-hosted-builder/actions-runner-libbpf.service rename to travis-ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.service diff --git a/travis-ci/vmtest/s390x-self-hosted-builder/fs/usr/bin/actions-runner b/travis-ci/rootfs/s390x-self-hosted-builder/fs/usr/bin/actions-runner similarity index 100% rename from travis-ci/vmtest/s390x-self-hosted-builder/fs/usr/bin/actions-runner rename to travis-ci/rootfs/s390x-self-hosted-builder/fs/usr/bin/actions-runner diff --git a/travis-ci/vmtest/s390x-self-hosted-builder/fs/usr/bin/entrypoint b/travis-ci/rootfs/s390x-self-hosted-builder/fs/usr/bin/entrypoint similarity index 100% rename from travis-ci/vmtest/s390x-self-hosted-builder/fs/usr/bin/entrypoint rename to travis-ci/rootfs/s390x-self-hosted-builder/fs/usr/bin/entrypoint diff --git a/travis-ci/vmtest/s390x-self-hosted-builder/qemu-user-static.service b/travis-ci/rootfs/s390x-self-hosted-builder/qemu-user-static.service similarity index 100% rename from travis-ci/vmtest/s390x-self-hosted-builder/qemu-user-static.service rename to travis-ci/rootfs/s390x-self-hosted-builder/qemu-user-static.service diff --git a/travis-ci/vmtest/build_pahole.sh b/travis-ci/vmtest/build_pahole.sh deleted file mode 100755 index b1f54e7..0000000 --- a/travis-ci/vmtest/build_pahole.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -eu - -source $(cd $(dirname $0) && pwd)/helpers.sh - -CWD=$(pwd) -REPO_PATH=$1 -PAHOLE_ORIGIN=${PAHOLE_ORIGIN:-https://git.kernel.org/pub/scm/devel/pahole/pahole.git} -PAHOLE_BRANCH=${PAHOLE_BRANCH:-master} - -travis_fold start build_pahole "Building pahole ${PAHOLE_ORIGIN} ${PAHOLE_BRANCH}" - -mkdir -p ${REPO_PATH} -cd ${REPO_PATH} -git init -git remote add origin ${PAHOLE_ORIGIN} -git fetch origin -git checkout ${PAHOLE_BRANCH} - -# temporary work-around to bump pahole to 1.22 before it is officially released -sed -i 's/DDWARVES_MINOR_VERSION=21/DDWARVES_MINOR_VERSION=22/' CMakeLists.txt - -mkdir -p build -cd build -cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -D__LIB=lib .. -make -j$((4*$(nproc))) all -sudo make install - -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib -ldd $(which pahole) -pahole --version - -travis_fold end build_pahole diff --git a/travis-ci/vmtest/checkout_latest_kernel.sh b/travis-ci/vmtest/checkout_latest_kernel.sh deleted file mode 100755 index 6cc3c3a..0000000 --- a/travis-ci/vmtest/checkout_latest_kernel.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -set -eu - -source $(cd $(dirname $0) && pwd)/helpers.sh - -CWD=$(pwd) -LIBBPF_PATH=$(pwd) -REPO_PATH=$1 - -KERNEL_ORIGIN=${KERNEL_ORIGIN:-https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git} -KERNEL_BRANCH=${KERNEL_BRANCH:-CHECKPOINT} -if [[ "${KERNEL_BRANCH}" = 'CHECKPOINT' ]]; then - echo "using CHECKPOINT sha1" - LINUX_SHA=$(cat ${LIBBPF_PATH}/CHECKPOINT-COMMIT) -else - echo "using ${KERNEL_BRANCH} sha1" - LINUX_SHA=$(git ls-remote ${KERNEL_ORIGIN} ${KERNEL_BRANCH} | awk '{print $1}') -fi -SNAPSHOT_URL=${KERNEL_ORIGIN}/snapshot/bpf-next-${LINUX_SHA}.tar.gz - -echo REPO_PATH = ${REPO_PATH} - -echo KERNEL_ORIGIN = ${KERNEL_ORIGIN} -echo LINUX_SHA = ${LINUX_SHA} -echo SNAPSHOT_URL = ${SNAPSHOT_URL} - -if [ ! -d "${REPO_PATH}" ]; then - echo - travis_fold start pull_kernel_srcs "Fetching kernel sources" - - mkdir -p $(dirname "${REPO_PATH}") - cd $(dirname "${REPO_PATH}") - # attempt to fetch desired bpf-next repo snapshot - if wget -nv ${SNAPSHOT_URL} && tar xf bpf-next-${LINUX_SHA}.tar.gz --totals ; then - mv bpf-next-${LINUX_SHA} $(basename ${REPO_PATH}) - else - # but fallback to git fetch approach if that fails - mkdir -p $(basename ${REPO_PATH}) - cd $(basename ${REPO_PATH}) - git init - git remote add bpf-next ${KERNEL_ORIGIN} - # try shallow clone first - git fetch --depth 32 bpf-next - # check if desired SHA exists - if ! git cat-file -e ${LINUX_SHA}^{commit} ; then - # if not, fetch all of bpf-next; slow and painful - git fetch bpf-next - fi - git reset --hard ${LINUX_SHA} - fi - - travis_fold end pull_kernel_srcs -fi diff --git a/travis-ci/vmtest/configs/INDEX b/travis-ci/vmtest/configs/INDEX deleted file mode 100644 index b6cbf08..0000000 --- a/travis-ci/vmtest/configs/INDEX +++ /dev/null @@ -1,9 +0,0 @@ -INDEX https://libbpf-vmtest.s3-us-west-1.amazonaws.com/x86_64/INDEX -x86_64/libbpf-vmtest-rootfs-2020.09.27.tar.zst https://libbpf-vmtest.s3-us-west-1.amazonaws.com/x86_64/libbpf-vmtest-rootfs-2020.09.27.tar.zst -x86_64/vmlinux-4.9.0.zst https://libbpf-vmtest.s3-us-west-1.amazonaws.com/x86_64/vmlinux-4.9.0.zst -x86_64/vmlinux-5.5.0-rc6.zst https://libbpf-vmtest.s3-us-west-1.amazonaws.com/x86_64/vmlinux-5.5.0-rc6.zst -x86_64/vmlinux-5.5.0.zst https://libbpf-vmtest.s3-us-west-1.amazonaws.com/x86_64/vmlinux-5.5.0.zst -x86_64/vmlinuz-5.5.0-rc6 https://libbpf-vmtest.s3-us-west-1.amazonaws.com/x86_64/vmlinuz-5.5.0-rc6 -x86_64/vmlinuz-5.5.0 https://libbpf-vmtest.s3-us-west-1.amazonaws.com/x86_64/vmlinuz-5.5.0 -x86_64/vmlinuz-4.9.0 https://libbpf-vmtest.s3-us-west-1.amazonaws.com/x86_64/vmlinuz-4.9.0 -s390x/libbpf-vmtest-rootfs-2021.03.24.tar.zst https://libbpf-vmtest.s3-us-west-1.amazonaws.com/s390x/libbpf-vmtest-rootfs-2021.03.24.tar.zst diff --git a/travis-ci/vmtest/helpers.sh b/travis-ci/vmtest/helpers.sh old mode 100644 new mode 100755 index 116e72c..856660a --- a/travis-ci/vmtest/helpers.sh +++ b/travis-ci/vmtest/helpers.sh @@ -23,8 +23,6 @@ travis_fold() { fi } -ARCH=$(uname -m) - __print() { local TITLE="" if [[ -n $2 ]]; then diff --git a/travis-ci/vmtest/prepare_selftests.sh b/travis-ci/vmtest/prepare_selftests.sh deleted file mode 100755 index 2ca6c09..0000000 --- a/travis-ci/vmtest/prepare_selftests.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -eu - -source $(cd $(dirname $0) && pwd)/helpers.sh - -REPO_PATH=${1:-} - -if [[ ! -z "$REPO_PATH" ]]; then - ${VMTEST_ROOT}/checkout_latest_kernel.sh ${REPO_PATH} - cd ${REPO_PATH} -fi - -if [[ "${KERNEL}" = 'LATEST' ]]; then - travis_fold start build_kernel "Kernel build" - - cp "$VMTEST_ROOT"/configs/config-latest."$ARCH" .config - make -j $((4*$(nproc))) olddefconfig all >/dev/null - travis_fold end build_kernel -fi - diff --git a/travis-ci/vmtest/run.sh b/travis-ci/vmtest/run.sh deleted file mode 100755 index 2a765b0..0000000 --- a/travis-ci/vmtest/run.sh +++ /dev/null @@ -1,569 +0,0 @@ -#!/bin/bash - -set -uo pipefail -trap 'exit 2' ERR - -source $(cd $(dirname $0) && pwd)/helpers.sh - -usage () { - USAGE_STRING="usage: $0 [-k KERNELRELEASE|-b DIR] [[-r ROOTFSVERSION] [-fo]|-I] [-Si] [-d DIR] IMG - $0 [-k KERNELRELEASE] -l - $0 -h - -Run "${PROJECT_NAME}" tests in a virtual machine. - -This exits with status 0 on success, 1 if the virtual machine ran successfully -but tests failed, and 2 if we encountered a fatal error. - -This script uses sudo to work around a libguestfs bug. - -Arguments: - IMG path of virtual machine disk image to create - -Versions: - -k, --kernel=KERNELRELEASE - kernel release to test. This is a glob pattern; the - newest (sorted by version number) release that matches - the pattern is used (default: newest available release) - - -b, --build DIR use the kernel built in the given directory. This option - cannot be combined with -k - - -r, --rootfs=ROOTFSVERSION - version of root filesystem to use (default: newest - available version) - -Setup: - -f, --force overwrite IMG if it already exists - - -o, --one-shot one-shot mode. By default, this script saves a clean copy - of the downloaded root filesystem image and vmlinux and - makes a copy (reflinked, when possible) for executing the - virtual machine. This allows subsequent runs to skip - downloading these files. If this option is given, the - root filesystem image and vmlinux are always - re-downloaded and are not saved. This option implies -f - - -s, --setup-cmd setup commands run on VM boot. Whitespace characters - should be escaped with preceding '\'. - - -I, --skip-image skip creating the disk image; use the existing one at - IMG. This option cannot be combined with -r, -f, or -o - - -S, --skip-source skip copying the source files and init scripts - -Miscellaneous: - -i, --interactive interactive mode. Boot the virtual machine into an - interactive shell instead of automatically running tests - - -d, --dir=DIR working directory to use for downloading and caching - files (default: current working directory) - - -l, --list list available kernel releases instead of running tests. - The list may be filtered with -k - - -h, --help display this help message and exit" - - case "$1" in - out) - echo "$USAGE_STRING" - exit 0 - ;; - err) - echo "$USAGE_STRING" >&2 - exit 2 - ;; - esac -} - -TEMP=$(getopt -o 'k:b:r:fos:ISid:lh' --long 'kernel:,build:,rootfs:,force,one-shot,setup-cmd,skip-image,skip-source:,interactive,dir:,list,help' -n "$0" -- "$@") -eval set -- "$TEMP" -unset TEMP - -unset KERNELRELEASE -unset BUILDDIR -unset ROOTFSVERSION -unset IMG -unset SETUPCMD -FORCE=0 -ONESHOT=0 -SKIPIMG=0 -SKIPSOURCE=0 -APPEND="" -DIR="$PWD" -LIST=0 - -# by default will copy all files that aren't listed in git exclusions -# but it doesn't work for entire kernel tree very well -# so for full kernel tree you may need to SOURCE_FULLCOPY=0 -SOURCE_FULLCOPY=${SOURCE_FULLCOPY:-1} - -while true; do - case "$1" in - -k|--kernel) - KERNELRELEASE="$2" - shift 2 - ;; - -b|--build) - BUILDDIR="$2" - shift 2 - ;; - -r|--rootfs) - ROOTFSVERSION="$2" - shift 2 - ;; - -f|--force) - FORCE=1 - shift - ;; - -o|--one-shot) - ONESHOT=1 - FORCE=1 - shift - ;; - -s|--setup-cmd) - SETUPCMD="$2" - shift 2 - ;; - -I|--skip-image) - SKIPIMG=1 - shift - ;; - -S|--skip-source) - SKIPSOURCE=1 - shift - ;; - -i|--interactive) - APPEND=" single" - shift - ;; - -d|--dir) - DIR="$2" - shift 2 - ;; - -l|--list) - LIST=1 - ;; - -h|--help) - usage out - ;; - --) - shift - break - ;; - *) - usage err - ;; - esac -done -if [[ -v BUILDDIR ]]; then - if [[ -v KERNELRELEASE ]]; then - usage err - fi -elif [[ ! -v KERNELRELEASE ]]; then - KERNELRELEASE='*' -fi -if [[ $SKIPIMG -ne 0 && ( -v ROOTFSVERSION || $FORCE -ne 0 ) ]]; then - usage err -fi -if (( LIST )); then - if [[ $# -ne 0 || -v BUILDDIR || -v ROOTFSVERSION || $FORCE -ne 0 || - $SKIPIMG -ne 0 || $SKIPSOURCE -ne 0 || -n $APPEND ]]; then - usage err - fi -else - if [[ $# -ne 1 ]]; then - usage err - fi - IMG="${!OPTIND}" -fi -if [[ "${SOURCE_FULLCOPY}" == "1" ]]; then - img_size=2G -else - img_size=8G -fi - -unset URLS -cache_urls() { - if ! declare -p URLS &> /dev/null; then - # This URL contains a mapping from file names to URLs where - # those files can be downloaded. - declare -gA URLS - while IFS=$'\t' read -r name url; do - URLS["$name"]="$url" - done < <(cat "${VMTEST_ROOT}/configs/INDEX") - fi -} - -matching_kernel_releases() { - local pattern="$1" - { - for file in "${!URLS[@]}"; do - if [[ $file =~ ^${ARCH}/vmlinux-(.*).zst$ ]]; then - release="${BASH_REMATCH[1]}" - case "$release" in - $pattern) - # sort -V handles rc versions properly - # if we use "~" instead of "-". - echo "${release//-rc/~rc}" - ;; - esac - fi - done - } | sort -rV | sed 's/~rc/-rc/g' -} - -newest_rootfs_version() { - { - for file in "${!URLS[@]}"; do - if [[ $file =~ ^${ARCH}/${PROJECT_NAME}-vmtest-rootfs-(.*)\.tar\.zst$ ]]; then - echo "${BASH_REMATCH[1]}" - fi - done - } | sort -rV | head -1 -} - -download() { - local file="$1" - cache_urls - if [[ ! -v URLS[$file] ]]; then - echo "$file not found" >&2 - return 1 - fi - echo "Downloading $file..." >&2 - curl -Lf "${URLS[$file]}" "${@:2}" -} - -set_nocow() { - touch "$@" - chattr +C "$@" >/dev/null 2>&1 || true -} - -cp_img() { - set_nocow "$2" - cp --reflink=auto "$1" "$2" -} - -create_rootfs_img() { - local path="$1" - set_nocow "$path" - truncate -s "$img_size" "$path" - mkfs.ext4 -q "$path" -} - -download_rootfs() { - local rootfsversion="$1" - download "${ARCH}/${PROJECT_NAME}-vmtest-rootfs-$rootfsversion.tar.zst" | - zstd -d -} - -tar_in() { - local dst_path="$1" - # guestfish --remote does not forward file descriptors, which prevents - # us from using `tar-in -` or bash process substitution. We don't want - # to copy all the data into a temporary file, so use a FIFO. - tmp=$(mktemp -d) - mkfifo "$tmp/fifo" - cat >"$tmp/fifo" & - local cat_pid=$! - guestfish --remote tar-in "$tmp/fifo" "$dst_path" - wait "$cat_pid" - rm -r "$tmp" - tmp= -} - -if (( LIST )); then - cache_urls - matching_kernel_releases "$KERNELRELEASE" - exit 0 -fi - -if [[ $FORCE -eq 0 && $SKIPIMG -eq 0 && -e $IMG ]]; then - echo "$IMG already exists; use -f to overwrite it or -I to reuse it" >&2 - exit 1 -fi - -# Only go to the network if it's actually a glob pattern. -if [[ -v BUILDDIR ]]; then - KERNELRELEASE="$(make -C "$BUILDDIR" -s kernelrelease)" -elif [[ ! $KERNELRELEASE =~ ^([^\\*?[]|\\[*?[])*\\?$ ]]; then - # We need to cache the list of URLs outside of the command - # substitution, which happens in a subshell. - cache_urls - KERNELRELEASE="$(matching_kernel_releases "$KERNELRELEASE" | head -1)" - if [[ -z $KERNELRELEASE ]]; then - echo "No matching kernel release found" >&2 - exit 1 - fi -fi -if [[ $SKIPIMG -eq 0 && ! -v ROOTFSVERSION ]]; then - cache_urls - ROOTFSVERSION="$(newest_rootfs_version)" -fi - -travis_fold start vmlinux_setup "Preparing Linux image" - -echo "Kernel release: $KERNELRELEASE" >&2 -echo - -if (( SKIPIMG )); then - echo "Not extracting root filesystem" >&2 -else - echo "Root filesystem version: $ROOTFSVERSION" >&2 -fi -echo "Disk image: $IMG" >&2 - -tmp= -ARCH_DIR="$DIR/$ARCH" -mkdir -p "$ARCH_DIR" - -cleanup() { - if [[ -n $tmp ]]; then - rm -rf "$tmp" || true - fi - guestfish --remote exit 2>/dev/null || true -} -trap cleanup EXIT - -if [[ -v BUILDDIR ]]; then - vmlinuz="$BUILDDIR/$(make -C "$BUILDDIR" -s image_name)" -else - vmlinuz="${ARCH_DIR}/vmlinuz-${KERNELRELEASE}" - if [[ ! -e $vmlinuz ]]; then - tmp="$(mktemp "$vmlinuz.XXX.part")" - download "${ARCH}/vmlinuz-${KERNELRELEASE}" -o "$tmp" - mv "$tmp" "$vmlinuz" - tmp= - fi -fi - -# Mount and set up the rootfs image. Use a persistent guestfish session in -# order to avoid the startup overhead. -# Work around https://bugs.launchpad.net/fuel/+bug/1467579. -sudo chmod +r /boot/vmlinuz* -eval "$(guestfish --listen)" -if (( ONESHOT )); then - rm -f "$IMG" - create_rootfs_img "$IMG" - guestfish --remote \ - add "$IMG" label:img : \ - launch : \ - mount /dev/disk/guestfs/img / - download_rootfs "$ROOTFSVERSION" | tar_in / -else - if (( ! SKIPIMG )); then - rootfs_img="${ARCH_DIR}/${PROJECT_NAME}-vmtest-rootfs-${ROOTFSVERSION}.img" - - if [[ ! -e $rootfs_img ]]; then - tmp="$(mktemp "$rootfs_img.XXX.part")" - set_nocow "$tmp" - truncate -s "$img_size" "$tmp" - mkfs.ext4 -q "$tmp" - - # libguestfs supports hotplugging only with a libvirt - # backend, which we are not using here, so handle the - # temporary image in a separate session. - download_rootfs "$ROOTFSVERSION" | - guestfish -a "$tmp" tar-in - / - - mv "$tmp" "$rootfs_img" - tmp= - fi - - rm -f "$IMG" - cp_img "$rootfs_img" "$IMG" - fi - guestfish --remote \ - add "$IMG" label:img : \ - launch : \ - mount /dev/disk/guestfs/img / -fi - -# Install vmlinux. -vmlinux="/boot/vmlinux-${KERNELRELEASE}" -if [[ -v BUILDDIR || $ONESHOT -eq 0 ]]; then - if [[ -v BUILDDIR ]]; then - source_vmlinux="${BUILDDIR}/vmlinux" - else - source_vmlinux="${ARCH_DIR}/vmlinux-${KERNELRELEASE}" - if [[ ! -e $source_vmlinux ]]; then - tmp="$(mktemp "$source_vmlinux.XXX.part")" - download "${ARCH}/vmlinux-${KERNELRELEASE}.zst" | zstd -dfo "$tmp" - mv "$tmp" "$source_vmlinux" - tmp= - fi - fi -else - source_vmlinux="${ARCH_DIR}/vmlinux-${KERNELRELEASE}" - download "${ARCH}/vmlinux-${KERNELRELEASE}.zst" | zstd -d >"$source_vmlinux" -fi -echo "Copying vmlinux..." >&2 -guestfish --remote \ - upload "$source_vmlinux" "$vmlinux" : \ - chmod 644 "$vmlinux" - -travis_fold end vmlinux_setup - -REPO_PATH="${SELFTEST_REPO_PATH:-travis-ci/vmtest/bpf-next}" -LIBBPF_PATH="${REPO_ROOT}" \ - VMTEST_ROOT="${VMTEST_ROOT}" \ - REPO_PATH="${REPO_PATH}" \ - VMLINUX_BTF=$(realpath ${source_vmlinux}) ${VMTEST_ROOT}/build_selftests.sh - -travis_fold start bpftool_checks "Running bpftool checks..." -bpftool_exitstatus= -if [[ "${KERNEL}" = 'LATEST' ]]; then - # "&& true" does not change the return code (it is not executed if the - # Python script fails), but it prevents the trap on ERR set at the top - # of this file to trigger on failure. - "${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf/test_bpftool_synctypes.py" && true - bpftool_exitstatus=$? - if [[ "$bpftool_exitstatus" -eq 0 ]]; then - print_notice bpftool_checks "bpftool checks passed successfully." - else - print_error bpftool_checks "bpftool checks returned ${bpftool_exitstatus}." - fi - bpftool_exitstatus="bpftool:${bpftool_exitstatus}" -else - echo "bpftool checks skipped." -fi -travis_fold end bpftool_checks - -travis_fold start vm_init "Starting virtual machine..." - -if (( SKIPSOURCE )); then - echo "Not copying source files..." >&2 -else - echo "Copying source files..." >&2 - # Copy the source files in. - guestfish --remote \ - mkdir-p "/${PROJECT_NAME}" : \ - chmod 0755 "/${PROJECT_NAME}" - if [[ "${SOURCE_FULLCOPY}" == "1" ]]; then - git ls-files -z | tar --null --files-from=- -c | tar_in "/${PROJECT_NAME}" - else - guestfish --remote \ - mkdir-p "/${PROJECT_NAME}/selftests" : \ - chmod 0755 "/${PROJECT_NAME}/selftests" : \ - mkdir-p "/${PROJECT_NAME}/travis-ci" : \ - chmod 0755 "/${PROJECT_NAME}/travis-ci" - tree --du -shaC "${REPO_ROOT}/selftests/bpf" - tar -C "${REPO_ROOT}/selftests" -c bpf | tar_in "/${PROJECT_NAME}/selftests" - tar -C "${REPO_ROOT}/travis-ci" -c vmtest | tar_in "/${PROJECT_NAME}/travis-ci" - fi -fi - -tmp=$(mktemp) -cat <"$tmp" -"#!/bin/sh - -echo 'Skipping setup commands' -echo vm_start:0 > /exitstatus -chmod 644 /exitstatus -HERE - -# Create the init scripts. -if [[ ! -z SETUPCMD ]]; then - # Unescape whitespace characters. - setup_cmd=$(sed 's/\(\\\)\([[:space:]]\)/\2/g' <<< "${SETUPCMD}") - kernel="${KERNELRELEASE}" - if [[ -v BUILDDIR ]]; then kernel='latest'; fi - setup_envvars="export KERNEL=${kernel}" - cat <"$tmp" -#!/bin/sh -set -eux - -echo 'Running setup commands' -${setup_envvars} -set +e -${setup_cmd}; exitstatus=\$? -echo -e '$(travis_fold start collect_status "Collect status")' -set -e -# If setup command did not write its exit status to /exitstatus, do it now -if [[ ! -s /exitstatus ]]; then - echo setup_cmd:\$exitstatus > /exitstatus -fi -chmod 644 /exitstatus -echo -e '$(travis_fold end collect_status)' -echo -e '$(travis_fold start shutdown Shutdown)' -HERE -fi - -guestfish --remote \ - upload "$tmp" /etc/rcS.d/S50-run-tests : \ - chmod 755 /etc/rcS.d/S50-run-tests - -cat <"$tmp" -#!/bin/sh - -poweroff -HERE -guestfish --remote \ - upload "$tmp" /etc/rcS.d/S99-poweroff : \ - chmod 755 /etc/rcS.d/S99-poweroff -rm "$tmp" -tmp= - -guestfish --remote exit - -echo "Starting VM with $(nproc) CPUs..." - -case "$ARCH" in -s390x) - qemu="qemu-system-s390x" - console="ttyS1" - smp=2 - kvm_accel="-enable-kvm" - tcg_accel="-machine accel=tcg" - ;; -x86_64) - qemu="qemu-system-x86_64" - console="ttyS0,115200" - smp=$(nproc) - kvm_accel="-cpu kvm64 -enable-kvm" - tcg_accel="-cpu qemu64 -machine accel=tcg" - ;; -*) - echo "Unsupported architecture" - exit 1 - ;; -esac -if kvm-ok ; then - accel=$kvm_accel -else - accel=$tcg_accel -fi -"$qemu" -nodefaults -display none -serial mon:stdio \ - ${accel} -smp "$smp" -m 4G \ - -drive file="$IMG",format=raw,index=1,media=disk,if=virtio,cache=none \ - -kernel "$vmlinuz" -append "root=/dev/vda rw console=$console kernel.panic=-1 $APPEND" - -# Set exit status to 1 if at least one group test returned non-0 -exitfile="${bpftool_exitstatus}${bpftool_exitstatus:+\n}" -exitfile+="$(guestfish --ro -a "$IMG" -i cat /exitstatus 2>/dev/null)" -exitstatus="$(echo -e "$exitfile" | awk --field-separator ':' \ - 'BEGIN { s=0 } { if ($2) {s=1} } END { print s }')" - -if [[ "$exitstatus" =~ ^[0-9]+$ ]]; then - printf '\nTests exit status: %s\n' "$exitstatus" >&2 -else - printf '\nCould not read tests exit status ("%s")\n' "$exitstatus" >&2 - exitstatus=1 -fi - -travis_fold end shutdown - -# Final summary - Don't use a fold, keep it visible -echo -e "\033[1;33mTest Results:\033[0m" -echo -e "$exitfile" | while read result; do - testgroup=${result%:*} - status=${result#*:} - # Print final result for each group of tests - if [[ "$status" -eq 0 ]]; then - printf "%20s: \033[1;32mPASS\033[0m\n" "$testgroup" - else - printf "%20s: \033[1;31mFAIL\033[0m (returned %s)\n" "$testgroup" "$status" - fi -done - -exit "$exitstatus" diff --git a/travis-ci/vmtest/run_selftests.sh b/travis-ci/vmtest/run_selftests.sh index b400ad0..d1e8536 100755 --- a/travis-ci/vmtest/run_selftests.sh +++ b/travis-ci/vmtest/run_selftests.sh @@ -4,6 +4,8 @@ set -euo pipefail source $(cd $(dirname $0) && pwd)/helpers.sh +ARCH=$(uname -m) + STATUS_FILE=/exitstatus read_lists() { @@ -47,11 +49,11 @@ test_verifier() { travis_fold end vm_init -configs_path=libbpf/travis-ci/vmtest/configs +configs_path=${PROJECT_NAME}/vmtest/configs BLACKLIST=$(read_lists "$configs_path/blacklist/BLACKLIST-${KERNEL}" "$configs_path/blacklist/BLACKLIST-${KERNEL}.${ARCH}") WHITELIST=$(read_lists "$configs_path/whitelist/WHITELIST-${KERNEL}" "$configs_path/whitelist/WHITELIST-${KERNEL}.${ARCH}") -cd libbpf/selftests/bpf +cd ${PROJECT_NAME}/selftests/bpf test_progs diff --git a/travis-ci/vmtest/run_vmtest.sh b/travis-ci/vmtest/run_vmtest.sh deleted file mode 100755 index a846d46..0000000 --- a/travis-ci/vmtest/run_vmtest.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -set -eu - -source $(cd $(dirname $0) && pwd)/helpers.sh - -VMTEST_SETUPCMD="GITHUB_WORKFLOW=${GITHUB_WORKFLOW:-} PROJECT_NAME=${PROJECT_NAME} ./${PROJECT_NAME}/travis-ci/vmtest/run_selftests.sh" - -# if CHECKOUT_KERNEL is 1 code will consider that kernel code lives elsewhere -# if 0 it will consider that REPO_ROOT is a kernel tree -CHECKOUT_KERNEL=${CHECKOUT_KERNEL:-1} - -echo "KERNEL: $KERNEL" -echo - -# Build latest pahole -${VMTEST_ROOT}/build_pahole.sh travis-ci/vmtest/pahole - -travis_fold start install_clang "Installing Clang/LLVM" - -wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - -sudo add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal main" -sudo apt-get update -sudo apt-get install --allow-downgrades -y libc6=2.31-0ubuntu9.2 -sudo aptitude install -y g++ libelf-dev -sudo aptitude install -y clang-14 llvm-14 - -travis_fold end install_clang - -# Build selftests (and latest kernel, if necessary) - -if [[ "$CHECKOUT_KERNEL" == "1" ]]; then - ${VMTEST_ROOT}/prepare_selftests.sh travis-ci/vmtest/bpf-next -else - ${VMTEST_ROOT}/prepare_selftests.sh -fi - -travis_fold start adduser_to_kvm "Add user ${USER}" -sudo adduser "${USER}" kvm -travis_fold stop adduser_to_kvm - -# Escape whitespace characters. -setup_cmd=$(sed 's/\([[:space:]]\)/\\\1/g' <<< "${VMTEST_SETUPCMD}") - -if [[ "${KERNEL}" = 'LATEST' ]]; then - if [[ "$CHECKOUT_KERNEL" == "1" ]]; then - sudo -E sudo -E -u "${USER}" "${VMTEST_ROOT}/run.sh" -b travis-ci/vmtest/bpf-next -o -d ~ -s "${setup_cmd}" ~/root.img - else - sudo -E sudo -E -u "${USER}" "${VMTEST_ROOT}/run.sh" -b "${REPO_ROOT}" -o -d ~ -s "${setup_cmd}" ~/root.img - fi -else - sudo -E sudo -E -u "${USER}" "${VMTEST_ROOT}/run.sh" -k "${KERNEL}*" -o -d ~ -s "${setup_cmd}" ~/root.img -fi From 33ec2ca026d568c4820324752be09a51460b7005 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 9 Dec 2021 22:43:20 -0800 Subject: [PATCH 002/131] sync: improve patch application process by using patch command git apply -3 doesn't always leave conflicted files in the working directory. Use patch --merge instead, it seems to work better in more complicated situations. Signed-off-by: Andrii Nakryiko --- scripts/sync-kernel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-kernel.sh b/scripts/sync-kernel.sh index 9ea1d56..45b4212 100755 --- a/scripts/sync-kernel.sh +++ b/scripts/sync-kernel.sh @@ -264,7 +264,7 @@ git checkout -b ${LIBBPF_SYNC_TAG} for patch in $(ls -1 ${TMP_DIR}/patches | tail -n +2); do if ! git am -3 --committer-date-is-author-date "${TMP_DIR}/patches/${patch}"; then - if ! git apply -3 "${TMP_DIR}/patches/${patch}"; then + if ! patch -p1 --merge < "${TMP_DIR}/patches/${patch}"; then read -p "Applying ${TMP_DIR}/patches/${patch} failed, please resolve manually and press to proceed..." fi git am --continue From bafda72319619e532533a895f17d38301a3a6faf Mon Sep 17 00:00:00 2001 From: Mehrdad Arshad Rad Date: Sun, 28 Nov 2021 11:33:37 -0800 Subject: [PATCH 003/131] libbpf: Remove duplicate assignments There is a same action when load_attr.attach_btf_id is initialized. Signed-off-by: Mehrdad Arshad Rad Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211128193337.10628-1-arshad.rad@gmail.com --- src/libbpf.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libbpf.c b/src/libbpf.c index a102fe6..a8b1010 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -6559,7 +6559,6 @@ static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_prog load_attr.expected_attach_type = prog->expected_attach_type; if (kernel_supports(obj, FEAT_PROG_NAME)) prog_name = prog->name; - load_attr.attach_btf_id = prog->attach_btf_id; load_attr.attach_prog_fd = prog->attach_prog_fd; load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; load_attr.attach_btf_id = prog->attach_btf_id; From 73c8768db7f4ba538e2d03bd59a3484f2e32b8f1 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Mon, 29 Nov 2021 19:06:19 -0800 Subject: [PATCH 004/131] bpf: Add bpf_loop helper This patch adds the kernel-side and API changes for a new helper function, bpf_loop: long bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, u64 flags); where long (*callback_fn)(u32 index, void *ctx); bpf_loop invokes the "callback_fn" **nr_loops** times or until the callback_fn returns 1. The callback_fn can only return 0 or 1, and this is enforced by the verifier. The callback_fn index is zero-indexed. A few things to please note: ~ The "u64 flags" parameter is currently unused but is included in case a future use case for it arises. ~ In the kernel-side implementation of bpf_loop (kernel/bpf/bpf_iter.c), bpf_callback_t is used as the callback function cast. ~ A program can have nested bpf_loop calls but the program must still adhere to the verifier constraint of its stack depth (the stack depth cannot exceed MAX_BPF_STACK)) ~ Recursive callback_fns do not pass the verifier, due to the call stack for these being too deep. ~ The next patch will include the tests and benchmark Signed-off-by: Joanne Koong Signed-off-by: Alexei Starovoitov Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211130030622.4131246-2-joannekoong@fb.com --- include/uapi/linux/bpf.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index a69e4b0..211b43a 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4957,6 +4957,30 @@ union bpf_attr { * **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*. * **-EBUSY** if failed to try lock mmap_lock. * **-EINVAL** for invalid **flags**. + * + * long bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, u64 flags) + * Description + * For **nr_loops**, call **callback_fn** function + * with **callback_ctx** as the context parameter. + * The **callback_fn** should be a static function and + * the **callback_ctx** should be a pointer to the stack. + * The **flags** is used to control certain aspects of the helper. + * Currently, the **flags** must be 0. Currently, nr_loops is + * limited to 1 << 23 (~8 million) loops. + * + * long (\*callback_fn)(u32 index, void \*ctx); + * + * where **index** is the current index in the loop. The index + * is zero-indexed. + * + * If **callback_fn** returns 0, the helper will continue to the next + * loop. If return value is 1, the helper will skip the rest of + * the loops and return. Other return values are not used now, + * and will be rejected by the verifier. + * + * Return + * The number of loops performed, **-EINVAL** for invalid **flags**, + * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5140,6 +5164,7 @@ union bpf_attr { FN(skc_to_unix_sock), \ FN(kallsyms_lookup_name), \ FN(find_vma), \ + FN(loop), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper From b2b45a3131295ea7d1fc2cd3aeed4cdacadb8126 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Tue, 23 Nov 2021 05:27:32 +0530 Subject: [PATCH 005/131] libbpf: Avoid double stores for success/failure case of ksym relocations Instead, jump directly to success case stores in case ret >= 0, else do the default 0 value store and jump over the success case. This is better in terms of readability. Readjust the code for kfunc relocation as well to follow a similar pattern, also leads to easier to follow code now. Suggested-by: Alexei Starovoitov Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Andrii Nakryiko Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20211122235733.634914-3-memxor@gmail.com --- src/gen_loader.c | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/gen_loader.c b/src/gen_loader.c index b8e95d3..3fe45f1 100644 --- a/src/gen_loader.c +++ b/src/gen_loader.c @@ -687,27 +687,29 @@ static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo return; } kdesc->off = btf_fd_idx; - /* set a default value for imm */ + /* jump to success case */ + emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); + /* set value for imm, off as 0 */ emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); - /* skip success case store if ret < 0 */ - emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 1)); + emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); + /* skip success case for ret < 0 */ + emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10)); /* store btf_id into insn[insn_idx].imm */ emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); + /* obtain fd in BPF_REG_9 */ + emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7)); + emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); + /* jump to fd_array store if fd denotes module BTF */ + emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2)); + /* set the default value for off */ + emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); + /* skip BTF fd store for vmlinux BTF */ + emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4)); /* load fd_array slot pointer */ emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx))); - /* skip store of BTF fd if ret < 0 */ - emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 3)); /* store BTF fd in slot */ - emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7)); - emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0)); - /* set a default value for off */ - emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); - /* skip insn->off store if ret < 0 */ - emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 2)); - /* skip if vmlinux BTF */ - emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_9, 0, 1)); /* store index into insn[insn_idx].off */ emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx)); log: @@ -816,17 +818,20 @@ static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, emit_bpf_find_by_name_kind(gen, relo); if (!relo->is_weak) emit_check_err(gen); - /* set default values as 0 */ + /* jump to success case */ + emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); + /* set values for insn[insn_idx].imm, insn[insn_idx + 1].imm as 0 */ emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0)); - /* skip success case stores if ret < 0 */ - emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 4)); + /* skip success case for ret < 0 */ + emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4)); /* store btf_id into insn[insn_idx].imm */ emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); /* store btf_obj_fd into insn[insn_idx + 1].imm */ emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm))); + /* skip src_reg adjustment */ emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); clear_src_reg: /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */ From c4da8092ccdf420aaa3a179c59c6c8ee4cbe483b Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Tue, 23 Nov 2021 05:27:33 +0530 Subject: [PATCH 006/131] libbpf: Avoid reload of imm for weak, unresolved, repeating ksym Alexei pointed out that we can use BPF_REG_0 which already contains imm from move_blob2blob computation. Note that we now compare the second insn's imm, but this should not matter, since both will be zeroed out for the error case for the insn populated earlier. Suggested-by: Alexei Starovoitov Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Andrii Nakryiko Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20211122235733.634914-4-memxor@gmail.com --- src/gen_loader.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/gen_loader.c b/src/gen_loader.c index 3fe45f1..f62d616 100644 --- a/src/gen_loader.c +++ b/src/gen_loader.c @@ -808,9 +808,8 @@ static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, kdesc->insn + offsetof(struct bpf_insn, imm)); move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_8, offsetof(struct bpf_insn, imm))); - /* jump over src_reg adjustment if imm is not 0 */ - emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 3)); + /* jump over src_reg adjustment if imm is not 0, reuse BPF_REG_0 from move_blob2blob */ + emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3)); goto clear_src_reg; } /* remember insn offset, so we can copy BTF ID and FD later */ From b9bd1f868247a721686d2c0d2909f7d8238e99c3 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 1 Dec 2021 10:10:24 -0800 Subject: [PATCH 007/131] libbpf: Replace btf__type_by_id() with btf_type_by_id(). To prepare relo_core.c to be compiled in the kernel and the user space replace btf__type_by_id with btf_type_by_id. In libbpf btf__type_by_id and btf_type_by_id have different behavior. bpf_core_apply_relo_insn() needs behavior of uapi btf__type_by_id vs internal btf_type_by_id, but type_id range check is already done in bpf_core_apply_relo(), so it's safe to replace it everywhere. The kernel btf_type_by_id() does the check anyway. It doesn't hurt. Suggested-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-2-alexei.starovoitov@gmail.com --- src/btf.c | 2 +- src/libbpf_internal.h | 2 +- src/relo_core.c | 19 ++++++++----------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/btf.c b/src/btf.c index 8024fe3..0d7b16e 100644 --- a/src/btf.c +++ b/src/btf.c @@ -454,7 +454,7 @@ const struct btf *btf__base_btf(const struct btf *btf) } /* internal helper returning non-const pointer to a type */ -struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id) +struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id) { if (type_id == 0) return &btf_void; diff --git a/src/libbpf_internal.h b/src/libbpf_internal.h index 311905d..6f143e9 100644 --- a/src/libbpf_internal.h +++ b/src/libbpf_internal.h @@ -172,7 +172,7 @@ static inline void *libbpf_reallocarray(void *ptr, size_t nmemb, size_t size) struct btf; struct btf_type; -struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id); +struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id); const char *btf_kind_str(const struct btf_type *t); const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id); diff --git a/src/relo_core.c b/src/relo_core.c index b5b8956..c0904f4 100644 --- a/src/relo_core.c +++ b/src/relo_core.c @@ -51,7 +51,7 @@ static bool is_flex_arr(const struct btf *btf, return false; /* has to be the last member of enclosing struct */ - t = btf__type_by_id(btf, acc->type_id); + t = btf_type_by_id(btf, acc->type_id); return acc->idx == btf_vlen(t) - 1; } @@ -388,7 +388,7 @@ static int bpf_core_match_member(const struct btf *local_btf, return 0; local_id = local_acc->type_id; - local_type = btf__type_by_id(local_btf, local_id); + local_type = btf_type_by_id(local_btf, local_id); local_member = btf_members(local_type) + local_acc->idx; local_name = btf__name_by_offset(local_btf, local_member->name_off); @@ -580,7 +580,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, return -EUCLEAN; /* request instruction poisoning */ acc = &spec->spec[spec->len - 1]; - t = btf__type_by_id(spec->btf, acc->type_id); + t = btf_type_by_id(spec->btf, acc->type_id); /* a[n] accessor needs special handling */ if (!acc->name) { @@ -729,7 +729,7 @@ static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, case BPF_ENUMVAL_VALUE: if (!spec) return -EUCLEAN; /* request instruction poisoning */ - t = btf__type_by_id(spec->btf, spec->spec[0].type_id); + t = btf_type_by_id(spec->btf, spec->spec[0].type_id); e = btf_enum(t) + spec->spec[0].idx; *val = e->val; break; @@ -805,8 +805,8 @@ static int bpf_core_calc_relo(const char *prog_name, if (res->orig_sz != res->new_sz) { const struct btf_type *orig_t, *new_t; - orig_t = btf__type_by_id(local_spec->btf, res->orig_type_id); - new_t = btf__type_by_id(targ_spec->btf, res->new_type_id); + orig_t = btf_type_by_id(local_spec->btf, res->orig_type_id); + new_t = btf_type_by_id(targ_spec->btf, res->new_type_id); /* There are two use cases in which it's safe to * adjust load/store's mem size: @@ -1054,7 +1054,7 @@ static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec) int i; type_id = spec->root_type_id; - t = btf__type_by_id(spec->btf, type_id); + t = btf_type_by_id(spec->btf, type_id); s = btf__name_by_offset(spec->btf, t->name_off); libbpf_print(level, "[%u] %s %s", type_id, btf_kind_str(t), str_is_empty(s) ? "" : s); @@ -1158,10 +1158,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, int i, j, err; local_id = relo->type_id; - local_type = btf__type_by_id(local_btf, local_id); - if (!local_type) - return -EINVAL; - + local_type = btf_type_by_id(local_btf, local_id); local_name = btf__name_by_offset(local_btf, local_type->name_off); if (!local_name) return -EINVAL; From 2be7e6a8305b409b86054dbdc835d8a39aee9cbb Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 1 Dec 2021 10:10:26 -0800 Subject: [PATCH 008/131] bpf: Prepare relo_core.c for kernel duty. Make relo_core.c to be compiled for the kernel and for user space libbpf. Note the patch is reducing BPF_CORE_SPEC_MAX_LEN from 64 to 32. This is the maximum number of nested structs and arrays. For example: struct sample { int a; struct { int b[10]; }; }; struct sample *s = ...; int *y = &s->b[5]; This field access is encoded as "0:1:0:5" and spec len is 4. The follow up patch might bump it back to 64. Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-4-alexei.starovoitov@gmail.com --- src/relo_core.c | 76 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/src/relo_core.c b/src/relo_core.c index c0904f4..56dbe6d 100644 --- a/src/relo_core.c +++ b/src/relo_core.c @@ -1,6 +1,60 @@ // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) /* Copyright (c) 2019 Facebook */ +#ifdef __KERNEL__ +#include +#include +#include +#include +#include "relo_core.h" + +static const char *btf_kind_str(const struct btf_type *t) +{ + return btf_type_str(t); +} + +static bool is_ldimm64_insn(struct bpf_insn *insn) +{ + return insn->code == (BPF_LD | BPF_IMM | BPF_DW); +} + +static const struct btf_type * +skip_mods_and_typedefs(const struct btf *btf, u32 id, u32 *res_id) +{ + return btf_type_skip_modifiers(btf, id, res_id); +} + +static const char *btf__name_by_offset(const struct btf *btf, u32 offset) +{ + return btf_name_by_offset(btf, offset); +} + +static s64 btf__resolve_size(const struct btf *btf, u32 type_id) +{ + const struct btf_type *t; + int size; + + t = btf_type_by_id(btf, type_id); + t = btf_resolve_size(btf, t, &size); + if (IS_ERR(t)) + return PTR_ERR(t); + return size; +} + +enum libbpf_print_level { + LIBBPF_WARN, + LIBBPF_INFO, + LIBBPF_DEBUG, +}; + +#undef pr_warn +#undef pr_info +#undef pr_debug +#define pr_warn(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) +#define pr_info(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) +#define pr_debug(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) +#define libbpf_print(level, fmt, ...) bpf_log((void *)prog_name, fmt, ##__VA_ARGS__) +#else #include #include #include @@ -12,8 +66,9 @@ #include "btf.h" #include "str_error.h" #include "libbpf_internal.h" +#endif -#define BPF_CORE_SPEC_MAX_LEN 64 +#define BPF_CORE_SPEC_MAX_LEN 32 /* represents BPF CO-RE field or array element accessor */ struct bpf_core_accessor { @@ -150,7 +205,7 @@ static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind) * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access * string to specify enumerator's value index that need to be relocated. */ -static int bpf_core_parse_spec(const struct btf *btf, +static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, __u32 type_id, const char *spec_str, enum bpf_core_relo_kind relo_kind, @@ -272,8 +327,8 @@ static int bpf_core_parse_spec(const struct btf *btf, return sz; spec->bit_offset += access_idx * sz * 8; } else { - pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", - type_id, spec_str, i, id, btf_kind_str(t)); + pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", + prog_name, type_id, spec_str, i, id, btf_kind_str(t)); return -EINVAL; } } @@ -346,8 +401,6 @@ recur: targ_id = btf_array(targ_type)->type; goto recur; default: - pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n", - btf_kind(local_type), local_id, targ_id); return 0; } } @@ -1045,7 +1098,7 @@ poison: * [] () + => @, * where is a C-syntax view of recorded field access, e.g.: x.a[3].b */ -static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec) +static void bpf_core_dump_spec(const char *prog_name, int level, const struct bpf_core_spec *spec) { const struct btf_type *t; const struct btf_enum *e; @@ -1167,7 +1220,8 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, if (str_is_empty(spec_str)) return -EINVAL; - err = bpf_core_parse_spec(local_btf, local_id, spec_str, relo->kind, &local_spec); + err = bpf_core_parse_spec(prog_name, local_btf, local_id, spec_str, + relo->kind, &local_spec); if (err) { pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), @@ -1178,7 +1232,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); - bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec); + bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, &local_spec); libbpf_print(LIBBPF_DEBUG, "\n"); /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */ @@ -1204,14 +1258,14 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, if (err < 0) { pr_warn("prog '%s': relo #%d: error matching candidate #%d ", prog_name, relo_idx, i); - bpf_core_dump_spec(LIBBPF_WARN, &cand_spec); + bpf_core_dump_spec(prog_name, LIBBPF_WARN, &cand_spec); libbpf_print(LIBBPF_WARN, ": %d\n", err); return err; } pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog_name, relo_idx, err == 0 ? "non-matching" : "matching", i); - bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec); + bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, &cand_spec); libbpf_print(LIBBPF_DEBUG, "\n"); if (err == 0) From 8e43882e53d32aa60097f81b33c9bfb171fd85e9 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 1 Dec 2021 10:10:27 -0800 Subject: [PATCH 009/131] bpf: Define enum bpf_core_relo_kind as uapi. enum bpf_core_relo_kind is generated by llvm and processed by libbpf. It's a de-facto uapi. With CO-RE in the kernel the bpf_core_relo_kind values become uapi de-jure. Also rename them with BPF_CORE_ prefix to distinguish from conflicting names in bpf_core_read.h. The enums bpf_field_info_kind, bpf_type_id_kind, bpf_type_info_kind, bpf_enum_value_kind are passing different values from bpf program into llvm. Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-5-alexei.starovoitov@gmail.com --- include/uapi/linux/bpf.h | 19 +++++++++ src/libbpf.c | 2 +- src/relo_core.c | 84 ++++++++++++++++++++-------------------- src/relo_core.h | 18 +-------- 4 files changed, 63 insertions(+), 60 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 211b43a..9e66b18 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -6374,4 +6374,23 @@ enum { BTF_F_ZERO = (1ULL << 3), }; +/* bpf_core_relo_kind encodes which aspect of captured field/type/enum value + * has to be adjusted by relocations. It is emitted by llvm and passed to + * libbpf and later to the kernel. + */ +enum bpf_core_relo_kind { + BPF_CORE_FIELD_BYTE_OFFSET = 0, /* field byte offset */ + BPF_CORE_FIELD_BYTE_SIZE = 1, /* field size in bytes */ + BPF_CORE_FIELD_EXISTS = 2, /* field existence in target kernel */ + BPF_CORE_FIELD_SIGNED = 3, /* field signedness (0 - unsigned, 1 - signed) */ + BPF_CORE_FIELD_LSHIFT_U64 = 4, /* bitfield-specific left bitshift */ + BPF_CORE_FIELD_RSHIFT_U64 = 5, /* bitfield-specific right bitshift */ + BPF_CORE_TYPE_ID_LOCAL = 6, /* type ID in local BPF object */ + BPF_CORE_TYPE_ID_TARGET = 7, /* type ID in target kernel */ + BPF_CORE_TYPE_EXISTS = 8, /* type existence in target kernel */ + BPF_CORE_TYPE_SIZE = 9, /* type size in bytes */ + BPF_CORE_ENUMVAL_EXISTS = 10, /* enum value existence in target kernel */ + BPF_CORE_ENUMVAL_VALUE = 11, /* enum value integer value */ +}; + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/src/libbpf.c b/src/libbpf.c index a8b1010..6af8b7a 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -5523,7 +5523,7 @@ static int bpf_core_apply_relo(struct bpf_program *prog, return -ENOTSUP; } - if (relo->kind != BPF_TYPE_ID_LOCAL && + if (relo->kind != BPF_CORE_TYPE_ID_LOCAL && !hashmap__find(cand_cache, type_key, (void **)&cands)) { cands = bpf_core_find_cands(prog->obj, local_btf, local_id); if (IS_ERR(cands)) { diff --git a/src/relo_core.c b/src/relo_core.c index 56dbe6d..d194fb9 100644 --- a/src/relo_core.c +++ b/src/relo_core.c @@ -113,18 +113,18 @@ static bool is_flex_arr(const struct btf *btf, static const char *core_relo_kind_str(enum bpf_core_relo_kind kind) { switch (kind) { - case BPF_FIELD_BYTE_OFFSET: return "byte_off"; - case BPF_FIELD_BYTE_SIZE: return "byte_sz"; - case BPF_FIELD_EXISTS: return "field_exists"; - case BPF_FIELD_SIGNED: return "signed"; - case BPF_FIELD_LSHIFT_U64: return "lshift_u64"; - case BPF_FIELD_RSHIFT_U64: return "rshift_u64"; - case BPF_TYPE_ID_LOCAL: return "local_type_id"; - case BPF_TYPE_ID_TARGET: return "target_type_id"; - case BPF_TYPE_EXISTS: return "type_exists"; - case BPF_TYPE_SIZE: return "type_size"; - case BPF_ENUMVAL_EXISTS: return "enumval_exists"; - case BPF_ENUMVAL_VALUE: return "enumval_value"; + case BPF_CORE_FIELD_BYTE_OFFSET: return "byte_off"; + case BPF_CORE_FIELD_BYTE_SIZE: return "byte_sz"; + case BPF_CORE_FIELD_EXISTS: return "field_exists"; + case BPF_CORE_FIELD_SIGNED: return "signed"; + case BPF_CORE_FIELD_LSHIFT_U64: return "lshift_u64"; + case BPF_CORE_FIELD_RSHIFT_U64: return "rshift_u64"; + case BPF_CORE_TYPE_ID_LOCAL: return "local_type_id"; + case BPF_CORE_TYPE_ID_TARGET: return "target_type_id"; + case BPF_CORE_TYPE_EXISTS: return "type_exists"; + case BPF_CORE_TYPE_SIZE: return "type_size"; + case BPF_CORE_ENUMVAL_EXISTS: return "enumval_exists"; + case BPF_CORE_ENUMVAL_VALUE: return "enumval_value"; default: return "unknown"; } } @@ -132,12 +132,12 @@ static const char *core_relo_kind_str(enum bpf_core_relo_kind kind) static bool core_relo_is_field_based(enum bpf_core_relo_kind kind) { switch (kind) { - case BPF_FIELD_BYTE_OFFSET: - case BPF_FIELD_BYTE_SIZE: - case BPF_FIELD_EXISTS: - case BPF_FIELD_SIGNED: - case BPF_FIELD_LSHIFT_U64: - case BPF_FIELD_RSHIFT_U64: + case BPF_CORE_FIELD_BYTE_OFFSET: + case BPF_CORE_FIELD_BYTE_SIZE: + case BPF_CORE_FIELD_EXISTS: + case BPF_CORE_FIELD_SIGNED: + case BPF_CORE_FIELD_LSHIFT_U64: + case BPF_CORE_FIELD_RSHIFT_U64: return true; default: return false; @@ -147,10 +147,10 @@ static bool core_relo_is_field_based(enum bpf_core_relo_kind kind) static bool core_relo_is_type_based(enum bpf_core_relo_kind kind) { switch (kind) { - case BPF_TYPE_ID_LOCAL: - case BPF_TYPE_ID_TARGET: - case BPF_TYPE_EXISTS: - case BPF_TYPE_SIZE: + case BPF_CORE_TYPE_ID_LOCAL: + case BPF_CORE_TYPE_ID_TARGET: + case BPF_CORE_TYPE_EXISTS: + case BPF_CORE_TYPE_SIZE: return true; default: return false; @@ -160,8 +160,8 @@ static bool core_relo_is_type_based(enum bpf_core_relo_kind kind) static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind) { switch (kind) { - case BPF_ENUMVAL_EXISTS: - case BPF_ENUMVAL_VALUE: + case BPF_CORE_ENUMVAL_EXISTS: + case BPF_CORE_ENUMVAL_VALUE: return true; default: return false; @@ -624,7 +624,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, *field_sz = 0; - if (relo->kind == BPF_FIELD_EXISTS) { + if (relo->kind == BPF_CORE_FIELD_EXISTS) { *val = spec ? 1 : 0; return 0; } @@ -637,7 +637,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, /* a[n] accessor needs special handling */ if (!acc->name) { - if (relo->kind == BPF_FIELD_BYTE_OFFSET) { + if (relo->kind == BPF_CORE_FIELD_BYTE_OFFSET) { *val = spec->bit_offset / 8; /* remember field size for load/store mem size */ sz = btf__resolve_size(spec->btf, acc->type_id); @@ -645,7 +645,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, return -EINVAL; *field_sz = sz; *type_id = acc->type_id; - } else if (relo->kind == BPF_FIELD_BYTE_SIZE) { + } else if (relo->kind == BPF_CORE_FIELD_BYTE_SIZE) { sz = btf__resolve_size(spec->btf, acc->type_id); if (sz < 0) return -EINVAL; @@ -697,36 +697,36 @@ static int bpf_core_calc_field_relo(const char *prog_name, *validate = !bitfield; switch (relo->kind) { - case BPF_FIELD_BYTE_OFFSET: + case BPF_CORE_FIELD_BYTE_OFFSET: *val = byte_off; if (!bitfield) { *field_sz = byte_sz; *type_id = field_type_id; } break; - case BPF_FIELD_BYTE_SIZE: + case BPF_CORE_FIELD_BYTE_SIZE: *val = byte_sz; break; - case BPF_FIELD_SIGNED: + case BPF_CORE_FIELD_SIGNED: /* enums will be assumed unsigned */ *val = btf_is_enum(mt) || (btf_int_encoding(mt) & BTF_INT_SIGNED); if (validate) *validate = true; /* signedness is never ambiguous */ break; - case BPF_FIELD_LSHIFT_U64: + case BPF_CORE_FIELD_LSHIFT_U64: #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ *val = 64 - (bit_off + bit_sz - byte_off * 8); #else *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8); #endif break; - case BPF_FIELD_RSHIFT_U64: + case BPF_CORE_FIELD_RSHIFT_U64: *val = 64 - bit_sz; if (validate) *validate = true; /* right shift is never ambiguous */ break; - case BPF_FIELD_EXISTS: + case BPF_CORE_FIELD_EXISTS: default: return -EOPNOTSUPP; } @@ -747,20 +747,20 @@ static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, } switch (relo->kind) { - case BPF_TYPE_ID_TARGET: + case BPF_CORE_TYPE_ID_TARGET: *val = spec->root_type_id; break; - case BPF_TYPE_EXISTS: + case BPF_CORE_TYPE_EXISTS: *val = 1; break; - case BPF_TYPE_SIZE: + case BPF_CORE_TYPE_SIZE: sz = btf__resolve_size(spec->btf, spec->root_type_id); if (sz < 0) return -EINVAL; *val = sz; break; - case BPF_TYPE_ID_LOCAL: - /* BPF_TYPE_ID_LOCAL is handled specially and shouldn't get here */ + case BPF_CORE_TYPE_ID_LOCAL: + /* BPF_CORE_TYPE_ID_LOCAL is handled specially and shouldn't get here */ default: return -EOPNOTSUPP; } @@ -776,10 +776,10 @@ static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, const struct btf_enum *e; switch (relo->kind) { - case BPF_ENUMVAL_EXISTS: + case BPF_CORE_ENUMVAL_EXISTS: *val = spec ? 1 : 0; break; - case BPF_ENUMVAL_VALUE: + case BPF_CORE_ENUMVAL_VALUE: if (!spec) return -EUCLEAN; /* request instruction poisoning */ t = btf_type_by_id(spec->btf, spec->spec[0].type_id); @@ -1236,7 +1236,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, libbpf_print(LIBBPF_DEBUG, "\n"); /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */ - if (relo->kind == BPF_TYPE_ID_LOCAL) { + if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) { targ_res.validate = true; targ_res.poison = false; targ_res.orig_val = local_spec.root_type_id; @@ -1302,7 +1302,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, } /* - * For BPF_FIELD_EXISTS relo or when used BPF program has field + * For BPF_CORE_FIELD_EXISTS relo or when used BPF program has field * existence checks or kernel version/config checks, it's expected * that we might not find any candidates. In this case, if field * wasn't found in any candidate, the list of candidates shouldn't diff --git a/src/relo_core.h b/src/relo_core.h index 3b9f8f1..3d0b86e 100644 --- a/src/relo_core.h +++ b/src/relo_core.h @@ -4,23 +4,7 @@ #ifndef __RELO_CORE_H #define __RELO_CORE_H -/* bpf_core_relo_kind encodes which aspect of captured field/type/enum value - * has to be adjusted by relocations. - */ -enum bpf_core_relo_kind { - BPF_FIELD_BYTE_OFFSET = 0, /* field byte offset */ - BPF_FIELD_BYTE_SIZE = 1, /* field size in bytes */ - BPF_FIELD_EXISTS = 2, /* field existence in target kernel */ - BPF_FIELD_SIGNED = 3, /* field signedness (0 - unsigned, 1 - signed) */ - BPF_FIELD_LSHIFT_U64 = 4, /* bitfield-specific left bitshift */ - BPF_FIELD_RSHIFT_U64 = 5, /* bitfield-specific right bitshift */ - BPF_TYPE_ID_LOCAL = 6, /* type ID in local BPF object */ - BPF_TYPE_ID_TARGET = 7, /* type ID in target kernel */ - BPF_TYPE_EXISTS = 8, /* type existence in target kernel */ - BPF_TYPE_SIZE = 9, /* type size in bytes */ - BPF_ENUMVAL_EXISTS = 10, /* enum value existence in target kernel */ - BPF_ENUMVAL_VALUE = 11, /* enum value integer value */ -}; +#include /* The minimum bpf_core_relo checked by the loader * From d785a21c71aa00eab530dd840d8ee067acddfd00 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 1 Dec 2021 10:10:28 -0800 Subject: [PATCH 010/131] bpf: Pass a set of bpf_core_relo-s to prog_load command. struct bpf_core_relo is generated by llvm and processed by libbpf. It's a de-facto uapi. With CO-RE in the kernel the struct bpf_core_relo becomes uapi de-jure. Add an ability to pass a set of 'struct bpf_core_relo' to prog_load command and let the kernel perform CO-RE relocations. Note the struct bpf_line_info and struct bpf_func_info have the same layout when passed from LLVM to libbpf and from libbpf to the kernel except "insn_off" fields means "byte offset" when LLVM generates it. Then libbpf converts it to "insn index" to pass to the kernel. The struct bpf_core_relo's "insn_off" field is always "byte offset". Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-6-alexei.starovoitov@gmail.com --- include/uapi/linux/bpf.h | 59 +++++++++++++++++++++++++++++++++++++++- src/relo_core.h | 53 ------------------------------------ 2 files changed, 58 insertions(+), 54 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 9e66b18..c268712 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1342,8 +1342,10 @@ union bpf_attr { /* or valid module BTF object fd or 0 to attach to vmlinux */ __u32 attach_btf_obj_fd; }; - __u32 :32; /* pad */ + __u32 core_relo_cnt; /* number of bpf_core_relo */ __aligned_u64 fd_array; /* array of FDs */ + __aligned_u64 core_relos; + __u32 core_relo_rec_size; /* sizeof(struct bpf_core_relo) */ }; struct { /* anonymous struct used by BPF_OBJ_* commands */ @@ -6393,4 +6395,59 @@ enum bpf_core_relo_kind { BPF_CORE_ENUMVAL_VALUE = 11, /* enum value integer value */ }; +/* + * "struct bpf_core_relo" is used to pass relocation data form LLVM to libbpf + * and from libbpf to the kernel. + * + * CO-RE relocation captures the following data: + * - insn_off - instruction offset (in bytes) within a BPF program that needs + * its insn->imm field to be relocated with actual field info; + * - type_id - BTF type ID of the "root" (containing) entity of a relocatable + * type or field; + * - access_str_off - offset into corresponding .BTF string section. String + * interpretation depends on specific relocation kind: + * - for field-based relocations, string encodes an accessed field using + * a sequence of field and array indices, separated by colon (:). It's + * conceptually very close to LLVM's getelementptr ([0]) instruction's + * arguments for identifying offset to a field. + * - for type-based relocations, strings is expected to be just "0"; + * - for enum value-based relocations, string contains an index of enum + * value within its enum type; + * - kind - one of enum bpf_core_relo_kind; + * + * Example: + * struct sample { + * int a; + * struct { + * int b[10]; + * }; + * }; + * + * struct sample *s = ...; + * int *x = &s->a; // encoded as "0:0" (a is field #0) + * int *y = &s->b[5]; // encoded as "0:1:0:5" (anon struct is field #1, + * // b is field #0 inside anon struct, accessing elem #5) + * int *z = &s[10]->b; // encoded as "10:1" (ptr is used as an array) + * + * type_id for all relocs in this example will capture BTF type id of + * `struct sample`. + * + * Such relocation is emitted when using __builtin_preserve_access_index() + * Clang built-in, passing expression that captures field address, e.g.: + * + * bpf_probe_read(&dst, sizeof(dst), + * __builtin_preserve_access_index(&src->a.b.c)); + * + * In this case Clang will emit field relocation recording necessary data to + * be able to find offset of embedded `a.b.c` field within `src` struct. + * + * [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction + */ +struct bpf_core_relo { + __u32 insn_off; + __u32 type_id; + __u32 access_str_off; + enum bpf_core_relo_kind kind; +}; + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/src/relo_core.h b/src/relo_core.h index 3d0b86e..f410691 100644 --- a/src/relo_core.h +++ b/src/relo_core.h @@ -6,59 +6,6 @@ #include -/* The minimum bpf_core_relo checked by the loader - * - * CO-RE relocation captures the following data: - * - insn_off - instruction offset (in bytes) within a BPF program that needs - * its insn->imm field to be relocated with actual field info; - * - type_id - BTF type ID of the "root" (containing) entity of a relocatable - * type or field; - * - access_str_off - offset into corresponding .BTF string section. String - * interpretation depends on specific relocation kind: - * - for field-based relocations, string encodes an accessed field using - * a sequence of field and array indices, separated by colon (:). It's - * conceptually very close to LLVM's getelementptr ([0]) instruction's - * arguments for identifying offset to a field. - * - for type-based relocations, strings is expected to be just "0"; - * - for enum value-based relocations, string contains an index of enum - * value within its enum type; - * - * Example to provide a better feel. - * - * struct sample { - * int a; - * struct { - * int b[10]; - * }; - * }; - * - * struct sample *s = ...; - * int x = &s->a; // encoded as "0:0" (a is field #0) - * int y = &s->b[5]; // encoded as "0:1:0:5" (anon struct is field #1, - * // b is field #0 inside anon struct, accessing elem #5) - * int z = &s[10]->b; // encoded as "10:1" (ptr is used as an array) - * - * type_id for all relocs in this example will capture BTF type id of - * `struct sample`. - * - * Such relocation is emitted when using __builtin_preserve_access_index() - * Clang built-in, passing expression that captures field address, e.g.: - * - * bpf_probe_read(&dst, sizeof(dst), - * __builtin_preserve_access_index(&src->a.b.c)); - * - * In this case Clang will emit field relocation recording necessary data to - * be able to find offset of embedded `a.b.c` field within `src` struct. - * - * [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction - */ -struct bpf_core_relo { - __u32 insn_off; - __u32 type_id; - __u32 access_str_off; - enum bpf_core_relo_kind kind; -}; - struct bpf_core_cand { const struct btf *btf; const struct btf_type *t; From 20e7ed521a896b44e61d507bc8349328e5f6c670 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 1 Dec 2021 10:10:30 -0800 Subject: [PATCH 011/131] libbpf: Cleanup struct bpf_core_cand. Remove two redundant fields from struct bpf_core_cand. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-8-alexei.starovoitov@gmail.com --- src/libbpf.c | 30 +++++++++++++++++------------- src/relo_core.h | 2 -- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index 6af8b7a..761256d 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -5179,15 +5179,18 @@ static int bpf_core_add_cands(struct bpf_core_cand *local_cand, struct bpf_core_cand_list *cands) { struct bpf_core_cand *new_cands, *cand; - const struct btf_type *t; - const char *targ_name; + const struct btf_type *t, *local_t; + const char *targ_name, *local_name; size_t targ_essent_len; int n, i; + local_t = btf__type_by_id(local_cand->btf, local_cand->id); + local_name = btf__str_by_offset(local_cand->btf, local_t->name_off); + n = btf__type_cnt(targ_btf); for (i = targ_start_id; i < n; i++) { t = btf__type_by_id(targ_btf, i); - if (btf_kind(t) != btf_kind(local_cand->t)) + if (btf_kind(t) != btf_kind(local_t)) continue; targ_name = btf__name_by_offset(targ_btf, t->name_off); @@ -5198,12 +5201,12 @@ static int bpf_core_add_cands(struct bpf_core_cand *local_cand, if (targ_essent_len != local_essent_len) continue; - if (strncmp(local_cand->name, targ_name, local_essent_len) != 0) + if (strncmp(local_name, targ_name, local_essent_len) != 0) continue; pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", - local_cand->id, btf_kind_str(local_cand->t), - local_cand->name, i, btf_kind_str(t), targ_name, + local_cand->id, btf_kind_str(local_t), + local_name, i, btf_kind_str(t), targ_name, targ_btf_name); new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, sizeof(*cands->cands)); @@ -5212,8 +5215,6 @@ static int bpf_core_add_cands(struct bpf_core_cand *local_cand, cand = &new_cands[cands->len]; cand->btf = targ_btf; - cand->t = t; - cand->name = targ_name; cand->id = i; cands->cands = new_cands; @@ -5320,18 +5321,21 @@ bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 l struct bpf_core_cand local_cand = {}; struct bpf_core_cand_list *cands; const struct btf *main_btf; + const struct btf_type *local_t; + const char *local_name; size_t local_essent_len; int err, i; local_cand.btf = local_btf; - local_cand.t = btf__type_by_id(local_btf, local_type_id); - if (!local_cand.t) + local_cand.id = local_type_id; + local_t = btf__type_by_id(local_btf, local_type_id); + if (!local_t) return ERR_PTR(-EINVAL); - local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off); - if (str_is_empty(local_cand.name)) + local_name = btf__name_by_offset(local_btf, local_t->name_off); + if (str_is_empty(local_name)) return ERR_PTR(-EINVAL); - local_essent_len = bpf_core_essential_name_len(local_cand.name); + local_essent_len = bpf_core_essential_name_len(local_name); cands = calloc(1, sizeof(*cands)); if (!cands) diff --git a/src/relo_core.h b/src/relo_core.h index f410691..4f864b8 100644 --- a/src/relo_core.h +++ b/src/relo_core.h @@ -8,8 +8,6 @@ struct bpf_core_cand { const struct btf *btf; - const struct btf_type *t; - const char *name; __u32 id; }; From 5f887b332cabf771d4c2a4a80498ab16b2537c70 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 1 Dec 2021 10:10:32 -0800 Subject: [PATCH 012/131] libbpf: Use CO-RE in the kernel in light skeleton. Without lskel the CO-RE relocations are processed by libbpf before any other work is done. Instead, when lskel is needed, remember relocation as RELO_CORE kind. Then when loader prog is generated for a given bpf program pass CO-RE relos of that program to gen loader via bpf_gen__record_relo_core(). The gen loader will remember them as-is and pass it later as-is into the kernel. The normal libbpf flow is to process CO-RE early before call relos happen. In case of gen_loader the core relos have to be added to other relos to be copied together when bpf static function is appended in different places to other main bpf progs. During the copy the append_subprog_relos() will adjust insn_idx for normal relos and for RELO_CORE kind too. When that is done each struct reloc_desc has good relos for specific main prog. Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-10-alexei.starovoitov@gmail.com --- src/bpf_gen_internal.h | 3 ++ src/gen_loader.c | 41 ++++++++++++++-- src/libbpf.c | 109 +++++++++++++++++++++++++++++------------ 3 files changed, 120 insertions(+), 33 deletions(-) diff --git a/src/bpf_gen_internal.h b/src/bpf_gen_internal.h index 03cb408..5d514bd 100644 --- a/src/bpf_gen_internal.h +++ b/src/bpf_gen_internal.h @@ -39,6 +39,8 @@ struct bpf_gen { int error; struct ksym_relo_desc *relos; int relo_cnt; + struct bpf_core_relo *core_relos; + int core_relo_cnt; char attach_target[128]; int attach_kind; struct ksym_desc *ksyms; @@ -64,5 +66,6 @@ void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, bool is_typeless, int kind, int insn_idx); +void bpf_gen__record_relo_core(struct bpf_gen *gen, const struct bpf_core_relo *core_relo); #endif diff --git a/src/gen_loader.c b/src/gen_loader.c index f62d616..ad450a4 100644 --- a/src/gen_loader.c +++ b/src/gen_loader.c @@ -842,6 +842,22 @@ clear_src_reg: emit_ksym_relo_log(gen, relo, kdesc->ref); } +void bpf_gen__record_relo_core(struct bpf_gen *gen, + const struct bpf_core_relo *core_relo) +{ + struct bpf_core_relo *relos; + + relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos)); + if (!relos) { + gen->error = -ENOMEM; + return; + } + gen->core_relos = relos; + relos += gen->core_relo_cnt; + memcpy(relos, core_relo, sizeof(*relos)); + gen->core_relo_cnt++; +} + static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns) { int insn; @@ -874,6 +890,15 @@ static void emit_relos(struct bpf_gen *gen, int insns) emit_relo(gen, gen->relos + i, insns); } +static void cleanup_core_relo(struct bpf_gen *gen) +{ + if (!gen->core_relo_cnt) + return; + free(gen->core_relos); + gen->core_relo_cnt = 0; + gen->core_relos = NULL; +} + static void cleanup_relos(struct bpf_gen *gen, int insns) { int i, insn; @@ -901,6 +926,7 @@ static void cleanup_relos(struct bpf_gen *gen, int insns) gen->relo_cnt = 0; gen->relos = NULL; } + cleanup_core_relo(gen); } void bpf_gen__prog_load(struct bpf_gen *gen, @@ -908,12 +934,13 @@ void bpf_gen__prog_load(struct bpf_gen *gen, const char *license, struct bpf_insn *insns, size_t insn_cnt, struct bpf_prog_load_opts *load_attr, int prog_idx) { - int attr_size = offsetofend(union bpf_attr, fd_array); - int prog_load_attr, license_off, insns_off, func_info, line_info; + int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos; + int attr_size = offsetofend(union bpf_attr, core_relo_rec_size); union bpf_attr attr; memset(&attr, 0, attr_size); - pr_debug("gen: prog_load: type %d insns_cnt %zd\n", prog_type, insn_cnt); + pr_debug("gen: prog_load: type %d insns_cnt %zd progi_idx %d\n", + prog_type, insn_cnt, prog_idx); /* add license string to blob of bytes */ license_off = add_data(gen, license, strlen(license) + 1); /* add insns to blob of bytes */ @@ -937,6 +964,11 @@ void bpf_gen__prog_load(struct bpf_gen *gen, line_info = add_data(gen, load_attr->line_info, attr.line_info_cnt * attr.line_info_rec_size); + attr.core_relo_rec_size = sizeof(struct bpf_core_relo); + attr.core_relo_cnt = gen->core_relo_cnt; + core_relos = add_data(gen, gen->core_relos, + attr.core_relo_cnt * attr.core_relo_rec_size); + memcpy(attr.prog_name, prog_name, min((unsigned)strlen(prog_name), BPF_OBJ_NAME_LEN - 1)); prog_load_attr = add_data(gen, &attr, attr_size); @@ -953,6 +985,9 @@ void bpf_gen__prog_load(struct bpf_gen *gen, /* populate union bpf_attr with a pointer to line_info */ emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info); + /* populate union bpf_attr with a pointer to core_relos */ + emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos); + /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */ emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array); diff --git a/src/libbpf.c b/src/libbpf.c index 761256d..f680d5f 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -230,13 +230,19 @@ enum reloc_type { RELO_EXTERN_VAR, RELO_EXTERN_FUNC, RELO_SUBPROG_ADDR, + RELO_CORE, }; struct reloc_desc { enum reloc_type type; int insn_idx; - int map_idx; - int sym_off; + union { + const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */ + struct { + int map_idx; + int sym_off; + }; + }; }; struct bpf_sec_def; @@ -5485,6 +5491,24 @@ static void *u32_as_hash_key(__u32 x) return (void *)(uintptr_t)x; } +static int record_relo_core(struct bpf_program *prog, + const struct bpf_core_relo *core_relo, int insn_idx) +{ + struct reloc_desc *relos, *relo; + + relos = libbpf_reallocarray(prog->reloc_desc, + prog->nr_reloc + 1, sizeof(*relos)); + if (!relos) + return -ENOMEM; + relo = &relos[prog->nr_reloc]; + relo->type = RELO_CORE; + relo->insn_idx = insn_idx; + relo->core_relo = core_relo; + prog->reloc_desc = relos; + prog->nr_reloc++; + return 0; +} + static int bpf_core_apply_relo(struct bpf_program *prog, const struct bpf_core_relo *relo, int relo_idx, @@ -5521,10 +5545,12 @@ static int bpf_core_apply_relo(struct bpf_program *prog, return -EINVAL; if (prog->obj->gen_loader) { - pr_warn("// TODO core_relo: prog %td insn[%d] %s kind %d\n", + const char *spec_str = btf__name_by_offset(local_btf, relo->access_str_off); + + pr_debug("record_relo_core: prog %td insn[%d] %s %s %s final insn_idx %d\n", prog - prog->obj->programs, relo->insn_off / 8, - local_name, relo->kind); - return -ENOTSUP; + btf_kind_str(local_type), local_name, spec_str, insn_idx); + return record_relo_core(prog, relo, insn_idx); } if (relo->kind != BPF_CORE_TYPE_ID_LOCAL && @@ -5729,6 +5755,9 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) case RELO_CALL: /* handled already */ break; + case RELO_CORE: + /* will be handled by bpf_program_record_relos() */ + break; default: pr_warn("prog '%s': relo #%d: bad relo type %d\n", prog->name, i, relo->type); @@ -6169,6 +6198,35 @@ bpf_object__free_relocs(struct bpf_object *obj) } } +static int cmp_relocs(const void *_a, const void *_b) +{ + const struct reloc_desc *a = _a; + const struct reloc_desc *b = _b; + + if (a->insn_idx != b->insn_idx) + return a->insn_idx < b->insn_idx ? -1 : 1; + + /* no two relocations should have the same insn_idx, but ... */ + if (a->type != b->type) + return a->type < b->type ? -1 : 1; + + return 0; +} + +static void bpf_object__sort_relos(struct bpf_object *obj) +{ + int i; + + for (i = 0; i < obj->nr_programs; i++) { + struct bpf_program *p = &obj->programs[i]; + + if (!p->nr_reloc) + continue; + + qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); + } +} + static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) { @@ -6183,6 +6241,8 @@ bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) err); return err; } + if (obj->gen_loader) + bpf_object__sort_relos(obj); } /* Before relocating calls pre-process relocations and mark @@ -6387,21 +6447,6 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, return 0; } -static int cmp_relocs(const void *_a, const void *_b) -{ - const struct reloc_desc *a = _a; - const struct reloc_desc *b = _b; - - if (a->insn_idx != b->insn_idx) - return a->insn_idx < b->insn_idx ? -1 : 1; - - /* no two relocations should have the same insn_idx, but ... */ - if (a->type != b->type) - return a->type < b->type ? -1 : 1; - - return 0; -} - static int bpf_object__collect_relos(struct bpf_object *obj) { int i, err; @@ -6434,14 +6479,7 @@ static int bpf_object__collect_relos(struct bpf_object *obj) return err; } - for (i = 0; i < obj->nr_programs; i++) { - struct bpf_program *p = &obj->programs[i]; - - if (!p->nr_reloc) - continue; - - qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); - } + bpf_object__sort_relos(obj); return 0; } @@ -6683,7 +6721,7 @@ out: return ret; } -static int bpf_program__record_externs(struct bpf_program *prog) +static int bpf_program_record_relos(struct bpf_program *prog) { struct bpf_object *obj = prog->obj; int i; @@ -6705,6 +6743,17 @@ static int bpf_program__record_externs(struct bpf_program *prog) ext->is_weak, false, BTF_KIND_FUNC, relo->insn_idx); break; + case RELO_CORE: { + struct bpf_core_relo cr = { + .insn_off = relo->insn_idx * 8, + .type_id = relo->core_relo->type_id, + .access_str_off = relo->core_relo->access_str_off, + .kind = relo->core_relo->kind, + }; + + bpf_gen__record_relo_core(obj->gen_loader, &cr); + break; + } default: continue; } @@ -6744,7 +6793,7 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog prog->name, prog->instances.nr); } if (obj->gen_loader) - bpf_program__record_externs(prog); + bpf_program_record_relos(prog); err = bpf_object_load_prog_instance(obj, prog, prog->insns, prog->insns_cnt, license, kern_ver, &fd); From a7935b996f6287be4e0e5cdb75c13352398137dd Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 1 Dec 2021 10:10:33 -0800 Subject: [PATCH 013/131] libbpf: Support init of inner maps in light skeleton. Add ability to initialize inner maps in light skeleton. Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-11-alexei.starovoitov@gmail.com --- src/bpf_gen_internal.h | 1 + src/gen_loader.c | 27 +++++++++++++++++++++++++++ src/libbpf.c | 6 +++--- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/bpf_gen_internal.h b/src/bpf_gen_internal.h index 5d514bd..2233089 100644 --- a/src/bpf_gen_internal.h +++ b/src/bpf_gen_internal.h @@ -67,5 +67,6 @@ void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum b void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, bool is_typeless, int kind, int insn_idx); void bpf_gen__record_relo_core(struct bpf_gen *gen, const struct bpf_core_relo *core_relo); +void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int key, int inner_map_idx); #endif diff --git a/src/gen_loader.c b/src/gen_loader.c index ad450a4..c2836d8 100644 --- a/src/gen_loader.c +++ b/src/gen_loader.c @@ -1066,6 +1066,33 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, emit_check_err(gen); } +void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, + int inner_map_idx) +{ + int attr_size = offsetofend(union bpf_attr, flags); + int map_update_attr, key; + union bpf_attr attr; + + memset(&attr, 0, attr_size); + pr_debug("gen: populate_outer_map: outer %d key %d inner %d\n", + outer_map_idx, slot, inner_map_idx); + + key = add_data(gen, &slot, sizeof(slot)); + + map_update_attr = add_data(gen, &attr, attr_size); + move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, + blob_fd_array_off(gen, outer_map_idx)); + emit_rel_store(gen, attr_field(map_update_attr, key), key); + emit_rel_store(gen, attr_field(map_update_attr, value), + blob_fd_array_off(gen, inner_map_idx)); + + /* emit MAP_UPDATE_ELEM command */ + emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); + debug_ret(gen, "populate_outer_map outer %d key %d inner %d", + outer_map_idx, slot, inner_map_idx); + emit_check_err(gen); +} + void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) { int attr_size = offsetofend(union bpf_attr, map_fd); diff --git a/src/libbpf.c b/src/libbpf.c index f680d5f..09ba9e9 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -4971,9 +4971,9 @@ static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) fd = bpf_map__fd(targ_map); if (obj->gen_loader) { - pr_warn("// TODO map_update_elem: idx %td key %d value==map_idx %td\n", - map - obj->maps, i, targ_map - obj->maps); - return -ENOTSUP; + bpf_gen__populate_outer_map(obj->gen_loader, + map - obj->maps, i, + targ_map - obj->maps); } else { err = bpf_map_update_elem(map->fd, &i, &fd, 0); } From 01b2b45e8df3e65f210ca0205a7325233aecb002 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 1 Dec 2021 10:10:34 -0800 Subject: [PATCH 014/131] libbpf: Clean gen_loader's attach kind. The gen_loader has to clear attach_kind otherwise the programs without attach_btf_id will fail load if they follow programs with attach_btf_id. Fixes: 67234743736a ("libbpf: Generate loader program out of BPF ELF file.") Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-12-alexei.starovoitov@gmail.com --- src/gen_loader.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/gen_loader.c b/src/gen_loader.c index c2836d8..8ed02e8 100644 --- a/src/gen_loader.c +++ b/src/gen_loader.c @@ -1018,9 +1018,11 @@ void bpf_gen__prog_load(struct bpf_gen *gen, debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt); /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */ cleanup_relos(gen, insns_off); - if (gen->attach_kind) + if (gen->attach_kind) { emit_sys_close_blob(gen, attr_field(prog_load_attr, attach_btf_obj_fd)); + gen->attach_kind = 0; + } emit_check_err(gen); /* remember prog_fd in the stack, if successful */ emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, From ac1c007607f288d5368e3d3f4e4bcaca0510d759 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 1 Dec 2021 15:28:16 -0800 Subject: [PATCH 015/131] libbpf: Use __u32 fields in bpf_map_create_opts Corresponding Linux UAPI struct uses __u32, not int, so keep it consistent. Fixes: 992c4225419a ("libbpf: Unify low-level map creation APIs w/ new bpf_map_create()") Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211201232824.3166325-2-andrii@kernel.org --- src/bpf.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bpf.h b/src/bpf.h index 70b6f44..f79e5fb 100644 --- a/src/bpf.h +++ b/src/bpf.h @@ -43,12 +43,12 @@ struct bpf_map_create_opts { __u32 btf_value_type_id; __u32 btf_vmlinux_value_type_id; - int inner_map_fd; - int map_flags; + __u32 inner_map_fd; + __u32 map_flags; __u64 map_extra; - int numa_node; - int map_ifindex; + __u32 numa_node; + __u32 map_ifindex; }; #define bpf_map_create_opts__last_field map_ifindex From d761220e33b00a68eae993eb2355024fc45e9f80 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 1 Dec 2021 15:28:17 -0800 Subject: [PATCH 016/131] libbpf: Add API to get/set log_level at per-program level Add bpf_program__set_log_level() and bpf_program__log_level() to fetch and adjust log_level sent during BPF_PROG_LOAD command. This allows to selectively request more or less verbose output in BPF verifier log. Also bump libbpf version to 0.7 and make these APIs the first in v0.7. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211201232824.3166325-3-andrii@kernel.org --- src/libbpf.c | 14 ++++++++++++++ src/libbpf.h | 2 ++ src/libbpf.map | 6 ++++++ src/libbpf_version.h | 2 +- 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/libbpf.c b/src/libbpf.c index 09ba9e9..d22ff9d 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -8475,6 +8475,20 @@ int bpf_program__set_flags(struct bpf_program *prog, __u32 flags) return 0; } +__u32 bpf_program__log_level(const struct bpf_program *prog) +{ + return prog->log_level; +} + +int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level) +{ + if (prog->obj->loaded) + return libbpf_err(-EBUSY); + + prog->log_level = log_level; + return 0; +} + #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \ .sec = sec_pfx, \ .prog_type = BPF_PROG_TYPE_##ptype, \ diff --git a/src/libbpf.h b/src/libbpf.h index d02139f..148fa85 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -499,6 +499,8 @@ bpf_program__set_expected_attach_type(struct bpf_program *prog, LIBBPF_API __u32 bpf_program__flags(const struct bpf_program *prog); LIBBPF_API int bpf_program__set_flags(struct bpf_program *prog, __u32 flags); +LIBBPF_API __u32 bpf_program__log_level(const struct bpf_program *prog); +LIBBPF_API int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level); LIBBPF_API int bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd, diff --git a/src/libbpf.map b/src/libbpf.map index 623002b..715df3a 100644 --- a/src/libbpf.map +++ b/src/libbpf.map @@ -419,3 +419,9 @@ LIBBPF_0.6.0 { perf_buffer__new_raw; perf_buffer__new_raw_deprecated; } LIBBPF_0.5.0; + +LIBBPF_0.7.0 { + global: + bpf_program__log_level; + bpf_program__set_log_level; +}; diff --git a/src/libbpf_version.h b/src/libbpf_version.h index dd56d76..0fefefc 100644 --- a/src/libbpf_version.h +++ b/src/libbpf_version.h @@ -4,6 +4,6 @@ #define __LIBBPF_VERSION_H #define LIBBPF_MAJOR_VERSION 0 -#define LIBBPF_MINOR_VERSION 6 +#define LIBBPF_MINOR_VERSION 7 #endif /* __LIBBPF_VERSION_H */ From b633ace36684ded208f9d51ea79d9dac79ba001b Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 1 Dec 2021 15:28:24 -0800 Subject: [PATCH 017/131] libbpf: Deprecate bpf_prog_load_xattr() API bpf_prog_load_xattr() is high-level API that's named as a low-level BPF_PROG_LOAD wrapper APIs, but it actually operates on struct bpf_object. It's badly and confusingly misnamed as it will load all the progs insige bpf_object, returning prog_fd of the very first BPF program. It also has a bunch of ad-hoc things like log_level override, map_ifindex auto-setting, etc. All this can be expressed more explicitly and cleanly through existing libbpf APIs. This patch marks bpf_prog_load_xattr() for deprecation in libbpf v0.8 ([0]). [0] Closes: https://github.com/libbpf/libbpf/issues/308 Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211201232824.3166325-10-andrii@kernel.org --- src/libbpf.h | 1 + src/libbpf_common.h | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/libbpf.h b/src/libbpf.h index 148fa85..c0d62dd 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -682,6 +682,7 @@ struct bpf_prog_load_attr { int prog_flags; }; +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__open() and bpf_object__load() instead") LIBBPF_API int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, struct bpf_object **pobj, int *prog_fd); LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__open() and bpf_object__load() instead") diff --git a/src/libbpf_common.h b/src/libbpf_common.h index b21cefc..000e377 100644 --- a/src/libbpf_common.h +++ b/src/libbpf_common.h @@ -40,6 +40,11 @@ #else #define __LIBBPF_MARK_DEPRECATED_0_7(X) #endif +#if __LIBBPF_CURRENT_VERSION_GEQ(0, 8) +#define __LIBBPF_MARK_DEPRECATED_0_8(X) X +#else +#define __LIBBPF_MARK_DEPRECATED_0_8(X) +#endif /* This set of internal macros allows to do "function overloading" based on * number of arguments provided by used in backwards-compatible way during the From c3c540b402e9ce7eedd7acc40c4b851214c7288b Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 3 Dec 2021 10:28:36 -0800 Subject: [PATCH 018/131] libbpf: Reduce bpf_core_apply_relo_insn() stack usage. Reduce bpf_core_apply_relo_insn() stack usage and bump BPF_CORE_SPEC_MAX_LEN limit back to 64. Fixes: 29db4bea1d10 ("bpf: Prepare relo_core.c for kernel duty.") Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211203182836.16646-1-alexei.starovoitov@gmail.com --- src/libbpf.c | 4 +++- src/relo_core.c | 62 +++++++++++++++---------------------------------- src/relo_core.h | 30 +++++++++++++++++++++++- 3 files changed, 51 insertions(+), 45 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index d22ff9d..d46962d 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -5515,6 +5515,7 @@ static int bpf_core_apply_relo(struct bpf_program *prog, const struct btf *local_btf, struct hashmap *cand_cache) { + struct bpf_core_spec specs_scratch[3] = {}; const void *type_key = u32_as_hash_key(relo->type_id); struct bpf_core_cand_list *cands = NULL; const char *prog_name = prog->name; @@ -5569,7 +5570,8 @@ static int bpf_core_apply_relo(struct bpf_program *prog, } } - return bpf_core_apply_relo_insn(prog_name, insn, insn_idx, relo, relo_idx, local_btf, cands); + return bpf_core_apply_relo_insn(prog_name, insn, insn_idx, relo, + relo_idx, local_btf, cands, specs_scratch); } static int diff --git a/src/relo_core.c b/src/relo_core.c index d194fb9..32464f0 100644 --- a/src/relo_core.c +++ b/src/relo_core.c @@ -68,33 +68,6 @@ enum libbpf_print_level { #include "libbpf_internal.h" #endif -#define BPF_CORE_SPEC_MAX_LEN 32 - -/* represents BPF CO-RE field or array element accessor */ -struct bpf_core_accessor { - __u32 type_id; /* struct/union type or array element type */ - __u32 idx; /* field index or array index */ - const char *name; /* field name or NULL for array accessor */ -}; - -struct bpf_core_spec { - const struct btf *btf; - /* high-level spec: named fields and array indices only */ - struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN]; - /* original unresolved (no skip_mods_or_typedefs) root type ID */ - __u32 root_type_id; - /* CO-RE relocation kind */ - enum bpf_core_relo_kind relo_kind; - /* high-level spec length */ - int len; - /* raw, low-level spec: 1-to-1 with accessor spec string */ - int raw_spec[BPF_CORE_SPEC_MAX_LEN]; - /* raw spec length */ - int raw_len; - /* field bit offset represented by spec */ - __u32 bit_offset; -}; - static bool is_flex_arr(const struct btf *btf, const struct bpf_core_accessor *acc, const struct btf_array *arr) @@ -1200,9 +1173,12 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, const struct bpf_core_relo *relo, int relo_idx, const struct btf *local_btf, - struct bpf_core_cand_list *cands) + struct bpf_core_cand_list *cands, + struct bpf_core_spec *specs_scratch) { - struct bpf_core_spec local_spec, cand_spec, targ_spec = {}; + struct bpf_core_spec *local_spec = &specs_scratch[0]; + struct bpf_core_spec *cand_spec = &specs_scratch[1]; + struct bpf_core_spec *targ_spec = &specs_scratch[2]; struct bpf_core_relo_res cand_res, targ_res; const struct btf_type *local_type; const char *local_name; @@ -1221,7 +1197,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, return -EINVAL; err = bpf_core_parse_spec(prog_name, local_btf, local_id, spec_str, - relo->kind, &local_spec); + relo->kind, local_spec); if (err) { pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), @@ -1232,15 +1208,15 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); - bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, &local_spec); + bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, local_spec); libbpf_print(LIBBPF_DEBUG, "\n"); /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */ if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) { targ_res.validate = true; targ_res.poison = false; - targ_res.orig_val = local_spec.root_type_id; - targ_res.new_val = local_spec.root_type_id; + targ_res.orig_val = local_spec->root_type_id; + targ_res.new_val = local_spec->root_type_id; goto patch_insn; } @@ -1253,38 +1229,38 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, for (i = 0, j = 0; i < cands->len; i++) { - err = bpf_core_spec_match(&local_spec, cands->cands[i].btf, - cands->cands[i].id, &cand_spec); + err = bpf_core_spec_match(local_spec, cands->cands[i].btf, + cands->cands[i].id, cand_spec); if (err < 0) { pr_warn("prog '%s': relo #%d: error matching candidate #%d ", prog_name, relo_idx, i); - bpf_core_dump_spec(prog_name, LIBBPF_WARN, &cand_spec); + bpf_core_dump_spec(prog_name, LIBBPF_WARN, cand_spec); libbpf_print(LIBBPF_WARN, ": %d\n", err); return err; } pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog_name, relo_idx, err == 0 ? "non-matching" : "matching", i); - bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, &cand_spec); + bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, cand_spec); libbpf_print(LIBBPF_DEBUG, "\n"); if (err == 0) continue; - err = bpf_core_calc_relo(prog_name, relo, relo_idx, &local_spec, &cand_spec, &cand_res); + err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, cand_spec, &cand_res); if (err) return err; if (j == 0) { targ_res = cand_res; - targ_spec = cand_spec; - } else if (cand_spec.bit_offset != targ_spec.bit_offset) { + *targ_spec = *cand_spec; + } else if (cand_spec->bit_offset != targ_spec->bit_offset) { /* if there are many field relo candidates, they * should all resolve to the same bit offset */ pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n", - prog_name, relo_idx, cand_spec.bit_offset, - targ_spec.bit_offset); + prog_name, relo_idx, cand_spec->bit_offset, + targ_spec->bit_offset); return -EINVAL; } else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) { /* all candidates should result in the same relocation @@ -1328,7 +1304,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, prog_name, relo_idx); /* calculate single target relo result explicitly */ - err = bpf_core_calc_relo(prog_name, relo, relo_idx, &local_spec, NULL, &targ_res); + err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, NULL, &targ_res); if (err) return err; } diff --git a/src/relo_core.h b/src/relo_core.h index 4f864b8..1779981 100644 --- a/src/relo_core.h +++ b/src/relo_core.h @@ -17,11 +17,39 @@ struct bpf_core_cand_list { int len; }; +#define BPF_CORE_SPEC_MAX_LEN 64 + +/* represents BPF CO-RE field or array element accessor */ +struct bpf_core_accessor { + __u32 type_id; /* struct/union type or array element type */ + __u32 idx; /* field index or array index */ + const char *name; /* field name or NULL for array accessor */ +}; + +struct bpf_core_spec { + const struct btf *btf; + /* high-level spec: named fields and array indices only */ + struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN]; + /* original unresolved (no skip_mods_or_typedefs) root type ID */ + __u32 root_type_id; + /* CO-RE relocation kind */ + enum bpf_core_relo_kind relo_kind; + /* high-level spec length */ + int len; + /* raw, low-level spec: 1-to-1 with accessor spec string */ + int raw_spec[BPF_CORE_SPEC_MAX_LEN]; + /* raw spec length */ + int raw_len; + /* field bit offset represented by spec */ + __u32 bit_offset; +}; + int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, int insn_idx, const struct bpf_core_relo *relo, int relo_idx, const struct btf *local_btf, - struct bpf_core_cand_list *cands); + struct bpf_core_cand_list *cands, + struct bpf_core_spec *specs_scratch); int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, __u32 targ_id); From 04941813a53f76907d74dac3b862d64c2ba6e6d0 Mon Sep 17 00:00:00 2001 From: huangxuesen Date: Mon, 6 Dec 2021 09:47:16 +0800 Subject: [PATCH 019/131] libbpf: Fix trivial typo Fix typo in comment from 'bpf_skeleton_map' to 'bpf_map_skeleton' and from 'bpf_skeleton_prog' to 'bpf_prog_skeleton'. Signed-off-by: huangxuesen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/1638755236-3851199-1-git-send-email-hxseverything@gmail.com --- src/libbpf.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libbpf.h b/src/libbpf.h index c0d62dd..2fa046a 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -1038,11 +1038,11 @@ struct bpf_object_skeleton { struct bpf_object **obj; int map_cnt; - int map_skel_sz; /* sizeof(struct bpf_skeleton_map) */ + int map_skel_sz; /* sizeof(struct bpf_map_skeleton) */ struct bpf_map_skeleton *maps; int prog_cnt; - int prog_skel_sz; /* sizeof(struct bpf_skeleton_prog) */ + int prog_skel_sz; /* sizeof(struct bpf_prog_skeleton) */ struct bpf_prog_skeleton *progs; }; From 728b1721e5de7ff64a01cabbf17167a8e48a17b8 Mon Sep 17 00:00:00 2001 From: Grant Seltzer Date: Mon, 6 Dec 2021 15:37:09 -0500 Subject: [PATCH 020/131] libbpf: Add doc comments in libbpf.h This adds comments above functions in libbpf.h which document their uses. These comments are of a format that doxygen and sphinx can pick up and render. These are rendered by libbpf.readthedocs.org These doc comments are for: - bpf_object__open_file() - bpf_object__open_mem() - bpf_program__attach_uprobe() - bpf_program__attach_uprobe_opts() Signed-off-by: Grant Seltzer Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211206203709.332530-1-grantseltzer@gmail.com --- src/libbpf.h | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/libbpf.h b/src/libbpf.h index 2fa046a..4802c1e 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -112,8 +112,30 @@ struct bpf_object_open_opts { #define bpf_object_open_opts__last_field btf_custom_path LIBBPF_API struct bpf_object *bpf_object__open(const char *path); + +/** + * @brief **bpf_object__open_file()** creates a bpf_object by opening + * the BPF ELF object file pointed to by the passed path and loading it + * into memory. + * @param path BPF object file path + * @param opts options for how to load the bpf object, this parameter is + * optional and can be set to NULL + * @return pointer to the new bpf_object; or NULL is returned on error, + * error code is stored in errno + */ LIBBPF_API struct bpf_object * bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts); + +/** + * @brief **bpf_object__open_mem()** creates a bpf_object by reading + * the BPF objects raw bytes from a memory buffer containing a valid + * BPF ELF object file. + * @param obj_buf pointer to the buffer containing ELF file bytes + * @param obj_buf_sz number of bytes in the buffer + * @param opts options for how to load the bpf object + * @return pointer to the new bpf_object; or NULL is returned on error, + * error code is stored in errno + */ LIBBPF_API struct bpf_object * bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, const struct bpf_object_open_opts *opts); @@ -348,10 +370,41 @@ struct bpf_uprobe_opts { }; #define bpf_uprobe_opts__last_field retprobe +/** + * @brief **bpf_program__attach_uprobe()** attaches a BPF program + * to the userspace function which is found by binary path and + * offset. You can optionally specify a particular proccess to attach + * to. You can also optionally attach the program to the function + * exit instead of entry. + * + * @param prog BPF program to attach + * @param retprobe Attach to function exit + * @param pid Process ID to attach the uprobe to, 0 for self (own process), + * -1 for all processes + * @param binary_path Path to binary that contains the function symbol + * @param func_offset Offset within the binary of the function symbol + * @return Reference to the newly created BPF link; or NULL is returned on error, + * error code is stored in errno + */ LIBBPF_API struct bpf_link * bpf_program__attach_uprobe(const struct bpf_program *prog, bool retprobe, pid_t pid, const char *binary_path, size_t func_offset); + +/** + * @brief **bpf_program__attach_uprobe_opts()** is just like + * bpf_program__attach_uprobe() except with a options struct + * for various configurations. + * + * @param prog BPF program to attach + * @param pid Process ID to attach the uprobe to, 0 for self (own process), + * -1 for all processes + * @param binary_path Path to binary that contains the function symbol + * @param func_offset Offset within the binary of the function symbol + * @param opts Options for altering program attachment + * @return Reference to the newly created BPF link; or NULL is returned on error, + * error code is stored in errno + */ LIBBPF_API struct bpf_link * bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, const char *binary_path, size_t func_offset, From 896a3ae0d0f33c2152a3cb828e9d99b6746e3123 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 9 Dec 2021 11:38:29 -0800 Subject: [PATCH 021/131] libbpf: Fix bpf_prog_load() log_buf logic for log_level 0 To unify libbpf APIs behavior w.r.t. log_buf and log_level, fix bpf_prog_load() to follow the same logic as bpf_btf_load() and high-level bpf_object__load() API will follow in the subsequent patches: - if log_level is 0 and non-NULL log_buf is provided by a user, attempt load operation initially with no log_buf and log_level set; - if successful, we are done, return new FD; - on error, retry the load operation with log_level bumped to 1 and log_buf set; this way verbose logging will be requested only when we are sure that there is a failure, but will be fast in the common/expected success case. Of course, user can still specify log_level > 0 from the very beginning to force log collection. Suggested-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211209193840.1248570-2-andrii@kernel.org --- src/bpf.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/bpf.c b/src/bpf.c index 4e7836e..3dc8634 100644 --- a/src/bpf.c +++ b/src/bpf.c @@ -303,10 +303,6 @@ int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, if (log_level && !log_buf) return libbpf_err(-EINVAL); - attr.log_level = log_level; - attr.log_buf = ptr_to_u64(log_buf); - attr.log_size = log_size; - func_info_rec_size = OPTS_GET(opts, func_info_rec_size, 0); func_info = OPTS_GET(opts, func_info, NULL); attr.func_info_rec_size = func_info_rec_size; @@ -321,6 +317,12 @@ int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, attr.fd_array = ptr_to_u64(OPTS_GET(opts, fd_array, NULL)); + if (log_level) { + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = log_size; + attr.log_level = log_level; + } + fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts); if (fd >= 0) return fd; @@ -366,16 +368,17 @@ int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, goto done; } - if (log_level || !log_buf) - goto done; + if (log_level == 0 && log_buf) { + /* log_level == 0 with non-NULL log_buf requires retrying on error + * with log_level == 1 and log_buf/log_buf_size set, to get details of + * failure + */ + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = log_size; + attr.log_level = 1; - /* Try again with log */ - log_buf[0] = 0; - attr.log_buf = ptr_to_u64(log_buf); - attr.log_size = log_size; - attr.log_level = 1; - - fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts); + fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts); + } done: /* free() doesn't affect errno, so we don't need to restore it */ free(finfo); From 3c93f7ddb20f90f3f272ccbbe34339737bea0501 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 9 Dec 2021 11:38:30 -0800 Subject: [PATCH 022/131] libbpf: Add OPTS-based bpf_btf_load() API Similar to previous bpf_prog_load() and bpf_map_create() APIs, add bpf_btf_load() API which is taking optional OPTS struct. Schedule bpf_load_btf() for deprecation in v0.8 ([0]). This makes naming consistent with BPF_BTF_LOAD command, sets up an API for extensibility in the future, moves options parameters (log-related fields) into optional options, and also allows to pass log_level directly. It also removes log buffer auto-allocation logic from low-level API (consistent with bpf_prog_load() behavior), but preserves a special treatment of log_level == 0 with non-NULL log_buf, which matches low-level bpf_prog_load() and high-level libbpf APIs for BTF and program loading behaviors. [0] Closes: https://github.com/libbpf/libbpf/issues/419 Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211209193840.1248570-3-andrii@kernel.org --- src/bpf.c | 59 ++++++++++++++++++++++++++++++++++++++------- src/bpf.h | 19 +++++++++++++-- src/libbpf.map | 1 + src/libbpf_probes.c | 2 +- 4 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/bpf.c b/src/bpf.c index 3dc8634..6b2407e 100644 --- a/src/bpf.c +++ b/src/bpf.c @@ -1047,24 +1047,65 @@ int bpf_raw_tracepoint_open(const char *name, int prog_fd) return libbpf_err_errno(fd); } -int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size, - bool do_log) +int bpf_btf_load(const void *btf_data, size_t btf_size, const struct bpf_btf_load_opts *opts) { - union bpf_attr attr = {}; + const size_t attr_sz = offsetofend(union bpf_attr, btf_log_level); + union bpf_attr attr; + char *log_buf; + size_t log_size; + __u32 log_level; int fd; - attr.btf = ptr_to_u64(btf); + memset(&attr, 0, attr_sz); + + if (!OPTS_VALID(opts, bpf_btf_load_opts)) + return libbpf_err(-EINVAL); + + log_buf = OPTS_GET(opts, log_buf, NULL); + log_size = OPTS_GET(opts, log_size, 0); + log_level = OPTS_GET(opts, log_level, 0); + + if (log_size > UINT_MAX) + return libbpf_err(-EINVAL); + if (log_size && !log_buf) + return libbpf_err(-EINVAL); + + attr.btf = ptr_to_u64(btf_data); attr.btf_size = btf_size; + /* log_level == 0 and log_buf != NULL means "try loading without + * log_buf, but retry with log_buf and log_level=1 on error", which is + * consistent across low-level and high-level BTF and program loading + * APIs within libbpf and provides a sensible behavior in practice + */ + if (log_level) { + attr.btf_log_buf = ptr_to_u64(log_buf); + attr.btf_log_size = (__u32)log_size; + attr.btf_log_level = log_level; + } + + fd = sys_bpf_fd(BPF_BTF_LOAD, &attr, attr_sz); + if (fd < 0 && log_buf && log_level == 0) { + attr.btf_log_buf = ptr_to_u64(log_buf); + attr.btf_log_size = (__u32)log_size; + attr.btf_log_level = 1; + fd = sys_bpf_fd(BPF_BTF_LOAD, &attr, attr_sz); + } + return libbpf_err_errno(fd); +} + +int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size, bool do_log) +{ + LIBBPF_OPTS(bpf_btf_load_opts, opts); + int fd; retry: if (do_log && log_buf && log_buf_size) { - attr.btf_log_level = 1; - attr.btf_log_size = log_buf_size; - attr.btf_log_buf = ptr_to_u64(log_buf); + opts.log_buf = log_buf; + opts.log_size = log_buf_size; + opts.log_level = 1; } - fd = sys_bpf_fd(BPF_BTF_LOAD, &attr, sizeof(attr)); - + fd = bpf_btf_load(btf, btf_size, &opts); if (fd < 0 && !do_log && log_buf && log_buf_size) { do_log = true; goto retry; diff --git a/src/bpf.h b/src/bpf.h index f79e5fb..5f7d963 100644 --- a/src/bpf.h +++ b/src/bpf.h @@ -213,6 +213,23 @@ LIBBPF_API int bpf_verify_program(enum bpf_prog_type type, char *log_buf, size_t log_buf_sz, int log_level); +struct bpf_btf_load_opts { + size_t sz; /* size of this struct for forward/backward compatibility */ + + /* kernel log options */ + char *log_buf; + __u32 log_level; + __u32 log_size; +}; +#define bpf_btf_load_opts__last_field log_size + +LIBBPF_API int bpf_btf_load(const void *btf_data, size_t btf_size, + const struct bpf_btf_load_opts *opts); + +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_btf_load() instead") +LIBBPF_API int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, + __u32 log_buf_size, bool do_log); + LIBBPF_API int bpf_map_update_elem(int fd, const void *key, const void *value, __u64 flags); @@ -340,8 +357,6 @@ LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt); LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd); -LIBBPF_API int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, - __u32 log_buf_size, bool do_log); LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset, __u64 *probe_addr); diff --git a/src/libbpf.map b/src/libbpf.map index 715df3a..08cdfe8 100644 --- a/src/libbpf.map +++ b/src/libbpf.map @@ -422,6 +422,7 @@ LIBBPF_0.6.0 { LIBBPF_0.7.0 { global: + bpf_btf_load; bpf_program__log_level; bpf_program__set_log_level; }; diff --git a/src/libbpf_probes.c b/src/libbpf_probes.c index 41f2be4..4bdec69 100644 --- a/src/libbpf_probes.c +++ b/src/libbpf_probes.c @@ -164,7 +164,7 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len, memcpy(raw_btf + hdr.hdr_len, raw_types, hdr.type_len); memcpy(raw_btf + hdr.hdr_len + hdr.type_len, str_sec, hdr.str_len); - btf_fd = bpf_load_btf(raw_btf, btf_len, NULL, 0, false); + btf_fd = bpf_btf_load(raw_btf, btf_len, NULL); free(raw_btf); return btf_fd; From 0504b7ff2211e2d4d9bbe1b550f0f5eb42ad6581 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 9 Dec 2021 11:38:31 -0800 Subject: [PATCH 023/131] libbpf: Allow passing preallocated log_buf when loading BTF into kernel Add libbpf-internal btf_load_into_kernel() that allows to pass preallocated log_buf and custom log_level to be passed into kernel during BPF_BTF_LOAD call. When custom log_buf is provided, btf_load_into_kernel() won't attempt an retry with automatically allocated internal temporary buffer to capture BTF validation log. It's important to note the relation between log_buf and log_level, which slightly deviates from stricter kernel logic. From kernel's POV, if log_buf is specified, log_level has to be > 0, and vice versa. While kernel has good reasons to request such "sanity, this, in practice, is a bit unconvenient and restrictive for libbpf's high-level bpf_object APIs. So libbpf will allow to set non-NULL log_buf and log_level == 0. This is fine and means to attempt to load BTF without logging requested, but if it failes, retry the load with custom log_buf and log_level 1. Similar logic will be implemented for program loading. In practice this means that users can provide custom log buffer just in case error happens, but not really request slower verbose logging all the time. This is also consistent with libbpf behavior when custom log_buf is not set: libbpf first tries to load everything with log_level=0, and only if error happens allocates internal log buffer and retries with log_level=1. Also, while at it, make BTF validation log more obvious and follow the log pattern libbpf is using for dumping BPF verifier log during BPF_PROG_LOAD. BTF loading resulting in an error will look like this: libbpf: BTF loading error: -22 libbpf: -- BEGIN BTF LOAD LOG --- magic: 0xeb9f version: 1 flags: 0x0 hdr_len: 24 type_off: 0 type_len: 1040 str_off: 1040 str_len: 2063598257 btf_total_size: 1753 Total section length too long -- END BTF LOAD LOG -- libbpf: Error loading .BTF into kernel: -22. BTF is optional, ignoring. This makes it much easier to find relevant parts in libbpf log output. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211209193840.1248570-4-andrii@kernel.org --- src/btf.c | 82 ++++++++++++++++++++++++++++++------------- src/libbpf_internal.h | 1 + 2 files changed, 58 insertions(+), 25 deletions(-) diff --git a/src/btf.c b/src/btf.c index 0d7b16e..e171424 100644 --- a/src/btf.c +++ b/src/btf.c @@ -1124,54 +1124,86 @@ struct btf *btf__parse_split(const char *path, struct btf *base_btf) static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian); -int btf__load_into_kernel(struct btf *btf) +int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level) { - __u32 log_buf_size = 0, raw_size; - char *log_buf = NULL; + LIBBPF_OPTS(bpf_btf_load_opts, opts); + __u32 buf_sz = 0, raw_size; + char *buf = NULL, *tmp; void *raw_data; int err = 0; if (btf->fd >= 0) return libbpf_err(-EEXIST); + if (log_sz && !log_buf) + return libbpf_err(-EINVAL); -retry_load: - if (log_buf_size) { - log_buf = malloc(log_buf_size); - if (!log_buf) - return libbpf_err(-ENOMEM); - - *log_buf = 0; - } - + /* cache native raw data representation */ raw_data = btf_get_raw_data(btf, &raw_size, false); if (!raw_data) { err = -ENOMEM; goto done; } - /* cache native raw data representation */ btf->raw_size = raw_size; btf->raw_data = raw_data; - btf->fd = bpf_load_btf(raw_data, raw_size, log_buf, log_buf_size, false); - if (btf->fd < 0) { - if (!log_buf || errno == ENOSPC) { - log_buf_size = max((__u32)BPF_LOG_BUF_SIZE, - log_buf_size << 1); - free(log_buf); - goto retry_load; +retry_load: + /* if log_level is 0, we won't provide log_buf/log_size to the kernel, + * initially. Only if BTF loading fails, we bump log_level to 1 and + * retry, using either auto-allocated or custom log_buf. This way + * non-NULL custom log_buf provides a buffer just in case, but hopes + * for successful load and no need for log_buf. + */ + if (log_level) { + /* if caller didn't provide custom log_buf, we'll keep + * allocating our own progressively bigger buffers for BTF + * verification log + */ + if (!log_buf) { + buf_sz = max((__u32)BPF_LOG_BUF_SIZE, buf_sz * 2); + tmp = realloc(buf, buf_sz); + if (!tmp) { + err = -ENOMEM; + goto done; + } + buf = tmp; + buf[0] = '\0'; } + opts.log_buf = log_buf ? log_buf : buf; + opts.log_size = log_buf ? log_sz : buf_sz; + opts.log_level = log_level; + } + + btf->fd = bpf_btf_load(raw_data, raw_size, &opts); + if (btf->fd < 0) { + /* time to turn on verbose mode and try again */ + if (log_level == 0) { + log_level = 1; + goto retry_load; + } + /* only retry if caller didn't provide custom log_buf, but + * make sure we can never overflow buf_sz + */ + if (!log_buf && errno == ENOSPC && buf_sz <= UINT_MAX / 2) + goto retry_load; + err = -errno; - pr_warn("Error loading BTF: %s(%d)\n", strerror(errno), errno); - if (*log_buf) - pr_warn("%s\n", log_buf); - goto done; + pr_warn("BTF loading error: %d\n", err); + /* don't print out contents of custom log_buf */ + if (!log_buf && buf[0]) + pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf); } done: - free(log_buf); + free(buf); return libbpf_err(err); } + +int btf__load_into_kernel(struct btf *btf) +{ + return btf_load_into_kernel(btf, NULL, 0, 0); +} + int btf__load(struct btf *) __attribute__((alias("btf__load_into_kernel"))); int btf__fd(const struct btf *btf) diff --git a/src/libbpf_internal.h b/src/libbpf_internal.h index 6f143e9..355c410 100644 --- a/src/libbpf_internal.h +++ b/src/libbpf_internal.h @@ -277,6 +277,7 @@ int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz); int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); int libbpf__load_raw_btf(const char *raw_types, size_t types_len, const char *str_sec, size_t str_len); +int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level); struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf); void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, From dc1df243145b25f4487af37147f219a649aa768d Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 9 Dec 2021 11:38:32 -0800 Subject: [PATCH 024/131] libbpf: Allow passing user log setting through bpf_object_open_opts Allow users to provide their own custom log_buf, log_size, and log_level at bpf_object level through bpf_object_open_opts. This log_buf will be used during BTF loading. Subsequent patch will use same log_buf during BPF program loading, unless overriden at per-bpf_program level. When such custom log_buf is provided, libbpf won't be attempting retrying loading of BTF to try to provide its own log buffer to capture kernel's error log output. User is responsible to provide big enough buffer, otherwise they run a risk of getting -ENOSPC error from the bpf() syscall. See also comments in bpf_object_open_opts regarding log_level and log_buf interactions. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211209193840.1248570-5-andrii@kernel.org --- src/bpf.h | 3 ++- src/libbpf.c | 24 +++++++++++++++++++++++- src/libbpf.h | 41 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/src/bpf.h b/src/bpf.h index 5f7d963..94e553a 100644 --- a/src/bpf.h +++ b/src/bpf.h @@ -195,8 +195,9 @@ struct bpf_load_program_attr { /* Flags to direct loading requirements */ #define MAPS_RELAX_COMPAT 0x01 -/* Recommend log buffer size */ +/* Recommended log buffer size */ #define BPF_LOG_BUF_SIZE (UINT32_MAX >> 8) /* verifier maximum in kernels <= 5.1 */ + LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_load() instead") LIBBPF_API int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, char *log_buf, size_t log_buf_sz); diff --git a/src/libbpf.c b/src/libbpf.c index d46962d..227da8c 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -573,6 +573,11 @@ struct bpf_object { size_t btf_module_cnt; size_t btf_module_cap; + /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */ + char *log_buf; + size_t log_size; + __u32 log_level; + void *priv; bpf_object_clear_priv_t clear_priv; @@ -3017,7 +3022,9 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) */ btf__set_fd(kern_btf, 0); } else { - err = btf__load_into_kernel(kern_btf); + /* currently BPF_BTF_LOAD only supports log_level 1 */ + err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, + obj->log_level ? 1 : 0); } if (sanitize) { if (!err) { @@ -6932,6 +6939,9 @@ __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, struct bpf_object *obj; char tmp_name[64]; int err; + char *log_buf; + size_t log_size; + __u32 log_level; if (elf_version(EV_CURRENT) == EV_NONE) { pr_warn("failed to init libelf for %s\n", @@ -6954,10 +6964,22 @@ __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, pr_debug("loading object '%s' from buffer\n", obj_name); } + log_buf = OPTS_GET(opts, kernel_log_buf, NULL); + log_size = OPTS_GET(opts, kernel_log_size, 0); + log_level = OPTS_GET(opts, kernel_log_level, 0); + if (log_size > UINT_MAX) + return ERR_PTR(-EINVAL); + if (log_size && !log_buf) + return ERR_PTR(-EINVAL); + obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); if (IS_ERR(obj)) return obj; + obj->log_buf = log_buf; + obj->log_size = log_size; + obj->log_level = log_level; + btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); if (btf_tmp_path) { if (strlen(btf_tmp_path) >= PATH_MAX) { diff --git a/src/libbpf.h b/src/libbpf.h index 4802c1e..5c984c6 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -108,8 +108,47 @@ struct bpf_object_open_opts { * struct_ops, etc) will need actual kernel BTF at /sys/kernel/btf/vmlinux. */ const char *btf_custom_path; + /* Pointer to a buffer for storing kernel logs for applicable BPF + * commands. Valid kernel_log_size has to be specified as well and are + * passed-through to bpf() syscall. Keep in mind that kernel might + * fail operation with -ENOSPC error if provided buffer is too small + * to contain entire log output. + * See the comment below for kernel_log_level for interaction between + * log_buf and log_level settings. + * + * If specified, this log buffer will be passed for: + * - each BPF progral load (BPF_PROG_LOAD) attempt, unless overriden + * with bpf_program__set_log() on per-program level, to get + * BPF verifier log output. + * - during BPF object's BTF load into kernel (BPF_BTF_LOAD) to get + * BTF sanity checking log. + * + * Each BPF command (BPF_BTF_LOAD or BPF_PROG_LOAD) will overwrite + * previous contents, so if you need more fine-grained control, set + * per-program buffer with bpf_program__set_log_buf() to preserve each + * individual program's verification log. Keep using kernel_log_buf + * for BTF verification log, if necessary. + */ + char *kernel_log_buf; + size_t kernel_log_size; + /* + * Log level can be set independently from log buffer. Log_level=0 + * means that libbpf will attempt loading BTF or program without any + * logging requested, but will retry with either its own or custom log + * buffer, if provided, and log_level=1 on any error. + * And vice versa, setting log_level>0 will request BTF or prog + * loading with verbose log from the first attempt (and as such also + * for successfully loaded BTF or program), and the actual log buffer + * could be either libbpf's own auto-allocated log buffer, if + * kernel_log_buffer is NULL, or user-provided custom kernel_log_buf. + * If user didn't provide custom log buffer, libbpf will emit captured + * logs through its print callback. + */ + __u32 kernel_log_level; + + size_t :0; }; -#define bpf_object_open_opts__last_field btf_custom_path +#define bpf_object_open_opts__last_field kernel_log_level LIBBPF_API struct bpf_object *bpf_object__open(const char *path); From d9b3fae391240645f6493dd2cac6b9d4bf0ea7ba Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 9 Dec 2021 11:38:33 -0800 Subject: [PATCH 025/131] libbpf: Improve logging around BPF program loading Add missing "prog '%s': " prefixes in few places and use consistently markers for beginning and end of program load logs. Here's an example of log output: libbpf: prog 'handler': BPF program load failed: Permission denied libbpf: -- BEGIN PROG LOAD LOG --- arg#0 reference type('UNKNOWN ') size cannot be determined: -22 ; out1 = in1; 0: (18) r1 = 0xffffc9000cdcc000 2: (61) r1 = *(u32 *)(r1 +0) ... 81: (63) *(u32 *)(r4 +0) = r5 R1_w=map_value(id=0,off=16,ks=4,vs=20,imm=0) R4=map_value(id=0,off=400,ks=4,vs=16,imm=0) invalid access to map value, value_size=16 off=400 size=4 R4 min value is outside of the allowed memory range processed 63 insns (limit 1000000) max_states_per_insn 0 total_states 0 peak_states 0 mark_read 0 -- END PROG LOAD LOG -- libbpf: failed to load program 'handler' libbpf: failed to load object 'test_skeleton' The entire verifier log, including BEGIN and END markers are now always youtput during a single print callback call. This should make it much easier to post-process or parse it, if necessary. It's not an explicit API guarantee, but it can be reasonably expected to stay like that. Also __bpf_object__open is renamed to bpf_object_open() as it's always an adventure to find the exact function that implements bpf_object's open phase, so drop the double underscored and use internal libbpf naming convention. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211209193840.1248570-6-andrii@kernel.org --- src/libbpf.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index 227da8c..a14b960 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -6662,8 +6662,10 @@ retry_load: ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); if (ret >= 0) { - if (log_buf && load_attr.log_level) - pr_debug("verifier log:\n%s", log_buf); + if (log_buf && load_attr.log_level) { + pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", + prog->name, log_buf); + } if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) { struct bpf_map *map; @@ -6676,8 +6678,8 @@ retry_load: if (bpf_prog_bind_map(ret, bpf_map__fd(map), NULL)) { cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warn("prog '%s': failed to bind .rodata map: %s\n", - prog->name, cp); + pr_warn("prog '%s': failed to bind map '%s': %s\n", + prog->name, map->real_name, cp); /* Don't fail hard if can't bind rodata. */ } } @@ -6691,23 +6693,22 @@ retry_load: if (!log_buf || errno == ENOSPC) { log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size << 1); - free(log_buf); goto retry_load; } ret = errno ? -errno : -LIBBPF_ERRNO__LOAD; cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warn("load bpf program failed: %s\n", cp); + pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp); pr_perm_msg(ret); if (log_buf && log_buf[0] != '\0') { ret = -LIBBPF_ERRNO__VERIFY; - pr_warn("-- BEGIN DUMP LOG ---\n"); - pr_warn("\n%s\n", log_buf); - pr_warn("-- END LOG --\n"); - } else if (insns_cnt >= BPF_MAXINSNS) { - pr_warn("Program too large (%d insns), at most %d insns\n", - insns_cnt, BPF_MAXINSNS); + pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", + prog->name, log_buf); + } + if (insns_cnt >= BPF_MAXINSNS) { + pr_warn("prog '%s': program too large (%d insns), at most %d insns\n", + prog->name, insns_cnt, BPF_MAXINSNS); ret = -LIBBPF_ERRNO__PROG2BIG; } else if (prog->type != BPF_PROG_TYPE_KPROBE) { /* Wrong program type? */ @@ -6931,9 +6932,8 @@ static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object return 0; } -static struct bpf_object * -__bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, - const struct bpf_object_open_opts *opts) +static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz, + const struct bpf_object_open_opts *opts) { const char *obj_name, *kconfig, *btf_tmp_path; struct bpf_object *obj; @@ -7033,7 +7033,7 @@ __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) return NULL; pr_debug("loading %s\n", attr->file); - return __bpf_object__open(attr->file, NULL, 0, &opts); + return bpf_object_open(attr->file, NULL, 0, &opts); } struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) @@ -7059,7 +7059,7 @@ bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) pr_debug("loading %s\n", path); - return libbpf_ptr(__bpf_object__open(path, NULL, 0, opts)); + return libbpf_ptr(bpf_object_open(path, NULL, 0, opts)); } struct bpf_object * @@ -7069,7 +7069,7 @@ bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, if (!obj_buf || obj_buf_sz == 0) return libbpf_err_ptr(-EINVAL); - return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, opts)); + return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts)); } struct bpf_object * @@ -7086,7 +7086,7 @@ bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, if (!obj_buf || obj_buf_sz == 0) return errno = EINVAL, NULL; - return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, &opts)); + return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, &opts)); } static int bpf_object_unload(struct bpf_object *obj) From eb9d74e7ad8c52962eb503278d3129bfd81e16e5 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 9 Dec 2021 11:38:34 -0800 Subject: [PATCH 026/131] libbpf: Preserve kernel error code and remove kprobe prog type guessing Instead of rewriting error code returned by the kernel of prog load with libbpf-sepcific variants pass through the original error. There is now also no need to have a backup generic -LIBBPF_ERRNO__LOAD fallback error as bpf_prog_load() guarantees that errno will be properly set no matter what. Also drop a completely outdated and pretty useless BPF_PROG_TYPE_KPROBE guess logic. It's not necessary and neither it's helpful in modern BPF applications. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211209193840.1248570-7-andrii@kernel.org --- src/libbpf.c | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index a14b960..a788b78 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -6696,34 +6696,19 @@ retry_load: free(log_buf); goto retry_load; } - ret = errno ? -errno : -LIBBPF_ERRNO__LOAD; + + ret = -errno; cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp); pr_perm_msg(ret); if (log_buf && log_buf[0] != '\0') { - ret = -LIBBPF_ERRNO__VERIFY; pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", prog->name, log_buf); } if (insns_cnt >= BPF_MAXINSNS) { pr_warn("prog '%s': program too large (%d insns), at most %d insns\n", prog->name, insns_cnt, BPF_MAXINSNS); - ret = -LIBBPF_ERRNO__PROG2BIG; - } else if (prog->type != BPF_PROG_TYPE_KPROBE) { - /* Wrong program type? */ - int fd; - - load_attr.expected_attach_type = 0; - load_attr.log_buf = NULL; - load_attr.log_size = 0; - fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, prog_name, license, - insns, insns_cnt, &load_attr); - if (fd >= 0) { - close(fd); - ret = -LIBBPF_ERRNO__PROGTYPE; - goto out; - } } out: From 90910812b59d841904418fa7eaba388eadf68c2f Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 9 Dec 2021 11:38:35 -0800 Subject: [PATCH 027/131] libbpf: Add per-program log buffer setter and getter Allow to set user-provided log buffer on a per-program basis ([0]). This gives great deal of flexibility in terms of which programs are loaded with logging enabled and where corresponding logs go. Log buffer set with bpf_program__set_log_buf() overrides kernel_log_buf and kernel_log_size settings set at bpf_object open time through bpf_object_open_opts, if any. Adjust bpf_object_load_prog_instance() logic to not perform own log buf allocation and load retry if custom log buffer is provided by the user. [0] Closes: https://github.com/libbpf/libbpf/issues/418 Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211209193840.1248570-8-andrii@kernel.org --- src/libbpf.c | 92 ++++++++++++++++++++++++++++++++++++++++---------- src/libbpf.h | 7 ++++ src/libbpf.map | 2 ++ 3 files changed, 84 insertions(+), 17 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index a788b78..38c8d21 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -331,7 +331,11 @@ struct bpf_program { struct reloc_desc *reloc_desc; int nr_reloc; - int log_level; + + /* BPF verifier log settings */ + char *log_buf; + size_t log_size; + __u32 log_level; struct { int nr; @@ -713,6 +717,9 @@ bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, prog->instances.fds = NULL; prog->instances.nr = -1; + /* inherit object's log_level */ + prog->log_level = obj->log_level; + prog->sec_name = strdup(sec_name); if (!prog->sec_name) goto errout; @@ -6591,8 +6598,10 @@ static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_prog const char *prog_name = NULL; char *cp, errmsg[STRERR_BUFSIZE]; size_t log_buf_size = 0; - char *log_buf = NULL; + char *log_buf = NULL, *tmp; int btf_fd, ret, err; + bool own_log_buf = true; + __u32 log_level = prog->log_level; if (prog->type == BPF_PROG_TYPE_UNSPEC) { /* @@ -6627,7 +6636,7 @@ static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_prog load_attr.line_info_rec_size = prog->line_info_rec_size; load_attr.line_info_cnt = prog->line_info_cnt; } - load_attr.log_level = prog->log_level; + load_attr.log_level = log_level; load_attr.prog_flags = prog->prog_flags; load_attr.fd_array = obj->fd_array; @@ -6648,21 +6657,42 @@ static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_prog *prog_fd = -1; return 0; } -retry_load: - if (log_buf_size) { - log_buf = malloc(log_buf_size); - if (!log_buf) - return -ENOMEM; - *log_buf = 0; +retry_load: + /* if log_level is zero, we don't request logs initiallly even if + * custom log_buf is specified; if the program load fails, then we'll + * bump log_level to 1 and use either custom log_buf or we'll allocate + * our own and retry the load to get details on what failed + */ + if (log_level) { + if (prog->log_buf) { + log_buf = prog->log_buf; + log_buf_size = prog->log_size; + own_log_buf = false; + } else if (obj->log_buf) { + log_buf = obj->log_buf; + log_buf_size = obj->log_size; + own_log_buf = false; + } else { + log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2); + tmp = realloc(log_buf, log_buf_size); + if (!tmp) { + ret = -ENOMEM; + goto out; + } + log_buf = tmp; + log_buf[0] = '\0'; + own_log_buf = true; + } } load_attr.log_buf = log_buf; load_attr.log_size = log_buf_size; - ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); + load_attr.log_level = log_level; + ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); if (ret >= 0) { - if (log_buf && load_attr.log_level) { + if (log_level && own_log_buf) { pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", prog->name, log_buf); } @@ -6690,19 +6720,26 @@ retry_load: goto out; } - if (!log_buf || errno == ENOSPC) { - log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, - log_buf_size << 1); - free(log_buf); + if (log_level == 0) { + log_level = 1; goto retry_load; } + /* On ENOSPC, increase log buffer size and retry, unless custom + * log_buf is specified. + * Be careful to not overflow u32, though. Kernel's log buf size limit + * isn't part of UAPI so it can always be bumped to full 4GB. So don't + * multiply by 2 unless we are sure we'll fit within 32 bits. + * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2). + */ + if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2) + goto retry_load; ret = -errno; cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp); pr_perm_msg(ret); - if (log_buf && log_buf[0] != '\0') { + if (own_log_buf && log_buf && log_buf[0] != '\0') { pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", prog->name, log_buf); } @@ -6712,7 +6749,8 @@ retry_load: } out: - free(log_buf); + if (own_log_buf) + free(log_buf); return ret; } @@ -8498,6 +8536,26 @@ int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level) return 0; } +const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size) +{ + *log_size = prog->log_size; + return prog->log_buf; +} + +int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size) +{ + if (log_size && !log_buf) + return -EINVAL; + if (prog->log_size > UINT_MAX) + return -EINVAL; + if (prog->obj->loaded) + return -EBUSY; + + prog->log_buf = log_buf; + prog->log_size = log_size; + return 0; +} + #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \ .sec = sec_pfx, \ .prog_type = BPF_PROG_TYPE_##ptype, \ diff --git a/src/libbpf.h b/src/libbpf.h index 5c984c6..dacde55 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -591,8 +591,15 @@ bpf_program__set_expected_attach_type(struct bpf_program *prog, LIBBPF_API __u32 bpf_program__flags(const struct bpf_program *prog); LIBBPF_API int bpf_program__set_flags(struct bpf_program *prog, __u32 flags); + +/* Per-program log level and log buffer getters/setters. + * See bpf_object_open_opts comments regarding log_level and log_buf + * interactions. + */ LIBBPF_API __u32 bpf_program__log_level(const struct bpf_program *prog); LIBBPF_API int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level); +LIBBPF_API const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size); +LIBBPF_API int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size); LIBBPF_API int bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd, diff --git a/src/libbpf.map b/src/libbpf.map index 08cdfe8..4d483af 100644 --- a/src/libbpf.map +++ b/src/libbpf.map @@ -423,6 +423,8 @@ LIBBPF_0.6.0 { LIBBPF_0.7.0 { global: bpf_btf_load; + bpf_program__log_buf; bpf_program__log_level; + bpf_program__set_log_buf; bpf_program__set_log_level; }; From f13e766fa43b2a0202a771a63ba22e3092f48d6b Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 9 Dec 2021 11:38:36 -0800 Subject: [PATCH 028/131] libbpf: Deprecate bpf_object__load_xattr() Deprecate non-extensible bpf_object__load_xattr() in v0.8 ([0]). With log_level control through bpf_object_open_opts or bpf_program__set_log_level(), we are finally at the point where bpf_object__load_xattr() doesn't provide any functionality that can't be accessed through other (better) ways. The other feature, target_btf_path, is also controllable through bpf_object_open_opts. [0] Closes: https://github.com/libbpf/libbpf/issues/289 Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211209193840.1248570-9-andrii@kernel.org --- src/libbpf.c | 23 ++++++++++------------- src/libbpf.h | 1 + 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index 38c8d21..64bc680 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -7462,14 +7462,10 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, return 0; } -int bpf_object__load_xattr(struct bpf_object_load_attr *attr) +static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path) { - struct bpf_object *obj; int err, i; - if (!attr) - return libbpf_err(-EINVAL); - obj = attr->obj; if (!obj) return libbpf_err(-EINVAL); @@ -7479,7 +7475,7 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr) } if (obj->gen_loader) - bpf_gen__init(obj->gen_loader, attr->log_level, obj->nr_programs, obj->nr_maps); + bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); err = bpf_object__probe_loading(obj); err = err ? : bpf_object__load_vmlinux_btf(obj, false); @@ -7488,8 +7484,8 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr) err = err ? : bpf_object__sanitize_maps(obj); err = err ? : bpf_object__init_kern_struct_ops_maps(obj); err = err ? : bpf_object__create_maps(obj); - err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : attr->target_btf_path); - err = err ? : bpf_object__load_progs(obj, attr->log_level); + err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path); + err = err ? : bpf_object__load_progs(obj, extra_log_level); err = err ? : bpf_object_init_prog_arrays(obj); if (obj->gen_loader) { @@ -7534,13 +7530,14 @@ out: return libbpf_err(err); } +int bpf_object__load_xattr(struct bpf_object_load_attr *attr) +{ + return bpf_object_load(attr->obj, attr->log_level, attr->target_btf_path); +} + int bpf_object__load(struct bpf_object *obj) { - struct bpf_object_load_attr attr = { - .obj = obj, - }; - - return bpf_object__load_xattr(&attr); + return bpf_object_load(obj, 0, NULL); } static int make_parent_dir(const char *path) diff --git a/src/libbpf.h b/src/libbpf.h index dacde55..a8b894d 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -214,6 +214,7 @@ struct bpf_object_load_attr { /* Load/unload object into/from kernel */ LIBBPF_API int bpf_object__load(struct bpf_object *obj); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__load() instead") LIBBPF_API int bpf_object__load_xattr(struct bpf_object_load_attr *attr); LIBBPF_DEPRECATED_SINCE(0, 6, "bpf_object__unload() is deprecated, use bpf_object__close() instead") LIBBPF_API int bpf_object__unload(struct bpf_object *obj); From c7dedfe23fca6434af9e28a08d70c54d757f19b6 Mon Sep 17 00:00:00 2001 From: Vincent Minet Date: Fri, 10 Dec 2021 07:31:12 +0100 Subject: [PATCH 029/131] libbpf: Fix typo in btf__dedup@LIBBPF_0.0.2 definition The btf__dedup_deprecated name was misspelled in the definition of the compat symbol for btf__dedup. This leads it to be missing from the shared library. This fixes it. Fixes: 957d350a8b94 ("libbpf: Turn btf_dedup_opts into OPTS-based struct") Signed-off-by: Vincent Minet Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211210063112.80047-1-vincent@vincent-minet.net --- src/btf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/btf.c b/src/btf.c index e171424..9aa19c8 100644 --- a/src/btf.c +++ b/src/btf.c @@ -3107,7 +3107,7 @@ done: return libbpf_err(err); } -COMPAT_VERSION(bpf__dedup_deprecated, btf__dedup, LIBBPF_0.0.2) +COMPAT_VERSION(btf__dedup_deprecated, btf__dedup, LIBBPF_0.0.2) int btf__dedup_deprecated(struct btf *btf, struct btf_ext *btf_ext, const void *unused_opts) { LIBBPF_OPTS(btf_dedup_opts, opts, .btf_ext = btf_ext); From bb14c6f5b58117b6b28e77777246dc406ac1cc79 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Fri, 10 Dec 2021 17:39:57 +0800 Subject: [PATCH 030/131] libbpf: Add "bool skipped" to struct bpf_map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix error: "failed to pin map: Bad file descriptor, path: /sys/fs/bpf/_rodata_str1_1." In the old kernel, the global data map will not be created, see [0]. So we should skip the pinning of the global data map to avoid bpf_object__pin_maps returning error. Therefore, when the map is not created, we mark “map->skipped" as true and then check during relocation and during pinning. Fixes: 16e0c35c6f7a ("libbpf: Load global data maps lazily on legacy kernels") Signed-off-by: Shuyi Cheng Signed-off-by: Andrii Nakryiko --- src/libbpf.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index 64bc680..902f1ad 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -431,6 +431,7 @@ struct bpf_map { char *pin_path; bool pinned; bool reused; + bool skipped; __u64 map_extra; }; @@ -5087,8 +5088,10 @@ bpf_object__create_maps(struct bpf_object *obj) * kernels. */ if (bpf_map__is_internal(map) && - !kernel_supports(obj, FEAT_GLOBAL_DATA)) + !kernel_supports(obj, FEAT_GLOBAL_DATA)) { + map->skipped = true; continue; + } retried = false; retry: @@ -5717,8 +5720,7 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) } else { const struct bpf_map *map = &obj->maps[relo->map_idx]; - if (bpf_map__is_internal(map) && - !kernel_supports(obj, FEAT_GLOBAL_DATA)) { + if (map->skipped) { pr_warn("prog '%s': relo #%d: kernel doesn't support global data\n", prog->name, i); return -ENOTSUP; @@ -7926,6 +7928,9 @@ int bpf_object__pin_maps(struct bpf_object *obj, const char *path) char *pin_path = NULL; char buf[PATH_MAX]; + if (map->skipped) + continue; + if (path) { int len; From a0ddf21c92270e89138879fca9e15b8b18ec31fb Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 22 Nov 2021 10:48:10 -0800 Subject: [PATCH 031/131] tools: sync uapi/linux/if_link.h header This file has not been updated for a while. Sync it before BIG TCP patch series. Signed-off-by: Eric Dumazet Link: https://lore.kernel.org/r/20211122184810.769159-1-eric.dumazet@gmail.com Signed-off-by: Jakub Kicinski --- include/uapi/linux/if_link.h | 293 +++++++++++++++++++++++++++++++---- 1 file changed, 259 insertions(+), 34 deletions(-) diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index b3610fd..eebd389 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -7,24 +7,23 @@ /* This struct should be in sync with struct rtnl_link_stats64 */ struct rtnl_link_stats { - __u32 rx_packets; /* total packets received */ - __u32 tx_packets; /* total packets transmitted */ - __u32 rx_bytes; /* total bytes received */ - __u32 tx_bytes; /* total bytes transmitted */ - __u32 rx_errors; /* bad packets received */ - __u32 tx_errors; /* packet transmit problems */ - __u32 rx_dropped; /* no space in linux buffers */ - __u32 tx_dropped; /* no space available in linux */ - __u32 multicast; /* multicast packets received */ + __u32 rx_packets; + __u32 tx_packets; + __u32 rx_bytes; + __u32 tx_bytes; + __u32 rx_errors; + __u32 tx_errors; + __u32 rx_dropped; + __u32 tx_dropped; + __u32 multicast; __u32 collisions; - /* detailed rx_errors: */ __u32 rx_length_errors; - __u32 rx_over_errors; /* receiver ring buff overflow */ - __u32 rx_crc_errors; /* recved pkt with crc error */ - __u32 rx_frame_errors; /* recv'd frame alignment error */ - __u32 rx_fifo_errors; /* recv'r fifo overrun */ - __u32 rx_missed_errors; /* receiver missed packet */ + __u32 rx_over_errors; + __u32 rx_crc_errors; + __u32 rx_frame_errors; + __u32 rx_fifo_errors; + __u32 rx_missed_errors; /* detailed tx_errors */ __u32 tx_aborted_errors; @@ -37,29 +36,201 @@ struct rtnl_link_stats { __u32 rx_compressed; __u32 tx_compressed; - __u32 rx_nohandler; /* dropped, no handler found */ + __u32 rx_nohandler; }; -/* The main device statistics structure */ +/** + * struct rtnl_link_stats64 - The main device statistics structure. + * + * @rx_packets: Number of good packets received by the interface. + * For hardware interfaces counts all good packets received from the device + * by the host, including packets which host had to drop at various stages + * of processing (even in the driver). + * + * @tx_packets: Number of packets successfully transmitted. + * For hardware interfaces counts packets which host was able to successfully + * hand over to the device, which does not necessarily mean that packets + * had been successfully transmitted out of the device, only that device + * acknowledged it copied them out of host memory. + * + * @rx_bytes: Number of good received bytes, corresponding to @rx_packets. + * + * For IEEE 802.3 devices should count the length of Ethernet Frames + * excluding the FCS. + * + * @tx_bytes: Number of good transmitted bytes, corresponding to @tx_packets. + * + * For IEEE 802.3 devices should count the length of Ethernet Frames + * excluding the FCS. + * + * @rx_errors: Total number of bad packets received on this network device. + * This counter must include events counted by @rx_length_errors, + * @rx_crc_errors, @rx_frame_errors and other errors not otherwise + * counted. + * + * @tx_errors: Total number of transmit problems. + * This counter must include events counter by @tx_aborted_errors, + * @tx_carrier_errors, @tx_fifo_errors, @tx_heartbeat_errors, + * @tx_window_errors and other errors not otherwise counted. + * + * @rx_dropped: Number of packets received but not processed, + * e.g. due to lack of resources or unsupported protocol. + * For hardware interfaces this counter may include packets discarded + * due to L2 address filtering but should not include packets dropped + * by the device due to buffer exhaustion which are counted separately in + * @rx_missed_errors (since procfs folds those two counters together). + * + * @tx_dropped: Number of packets dropped on their way to transmission, + * e.g. due to lack of resources. + * + * @multicast: Multicast packets received. + * For hardware interfaces this statistic is commonly calculated + * at the device level (unlike @rx_packets) and therefore may include + * packets which did not reach the host. + * + * For IEEE 802.3 devices this counter may be equivalent to: + * + * - 30.3.1.1.21 aMulticastFramesReceivedOK + * + * @collisions: Number of collisions during packet transmissions. + * + * @rx_length_errors: Number of packets dropped due to invalid length. + * Part of aggregate "frame" errors in `/proc/net/dev`. + * + * For IEEE 802.3 devices this counter should be equivalent to a sum + * of the following attributes: + * + * - 30.3.1.1.23 aInRangeLengthErrors + * - 30.3.1.1.24 aOutOfRangeLengthField + * - 30.3.1.1.25 aFrameTooLongErrors + * + * @rx_over_errors: Receiver FIFO overflow event counter. + * + * Historically the count of overflow events. Such events may be + * reported in the receive descriptors or via interrupts, and may + * not correspond one-to-one with dropped packets. + * + * The recommended interpretation for high speed interfaces is - + * number of packets dropped because they did not fit into buffers + * provided by the host, e.g. packets larger than MTU or next buffer + * in the ring was not available for a scatter transfer. + * + * Part of aggregate "frame" errors in `/proc/net/dev`. + * + * This statistics was historically used interchangeably with + * @rx_fifo_errors. + * + * This statistic corresponds to hardware events and is not commonly used + * on software devices. + * + * @rx_crc_errors: Number of packets received with a CRC error. + * Part of aggregate "frame" errors in `/proc/net/dev`. + * + * For IEEE 802.3 devices this counter must be equivalent to: + * + * - 30.3.1.1.6 aFrameCheckSequenceErrors + * + * @rx_frame_errors: Receiver frame alignment errors. + * Part of aggregate "frame" errors in `/proc/net/dev`. + * + * For IEEE 802.3 devices this counter should be equivalent to: + * + * - 30.3.1.1.7 aAlignmentErrors + * + * @rx_fifo_errors: Receiver FIFO error counter. + * + * Historically the count of overflow events. Those events may be + * reported in the receive descriptors or via interrupts, and may + * not correspond one-to-one with dropped packets. + * + * This statistics was used interchangeably with @rx_over_errors. + * Not recommended for use in drivers for high speed interfaces. + * + * This statistic is used on software devices, e.g. to count software + * packet queue overflow (can) or sequencing errors (GRE). + * + * @rx_missed_errors: Count of packets missed by the host. + * Folded into the "drop" counter in `/proc/net/dev`. + * + * Counts number of packets dropped by the device due to lack + * of buffer space. This usually indicates that the host interface + * is slower than the network interface, or host is not keeping up + * with the receive packet rate. + * + * This statistic corresponds to hardware events and is not used + * on software devices. + * + * @tx_aborted_errors: + * Part of aggregate "carrier" errors in `/proc/net/dev`. + * For IEEE 802.3 devices capable of half-duplex operation this counter + * must be equivalent to: + * + * - 30.3.1.1.11 aFramesAbortedDueToXSColls + * + * High speed interfaces may use this counter as a general device + * discard counter. + * + * @tx_carrier_errors: Number of frame transmission errors due to loss + * of carrier during transmission. + * Part of aggregate "carrier" errors in `/proc/net/dev`. + * + * For IEEE 802.3 devices this counter must be equivalent to: + * + * - 30.3.1.1.13 aCarrierSenseErrors + * + * @tx_fifo_errors: Number of frame transmission errors due to device + * FIFO underrun / underflow. This condition occurs when the device + * begins transmission of a frame but is unable to deliver the + * entire frame to the transmitter in time for transmission. + * Part of aggregate "carrier" errors in `/proc/net/dev`. + * + * @tx_heartbeat_errors: Number of Heartbeat / SQE Test errors for + * old half-duplex Ethernet. + * Part of aggregate "carrier" errors in `/proc/net/dev`. + * + * For IEEE 802.3 devices possibly equivalent to: + * + * - 30.3.2.1.4 aSQETestErrors + * + * @tx_window_errors: Number of frame transmission errors due + * to late collisions (for Ethernet - after the first 64B of transmission). + * Part of aggregate "carrier" errors in `/proc/net/dev`. + * + * For IEEE 802.3 devices this counter must be equivalent to: + * + * - 30.3.1.1.10 aLateCollisions + * + * @rx_compressed: Number of correctly received compressed packets. + * This counters is only meaningful for interfaces which support + * packet compression (e.g. CSLIP, PPP). + * + * @tx_compressed: Number of transmitted compressed packets. + * This counters is only meaningful for interfaces which support + * packet compression (e.g. CSLIP, PPP). + * + * @rx_nohandler: Number of packets received on the interface + * but dropped by the networking stack because the device is + * not designated to receive packets (e.g. backup link in a bond). + */ struct rtnl_link_stats64 { - __u64 rx_packets; /* total packets received */ - __u64 tx_packets; /* total packets transmitted */ - __u64 rx_bytes; /* total bytes received */ - __u64 tx_bytes; /* total bytes transmitted */ - __u64 rx_errors; /* bad packets received */ - __u64 tx_errors; /* packet transmit problems */ - __u64 rx_dropped; /* no space in linux buffers */ - __u64 tx_dropped; /* no space available in linux */ - __u64 multicast; /* multicast packets received */ + __u64 rx_packets; + __u64 tx_packets; + __u64 rx_bytes; + __u64 tx_bytes; + __u64 rx_errors; + __u64 tx_errors; + __u64 rx_dropped; + __u64 tx_dropped; + __u64 multicast; __u64 collisions; /* detailed rx_errors: */ __u64 rx_length_errors; - __u64 rx_over_errors; /* receiver ring buff overflow */ - __u64 rx_crc_errors; /* recved pkt with crc error */ - __u64 rx_frame_errors; /* recv'd frame alignment error */ - __u64 rx_fifo_errors; /* recv'r fifo overrun */ - __u64 rx_missed_errors; /* receiver missed packet */ + __u64 rx_over_errors; + __u64 rx_crc_errors; + __u64 rx_frame_errors; + __u64 rx_fifo_errors; + __u64 rx_missed_errors; /* detailed tx_errors */ __u64 tx_aborted_errors; @@ -71,8 +242,7 @@ struct rtnl_link_stats64 { /* for cslip etc */ __u64 rx_compressed; __u64 tx_compressed; - - __u64 rx_nohandler; /* dropped, no handler found */ + __u64 rx_nohandler; }; /* The struct should be in sync with struct ifmap */ @@ -170,12 +340,29 @@ enum { IFLA_PROP_LIST, IFLA_ALT_IFNAME, /* Alternative ifname */ IFLA_PERM_ADDRESS, + IFLA_PROTO_DOWN_REASON, + + /* device (sysfs) name as parent, used instead + * of IFLA_LINK where there's no parent netdev + */ + IFLA_PARENT_DEV_NAME, + IFLA_PARENT_DEV_BUS_NAME, + __IFLA_MAX }; #define IFLA_MAX (__IFLA_MAX - 1) +enum { + IFLA_PROTO_DOWN_REASON_UNSPEC, + IFLA_PROTO_DOWN_REASON_MASK, /* u32, mask for reason bits */ + IFLA_PROTO_DOWN_REASON_VALUE, /* u32, reason bit value */ + + __IFLA_PROTO_DOWN_REASON_CNT, + IFLA_PROTO_DOWN_REASON_MAX = __IFLA_PROTO_DOWN_REASON_CNT - 1 +}; + /* backwards compatibility for userspace */ #ifndef __KERNEL__ #define IFLA_RTA(r) ((struct rtattr*)(((char*)(r)) + NLMSG_ALIGN(sizeof(struct ifinfomsg)))) @@ -293,6 +480,7 @@ enum { IFLA_BR_MCAST_MLD_VERSION, IFLA_BR_VLAN_STATS_PER_PORT, IFLA_BR_MULTI_BOOLOPT, + IFLA_BR_MCAST_QUERIER_STATE, __IFLA_BR_MAX, }; @@ -346,6 +534,8 @@ enum { IFLA_BRPORT_BACKUP_PORT, IFLA_BRPORT_MRP_RING_OPEN, IFLA_BRPORT_MRP_IN_OPEN, + IFLA_BRPORT_MCAST_EHT_HOSTS_LIMIT, + IFLA_BRPORT_MCAST_EHT_HOSTS_CNT, __IFLA_BRPORT_MAX }; #define IFLA_BRPORT_MAX (__IFLA_BRPORT_MAX - 1) @@ -433,6 +623,7 @@ enum macvlan_macaddr_mode { }; #define MACVLAN_FLAG_NOPROMISC 1 +#define MACVLAN_FLAG_NODST 2 /* skip dst macvlan if matching src macvlan */ /* VRF section */ enum { @@ -597,6 +788,18 @@ enum ifla_geneve_df { GENEVE_DF_MAX = __GENEVE_DF_END - 1, }; +/* Bareudp section */ +enum { + IFLA_BAREUDP_UNSPEC, + IFLA_BAREUDP_PORT, + IFLA_BAREUDP_ETHERTYPE, + IFLA_BAREUDP_SRCPORT_MIN, + IFLA_BAREUDP_MULTIPROTO_MODE, + __IFLA_BAREUDP_MAX +}; + +#define IFLA_BAREUDP_MAX (__IFLA_BAREUDP_MAX - 1) + /* PPP section */ enum { IFLA_PPP_UNSPEC, @@ -899,7 +1102,14 @@ enum { #define IFLA_IPOIB_MAX (__IFLA_IPOIB_MAX - 1) -/* HSR section */ +/* HSR/PRP section, both uses same interface */ + +/* Different redundancy protocols for hsr device */ +enum { + HSR_PROTOCOL_HSR, + HSR_PROTOCOL_PRP, + HSR_PROTOCOL_MAX, +}; enum { IFLA_HSR_UNSPEC, @@ -909,6 +1119,9 @@ enum { IFLA_HSR_SUPERVISION_ADDR, /* Supervision frame multicast addr */ IFLA_HSR_SEQ_NR, IFLA_HSR_VERSION, /* HSR version */ + IFLA_HSR_PROTOCOL, /* Indicate different protocol than + * HSR. For example PRP. + */ __IFLA_HSR_MAX, }; @@ -1033,6 +1246,8 @@ enum { #define RMNET_FLAGS_INGRESS_MAP_COMMANDS (1U << 1) #define RMNET_FLAGS_INGRESS_MAP_CKSUMV4 (1U << 2) #define RMNET_FLAGS_EGRESS_MAP_CKSUMV4 (1U << 3) +#define RMNET_FLAGS_INGRESS_MAP_CKSUMV5 (1U << 4) +#define RMNET_FLAGS_EGRESS_MAP_CKSUMV5 (1U << 5) enum { IFLA_RMNET_UNSPEC, @@ -1048,4 +1263,14 @@ struct ifla_rmnet_flags { __u32 mask; }; +/* MCTP section */ + +enum { + IFLA_MCTP_UNSPEC, + IFLA_MCTP_NET, + __IFLA_MCTP_MAX, +}; + +#define IFLA_MCTP_MAX (__IFLA_MCTP_MAX - 1) + #endif /* _UAPI_LINUX_IF_LINK_H */ From 1f83414ea4f8bd1f77e1f8287bb10cf0facdb4f0 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 10 Dec 2021 11:56:03 -0800 Subject: [PATCH 032/131] sync: auto-generate latest BPF helpers Latest changes to BPF helper definitions. --- src/bpf_helper_defs.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/bpf_helper_defs.h b/src/bpf_helper_defs.h index 8a4edf6..fc8371d 100644 --- a/src/bpf_helper_defs.h +++ b/src/bpf_helper_defs.h @@ -4136,4 +4136,32 @@ static long (*bpf_kallsyms_lookup_name)(const char *name, int name_sz, int flags */ static long (*bpf_find_vma)(struct task_struct *task, __u64 addr, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 180; +/* + * bpf_loop + * + * For **nr_loops**, call **callback_fn** function + * with **callback_ctx** as the context parameter. + * The **callback_fn** should be a static function and + * the **callback_ctx** should be a pointer to the stack. + * The **flags** is used to control certain aspects of the helper. + * Currently, the **flags** must be 0. Currently, nr_loops is + * limited to 1 << 23 (~8 million) loops. + * + * long (\*callback_fn)(u32 index, void \*ctx); + * + * where **index** is the current index in the loop. The index + * is zero-indexed. + * + * If **callback_fn** returns 0, the helper will continue to the next + * loop. If return value is 1, the helper will skip the rest of + * the loops and return. Other return values are not used now, + * and will be rejected by the verifier. + * + * + * Returns + * The number of loops performed, **-EINVAL** for invalid **flags**, + * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. + */ +static long (*bpf_loop)(__u32 nr_loops, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 181; + From bcf58fc7a52b13bd34dc25d4f5c48ecfd1106413 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 10 Dec 2021 11:56:03 -0800 Subject: [PATCH 033/131] sync: latest libbpf changes from kernel Syncing latest libbpf commits from kernel repository. Baseline bpf-next commit: 43174f0d4597325cb91f1f1f55263eb6e6101036 Checkpoint bpf-next commit: 229fae38d0fc0d6ff58d57cbeb1432da55e58d4f Baseline bpf commit: c0d95d3380ee099d735e08618c0d599e72f6c8b0 Checkpoint bpf commit: 0be2516f865f5a876837184a8385163ff64a5889 Alexei Starovoitov (8): libbpf: Replace btf__type_by_id() with btf_type_by_id(). bpf: Prepare relo_core.c for kernel duty. bpf: Define enum bpf_core_relo_kind as uapi. bpf: Pass a set of bpf_core_relo-s to prog_load command. libbpf: Use CO-RE in the kernel in light skeleton. libbpf: Support init of inner maps in light skeleton. libbpf: Clean gen_loader's attach kind. libbpf: Reduce bpf_core_apply_relo_insn() stack usage. Andrii Nakryiko (12): libbpf: Cleanup struct bpf_core_cand. libbpf: Use __u32 fields in bpf_map_create_opts libbpf: Add API to get/set log_level at per-program level libbpf: Deprecate bpf_prog_load_xattr() API libbpf: Fix bpf_prog_load() log_buf logic for log_level 0 libbpf: Add OPTS-based bpf_btf_load() API libbpf: Allow passing preallocated log_buf when loading BTF into kernel libbpf: Allow passing user log setting through bpf_object_open_opts libbpf: Improve logging around BPF program loading libbpf: Preserve kernel error code and remove kprobe prog type guessing libbpf: Add per-program log buffer setter and getter libbpf: Deprecate bpf_object__load_xattr() Eric Dumazet (1): tools: sync uapi/linux/if_link.h header Grant Seltzer (1): libbpf: Add doc comments in libbpf.h Joanne Koong (1): bpf: Add bpf_loop helper Kumar Kartikeya Dwivedi (2): libbpf: Avoid double stores for success/failure case of ksym relocations libbpf: Avoid reload of imm for weak, unresolved, repeating ksym Mehrdad Arshad Rad (1): libbpf: Remove duplicate assignments Shuyi Cheng (1): libbpf: Add "bool skipped" to struct bpf_map Vincent Minet (1): libbpf: Fix typo in btf__dedup@LIBBPF_0.0.2 definition huangxuesen (1): libbpf: Fix trivial typo include/uapi/linux/bpf.h | 103 +++++++++- include/uapi/linux/if_link.h | 293 +++++++++++++++++++++++---- src/bpf.c | 88 ++++++--- src/bpf.h | 30 ++- src/bpf_gen_internal.h | 4 + src/btf.c | 82 +++++--- src/gen_loader.c | 114 ++++++++--- src/libbpf.c | 371 ++++++++++++++++++++++++----------- src/libbpf.h | 109 +++++++++- src/libbpf.map | 9 + src/libbpf_common.h | 5 + src/libbpf_internal.h | 3 +- src/libbpf_probes.c | 2 +- src/libbpf_version.h | 2 +- src/relo_core.c | 231 ++++++++++++---------- src/relo_core.h | 103 +++------- 16 files changed, 1138 insertions(+), 411 deletions(-) -- 2.30.2 --- BPF-CHECKPOINT-COMMIT | 2 +- CHECKPOINT-COMMIT | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BPF-CHECKPOINT-COMMIT b/BPF-CHECKPOINT-COMMIT index a3eccad..561199b 100644 --- a/BPF-CHECKPOINT-COMMIT +++ b/BPF-CHECKPOINT-COMMIT @@ -1 +1 @@ -c0d95d3380ee099d735e08618c0d599e72f6c8b0 +0be2516f865f5a876837184a8385163ff64a5889 diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index bf9b613..47d6510 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -43174f0d4597325cb91f1f1f55263eb6e6101036 +229fae38d0fc0d6ff58d57cbeb1432da55e58d4f From 5eb804a2dbf72e8f048afe8d4d8780a5e227ec21 Mon Sep 17 00:00:00 2001 From: Hangbin Liu Date: Tue, 30 Nov 2021 12:29:47 +0800 Subject: [PATCH 034/131] Bonding: add arp_missed_max option Currently, we use hard code number to verify if we are in the arp_interval timeslice. But some user may want to reduce/extend the verify timeslice. With the similar team option 'missed_max' the uers could change that number based on their own environment. Acked-by: Jay Vosburgh Signed-off-by: Hangbin Liu Signed-off-by: David S. Miller --- include/uapi/linux/if_link.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index eebd389..4ac53b3 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -858,6 +858,7 @@ enum { IFLA_BOND_TLB_DYNAMIC_LB, IFLA_BOND_PEER_NOTIF_DELAY, IFLA_BOND_AD_LACP_ACTIVE, + IFLA_BOND_MISSED_MAX, __IFLA_BOND_MAX, }; From 16bb788578a26ee2aed9ecd18f409fc63802cf1d Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 11 Dec 2021 17:16:19 -0800 Subject: [PATCH 035/131] libbpf: Fix gen_loader assumption on number of programs. libbpf's obj->nr_programs includes static and global functions. That number could be higher than the actual number of bpf programs going be loaded by gen_loader. Passing larger nr_programs to bpf_gen__init() doesn't hurt. Those exra stack slots will stay as zero. bpf_gen__finish() needs to check that actual number of progs that gen_loader saw is less than or equal to obj->nr_programs. Fixes: ba05fd36b851 ("libbpf: Perform map fd cleanup for gen_loader in case of error") Signed-off-by: Alexei Starovoitov --- src/gen_loader.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/gen_loader.c b/src/gen_loader.c index 8ed02e8..c5859c2 100644 --- a/src/gen_loader.c +++ b/src/gen_loader.c @@ -371,8 +371,9 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) { int i; - if (nr_progs != gen->nr_progs || nr_maps != gen->nr_maps) { - pr_warn("progs/maps mismatch\n"); + if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { + pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n", + nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); gen->error = -EFAULT; return gen->error; } From 04804b471056ecef693e08ba8884c83573d76ff8 Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Fri, 10 Dec 2021 22:16:49 +0800 Subject: [PATCH 036/131] bpf: Add bpf_strncmp helper The helper compares two strings: one string is a null-terminated read-only string, and another string has const max storage size but doesn't need to be null-terminated. It can be used to compare file name in tracing or LSM program. Signed-off-by: Hou Tao Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211210141652.877186-2-houtao1@huawei.com --- include/uapi/linux/bpf.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index c268712..2820c77 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4983,6 +4983,16 @@ union bpf_attr { * Return * The number of loops performed, **-EINVAL** for invalid **flags**, * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. + * + * long bpf_strncmp(const char *s1, u32 s1_sz, const char *s2) + * Description + * Do strncmp() between **s1** and **s2**. **s1** doesn't need + * to be null-terminated and **s1_sz** is the maximum storage + * size of **s1**. **s2** must be a read-only string. + * Return + * An integer less than, equal to, or greater than zero + * if the first **s1_sz** bytes of **s1** is found to be + * less than, to match, or be greater than **s2**. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5167,6 +5177,7 @@ union bpf_attr { FN(kallsyms_lookup_name), \ FN(find_vma), \ FN(loop), \ + FN(strncmp), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper From ac20634cdc9588a123b1ffbf0fc9b05a6a81edfb Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Sun, 12 Dec 2021 17:07:06 -0800 Subject: [PATCH 037/131] libbpf: Don't validate TYPE_ID relo's original imm value During linking, type IDs in the resulting linked BPF object file can change, and so ldimm64 instructions corresponding to BPF_CORE_TYPE_ID_TARGET and BPF_CORE_TYPE_ID_LOCAL CO-RE relos can get their imm value out of sync with actual CO-RE relocation information that's updated by BPF linker properly during linking process. We could teach BPF linker to adjust such instructions, but it feels a bit too much for linker to re-implement good chunk of bpf_core_patch_insns logic just for this. This is a redundant safety check for TYPE_ID relocations, as the real validation is in matching CO-RE specs, so if that works fine, it's very unlikely that there is something wrong with the instruction itself. So, instead, teach libbpf (and kernel) to ignore insn->imm for BPF_CORE_TYPE_ID_TARGET and BPF_CORE_TYPE_ID_LOCAL relos. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211213010706.100231-1-andrii@kernel.org --- src/relo_core.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/relo_core.c b/src/relo_core.c index 32464f0..c770483 100644 --- a/src/relo_core.c +++ b/src/relo_core.c @@ -709,10 +709,14 @@ static int bpf_core_calc_field_relo(const char *prog_name, static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, const struct bpf_core_spec *spec, - __u32 *val) + __u32 *val, bool *validate) { __s64 sz; + /* by default, always check expected value in bpf_insn */ + if (validate) + *validate = true; + /* type-based relos return zero when target type is not found */ if (!spec) { *val = 0; @@ -722,6 +726,11 @@ static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, switch (relo->kind) { case BPF_CORE_TYPE_ID_TARGET: *val = spec->root_type_id; + /* type ID, embedded in bpf_insn, might change during linking, + * so enforcing it is pointless + */ + if (validate) + *validate = false; break; case BPF_CORE_TYPE_EXISTS: *val = 1; @@ -861,8 +870,8 @@ static int bpf_core_calc_relo(const char *prog_name, res->fail_memsz_adjust = true; } } else if (core_relo_is_type_based(relo->kind)) { - err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val); - err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val); + err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val, &res->validate); + err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val, NULL); } else if (core_relo_is_enumval_based(relo->kind)) { err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val); err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val); @@ -1213,7 +1222,8 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */ if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) { - targ_res.validate = true; + /* bpf_insn's imm value could get out of sync during linking */ + targ_res.validate = false; targ_res.poison = false; targ_res.orig_val = local_spec->root_type_id; targ_res.new_val = local_spec->root_type_id; @@ -1227,7 +1237,6 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, return -EOPNOTSUPP; } - for (i = 0, j = 0; i < cands->len; i++) { err = bpf_core_spec_match(local_spec, cands->cands[i].btf, cands->cands[i].id, cand_spec); From 2b0d408764992b39a4981e1886cc2ba3da2e75d9 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Wed, 8 Dec 2021 20:32:44 +0100 Subject: [PATCH 038/131] bpf: Add get_func_[arg|ret|arg_cnt] helpers Adding following helpers for tracing programs: Get n-th argument of the traced function: long bpf_get_func_arg(void *ctx, u32 n, u64 *value) Get return value of the traced function: long bpf_get_func_ret(void *ctx, u64 *value) Get arguments count of the traced function: long bpf_get_func_arg_cnt(void *ctx) The trampoline now stores number of arguments on ctx-8 address, so it's easy to verify argument index and find return value argument's position. Moving function ip address on the trampoline stack behind the number of functions arguments, so it's now stored on ctx-16 address if it's needed. All helpers above are inlined by verifier. Also bit unrelated small change - using newly added function bpf_prog_has_trampoline in check_get_func_ip. Signed-off-by: Jiri Olsa Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211208193245.172141-5-jolsa@kernel.org --- include/uapi/linux/bpf.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 2820c77..b0383d3 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4993,6 +4993,31 @@ union bpf_attr { * An integer less than, equal to, or greater than zero * if the first **s1_sz** bytes of **s1** is found to be * less than, to match, or be greater than **s2**. + * + * long bpf_get_func_arg(void *ctx, u32 n, u64 *value) + * Description + * Get **n**-th argument (zero based) of the traced function (for tracing programs) + * returned in **value**. + * + * Return + * 0 on success. + * **-EINVAL** if n >= arguments count of traced function. + * + * long bpf_get_func_ret(void *ctx, u64 *value) + * Description + * Get return value of the traced function (for tracing programs) + * in **value**. + * + * Return + * 0 on success. + * **-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN. + * + * long bpf_get_func_arg_cnt(void *ctx) + * Description + * Get number of arguments of the traced function (for tracing programs). + * + * Return + * The number of arguments of the traced function. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5178,6 +5203,9 @@ union bpf_attr { FN(find_vma), \ FN(loop), \ FN(strncmp), \ + FN(get_func_arg), \ + FN(get_func_ret), \ + FN(get_func_arg_cnt), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper From 6894f573d2c2a550442116e94883a9e520e2c5a8 Mon Sep 17 00:00:00 2001 From: Grant Seltzer Date: Thu, 9 Dec 2021 18:22:22 -0500 Subject: [PATCH 039/131] libbpf: Add doc comments for bpf_program__(un)pin() This adds doc comments for the two bpf_program pinning functions, bpf_program__pin() and bpf_program__unpin() Signed-off-by: Grant Seltzer Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211209232222.541733-1-grantseltzer@gmail.com --- src/libbpf.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/libbpf.h b/src/libbpf.h index a8b894d..3f5ff2c 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -339,7 +339,31 @@ LIBBPF_DEPRECATED_SINCE(0, 7, "multi-instance bpf_program support is deprecated" LIBBPF_API int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, int instance); + +/** + * @brief **bpf_program__pin()** pins the BPF program to a file + * in the BPF FS specified by a path. This increments the programs + * reference count, allowing it to stay loaded after the process + * which loaded it has exited. + * + * @param prog BPF program to pin, must already be loaded + * @param path file path in a BPF file system + * @return 0, on success; negative error code, otherwise + */ LIBBPF_API int bpf_program__pin(struct bpf_program *prog, const char *path); + +/** + * @brief **bpf_program__unpin()** unpins the BPF program from a file + * in the BPFFS specified by a path. This decrements the programs + * reference count. + * + * The file pinning the BPF program can also be unlinked by a different + * process in which case this function will return an error. + * + * @param prog BPF program to unpin + * @param path file path to the pin in a BPF file system + * @return 0, on success; negative error code, otherwise + */ LIBBPF_API int bpf_program__unpin(struct bpf_program *prog, const char *path); LIBBPF_API void bpf_program__unload(struct bpf_program *prog); From df5689f1c83a991336c10cdd4fd98e7b1afcbe0d Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 13 Dec 2021 17:00:32 -0800 Subject: [PATCH 040/131] libbpf: Fix potential uninit memory read In case of BPF_CORE_TYPE_ID_LOCAL we fill out target result explicitly. But targ_res itself isn't initialized in such a case, and subsequent call to bpf_core_patch_insn() might read uninitialized field (like fail_memsz_adjust in this case). So ensure that targ_res is zero-initialized for BPF_CORE_TYPE_ID_LOCAL case. This was reported by Coverity static analyzer. Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211214010032.3843804-1-andrii@kernel.org --- src/relo_core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/relo_core.c b/src/relo_core.c index c770483..910865e 100644 --- a/src/relo_core.c +++ b/src/relo_core.c @@ -1223,6 +1223,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */ if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) { /* bpf_insn's imm value could get out of sync during linking */ + memset(&targ_res, 0, sizeof(targ_res)); targ_res.validate = false; targ_res.poison = false; targ_res.orig_val = local_spec->root_type_id; From a4e725f8f5228b5b2c5246e81b17177ba07ea461 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 10 Dec 2021 16:40:43 -0800 Subject: [PATCH 041/131] libbpf: Add sane strncpy alternative and use it internally strncpy() has a notoriously error-prone semantics which makes GCC complain about it a lot (and quite often completely completely falsely at that). Instead of pleasing GCC all the time (-Wno-stringop-truncation is unfortunately only supported by GCC, so it's a bit too messy to just enable it in Makefile), add libbpf-internal libbpf_strlcpy() helper which follows what FreeBSD's strlcpy() does and what most people would expect from strncpy(): copies up to N-1 first bytes from source string into destination string and ensures zero-termination afterwards. Replace all the relevant uses of strncpy/strncat/memcpy in libbpf with libbpf_strlcpy(). This also fixes the issue reported by Emmanuel Deloget in xsk.c where memcpy() could access source string beyond its end. Fixes: 2f6324a3937f8 (libbpf: Support shared umems between queues and devices) Reported-by: Emmanuel Deloget Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211211004043.2374068-1-andrii@kernel.org --- src/bpf.c | 4 ++-- src/btf_dump.c | 4 ++-- src/gen_loader.c | 6 ++---- src/libbpf.c | 8 +++----- src/libbpf_internal.h | 19 +++++++++++++++++++ src/xsk.c | 9 +++------ 6 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/bpf.c b/src/bpf.c index 6b2407e..1f84d70 100644 --- a/src/bpf.c +++ b/src/bpf.c @@ -112,7 +112,7 @@ int bpf_map_create(enum bpf_map_type map_type, attr.map_type = map_type; if (map_name) - strncat(attr.map_name, map_name, sizeof(attr.map_name) - 1); + libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); attr.key_size = key_size; attr.value_size = value_size; attr.max_entries = max_entries; @@ -271,7 +271,7 @@ int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, attr.kern_version = OPTS_GET(opts, kern_version, 0); if (prog_name) - strncat(attr.prog_name, prog_name, sizeof(attr.prog_name) - 1); + libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); attr.license = ptr_to_u64(license); if (insn_cnt > UINT_MAX) diff --git a/src/btf_dump.c b/src/btf_dump.c index f06a1d3..b9a3260 100644 --- a/src/btf_dump.c +++ b/src/btf_dump.c @@ -2321,8 +2321,8 @@ int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, if (!opts->indent_str) d->typed_dump->indent_str[0] = '\t'; else - strncat(d->typed_dump->indent_str, opts->indent_str, - sizeof(d->typed_dump->indent_str) - 1); + libbpf_strlcpy(d->typed_dump->indent_str, opts->indent_str, + sizeof(d->typed_dump->indent_str)); d->typed_dump->compact = OPTS_GET(opts, compact, false); d->typed_dump->skip_names = OPTS_GET(opts, skip_names, false); diff --git a/src/gen_loader.c b/src/gen_loader.c index c5859c2..8ecef10 100644 --- a/src/gen_loader.c +++ b/src/gen_loader.c @@ -463,8 +463,7 @@ void bpf_gen__map_create(struct bpf_gen *gen, attr.map_flags = map_attr->map_flags; attr.map_extra = map_attr->map_extra; if (map_name) - memcpy(attr.map_name, map_name, - min((unsigned)strlen(map_name), BPF_OBJ_NAME_LEN - 1)); + libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); attr.numa_node = map_attr->numa_node; attr.map_ifindex = map_attr->map_ifindex; attr.max_entries = max_entries; @@ -970,8 +969,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, core_relos = add_data(gen, gen->core_relos, attr.core_relo_cnt * attr.core_relo_rec_size); - memcpy(attr.prog_name, prog_name, - min((unsigned)strlen(prog_name), BPF_OBJ_NAME_LEN - 1)); + libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); prog_load_attr = add_data(gen, &attr, attr_size); /* populate union bpf_attr with a pointer to license */ diff --git a/src/libbpf.c b/src/libbpf.c index 902f1ad..c6d84b0 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -1201,12 +1201,10 @@ static struct bpf_object *bpf_object__new(const char *path, strcpy(obj->path, path); if (obj_name) { - strncpy(obj->name, obj_name, sizeof(obj->name) - 1); - obj->name[sizeof(obj->name) - 1] = 0; + libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); } else { /* Using basename() GNU version which doesn't modify arg. */ - strncpy(obj->name, basename((void *)path), - sizeof(obj->name) - 1); + libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); end = strchr(obj->name, '.'); if (end) *end = 0; @@ -1358,7 +1356,7 @@ static int bpf_object__check_endianness(struct bpf_object *obj) static int bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) { - memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); + libbpf_strlcpy(obj->license, data, sizeof(obj->license)); pr_debug("license of %s is %s\n", obj->path, obj->license); return 0; } diff --git a/src/libbpf_internal.h b/src/libbpf_internal.h index 355c410..5e8166a 100644 --- a/src/libbpf_internal.h +++ b/src/libbpf_internal.h @@ -169,6 +169,25 @@ static inline void *libbpf_reallocarray(void *ptr, size_t nmemb, size_t size) return realloc(ptr, total); } +/* Copy up to sz - 1 bytes from zero-terminated src string and ensure that dst + * is zero-terminated string no matter what (unless sz == 0, in which case + * it's a no-op). It's conceptually close to FreeBSD's strlcpy(), but differs + * in what is returned. Given this is internal helper, it's trivial to extend + * this, when necessary. Use this instead of strncpy inside libbpf source code. + */ +static inline void libbpf_strlcpy(char *dst, const char *src, size_t sz) +{ + size_t i; + + if (sz == 0) + return; + + sz--; + for (i = 0; i < sz && src[i]; i++) + dst[i] = src[i]; + dst[i] = '\0'; +} + struct btf; struct btf_type; diff --git a/src/xsk.c b/src/xsk.c index e8d94c6..edafe56 100644 --- a/src/xsk.c +++ b/src/xsk.c @@ -548,8 +548,7 @@ static int xsk_get_max_queues(struct xsk_socket *xsk) return -errno; ifr.ifr_data = (void *)&channels; - memcpy(ifr.ifr_name, ctx->ifname, IFNAMSIZ - 1); - ifr.ifr_name[IFNAMSIZ - 1] = '\0'; + libbpf_strlcpy(ifr.ifr_name, ctx->ifname, IFNAMSIZ); err = ioctl(fd, SIOCETHTOOL, &ifr); if (err && errno != EOPNOTSUPP) { ret = -errno; @@ -768,8 +767,7 @@ static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk) } ctx->ifindex = ifindex; - memcpy(ctx->ifname, ifname, IFNAMSIZ -1); - ctx->ifname[IFNAMSIZ - 1] = 0; + libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ); xsk->ctx = ctx; xsk->ctx->has_bpf_link = xsk_probe_bpf_link(); @@ -951,8 +949,7 @@ static struct xsk_ctx *xsk_create_ctx(struct xsk_socket *xsk, ctx->refcount = 1; ctx->umem = umem; ctx->queue_id = queue_id; - memcpy(ctx->ifname, ifname, IFNAMSIZ - 1); - ctx->ifname[IFNAMSIZ - 1] = '\0'; + libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ); ctx->fill = fill; ctx->comp = comp; From 216eaa760edf06f3521d5d49a419515607c1f38e Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 14 Dec 2021 11:59:03 -0800 Subject: [PATCH 042/131] libbpf: Auto-bump RLIMIT_MEMLOCK if kernel needs it for BPF The need to increase RLIMIT_MEMLOCK to do anything useful with BPF is one of the first extremely frustrating gotchas that all new BPF users go through and in some cases have to learn it a very hard way. Luckily, starting with upstream Linux kernel version 5.11, BPF subsystem dropped the dependency on memlock and uses memcg-based memory accounting instead. Unfortunately, detecting memcg-based BPF memory accounting is far from trivial (as can be evidenced by this patch), so in practice most BPF applications still do unconditional RLIMIT_MEMLOCK increase. As we move towards libbpf 1.0, it would be good to allow users to forget about RLIMIT_MEMLOCK vs memcg and let libbpf do the sensible adjustment automatically. This patch paves the way forward in this matter. Libbpf will do feature detection of memcg-based accounting, and if detected, will do nothing. But if the kernel is too old, just like BCC, libbpf will automatically increase RLIMIT_MEMLOCK on behalf of user application ([0]). As this is technically a breaking change, during the transition period applications have to opt into libbpf 1.0 mode by setting LIBBPF_STRICT_AUTO_RLIMIT_MEMLOCK bit when calling libbpf_set_strict_mode(). Libbpf allows to control the exact amount of set RLIMIT_MEMLOCK limit with libbpf_set_memlock_rlim_max() API. Passing 0 will make libbpf do nothing with RLIMIT_MEMLOCK. libbpf_set_memlock_rlim_max() has to be called before the first bpf_prog_load(), bpf_btf_load(), or bpf_object__load() call, otherwise it has no effect and will return -EBUSY. [0] Closes: https://github.com/libbpf/libbpf/issues/369 Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211214195904.1785155-2-andrii@kernel.org --- src/bpf.c | 81 +++++++++++++++++++++++++++++++++++++++++++ src/bpf.h | 2 ++ src/libbpf.c | 47 +++++-------------------- src/libbpf.map | 1 + src/libbpf_internal.h | 39 +++++++++++++++++++++ src/libbpf_legacy.h | 12 ++++++- 6 files changed, 143 insertions(+), 39 deletions(-) diff --git a/src/bpf.c b/src/bpf.c index 1f84d70..9b64eed 100644 --- a/src/bpf.c +++ b/src/bpf.c @@ -28,7 +28,9 @@ #include #include #include +#include #include +#include #include "bpf.h" #include "libbpf.h" #include "libbpf_internal.h" @@ -94,6 +96,77 @@ static inline int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int return fd; } +/* Probe whether kernel switched from memlock-based (RLIMIT_MEMLOCK) to + * memcg-based memory accounting for BPF maps and progs. This was done in [0]. + * We use the support for bpf_ktime_get_coarse_ns() helper, which was added in + * the same 5.11 Linux release ([1]), to detect memcg-based accounting for BPF. + * + * [0] https://lore.kernel.org/bpf/20201201215900.3569844-1-guro@fb.com/ + * [1] d05512618056 ("bpf: Add bpf_ktime_get_coarse_ns helper") + */ +int probe_memcg_account(void) +{ + const size_t prog_load_attr_sz = offsetofend(union bpf_attr, attach_btf_obj_fd); + struct bpf_insn insns[] = { + BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns), + BPF_EXIT_INSN(), + }; + size_t insn_cnt = sizeof(insns) / sizeof(insns[0]); + union bpf_attr attr; + int prog_fd; + + /* attempt loading freplace trying to use custom BTF */ + memset(&attr, 0, prog_load_attr_sz); + attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = insn_cnt; + attr.license = ptr_to_u64("GPL"); + + prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, prog_load_attr_sz); + if (prog_fd >= 0) { + close(prog_fd); + return 1; + } + return 0; +} + +static bool memlock_bumped; +static rlim_t memlock_rlim = RLIM_INFINITY; + +int libbpf_set_memlock_rlim(size_t memlock_bytes) +{ + if (memlock_bumped) + return libbpf_err(-EBUSY); + + memlock_rlim = memlock_bytes; + return 0; +} + +int bump_rlimit_memlock(void) +{ + struct rlimit rlim; + + /* this the default in libbpf 1.0, but for now user has to opt-in explicitly */ + if (!(libbpf_mode & LIBBPF_STRICT_AUTO_RLIMIT_MEMLOCK)) + return 0; + + /* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */ + if (memlock_bumped || kernel_supports(NULL, FEAT_MEMCG_ACCOUNT)) + return 0; + + memlock_bumped = true; + + /* zero memlock_rlim_max disables auto-bumping RLIMIT_MEMLOCK */ + if (memlock_rlim == 0) + return 0; + + rlim.rlim_cur = rlim.rlim_max = memlock_rlim; + if (setrlimit(RLIMIT_MEMLOCK, &rlim)) + return -errno; + + return 0; +} + int bpf_map_create(enum bpf_map_type map_type, const char *map_name, __u32 key_size, @@ -105,6 +178,8 @@ int bpf_map_create(enum bpf_map_type map_type, union bpf_attr attr; int fd; + bump_rlimit_memlock(); + memset(&attr, 0, attr_sz); if (!OPTS_VALID(opts, bpf_map_create_opts)) @@ -251,6 +326,8 @@ int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, union bpf_attr attr; char *log_buf; + bump_rlimit_memlock(); + if (!OPTS_VALID(opts, bpf_prog_load_opts)) return libbpf_err(-EINVAL); @@ -456,6 +533,8 @@ int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns, union bpf_attr attr; int fd; + bump_rlimit_memlock(); + memset(&attr, 0, sizeof(attr)); attr.prog_type = type; attr.insn_cnt = (__u32)insns_cnt; @@ -1056,6 +1135,8 @@ int bpf_btf_load(const void *btf_data, size_t btf_size, const struct bpf_btf_loa __u32 log_level; int fd; + bump_rlimit_memlock(); + memset(&attr, 0, attr_sz); if (!OPTS_VALID(opts, bpf_btf_load_opts)) diff --git a/src/bpf.h b/src/bpf.h index 94e553a..00619f6 100644 --- a/src/bpf.h +++ b/src/bpf.h @@ -35,6 +35,8 @@ extern "C" { #endif +int libbpf_set_memlock_rlim(size_t memlock_bytes); + struct bpf_map_create_opts { size_t sz; /* size of this struct for forward/backward compatibility */ diff --git a/src/libbpf.c b/src/libbpf.c index c6d84b0..684c34c 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -187,42 +187,6 @@ const char *libbpf_version_string(void) #undef __S } -enum kern_feature_id { - /* v4.14: kernel support for program & map names. */ - FEAT_PROG_NAME, - /* v5.2: kernel support for global data sections. */ - FEAT_GLOBAL_DATA, - /* BTF support */ - FEAT_BTF, - /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ - FEAT_BTF_FUNC, - /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ - FEAT_BTF_DATASEC, - /* BTF_FUNC_GLOBAL is supported */ - FEAT_BTF_GLOBAL_FUNC, - /* BPF_F_MMAPABLE is supported for arrays */ - FEAT_ARRAY_MMAP, - /* kernel support for expected_attach_type in BPF_PROG_LOAD */ - FEAT_EXP_ATTACH_TYPE, - /* bpf_probe_read_{kernel,user}[_str] helpers */ - FEAT_PROBE_READ_KERN, - /* BPF_PROG_BIND_MAP is supported */ - FEAT_PROG_BIND_MAP, - /* Kernel support for module BTFs */ - FEAT_MODULE_BTF, - /* BTF_KIND_FLOAT support */ - FEAT_BTF_FLOAT, - /* BPF perf link support */ - FEAT_PERF_LINK, - /* BTF_KIND_DECL_TAG support */ - FEAT_BTF_DECL_TAG, - /* BTF_KIND_TYPE_TAG support */ - FEAT_BTF_TYPE_TAG, - __FEAT_CNT, -}; - -static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); - enum reloc_type { RELO_LD64, RELO_CALL, @@ -4352,6 +4316,10 @@ bpf_object__probe_loading(struct bpf_object *obj) if (obj->gen_loader) return 0; + ret = bump_rlimit_memlock(); + if (ret) + pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret); + /* make sure basic loading works */ ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) @@ -4718,14 +4686,17 @@ static struct kern_feature_desc { [FEAT_BTF_TYPE_TAG] = { "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag, }, + [FEAT_MEMCG_ACCOUNT] = { + "memcg-based memory accounting", probe_memcg_account, + }, }; -static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) +bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) { struct kern_feature_desc *feat = &feature_probes[feat_id]; int ret; - if (obj->gen_loader) + if (obj && obj->gen_loader) /* To generate loader program assume the latest kernel * to avoid doing extra prog_load, map_create syscalls. */ diff --git a/src/libbpf.map b/src/libbpf.map index 4d483af..b3938b3 100644 --- a/src/libbpf.map +++ b/src/libbpf.map @@ -427,4 +427,5 @@ LIBBPF_0.7.0 { bpf_program__log_level; bpf_program__set_log_buf; bpf_program__set_log_level; + libbpf_set_memlock_rlim_max; }; diff --git a/src/libbpf_internal.h b/src/libbpf_internal.h index 5e8166a..5dbe4f4 100644 --- a/src/libbpf_internal.h +++ b/src/libbpf_internal.h @@ -291,6 +291,45 @@ static inline bool libbpf_validate_opts(const char *opts, (opts)->sz - __off); \ }) +enum kern_feature_id { + /* v4.14: kernel support for program & map names. */ + FEAT_PROG_NAME, + /* v5.2: kernel support for global data sections. */ + FEAT_GLOBAL_DATA, + /* BTF support */ + FEAT_BTF, + /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ + FEAT_BTF_FUNC, + /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ + FEAT_BTF_DATASEC, + /* BTF_FUNC_GLOBAL is supported */ + FEAT_BTF_GLOBAL_FUNC, + /* BPF_F_MMAPABLE is supported for arrays */ + FEAT_ARRAY_MMAP, + /* kernel support for expected_attach_type in BPF_PROG_LOAD */ + FEAT_EXP_ATTACH_TYPE, + /* bpf_probe_read_{kernel,user}[_str] helpers */ + FEAT_PROBE_READ_KERN, + /* BPF_PROG_BIND_MAP is supported */ + FEAT_PROG_BIND_MAP, + /* Kernel support for module BTFs */ + FEAT_MODULE_BTF, + /* BTF_KIND_FLOAT support */ + FEAT_BTF_FLOAT, + /* BPF perf link support */ + FEAT_PERF_LINK, + /* BTF_KIND_DECL_TAG support */ + FEAT_BTF_DECL_TAG, + /* BTF_KIND_TYPE_TAG support */ + FEAT_BTF_TYPE_TAG, + /* memcg-based accounting for BPF maps and progs */ + FEAT_MEMCG_ACCOUNT, + __FEAT_CNT, +}; + +int probe_memcg_account(void); +bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); +int bump_rlimit_memlock(void); int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz); int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); diff --git a/src/libbpf_legacy.h b/src/libbpf_legacy.h index bb03c56..79131f7 100644 --- a/src/libbpf_legacy.h +++ b/src/libbpf_legacy.h @@ -45,7 +45,6 @@ enum libbpf_strict_mode { * (positive) error code. */ LIBBPF_STRICT_DIRECT_ERRS = 0x02, - /* * Enforce strict BPF program section (SEC()) names. * E.g., while prefiously SEC("xdp_whatever") or SEC("perf_event_blah") were @@ -63,6 +62,17 @@ enum libbpf_strict_mode { * Clients can maintain it on their own if it is valuable for them. */ LIBBPF_STRICT_NO_OBJECT_LIST = 0x08, + /* + * Automatically bump RLIMIT_MEMLOCK using setrlimit() before the + * first BPF program or map creation operation. This is done only if + * kernel is too old to support memcg-based memory accounting for BPF + * subsystem. By default, RLIMIT_MEMLOCK limit is set to RLIM_INFINITY, + * but it can be overriden with libbpf_set_memlock_rlim_max() API. + * Note that libbpf_set_memlock_rlim_max() needs to be called before + * the very first bpf_prog_load(), bpf_map_create() or bpf_object__load() + * operation. + */ + LIBBPF_STRICT_AUTO_RLIMIT_MEMLOCK = 0x10, __LIBBPF_STRICT_LAST, }; From 7152ecf163cffb7b6f2e5c1cc7f3dbd42a47a2ed Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Mon, 13 Dec 2021 19:59:31 -0800 Subject: [PATCH 043/131] libbpf: Mark bpf_object__find_program_by_title API deprecated. Deprecate this API since v0.7. All callers should move to bpf_object__find_program_by_name if possible, otherwise use bpf_object__for_each_program to find a program out from a given section. [0] Closes: https://github.com/libbpf/libbpf/issues/292 Signed-off-by: Kui-Feng Lee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211214035931.1148209-5-kuifeng@fb.com --- src/libbpf.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libbpf.h b/src/libbpf.h index 3f5ff2c..42b2f36 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -227,6 +227,7 @@ struct btf; LIBBPF_API struct btf *bpf_object__btf(const struct bpf_object *obj); LIBBPF_API int bpf_object__btf_fd(const struct bpf_object *obj); +LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__find_program_by_name() instead") LIBBPF_API struct bpf_program * bpf_object__find_program_by_title(const struct bpf_object *obj, const char *title); From 266e897ad21be112d50127df63c056182a9962ef Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 14 Dec 2021 15:08:35 -0800 Subject: [PATCH 044/131] sync: auto-generate latest BPF helpers Latest changes to BPF helper definitions. --- src/bpf_helper_defs.h | 51 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/bpf_helper_defs.h b/src/bpf_helper_defs.h index fc8371d..1c70294 100644 --- a/src/bpf_helper_defs.h +++ b/src/bpf_helper_defs.h @@ -4164,4 +4164,55 @@ static long (*bpf_find_vma)(struct task_struct *task, __u64 addr, void *callback */ static long (*bpf_loop)(__u32 nr_loops, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 181; +/* + * bpf_strncmp + * + * Do strncmp() between **s1** and **s2**. **s1** doesn't need + * to be null-terminated and **s1_sz** is the maximum storage + * size of **s1**. **s2** must be a read-only string. + * + * Returns + * An integer less than, equal to, or greater than zero + * if the first **s1_sz** bytes of **s1** is found to be + * less than, to match, or be greater than **s2**. + */ +static long (*bpf_strncmp)(const char *s1, __u32 s1_sz, const char *s2) = (void *) 182; + +/* + * bpf_get_func_arg + * + * Get **n**-th argument (zero based) of the traced function (for tracing programs) + * returned in **value**. + * + * + * Returns + * 0 on success. + * **-EINVAL** if n >= arguments count of traced function. + */ +static long (*bpf_get_func_arg)(void *ctx, __u32 n, __u64 *value) = (void *) 183; + +/* + * bpf_get_func_ret + * + * Get return value of the traced function (for tracing programs) + * in **value**. + * + * + * Returns + * 0 on success. + * **-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN. + */ +static long (*bpf_get_func_ret)(void *ctx, __u64 *value) = (void *) 184; + +/* + * bpf_get_func_arg_cnt + * + * Get number of arguments of the traced function (for tracing programs). + * + * + * Returns + * The number of arguments of the traced function. + */ +static long (*bpf_get_func_arg_cnt)(void *ctx) = (void *) 185; + From 61acde230836b48d6797f3b0bf3918a36987d459 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 14 Dec 2021 15:08:35 -0800 Subject: [PATCH 045/131] sync: latest libbpf changes from kernel Syncing latest libbpf commits from kernel repository. Baseline bpf-next commit: 229fae38d0fc0d6ff58d57cbeb1432da55e58d4f Checkpoint bpf-next commit: a34efe503bc55c5732e328e5191ad549eb899f31 Baseline bpf commit: 0be2516f865f5a876837184a8385163ff64a5889 Checkpoint bpf commit: f7abc4c8df8c7930d0b9c56d9abee9a1fca635e9 Alexei Starovoitov (1): libbpf: Fix gen_loader assumption on number of programs. Andrii Nakryiko (4): libbpf: Don't validate TYPE_ID relo's original imm value libbpf: Fix potential uninit memory read libbpf: Add sane strncpy alternative and use it internally libbpf: Auto-bump RLIMIT_MEMLOCK if kernel needs it for BPF Grant Seltzer (1): libbpf: Add doc comments for bpf_program__(un)pin() Hangbin Liu (1): Bonding: add arp_missed_max option Hou Tao (1): bpf: Add bpf_strncmp helper Jiri Olsa (1): bpf: Add get_func_[arg|ret|arg_cnt] helpers Kui-Feng Lee (1): libbpf: Mark bpf_object__find_program_by_title API deprecated. include/uapi/linux/bpf.h | 39 +++++++++++++++++ include/uapi/linux/if_link.h | 1 + src/bpf.c | 85 +++++++++++++++++++++++++++++++++++- src/bpf.h | 2 + src/btf_dump.c | 4 +- src/gen_loader.c | 12 +++-- src/libbpf.c | 55 +++++------------------ src/libbpf.h | 25 +++++++++++ src/libbpf.map | 1 + src/libbpf_internal.h | 58 ++++++++++++++++++++++++ src/libbpf_legacy.h | 12 ++++- src/relo_core.c | 20 ++++++--- src/xsk.c | 9 ++-- 13 files changed, 260 insertions(+), 63 deletions(-) -- 2.30.2 --- BPF-CHECKPOINT-COMMIT | 2 +- CHECKPOINT-COMMIT | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BPF-CHECKPOINT-COMMIT b/BPF-CHECKPOINT-COMMIT index 561199b..d48ec9f 100644 --- a/BPF-CHECKPOINT-COMMIT +++ b/BPF-CHECKPOINT-COMMIT @@ -1 +1 @@ -0be2516f865f5a876837184a8385163ff64a5889 +f7abc4c8df8c7930d0b9c56d9abee9a1fca635e9 diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index 47d6510..ba87459 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -229fae38d0fc0d6ff58d57cbeb1432da55e58d4f +a34efe503bc55c5732e328e5191ad549eb899f31 From 19656636a9b9a2de1f71fa3135709295c16701cc Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 14 Dec 2021 16:26:48 -0800 Subject: [PATCH 046/131] vmtest: blacklist bpf_loop and get_func_args_test for s390x bpf_loop is using arch-specific sys_nanosleep attach function. get_func_args_test relies on BPF trampoline. Signed-off-by: Andrii Nakryiko --- travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x | 2 ++ 1 file changed, 2 insertions(+) diff --git a/travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x b/travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x index 5b0a7fb..11e3f5f 100644 --- a/travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x +++ b/travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x @@ -3,6 +3,7 @@ atomics # attach(add): actual -524 <= expected bpf_iter_setsockopt # JIT does not support calling kernel function (kfunc) bloom_filter_map # failed to find kernel BTF type ID of '__x64_sys_getpgid': -3 (?) bpf_tcp_ca # JIT does not support calling kernel function (kfunc) +bpf_loop # attaches to __x64_sys_nanosleep core_read_macros # unknown func bpf_probe_read#4 (overlapping) d_path # failed to auto-attach program 'prog_stat': -524 (trampoline) dummy_st_ops # test_run unexpected error: -524 (errno 524) (trampoline) @@ -12,6 +13,7 @@ fexit_bpf2bpf # freplace_attach_trace unexpected erro fexit_sleep # fexit_skel_load fexit skeleton failed (trampoline) fexit_stress # fexit attach failed prog 0 failed: -524 (trampoline) fexit_test # fexit_first_attach unexpected error: -524 (trampoline) +get_func_args_test # trampoline get_func_ip_test # get_func_ip_test__attach unexpected error: -524 (trampoline) get_stack_raw_tp # user_stack corrupted user stack (no backchain userspace) kfree_skb # attach fentry unexpected error: -524 (trampoline) From dc49f2d07b7c924f6b6f3595b27a8a326ed483d3 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 16 Dec 2021 14:30:46 -0800 Subject: [PATCH 047/131] ci: add LIRC kernel config Add necessary kernel config values to make BPF_PROG_TYPE_LIRC_MODE2 programs work. Signed-off-by: Andrii Nakryiko --- travis-ci/vmtest/configs/config-latest.x86_64 | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/travis-ci/vmtest/configs/config-latest.x86_64 b/travis-ci/vmtest/configs/config-latest.x86_64 index 481555e..a577b17 100644 --- a/travis-ci/vmtest/configs/config-latest.x86_64 +++ b/travis-ci/vmtest/configs/config-latest.x86_64 @@ -1791,7 +1791,14 @@ CONFIG_BCMA_POSSIBLE=y # end of Multifunction device drivers # CONFIG_REGULATOR is not set -# CONFIG_RC_CORE is not set +CONFIG_RC_CORE=y +# CONFIG_RC_MAP is not set +CONFIG_LIRC=y +CONFIG_BPF_LIRC_MODE2=y +# CONFIG_RC_DECODERS is not set +CONFIG_RC_DEVICES=y +CONFIG_RC_LOOPBACK=y +# CONFIG_IR_SERIAL is not set # CONFIG_MEDIA_CEC_SUPPORT is not set # CONFIG_MEDIA_SUPPORT is not set From 8e706ddc6c0e33a7996cf3122f5c31ab5123a139 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 14 Dec 2021 15:20:54 -0800 Subject: [PATCH 048/131] libbpf: Avoid reading past ELF data section end when copying license Fix possible read beyond ELF "license" data section if the license string is not properly zero-terminated. Use the fact that libbpf_strlcpy never accesses the (N-1)st byte of the source string because it's replaced with '\0' anyways. If this happens, it's a violation of contract between libbpf and a user, but not handling this more robustly upsets CIFuzz, so given the fix is trivial, let's fix the potential issue. Fixes: 9fc205b413b3 ("libbpf: Add sane strncpy alternative and use it internally") Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211214232054.3458774-1-andrii@kernel.org --- src/libbpf.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libbpf.c b/src/libbpf.c index 684c34c..cf862a1 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -1320,7 +1320,10 @@ static int bpf_object__check_endianness(struct bpf_object *obj) static int bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) { - libbpf_strlcpy(obj->license, data, sizeof(obj->license)); + /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't + * go over allowed ELF data section buffer + */ + libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license))); pr_debug("license of %s is %s\n", obj->path, obj->license); return 0; } From 168cf9b8ae24afd779f2976c76c6a8628540a3e4 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 17 Dec 2021 09:12:00 -0800 Subject: [PATCH 049/131] libbpf: Rework feature-probing APIs Create three extensible alternatives to inconsistently named feature-probing APIs: - libbpf_probe_bpf_prog_type() instead of bpf_probe_prog_type(); - libbpf_probe_bpf_map_type() instead of bpf_probe_map_type(); - libbpf_probe_bpf_helper() instead of bpf_probe_helper(). Set up return values such that libbpf can report errors (e.g., if some combination of input arguments isn't possible to validate, etc), in addition to whether the feature is supported (return value 1) or not supported (return value 0). Also schedule deprecation of those three APIs. Also schedule deprecation of bpf_probe_large_insn_limit(). Also fix all the existing detection logic for various program and map types that never worked: - BPF_PROG_TYPE_LIRC_MODE2; - BPF_PROG_TYPE_TRACING; - BPF_PROG_TYPE_LSM; - BPF_PROG_TYPE_EXT; - BPF_PROG_TYPE_SYSCALL; - BPF_PROG_TYPE_STRUCT_OPS; - BPF_MAP_TYPE_STRUCT_OPS; - BPF_MAP_TYPE_BLOOM_FILTER. Above prog/map types needed special setups and detection logic to work. Subsequent patch adds selftests that will make sure that all the detection logic keeps working for all current and future program and map types, avoiding otherwise inevitable bit rot. [0] Closes: https://github.com/libbpf/libbpf/issues/312 Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Acked-by: Dave Marchevsky Cc: Julia Kartseva Link: https://lore.kernel.org/bpf/20211217171202.3352835-2-andrii@kernel.org --- src/libbpf.h | 52 +++++++++- src/libbpf.map | 3 + src/libbpf_probes.c | 237 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 237 insertions(+), 55 deletions(-) diff --git a/src/libbpf.h b/src/libbpf.h index 42b2f36..85dfef8 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -1052,13 +1052,57 @@ bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo, * user, causing subsequent probes to fail. In this case, the caller may want * to adjust that limit with setrlimit(). */ -LIBBPF_API bool bpf_probe_prog_type(enum bpf_prog_type prog_type, - __u32 ifindex); +LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_prog_type() instead") +LIBBPF_API bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex); +LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_map_type() instead") LIBBPF_API bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex); -LIBBPF_API bool bpf_probe_helper(enum bpf_func_id id, - enum bpf_prog_type prog_type, __u32 ifindex); +LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_helper() instead") +LIBBPF_API bool bpf_probe_helper(enum bpf_func_id id, enum bpf_prog_type prog_type, __u32 ifindex); +LIBBPF_DEPRECATED_SINCE(0, 8, "implement your own or use bpftool for feature detection") LIBBPF_API bool bpf_probe_large_insn_limit(__u32 ifindex); +/** + * @brief **libbpf_probe_bpf_prog_type()** detects if host kernel supports + * BPF programs of a given type. + * @param prog_type BPF program type to detect kernel support for + * @param opts reserved for future extensibility, should be NULL + * @return 1, if given program type is supported; 0, if given program type is + * not supported; negative error code if feature detection failed or can't be + * performed + * + * Make sure the process has required set of CAP_* permissions (or runs as + * root) when performing feature checking. + */ +LIBBPF_API int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts); +/** + * @brief **libbpf_probe_bpf_map_type()** detects if host kernel supports + * BPF maps of a given type. + * @param map_type BPF map type to detect kernel support for + * @param opts reserved for future extensibility, should be NULL + * @return 1, if given map type is supported; 0, if given map type is + * not supported; negative error code if feature detection failed or can't be + * performed + * + * Make sure the process has required set of CAP_* permissions (or runs as + * root) when performing feature checking. + */ +LIBBPF_API int libbpf_probe_bpf_map_type(enum bpf_map_type map_type, const void *opts); +/** + * @brief **libbpf_probe_bpf_helper()** detects if host kernel supports the + * use of a given BPF helper from specified BPF program type. + * @param prog_type BPF program type used to check the support of BPF helper + * @param helper_id BPF helper ID (enum bpf_func_id) to check support for + * @param opts reserved for future extensibility, should be NULL + * @return 1, if given combination of program type and helper is supported; 0, + * if the combination is not supported; negative error code if feature + * detection for provided input arguments failed or can't be performed + * + * Make sure the process has required set of CAP_* permissions (or runs as + * root) when performing feature checking. + */ +LIBBPF_API int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, + enum bpf_func_id helper_id, const void *opts); + /* * Get bpf_prog_info in continuous memory * diff --git a/src/libbpf.map b/src/libbpf.map index b3938b3..5297839 100644 --- a/src/libbpf.map +++ b/src/libbpf.map @@ -427,5 +427,8 @@ LIBBPF_0.7.0 { bpf_program__log_level; bpf_program__set_log_buf; bpf_program__set_log_level; + libbpf_probe_bpf_helper; + libbpf_probe_bpf_map_type; + libbpf_probe_bpf_prog_type; libbpf_set_memlock_rlim_max; }; diff --git a/src/libbpf_probes.c b/src/libbpf_probes.c index 4bdec69..9d39852 100644 --- a/src/libbpf_probes.c +++ b/src/libbpf_probes.c @@ -64,12 +64,20 @@ static int get_kernel_version(void) return (version << 16) + (subversion << 8) + patchlevel; } -static void -probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, - size_t insns_cnt, char *buf, size_t buf_len, __u32 ifindex) +static int probe_prog_load(enum bpf_prog_type prog_type, + const struct bpf_insn *insns, size_t insns_cnt, + char *log_buf, size_t log_buf_sz, + __u32 ifindex) { - LIBBPF_OPTS(bpf_prog_load_opts, opts); - int fd; + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .log_buf = log_buf, + .log_size = log_buf_sz, + .log_level = log_buf ? 1 : 0, + .prog_ifindex = ifindex, + ); + int fd, err, exp_err = 0; + const char *exp_msg = NULL; + char buf[4096]; switch (prog_type) { case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: @@ -84,6 +92,38 @@ probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, case BPF_PROG_TYPE_KPROBE: opts.kern_version = get_kernel_version(); break; + case BPF_PROG_TYPE_LIRC_MODE2: + opts.expected_attach_type = BPF_LIRC_MODE2; + break; + case BPF_PROG_TYPE_TRACING: + case BPF_PROG_TYPE_LSM: + opts.log_buf = buf; + opts.log_size = sizeof(buf); + opts.log_level = 1; + if (prog_type == BPF_PROG_TYPE_TRACING) + opts.expected_attach_type = BPF_TRACE_FENTRY; + else + opts.expected_attach_type = BPF_MODIFY_RETURN; + opts.attach_btf_id = 1; + + exp_err = -EINVAL; + exp_msg = "attach_btf_id 1 is not a function"; + break; + case BPF_PROG_TYPE_EXT: + opts.log_buf = buf; + opts.log_size = sizeof(buf); + opts.log_level = 1; + opts.attach_btf_id = 1; + + exp_err = -EINVAL; + exp_msg = "Cannot replace kernel functions"; + break; + case BPF_PROG_TYPE_SYSCALL: + opts.prog_flags = BPF_F_SLEEPABLE; + break; + case BPF_PROG_TYPE_STRUCT_OPS: + exp_err = -524; /* -ENOTSUPP */ + break; case BPF_PROG_TYPE_UNSPEC: case BPF_PROG_TYPE_SOCKET_FILTER: case BPF_PROG_TYPE_SCHED_CLS: @@ -103,25 +143,42 @@ probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, case BPF_PROG_TYPE_RAW_TRACEPOINT: case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: case BPF_PROG_TYPE_LWT_SEG6LOCAL: - case BPF_PROG_TYPE_LIRC_MODE2: case BPF_PROG_TYPE_SK_REUSEPORT: case BPF_PROG_TYPE_FLOW_DISSECTOR: case BPF_PROG_TYPE_CGROUP_SYSCTL: - case BPF_PROG_TYPE_TRACING: - case BPF_PROG_TYPE_STRUCT_OPS: - case BPF_PROG_TYPE_EXT: - case BPF_PROG_TYPE_LSM: - default: break; + default: + return -EOPNOTSUPP; } - opts.prog_ifindex = ifindex; - opts.log_buf = buf; - opts.log_size = buf_len; - - fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insns_cnt, NULL); + fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insns_cnt, &opts); + err = -errno; if (fd >= 0) close(fd); + if (exp_err) { + if (fd >= 0 || err != exp_err) + return 0; + if (exp_msg && !strstr(buf, exp_msg)) + return 0; + return 1; + } + return fd >= 0 ? 1 : 0; +} + +int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN() + }; + const size_t insn_cnt = ARRAY_SIZE(insns); + int ret; + + if (opts) + return libbpf_err(-EINVAL); + + ret = probe_prog_load(prog_type, insns, insn_cnt, NULL, 0, 0); + return libbpf_err(ret); } bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex) @@ -131,12 +188,16 @@ bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex) BPF_EXIT_INSN() }; + /* prefer libbpf_probe_bpf_prog_type() unless offload is requested */ + if (ifindex == 0) + return libbpf_probe_bpf_prog_type(prog_type, NULL) == 1; + if (ifindex && prog_type == BPF_PROG_TYPE_SCHED_CLS) /* nfp returns -EINVAL on exit(0) with TC offload */ insns[0].imm = 2; errno = 0; - probe_load(prog_type, insns, ARRAY_SIZE(insns), NULL, 0, ifindex); + probe_prog_load(prog_type, insns, ARRAY_SIZE(insns), NULL, 0, ifindex); return errno != EINVAL && errno != EOPNOTSUPP; } @@ -197,16 +258,18 @@ static int load_local_storage_btf(void) strs, sizeof(strs)); } -bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) +static int probe_map_create(enum bpf_map_type map_type, __u32 ifindex) { - int key_size, value_size, max_entries, map_flags; + LIBBPF_OPTS(bpf_map_create_opts, opts); + int key_size, value_size, max_entries; __u32 btf_key_type_id = 0, btf_value_type_id = 0; - int fd = -1, btf_fd = -1, fd_inner; + int fd = -1, btf_fd = -1, fd_inner = -1, exp_err = 0, err; + + opts.map_ifindex = ifindex; key_size = sizeof(__u32); value_size = sizeof(__u32); max_entries = 1; - map_flags = 0; switch (map_type) { case BPF_MAP_TYPE_STACK_TRACE: @@ -215,7 +278,7 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) case BPF_MAP_TYPE_LPM_TRIE: key_size = sizeof(__u64); value_size = sizeof(__u64); - map_flags = BPF_F_NO_PREALLOC; + opts.map_flags = BPF_F_NO_PREALLOC; break; case BPF_MAP_TYPE_CGROUP_STORAGE: case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: @@ -234,17 +297,25 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) btf_value_type_id = 3; value_size = 8; max_entries = 0; - map_flags = BPF_F_NO_PREALLOC; + opts.map_flags = BPF_F_NO_PREALLOC; btf_fd = load_local_storage_btf(); if (btf_fd < 0) - return false; + return btf_fd; break; case BPF_MAP_TYPE_RINGBUF: key_size = 0; value_size = 0; max_entries = 4096; break; - case BPF_MAP_TYPE_UNSPEC: + case BPF_MAP_TYPE_STRUCT_OPS: + /* we'll get -ENOTSUPP for invalid BTF type ID for struct_ops */ + opts.btf_vmlinux_value_type_id = 1; + exp_err = -524; /* -ENOTSUPP */ + break; + case BPF_MAP_TYPE_BLOOM_FILTER: + key_size = 0; + max_entries = 1; + break; case BPF_MAP_TYPE_HASH: case BPF_MAP_TYPE_ARRAY: case BPF_MAP_TYPE_PROG_ARRAY: @@ -263,49 +334,114 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) case BPF_MAP_TYPE_XSKMAP: case BPF_MAP_TYPE_SOCKHASH: case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: - case BPF_MAP_TYPE_STRUCT_OPS: - default: break; + case BPF_MAP_TYPE_UNSPEC: + default: + return -EOPNOTSUPP; } if (map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS || map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { - LIBBPF_OPTS(bpf_map_create_opts, opts); - /* TODO: probe for device, once libbpf has a function to create * map-in-map for offload */ if (ifindex) - return false; + goto cleanup; fd_inner = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(__u32), sizeof(__u32), 1, NULL); if (fd_inner < 0) - return false; + goto cleanup; opts.inner_map_fd = fd_inner; - fd = bpf_map_create(map_type, NULL, sizeof(__u32), sizeof(__u32), 1, &opts); - close(fd_inner); - } else { - LIBBPF_OPTS(bpf_map_create_opts, opts); - - /* Note: No other restriction on map type probes for offload */ - opts.map_flags = map_flags; - opts.map_ifindex = ifindex; - if (btf_fd >= 0) { - opts.btf_fd = btf_fd; - opts.btf_key_type_id = btf_key_type_id; - opts.btf_value_type_id = btf_value_type_id; - } - - fd = bpf_map_create(map_type, NULL, key_size, value_size, max_entries, &opts); } + + if (btf_fd >= 0) { + opts.btf_fd = btf_fd; + opts.btf_key_type_id = btf_key_type_id; + opts.btf_value_type_id = btf_value_type_id; + } + + fd = bpf_map_create(map_type, NULL, key_size, value_size, max_entries, &opts); + err = -errno; + +cleanup: if (fd >= 0) close(fd); + if (fd_inner >= 0) + close(fd_inner); if (btf_fd >= 0) close(btf_fd); - return fd >= 0; + if (exp_err) + return fd < 0 && err == exp_err ? 1 : 0; + else + return fd >= 0 ? 1 : 0; +} + +int libbpf_probe_bpf_map_type(enum bpf_map_type map_type, const void *opts) +{ + int ret; + + if (opts) + return libbpf_err(-EINVAL); + + ret = probe_map_create(map_type, 0); + return libbpf_err(ret); +} + +bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) +{ + return probe_map_create(map_type, ifindex) == 1; +} + +int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, enum bpf_func_id helper_id, + const void *opts) +{ + struct bpf_insn insns[] = { + BPF_EMIT_CALL((__u32)helper_id), + BPF_EXIT_INSN(), + }; + const size_t insn_cnt = ARRAY_SIZE(insns); + char buf[4096]; + int ret; + + if (opts) + return libbpf_err(-EINVAL); + + /* we can't successfully load all prog types to check for BPF helper + * support, so bail out with -EOPNOTSUPP error + */ + switch (prog_type) { + case BPF_PROG_TYPE_TRACING: + case BPF_PROG_TYPE_EXT: + case BPF_PROG_TYPE_LSM: + case BPF_PROG_TYPE_STRUCT_OPS: + return -EOPNOTSUPP; + default: + break; + } + + buf[0] = '\0'; + ret = probe_prog_load(prog_type, insns, insn_cnt, buf, sizeof(buf), 0); + if (ret < 0) + return libbpf_err(ret); + + /* If BPF verifier doesn't recognize BPF helper ID (enum bpf_func_id) + * at all, it will emit something like "invalid func unknown#181". + * If BPF verifier recognizes BPF helper but it's not supported for + * given BPF program type, it will emit "unknown func bpf_sys_bpf#166". + * In both cases, provided combination of BPF program type and BPF + * helper is not supported by the kernel. + * In all other cases, probe_prog_load() above will either succeed (e.g., + * because BPF helper happens to accept no input arguments or it + * accepts one input argument and initial PTR_TO_CTX is fine for + * that), or we'll get some more specific BPF verifier error about + * some unsatisfied conditions. + */ + if (ret == 0 && (strstr(buf, "invalid func ") || strstr(buf, "unknown func "))) + return 0; + return 1; /* assume supported */ } bool bpf_probe_helper(enum bpf_func_id id, enum bpf_prog_type prog_type, @@ -318,8 +454,7 @@ bool bpf_probe_helper(enum bpf_func_id id, enum bpf_prog_type prog_type, char buf[4096] = {}; bool res; - probe_load(prog_type, insns, ARRAY_SIZE(insns), buf, sizeof(buf), - ifindex); + probe_prog_load(prog_type, insns, ARRAY_SIZE(insns), buf, sizeof(buf), ifindex); res = !grep(buf, "invalid func ") && !grep(buf, "unknown func "); if (ifindex) { @@ -351,8 +486,8 @@ bool bpf_probe_large_insn_limit(__u32 ifindex) insns[BPF_MAXINSNS] = BPF_EXIT_INSN(); errno = 0; - probe_load(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns), NULL, 0, - ifindex); + probe_prog_load(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns), NULL, 0, + ifindex); return errno != E2BIG && errno != EINVAL; } From 96268bf0c2b73b3ba91172a74179c2272870b8f8 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 17 Dec 2021 16:28:21 -0800 Subject: [PATCH 050/131] sync: latest libbpf changes from kernel Syncing latest libbpf commits from kernel repository. Baseline bpf-next commit: a34efe503bc55c5732e328e5191ad549eb899f31 Checkpoint bpf-next commit: e967a20a8fabc6442a78e2e2059e63a4bb6aed08 Baseline bpf commit: f7abc4c8df8c7930d0b9c56d9abee9a1fca635e9 Checkpoint bpf commit: 819d11507f6637731947836e6308f5966d64cf9d Andrii Nakryiko (2): libbpf: Avoid reading past ELF data section end when copying license libbpf: Rework feature-probing APIs src/libbpf.c | 5 +- src/libbpf.h | 52 +++++++++- src/libbpf.map | 3 + src/libbpf_probes.c | 235 ++++++++++++++++++++++++++++++++++---------- 4 files changed, 240 insertions(+), 55 deletions(-) -- 2.30.2 --- BPF-CHECKPOINT-COMMIT | 2 +- CHECKPOINT-COMMIT | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BPF-CHECKPOINT-COMMIT b/BPF-CHECKPOINT-COMMIT index d48ec9f..417cd69 100644 --- a/BPF-CHECKPOINT-COMMIT +++ b/BPF-CHECKPOINT-COMMIT @@ -1 +1 @@ -f7abc4c8df8c7930d0b9c56d9abee9a1fca635e9 +819d11507f6637731947836e6308f5966d64cf9d diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index ba87459..b364dc6 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -a34efe503bc55c5732e328e5191ad549eb899f31 +e967a20a8fabc6442a78e2e2059e63a4bb6aed08 From 9fab7c81ecd462ce6aa1eabd24b230f0fc6a14c7 Mon Sep 17 00:00:00 2001 From: Yucong Sun Date: Mon, 20 Dec 2021 13:12:47 -0800 Subject: [PATCH 051/131] ci: Add a step to patch kernel with temporary fixes Apply a custom set of patches against bpf-next kernel tree before building vmlinux image. --- .github/actions/vmtest/action.yml | 5 +++ travis-ci/diffs/.do_not_use_dot_patch_here | 0 .../001-fix-oob-write-in-test_verifier.diff | 35 +++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 travis-ci/diffs/.do_not_use_dot_patch_here create mode 100644 travis-ci/diffs/001-fix-oob-write-in-test_verifier.diff diff --git a/.github/actions/vmtest/action.yml b/.github/actions/vmtest/action.yml index 2f86607..1bb52c6 100644 --- a/.github/actions/vmtest/action.yml +++ b/.github/actions/vmtest/action.yml @@ -33,6 +33,11 @@ runs: repo: 'https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git' rev: ${{ env.CHECKPOINT }} dest: '${{ github.workspace }}/.kernel' + - name: Patch kernel source + uses: libbpf/ci/patch-kernel@master + with: + patches-root: '${{ github.workspace }}/travis-ci/diffs' + repo-root: '.kernel' - name: Prepare to build BPF selftests shell: bash run: | diff --git a/travis-ci/diffs/.do_not_use_dot_patch_here b/travis-ci/diffs/.do_not_use_dot_patch_here new file mode 100644 index 0000000..e69de29 diff --git a/travis-ci/diffs/001-fix-oob-write-in-test_verifier.diff b/travis-ci/diffs/001-fix-oob-write-in-test_verifier.diff new file mode 100644 index 0000000..eb5ef26 --- /dev/null +++ b/travis-ci/diffs/001-fix-oob-write-in-test_verifier.diff @@ -0,0 +1,35 @@ +From: Kumar Kartikeya Dwivedi +To: bpf@vger.kernel.org +Cc: Alexei Starovoitov , + Daniel Borkmann , + Andrii Nakryiko +Subject: [PATCH bpf-next] selftests/bpf: Fix OOB write in test_verifier +Date: Tue, 14 Dec 2021 07:18:00 +0530 [thread overview] +Message-ID: <20211214014800.78762-1-memxor@gmail.com> (raw) + +The commit referenced below added fixup_map_timer support (to create a +BPF map containing timers), but failed to increase the size of the +map_fds array, leading to out of bounds write. Fix this by changing +MAX_NR_MAPS to 22. + +Fixes: e60e6962c503 ("selftests/bpf: Add tests for restricted helpers") +Signed-off-by: Kumar Kartikeya Dwivedi +--- + tools/testing/selftests/bpf/test_verifier.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c +index ad5d30bafd93..33e2ecb3bef9 100644 +--- a/tools/testing/selftests/bpf/test_verifier.c ++++ b/tools/testing/selftests/bpf/test_verifier.c +@@ -54,7 +54,7 @@ + #define MAX_INSNS BPF_MAXINSNS + #define MAX_TEST_INSNS 1000000 + #define MAX_FIXUPS 8 +-#define MAX_NR_MAPS 21 ++#define MAX_NR_MAPS 22 + #define MAX_TEST_RUNS 8 + #define POINTER_VALUE 0xcafe4all + #define TEST_DATA_LEN 64 +-- +2.34.1 From 287d0d097b05c18e6aec86ecf8d89dd816a9dd31 Mon Sep 17 00:00:00 2001 From: grantseltzer Date: Mon, 20 Dec 2021 11:59:56 -0500 Subject: [PATCH 052/131] Add documentation for error checking in API Signed-off-by: grantseltzer --- docs/api.rst | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index 4ad9f34..ea181af 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -6,7 +6,49 @@ LIBBPF API -================== +========== + +Error Handling +-------------- + +When libbpf is used in "libbpf 1.0 mode", API functions can return errors in one of two ways. + +You can set "libbpf 1.0" mode with the following line: + +.. code-block:: + + libbpf_set_strict_mode(LIBBPF_STRICT_DIRECT_ERRS | LIBBPF_STRICT_CLEAN_PTRS); + +If the function returns an error code directly, it uses 0 to indicate success +and a negative error code to indicate what caused the error. In this case the +error code should be checked directly from the return, you do not need to check +errno. + +For example: + +.. code-block:: + + err = some_libbpf_api_with_error_return(...); + if (err < 0) { + /* Handle error accordingly */ + } + +If the function returns a pointer, it will return NULL to indicate there was +an error. In this case errno should be checked for the error code. + +For example: + +.. code-block:: + + ptr = some_libbpf_api_returning_ptr(); + if (!ptr) { + /* note no minus sign for EINVAL and E2BIG below */ + if (errno = EINVAL) { + /* handle EINVAL error */ + } else if (errno == E2BIG) { + /* handle E2BIG error */ + } + } libbpf.h -------- From 1321a8bb497584b22742ea69461811d4a6033a59 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 23 Dec 2021 14:17:35 +0100 Subject: [PATCH 053/131] libbpf: Do not use btf_dump__new() macro in C++ mode As reported in here [0], C++ compilers don't support __builtin_types_compatible_p(), so at least don't screw up compilation for them and let C++ users pick btf_dump__new vs btf_dump__new_deprecated explicitly. [0] https://github.com/libbpf/libbpf/issues/283#issuecomment-986100727 Fixes: 6084f5dc928f ("libbpf: Ensure btf_dump__new() and btf_dump_opts are future-proof") Signed-off-by: Jiri Olsa Signed-off-by: Andrii Nakryiko Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20211223131736.483956-1-jolsa@kernel.org --- src/btf.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/btf.h b/src/btf.h index 742a2bf..061839f 100644 --- a/src/btf.h +++ b/src/btf.h @@ -313,12 +313,18 @@ LIBBPF_API struct btf_dump *btf_dump__new_deprecated(const struct btf *btf, * * The rest works just like in case of ___libbpf_override() usage with symbol * versioning. + * + * C++ compilers don't support __builtin_types_compatible_p(), so at least + * don't screw up compilation for them and let C++ users pick btf_dump__new + * vs btf_dump__new_deprecated explicitly. */ +#ifndef __cplusplus #define btf_dump__new(a1, a2, a3, a4) __builtin_choose_expr( \ __builtin_types_compatible_p(typeof(a4), btf_dump_printf_fn_t) || \ __builtin_types_compatible_p(typeof(a4), void(void *, const char *, va_list)), \ btf_dump__new_deprecated((void *)a1, (void *)a2, (void *)a3, (void *)a4), \ btf_dump__new((void *)a1, (void *)a2, (void *)a3, (void *)a4)) +#endif LIBBPF_API void btf_dump__free(struct btf_dump *d); From af2da673d8bd100b26e54099e8c690e9996d3a5a Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 23 Dec 2021 20:00:21 -0800 Subject: [PATCH 054/131] sync: latest libbpf changes from kernel Syncing latest libbpf commits from kernel repository. Baseline bpf-next commit: e967a20a8fabc6442a78e2e2059e63a4bb6aed08 Checkpoint bpf-next commit: ecf45e60a62dfeb65658abac02f0bdb45b786911 Baseline bpf commit: 819d11507f6637731947836e6308f5966d64cf9d Checkpoint bpf commit: 819d11507f6637731947836e6308f5966d64cf9d Jiri Olsa (1): libbpf: Do not use btf_dump__new() macro in C++ mode src/btf.h | 6 ++++++ 1 file changed, 6 insertions(+) -- 2.30.2 --- CHECKPOINT-COMMIT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index b364dc6..4cc3273 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -e967a20a8fabc6442a78e2e2059e63a4bb6aed08 +ecf45e60a62dfeb65658abac02f0bdb45b786911 From bf7aacea49ee3f50542756cedba8095ca772b6ec Mon Sep 17 00:00:00 2001 From: grantseltzer Date: Mon, 3 Jan 2022 13:34:33 -0500 Subject: [PATCH 055/131] Fix comparison operator in API documentation --- docs/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index ea181af..7a8e709 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -43,7 +43,7 @@ For example: ptr = some_libbpf_api_returning_ptr(); if (!ptr) { /* note no minus sign for EINVAL and E2BIG below */ - if (errno = EINVAL) { + if (errno == EINVAL) { /* handle EINVAL error */ } else if (errno == E2BIG) { /* handle E2BIG error */ From ceba6a788a5c8d312c961141c731f69ad43aa85b Mon Sep 17 00:00:00 2001 From: Chris Tarazi Date: Wed, 29 Dec 2021 15:31:28 -0800 Subject: [PATCH 056/131] travis-ci/rootfs: Fix mount(8) invocation for Arch Linux Given that the rootfs for Arch Linux uses the busybox variant of mount(8), the `-l` doesn't exist on that binary and gives the following error msg with version v1.34.1 of busybox when invoking mkrootfs_arch.sh: ``` ... [ 0.781471] random: fast init done starting pid 72, tty '': '/etc/init.d/rcS' + for path in /etc/rcS.d/S* + '[' -x /etc/rcS.d/S10-mount ']' + /etc/rcS.d/S10-mount + /bin/mount proc /proc -t proc ++ /bin/mount -l -t devtmpfs /bin/mount: unrecognized option: l ... ``` This prevented me from generating a rootfs. This is fixed by removing the `-l`, as plainly invoking `mount -t devtmpfs` returns the same output with `mount -l ...` on the non-busybox variant (on Arch Linux, it comes from the `utils-linux` package). After this change, I was able to run `./tools/testing/selftests/bpf/vmtest.sh -i` (from the kernel src; with modification to the script to pick up my locally-generated .zstd of the new rootfs) and it worked. Signed-off-by: Chris Tarazi --- travis-ci/rootfs/mkrootfs_tweak.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-ci/rootfs/mkrootfs_tweak.sh b/travis-ci/rootfs/mkrootfs_tweak.sh index b0632d8..2aafca1 100755 --- a/travis-ci/rootfs/mkrootfs_tweak.sh +++ b/travis-ci/rootfs/mkrootfs_tweak.sh @@ -25,7 +25,7 @@ set -eux /bin/mount proc /proc -t proc # Mount devtmpfs if not mounted -if [[ -z $(/bin/mount -l -t devtmpfs) ]]; then +if [[ -z $(/bin/mount -t devtmpfs) ]]; then /bin/mount devtmpfs /dev -t devtmpfs fi From 7c382f0df9bcdda688cfed372e5c42eeee23b50c Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Tue, 28 Dec 2021 17:00:10 +0530 Subject: [PATCH 057/131] ci: Add conntrack kernel config Add necessary kernel config values to test BPF kfunc functionality for netfilter's conntrack subsystem. Signed-off-by: Kumar Kartikeya Dwivedi --- travis-ci/vmtest/configs/config-latest.s390x | 4 +++- travis-ci/vmtest/configs/config-latest.x86_64 | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/travis-ci/vmtest/configs/config-latest.s390x b/travis-ci/vmtest/configs/config-latest.s390x index aa4411d..ea6de71 100644 --- a/travis-ci/vmtest/configs/config-latest.s390x +++ b/travis-ci/vmtest/configs/config-latest.s390x @@ -782,7 +782,7 @@ CONFIG_NETFILTER_NETLINK=y # CONFIG_NETFILTER_NETLINK_QUEUE is not set # CONFIG_NETFILTER_NETLINK_LOG is not set # CONFIG_NETFILTER_NETLINK_OSF is not set -# CONFIG_NF_CONNTRACK is not set +CONFIG_NF_CONNTRACK=y # CONFIG_NF_LOG_SYSLOG is not set CONFIG_NF_TABLES=y # CONFIG_NF_TABLES_INET is not set @@ -871,6 +871,7 @@ CONFIG_NETFILTER_XT_MATCH_BPF=y # # IP: Netfilter Configuration # +CONFIG_NF_DEFRAG_IPV4=y # CONFIG_NF_SOCKET_IPV4 is not set # CONFIG_NF_TPROXY_IPV4 is not set # CONFIG_NF_TABLES_IPV4 is not set @@ -903,6 +904,7 @@ CONFIG_IP_NF_FILTER=y # CONFIG_IP6_NF_IPTABLES is not set # end of IPv6: Netfilter Configuration +CONFIG_NF_DEFRAG_IPV6=y CONFIG_BPFILTER=y CONFIG_BPFILTER_UMH=y # CONFIG_IP_DCCP is not set diff --git a/travis-ci/vmtest/configs/config-latest.x86_64 b/travis-ci/vmtest/configs/config-latest.x86_64 index a577b17..adc49e4 100644 --- a/travis-ci/vmtest/configs/config-latest.x86_64 +++ b/travis-ci/vmtest/configs/config-latest.x86_64 @@ -975,7 +975,7 @@ CONFIG_NETFILTER_NETLINK=y CONFIG_NETFILTER_NETLINK_QUEUE=y CONFIG_NETFILTER_NETLINK_LOG=y # CONFIG_NETFILTER_NETLINK_OSF is not set -# CONFIG_NF_CONNTRACK is not set +CONFIG_NF_CONNTRACK=y # CONFIG_NF_LOG_NETDEV is not set # CONFIG_NF_TABLES is not set CONFIG_NETFILTER_XTABLES=y @@ -1048,6 +1048,7 @@ CONFIG_NETFILTER_XT_MATCH_STATISTIC=y # # IP: Netfilter Configuration # +CONFIG_NF_DEFRAG_IPV4=y # CONFIG_NF_SOCKET_IPV4 is not set # CONFIG_NF_TPROXY_IPV4 is not set # CONFIG_NF_DUP_IPV4 is not set @@ -1089,6 +1090,7 @@ CONFIG_IP6_NF_IPTABLES=y # CONFIG_IP6_NF_SECURITY is not set # end of IPv6: Netfilter Configuration +CONFIG_NF_DEFRAG_IPV6=y CONFIG_BPFILTER=y CONFIG_BPFILTER_UMH=m # CONFIG_IP_DCCP is not set From 0557ad0a9c6e900abf3a4d0fd2a47fefe721e44c Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 22 Dec 2021 13:39:23 -0800 Subject: [PATCH 058/131] libbpf: Normalize PT_REGS_xxx() macro definitions Refactor PT_REGS macros definitions in bpf_tracing.h to avoid excessive duplication. We currently have classic PT_REGS_xxx() and CO-RE-enabled PT_REGS_xxx_CORE(). We are about to add also _SYSCALL variants, which would require excessive copying of all the per-architecture definitions. Instead, separate architecture-specific field/register names from the final macro that utilize them. That way for upcoming _SYSCALL variants we'll be able to just define x86_64 exception and otherwise have one common set of _SYSCALL macro definitions common for all architectures. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Tested-by: Ilya Leoshkevich Acked-by: Yonghong Song Acked-by: Ilya Leoshkevich Link: https://lore.kernel.org/bpf/20211222213924.1869758-1-andrii@kernel.org --- src/bpf_tracing.h | 371 +++++++++++++++++++--------------------------- 1 file changed, 149 insertions(+), 222 deletions(-) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index db05a59..20fe06d 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -66,277 +66,204 @@ #if defined(__KERNEL__) || defined(__VMLINUX_H__) -#define PT_REGS_PARM1(x) ((x)->di) -#define PT_REGS_PARM2(x) ((x)->si) -#define PT_REGS_PARM3(x) ((x)->dx) -#define PT_REGS_PARM4(x) ((x)->cx) -#define PT_REGS_PARM5(x) ((x)->r8) -#define PT_REGS_RET(x) ((x)->sp) -#define PT_REGS_FP(x) ((x)->bp) -#define PT_REGS_RC(x) ((x)->ax) -#define PT_REGS_SP(x) ((x)->sp) -#define PT_REGS_IP(x) ((x)->ip) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), di) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), si) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), dx) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), cx) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), r8) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), sp) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), bp) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), ax) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), sp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), ip) +#define __PT_PARM1_REG di +#define __PT_PARM2_REG si +#define __PT_PARM3_REG dx +#define __PT_PARM4_REG cx +#define __PT_PARM5_REG r8 +#define __PT_RET_REG sp +#define __PT_FP_REG bp +#define __PT_RC_REG ax +#define __PT_SP_REG sp +#define __PT_IP_REG ip #else #ifdef __i386__ + +#define __PT_PARM1_REG eax +#define __PT_PARM2_REG edx +#define __PT_PARM3_REG ecx /* i386 kernel is built with -mregparm=3 */ -#define PT_REGS_PARM1(x) ((x)->eax) -#define PT_REGS_PARM2(x) ((x)->edx) -#define PT_REGS_PARM3(x) ((x)->ecx) -#define PT_REGS_PARM4(x) 0 -#define PT_REGS_PARM5(x) 0 -#define PT_REGS_RET(x) ((x)->esp) -#define PT_REGS_FP(x) ((x)->ebp) -#define PT_REGS_RC(x) ((x)->eax) -#define PT_REGS_SP(x) ((x)->esp) -#define PT_REGS_IP(x) ((x)->eip) +#define __PT_PARM4_REG __unsupported__ +#define __PT_PARM5_REG __unsupported__ +#define __PT_RET_REG esp +#define __PT_FP_REG ebp +#define __PT_RC_REG eax +#define __PT_SP_REG esp +#define __PT_IP_REG eip -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), eax) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), edx) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), ecx) -#define PT_REGS_PARM4_CORE(x) 0 -#define PT_REGS_PARM5_CORE(x) 0 -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), esp) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), ebp) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), eax) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), esp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), eip) +#else /* __i386__ */ -#else +#define __PT_PARM1_REG rdi +#define __PT_PARM2_REG rsi +#define __PT_PARM3_REG rdx +#define __PT_PARM4_REG rcx +#define __PT_PARM5_REG r8 +#define __PT_RET_REG rsp +#define __PT_FP_REG rbp +#define __PT_RC_REG rax +#define __PT_SP_REG rsp +#define __PT_IP_REG rip -#define PT_REGS_PARM1(x) ((x)->rdi) -#define PT_REGS_PARM2(x) ((x)->rsi) -#define PT_REGS_PARM3(x) ((x)->rdx) -#define PT_REGS_PARM4(x) ((x)->rcx) -#define PT_REGS_PARM5(x) ((x)->r8) -#define PT_REGS_RET(x) ((x)->rsp) -#define PT_REGS_FP(x) ((x)->rbp) -#define PT_REGS_RC(x) ((x)->rax) -#define PT_REGS_SP(x) ((x)->rsp) -#define PT_REGS_IP(x) ((x)->rip) +#endif /* __i386__ */ -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), rdi) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), rsi) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), rdx) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), rcx) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), r8) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), rsp) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), rbp) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), rax) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), rsp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), rip) - -#endif -#endif +#endif /* __KERNEL__ || __VMLINUX_H__ */ #elif defined(bpf_target_s390) /* s390 provides user_pt_regs instead of struct pt_regs to userspace */ -struct pt_regs; -#define PT_REGS_S390 const volatile user_pt_regs -#define PT_REGS_PARM1(x) (((PT_REGS_S390 *)(x))->gprs[2]) -#define PT_REGS_PARM2(x) (((PT_REGS_S390 *)(x))->gprs[3]) -#define PT_REGS_PARM3(x) (((PT_REGS_S390 *)(x))->gprs[4]) -#define PT_REGS_PARM4(x) (((PT_REGS_S390 *)(x))->gprs[5]) -#define PT_REGS_PARM5(x) (((PT_REGS_S390 *)(x))->gprs[6]) -#define PT_REGS_RET(x) (((PT_REGS_S390 *)(x))->gprs[14]) -/* Works only with CONFIG_FRAME_POINTER */ -#define PT_REGS_FP(x) (((PT_REGS_S390 *)(x))->gprs[11]) -#define PT_REGS_RC(x) (((PT_REGS_S390 *)(x))->gprs[2]) -#define PT_REGS_SP(x) (((PT_REGS_S390 *)(x))->gprs[15]) -#define PT_REGS_IP(x) (((PT_REGS_S390 *)(x))->psw.addr) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[2]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[3]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[4]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[5]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[6]) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[14]) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[11]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[2]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[15]) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), psw.addr) +#define __PT_REGS_CAST(x) ((const user_pt_regs *)(x)) +#define __PT_PARM1_REG gprs[2] +#define __PT_PARM2_REG gprs[3] +#define __PT_PARM3_REG gprs[4] +#define __PT_PARM4_REG gprs[5] +#define __PT_PARM5_REG gprs[6] +#define __PT_RET_REG grps[14] +#define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_RC_REG gprs[2] +#define __PT_SP_REG gprs[15] +#define __PT_IP_REG psw.addr #elif defined(bpf_target_arm) -#define PT_REGS_PARM1(x) ((x)->uregs[0]) -#define PT_REGS_PARM2(x) ((x)->uregs[1]) -#define PT_REGS_PARM3(x) ((x)->uregs[2]) -#define PT_REGS_PARM4(x) ((x)->uregs[3]) -#define PT_REGS_PARM5(x) ((x)->uregs[4]) -#define PT_REGS_RET(x) ((x)->uregs[14]) -#define PT_REGS_FP(x) ((x)->uregs[11]) /* Works only with CONFIG_FRAME_POINTER */ -#define PT_REGS_RC(x) ((x)->uregs[0]) -#define PT_REGS_SP(x) ((x)->uregs[13]) -#define PT_REGS_IP(x) ((x)->uregs[12]) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), uregs[0]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), uregs[1]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), uregs[2]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), uregs[3]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), uregs[4]) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), uregs[14]) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), uregs[11]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), uregs[0]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), uregs[13]) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), uregs[12]) +#define __PT_PARM1_REG uregs[0] +#define __PT_PARM2_REG uregs[1] +#define __PT_PARM3_REG uregs[2] +#define __PT_PARM4_REG uregs[3] +#define __PT_PARM5_REG uregs[4] +#define __PT_RET_REG uregs[14] +#define __PT_FP_REG uregs[11] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_RC_REG uregs[0] +#define __PT_SP_REG uregs[13] +#define __PT_IP_REG uregs[12] #elif defined(bpf_target_arm64) /* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */ -struct pt_regs; -#define PT_REGS_ARM64 const volatile struct user_pt_regs -#define PT_REGS_PARM1(x) (((PT_REGS_ARM64 *)(x))->regs[0]) -#define PT_REGS_PARM2(x) (((PT_REGS_ARM64 *)(x))->regs[1]) -#define PT_REGS_PARM3(x) (((PT_REGS_ARM64 *)(x))->regs[2]) -#define PT_REGS_PARM4(x) (((PT_REGS_ARM64 *)(x))->regs[3]) -#define PT_REGS_PARM5(x) (((PT_REGS_ARM64 *)(x))->regs[4]) -#define PT_REGS_RET(x) (((PT_REGS_ARM64 *)(x))->regs[30]) -/* Works only with CONFIG_FRAME_POINTER */ -#define PT_REGS_FP(x) (((PT_REGS_ARM64 *)(x))->regs[29]) -#define PT_REGS_RC(x) (((PT_REGS_ARM64 *)(x))->regs[0]) -#define PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->sp) -#define PT_REGS_IP(x) (((PT_REGS_ARM64 *)(x))->pc) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[0]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[1]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[2]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[3]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[4]) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[30]) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[29]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[0]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), sp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), pc) +#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x)) +#define __PT_PARM1_REG regs[0] +#define __PT_PARM2_REG regs[1] +#define __PT_PARM3_REG regs[2] +#define __PT_PARM4_REG regs[3] +#define __PT_PARM5_REG regs[4] +#define __PT_RET_REG regs[30] +#define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_RC_REG regs[0] +#define __PT_SP_REG sp +#define __PT_IP_REG pc #elif defined(bpf_target_mips) -#define PT_REGS_PARM1(x) ((x)->regs[4]) -#define PT_REGS_PARM2(x) ((x)->regs[5]) -#define PT_REGS_PARM3(x) ((x)->regs[6]) -#define PT_REGS_PARM4(x) ((x)->regs[7]) -#define PT_REGS_PARM5(x) ((x)->regs[8]) -#define PT_REGS_RET(x) ((x)->regs[31]) -#define PT_REGS_FP(x) ((x)->regs[30]) /* Works only with CONFIG_FRAME_POINTER */ -#define PT_REGS_RC(x) ((x)->regs[2]) -#define PT_REGS_SP(x) ((x)->regs[29]) -#define PT_REGS_IP(x) ((x)->cp0_epc) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), regs[4]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), regs[5]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), regs[6]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), regs[7]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), regs[8]) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), regs[31]) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), regs[30]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), regs[2]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), regs[29]) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), cp0_epc) +#define __PT_PARM1_REG regs[4] +#define __PT_PARM2_REG regs[5] +#define __PT_PARM3_REG regs[6] +#define __PT_PARM4_REG regs[7] +#define __PT_PARM5_REG regs[8] +#define __PT_RET_REG regs[31] +#define __PT_FP_REG regs[30] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_RC_REG regs[2] +#define __PT_SP_REG regs[29] +#define __PT_IP_REG cp0_epc #elif defined(bpf_target_powerpc) -#define PT_REGS_PARM1(x) ((x)->gpr[3]) -#define PT_REGS_PARM2(x) ((x)->gpr[4]) -#define PT_REGS_PARM3(x) ((x)->gpr[5]) -#define PT_REGS_PARM4(x) ((x)->gpr[6]) -#define PT_REGS_PARM5(x) ((x)->gpr[7]) -#define PT_REGS_RC(x) ((x)->gpr[3]) -#define PT_REGS_SP(x) ((x)->sp) -#define PT_REGS_IP(x) ((x)->nip) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), gpr[3]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), gpr[4]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), gpr[5]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), gpr[6]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), gpr[7]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), gpr[3]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), sp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), nip) +#define __PT_PARM1_REG gpr[3] +#define __PT_PARM2_REG gpr[4] +#define __PT_PARM3_REG gpr[5] +#define __PT_PARM4_REG gpr[6] +#define __PT_PARM5_REG gpr[7] +#define __PT_RET_REG regs[31] +#define __PT_FP_REG __unsupported__ +#define __PT_RC_REG gpr[3] +#define __PT_SP_REG sp +#define __PT_IP_REG nip #elif defined(bpf_target_sparc) -#define PT_REGS_PARM1(x) ((x)->u_regs[UREG_I0]) -#define PT_REGS_PARM2(x) ((x)->u_regs[UREG_I1]) -#define PT_REGS_PARM3(x) ((x)->u_regs[UREG_I2]) -#define PT_REGS_PARM4(x) ((x)->u_regs[UREG_I3]) -#define PT_REGS_PARM5(x) ((x)->u_regs[UREG_I4]) -#define PT_REGS_RET(x) ((x)->u_regs[UREG_I7]) -#define PT_REGS_RC(x) ((x)->u_regs[UREG_I0]) -#define PT_REGS_SP(x) ((x)->u_regs[UREG_FP]) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I0]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I1]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I2]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I3]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I4]) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I7]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I0]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), u_regs[UREG_FP]) - +#define __PT_PARM1_REG u_regs[UREG_I0] +#define __PT_PARM2_REG u_regs[UREG_I1] +#define __PT_PARM3_REG u_regs[UREG_I2] +#define __PT_PARM4_REG u_regs[UREG_I3] +#define __PT_PARM5_REG u_regs[UREG_I4] +#define __PT_RET_REG u_regs[UREG_I7] +#define __PT_FP_REG __unsupported__ +#define __PT_RC_REG u_regs[UREG_I0] +#define __PT_SP_REG u_regs[UREG_FP] /* Should this also be a bpf_target check for the sparc case? */ #if defined(__arch64__) -#define PT_REGS_IP(x) ((x)->tpc) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), tpc) +#define __PT_IP_REG tpc #else -#define PT_REGS_IP(x) ((x)->pc) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), pc) +#define __PT_IP_REG pc #endif #elif defined(bpf_target_riscv) -struct pt_regs; -#define PT_REGS_RV const volatile struct user_regs_struct -#define PT_REGS_PARM1(x) (((PT_REGS_RV *)(x))->a0) -#define PT_REGS_PARM2(x) (((PT_REGS_RV *)(x))->a1) -#define PT_REGS_PARM3(x) (((PT_REGS_RV *)(x))->a2) -#define PT_REGS_PARM4(x) (((PT_REGS_RV *)(x))->a3) -#define PT_REGS_PARM5(x) (((PT_REGS_RV *)(x))->a4) -#define PT_REGS_RET(x) (((PT_REGS_RV *)(x))->ra) -#define PT_REGS_FP(x) (((PT_REGS_RV *)(x))->s5) -#define PT_REGS_RC(x) (((PT_REGS_RV *)(x))->a5) -#define PT_REGS_SP(x) (((PT_REGS_RV *)(x))->sp) -#define PT_REGS_IP(x) (((PT_REGS_RV *)(x))->epc) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a0) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a1) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a2) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a3) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a4) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), ra) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), fp) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a5) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), sp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), epc) +#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x)) +#define __PT_PARM1_REG a0 +#define __PT_PARM2_REG a1 +#define __PT_PARM3_REG a2 +#define __PT_PARM4_REG a3 +#define __PT_PARM5_REG a4 +#define __PT_RET_REG ra +#define __PT_FP_REG fp +#define __PT_RC_REG a5 +#define __PT_SP_REG sp +#define __PT_IP_REG epc #endif +#if defined(bpf_target_defined) + +struct pt_regs; + +/* allow some architecutres to override `struct pt_regs` */ +#ifndef __PT_REGS_CAST +#define __PT_REGS_CAST(x) (x) +#endif + +#define PT_REGS_PARM1(x) (__PT_REGS_CAST(x)->__PT_PARM1_REG) +#define PT_REGS_PARM2(x) (__PT_REGS_CAST(x)->__PT_PARM2_REG) +#define PT_REGS_PARM3(x) (__PT_REGS_CAST(x)->__PT_PARM3_REG) +#define PT_REGS_PARM4(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG) +#define PT_REGS_PARM5(x) (__PT_REGS_CAST(x)->__PT_PARM5_REG) +#define PT_REGS_RET(x) (__PT_REGS_CAST(x)->__PT_RET_REG) +#define PT_REGS_FP(x) (__PT_REGS_CAST(x)->__PT_FP_REG) +#define PT_REGS_RC(x) (__PT_REGS_CAST(x)->__PT_RC_REG) +#define PT_REGS_SP(x) (__PT_REGS_CAST(x)->__PT_SP_REG) +#define PT_REGS_IP(x) (__PT_REGS_CAST(x)->__PT_IP_REG) + +#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_REG) +#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_REG) +#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_REG) +#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG) +#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_REG) +#define PT_REGS_RET_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RET_REG) +#define PT_REGS_FP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_FP_REG) +#define PT_REGS_RC_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RC_REG) +#define PT_REGS_SP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_SP_REG) +#define PT_REGS_IP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_IP_REG) + #if defined(bpf_target_powerpc) + #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; }) #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP + #elif defined(bpf_target_sparc) + #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP -#elif defined(bpf_target_defined) + +#else + #define BPF_KPROBE_READ_RET_IP(ip, ctx) \ ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) #define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \ - ({ bpf_probe_read_kernel(&(ip), sizeof(ip), \ - (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) + ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) + #endif -#if !defined(bpf_target_defined) +#else /* defined(bpf_target_defined) */ #define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_PARM2(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) @@ -363,7 +290,7 @@ struct pt_regs; #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#endif /* !defined(bpf_target_defined) */ +#endif /* defined(bpf_target_defined) */ #ifndef ___bpf_concat #define ___bpf_concat(a, b) a ## b From 3f592a59d7f67139b0df5b1652672de8aec85781 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 22 Dec 2021 13:39:24 -0800 Subject: [PATCH 059/131] libbpf: Use 100-character limit to make bpf_tracing.h easier to read Improve bpf_tracing.h's macro definition readability by keeping them single-line and better aligned. This makes it easier to follow all those variadic patterns. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20211222213924.1869758-2-andrii@kernel.org --- src/bpf_tracing.h | 54 +++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index 20fe06d..90f56b0 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -302,25 +302,23 @@ struct pt_regs; #define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N #endif #ifndef ___bpf_narg -#define ___bpf_narg(...) \ - ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#define ___bpf_narg(...) ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) #endif -#define ___bpf_ctx_cast0() ctx -#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), (void *)ctx[0] -#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), (void *)ctx[1] -#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), (void *)ctx[2] -#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), (void *)ctx[3] -#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), (void *)ctx[4] -#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), (void *)ctx[5] -#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), (void *)ctx[6] -#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), (void *)ctx[7] -#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), (void *)ctx[8] +#define ___bpf_ctx_cast0() ctx +#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), (void *)ctx[0] +#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), (void *)ctx[1] +#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), (void *)ctx[2] +#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), (void *)ctx[3] +#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), (void *)ctx[4] +#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), (void *)ctx[5] +#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), (void *)ctx[6] +#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), (void *)ctx[7] +#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), (void *)ctx[8] #define ___bpf_ctx_cast10(x, args...) ___bpf_ctx_cast9(args), (void *)ctx[9] #define ___bpf_ctx_cast11(x, args...) ___bpf_ctx_cast10(args), (void *)ctx[10] #define ___bpf_ctx_cast12(x, args...) ___bpf_ctx_cast11(args), (void *)ctx[11] -#define ___bpf_ctx_cast(args...) \ - ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args) +#define ___bpf_ctx_cast(args...) ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args) /* * BPF_PROG is a convenience wrapper for generic tp_btf/fentry/fexit and @@ -353,19 +351,13 @@ ____##name(unsigned long long *ctx, ##args) struct pt_regs; -#define ___bpf_kprobe_args0() ctx -#define ___bpf_kprobe_args1(x) \ - ___bpf_kprobe_args0(), (void *)PT_REGS_PARM1(ctx) -#define ___bpf_kprobe_args2(x, args...) \ - ___bpf_kprobe_args1(args), (void *)PT_REGS_PARM2(ctx) -#define ___bpf_kprobe_args3(x, args...) \ - ___bpf_kprobe_args2(args), (void *)PT_REGS_PARM3(ctx) -#define ___bpf_kprobe_args4(x, args...) \ - ___bpf_kprobe_args3(args), (void *)PT_REGS_PARM4(ctx) -#define ___bpf_kprobe_args5(x, args...) \ - ___bpf_kprobe_args4(args), (void *)PT_REGS_PARM5(ctx) -#define ___bpf_kprobe_args(args...) \ - ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args) +#define ___bpf_kprobe_args0() ctx +#define ___bpf_kprobe_args1(x) ___bpf_kprobe_args0(), (void *)PT_REGS_PARM1(ctx) +#define ___bpf_kprobe_args2(x, args...) ___bpf_kprobe_args1(args), (void *)PT_REGS_PARM2(ctx) +#define ___bpf_kprobe_args3(x, args...) ___bpf_kprobe_args2(args), (void *)PT_REGS_PARM3(ctx) +#define ___bpf_kprobe_args4(x, args...) ___bpf_kprobe_args3(args), (void *)PT_REGS_PARM4(ctx) +#define ___bpf_kprobe_args5(x, args...) ___bpf_kprobe_args4(args), (void *)PT_REGS_PARM5(ctx) +#define ___bpf_kprobe_args(args...) ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args) /* * BPF_KPROBE serves the same purpose for kprobes as BPF_PROG for @@ -391,11 +383,9 @@ typeof(name(0)) name(struct pt_regs *ctx) \ static __attribute__((always_inline)) typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args) -#define ___bpf_kretprobe_args0() ctx -#define ___bpf_kretprobe_args1(x) \ - ___bpf_kretprobe_args0(), (void *)PT_REGS_RC(ctx) -#define ___bpf_kretprobe_args(args...) \ - ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args) +#define ___bpf_kretprobe_args0() ctx +#define ___bpf_kretprobe_args1(x) ___bpf_kretprobe_args0(), (void *)PT_REGS_RC(ctx) +#define ___bpf_kretprobe_args(args...) ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args) /* * BPF_KRETPROBE is similar to BPF_KPROBE, except, it only provides optional From 0171976dc5e565c051fc82acd0b417060a37bf8d Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 22 Dec 2021 15:10:03 -0800 Subject: [PATCH 060/131] libbpf: Improve LINUX_VERSION_CODE detection Ubuntu reports incorrect kernel version through uname(), which on older kernels leads to kprobe BPF programs failing to load due to the version check mismatch. Accommodate Ubuntu's quirks with LINUX_VERSION_CODE by using Ubuntu-specific /proc/version_code to fetch major/minor/patch versions to form LINUX_VERSION_CODE. While at it, consolide libbpf's kernel version detection code between libbpf.c and libbpf_probes.c. [0] Closes: https://github.com/libbpf/libbpf/issues/421 Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20211222231003.2334940-1-andrii@kernel.org --- src/libbpf.c | 27 ++++++++++++++++++++++++++- src/libbpf_internal.h | 2 ++ src/libbpf_probes.c | 16 ---------------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index cf862a1..9cb99d1 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -795,11 +795,36 @@ bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, return 0; } -static __u32 get_kernel_version(void) +__u32 get_kernel_version(void) { + /* On Ubuntu LINUX_VERSION_CODE doesn't correspond to info.release, + * but Ubuntu provides /proc/version_signature file, as described at + * https://ubuntu.com/kernel, with an example contents below, which we + * can use to get a proper LINUX_VERSION_CODE. + * + * Ubuntu 5.4.0-12.15-generic 5.4.8 + * + * In the above, 5.4.8 is what kernel is actually expecting, while + * uname() call will return 5.4.0 in info.release. + */ + const char *ubuntu_kver_file = "/proc/version_signature"; __u32 major, minor, patch; struct utsname info; + if (access(ubuntu_kver_file, R_OK) == 0) { + FILE *f; + + f = fopen(ubuntu_kver_file, "r"); + if (f) { + if (fscanf(f, "%*s %*s %d.%d.%d\n", &major, &minor, &patch) == 3) { + fclose(f); + return KERNEL_VERSION(major, minor, patch); + } + fclose(f); + } + /* something went wrong, fall back to uname() approach */ + } + uname(&info); if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3) return 0; diff --git a/src/libbpf_internal.h b/src/libbpf_internal.h index 5dbe4f4..1565679 100644 --- a/src/libbpf_internal.h +++ b/src/libbpf_internal.h @@ -188,6 +188,8 @@ static inline void libbpf_strlcpy(char *dst, const char *src, size_t sz) dst[i] = '\0'; } +__u32 get_kernel_version(void); + struct btf; struct btf_type; diff --git a/src/libbpf_probes.c b/src/libbpf_probes.c index 9d39852..97b06ce 100644 --- a/src/libbpf_probes.c +++ b/src/libbpf_probes.c @@ -48,22 +48,6 @@ static int get_vendor_id(int ifindex) return strtol(buf, NULL, 0); } -static int get_kernel_version(void) -{ - int version, subversion, patchlevel; - struct utsname utsn; - - /* Return 0 on failure, and attempt to probe with empty kversion */ - if (uname(&utsn)) - return 0; - - if (sscanf(utsn.release, "%d.%d.%d", - &version, &subversion, &patchlevel) != 3) - return 0; - - return (version << 16) + (subversion << 8) + patchlevel; -} - static int probe_prog_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, size_t insns_cnt, char *log_buf, size_t log_buf_sz, From 2e52e09bc26fbfff1f8d2c88843cfe3ad489ae1d Mon Sep 17 00:00:00 2001 From: Christy Lee Date: Wed, 29 Dec 2021 12:41:56 -0800 Subject: [PATCH 061/131] libbpf: Deprecate bpf_perf_event_read_simple() API With perf_buffer__poll() and perf_buffer__consume() APIs available, there is no reason to expose bpf_perf_event_read_simple() API to users. If users need custom perf buffer, they could re-implement the function. Mark bpf_perf_event_read_simple() and move the logic to a new static function so it can still be called by other functions in the same file. [0] Closes: https://github.com/libbpf/libbpf/issues/310 Signed-off-by: Christy Lee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211229204156.13569-1-christylee@fb.com --- src/libbpf.c | 22 ++++++++++++++-------- src/libbpf.h | 1 + 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index 9cb99d1..1d02ba7 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -10676,10 +10676,10 @@ struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map) return link; } -enum bpf_perf_event_ret -bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, - void **copy_mem, size_t *copy_size, - bpf_perf_event_print_t fn, void *private_data) +static enum bpf_perf_event_ret +perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, + void **copy_mem, size_t *copy_size, + bpf_perf_event_print_t fn, void *private_data) { struct perf_event_mmap_page *header = mmap_mem; __u64 data_head = ring_buffer_read_head(header); @@ -10724,6 +10724,12 @@ bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, return libbpf_err(ret); } +__attribute__((alias("perf_event_read_simple"))) +enum bpf_perf_event_ret +bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, + void **copy_mem, size_t *copy_size, + bpf_perf_event_print_t fn, void *private_data); + struct perf_buffer; struct perf_buffer_params { @@ -11132,10 +11138,10 @@ static int perf_buffer__process_records(struct perf_buffer *pb, { enum bpf_perf_event_ret ret; - ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, - pb->page_size, &cpu_buf->buf, - &cpu_buf->buf_size, - perf_buffer__process_record, cpu_buf); + ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size, + pb->page_size, &cpu_buf->buf, + &cpu_buf->buf_size, + perf_buffer__process_record, cpu_buf); if (ret != LIBBPF_PERF_EVENT_CONT) return ret; return 0; diff --git a/src/libbpf.h b/src/libbpf.h index 85dfef8..ddf1cc9 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -1026,6 +1026,7 @@ LIBBPF_API int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_i typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr, void *private_data); +LIBBPF_DEPRECATED_SINCE(0, 8, "use perf_buffer__poll() or perf_buffer__consume() instead") LIBBPF_API enum bpf_perf_event_ret bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, void **copy_mem, size_t *copy_size, From c2f2c26cb2dbe75051d7a98289783495b61ccafe Mon Sep 17 00:00:00 2001 From: Qiang Wang Date: Mon, 27 Dec 2021 21:07:12 +0800 Subject: [PATCH 062/131] libbpf: Use probe_name for legacy kprobe Fix a bug in commit 46ed5fc33db9, which wrongly used the func_name instead of probe_name to register legacy kprobe. Fixes: 46ed5fc33db9 ("libbpf: Refactor and simplify legacy kprobe code") Co-developed-by: Chengming Zhou Signed-off-by: Qiang Wang Signed-off-by: Chengming Zhou Signed-off-by: Andrii Nakryiko Tested-by: Hengqi Chen Reviewed-by: Hengqi Chen Link: https://lore.kernel.org/bpf/20211227130713.66933-1-wangqiang.wq.frank@bytedance.com --- src/libbpf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libbpf.c b/src/libbpf.c index 1d02ba7..26e49e6 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -10017,7 +10017,7 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), func_name, offset); - legacy_probe = strdup(func_name); + legacy_probe = strdup(probe_name); if (!legacy_probe) return libbpf_err_ptr(-ENOMEM); From b0c3d7133faa55fbcbc19f76f9cc9e183e178c97 Mon Sep 17 00:00:00 2001 From: Qiang Wang Date: Mon, 27 Dec 2021 21:07:13 +0800 Subject: [PATCH 063/131] libbpf: Support repeated legacy kprobes on same function If repeated legacy kprobes on same function in one process, libbpf will register using the same probe name and got -EBUSY error. So append index to the probe name format to fix this problem. Co-developed-by: Chengming Zhou Signed-off-by: Qiang Wang Signed-off-by: Chengming Zhou Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211227130713.66933-2-wangqiang.wq.frank@bytedance.com --- src/libbpf.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libbpf.c b/src/libbpf.c index 26e49e6..7f10dd5 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -9916,7 +9916,10 @@ static int append_to_file(const char *file, const char *fmt, ...) static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, const char *kfunc_name, size_t offset) { - snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), kfunc_name, offset); + static int index = 0; + + snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset, + __sync_fetch_and_add(&index, 1)); } static int add_kprobe_event_legacy(const char *probe_name, bool retprobe, From 8440112546c8c5bc6f5ef6107d51bb46be716011 Mon Sep 17 00:00:00 2001 From: Christy Lee Date: Tue, 4 Jan 2022 16:06:01 -0800 Subject: [PATCH 064/131] libbpf 1.0: Deprecate bpf_map__is_offload_neutral() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deprecate bpf_map__is_offload_neutral(). It’s most probably broken already. PERF_EVENT_ARRAY isn’t the only map that’s not suitable for hardware offloading. Applications can directly check map type instead. [0] Closes: https://github.com/libbpf/libbpf/issues/306 Signed-off-by: Christy Lee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220105000601.2090044-1-christylee@fb.com --- src/libbpf.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libbpf.h b/src/libbpf.h index ddf1cc9..88dd943 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -744,6 +744,7 @@ LIBBPF_API void *bpf_map__priv(const struct bpf_map *map); LIBBPF_API int bpf_map__set_initial_value(struct bpf_map *map, const void *data, size_t size); LIBBPF_API const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_map__type() instead") LIBBPF_API bool bpf_map__is_offload_neutral(const struct bpf_map *map); /** From 12932191c67f38aa83b73a09a3e3ec36d30b8536 Mon Sep 17 00:00:00 2001 From: Christy Lee Date: Tue, 4 Jan 2022 16:31:20 -0800 Subject: [PATCH 065/131] libbpf 1.0: Deprecate bpf_object__find_map_by_offset() API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit API created with simplistic assumptions about BPF map definitions. It hasn’t worked for a while, deprecate it in preparation for libbpf 1.0. [0] Closes: https://github.com/libbpf/libbpf/issues/302 Signed-off-by: Christy Lee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220105003120.2222673-1-christylee@fb.com --- src/libbpf.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libbpf.h b/src/libbpf.h index 88dd943..8b9bc5e 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -677,7 +677,8 @@ bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name); * Get bpf_map through the offset of corresponding struct bpf_map_def * in the BPF object file. */ -LIBBPF_API struct bpf_map * +LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__find_map_by_name() instead") +struct bpf_map * bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset); LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__next_map() instead") From 4449d7150965cfd8ce519419ddada652047981e0 Mon Sep 17 00:00:00 2001 From: Grant Seltzer Date: Thu, 6 Jan 2022 15:13:05 -0500 Subject: [PATCH 066/131] libbpf: Add documentation for bpf_map batch operations This adds documention for: - bpf_map_delete_batch() - bpf_map_lookup_batch() - bpf_map_lookup_and_delete_batch() - bpf_map_update_batch() This also updates the public API for the `keys` parameter of `bpf_map_delete_batch()`, and both the `keys` and `values` parameters of `bpf_map_update_batch()` to be constants. Signed-off-by: Grant Seltzer Signed-off-by: Andrii Nakryiko Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20220106201304.112675-1-grantseltzer@gmail.com --- src/bpf.c | 8 ++-- src/bpf.h | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 6 deletions(-) diff --git a/src/bpf.c b/src/bpf.c index 9b64eed..550b4cb 100644 --- a/src/bpf.c +++ b/src/bpf.c @@ -691,11 +691,11 @@ static int bpf_map_batch_common(int cmd, int fd, void *in_batch, return libbpf_err_errno(ret); } -int bpf_map_delete_batch(int fd, void *keys, __u32 *count, +int bpf_map_delete_batch(int fd, const void *keys, __u32 *count, const struct bpf_map_batch_opts *opts) { return bpf_map_batch_common(BPF_MAP_DELETE_BATCH, fd, NULL, - NULL, keys, NULL, count, opts); + NULL, (void *)keys, NULL, count, opts); } int bpf_map_lookup_batch(int fd, void *in_batch, void *out_batch, void *keys, @@ -715,11 +715,11 @@ int bpf_map_lookup_and_delete_batch(int fd, void *in_batch, void *out_batch, count, opts); } -int bpf_map_update_batch(int fd, void *keys, void *values, __u32 *count, +int bpf_map_update_batch(int fd, const void *keys, const void *values, __u32 *count, const struct bpf_map_batch_opts *opts) { return bpf_map_batch_common(BPF_MAP_UPDATE_BATCH, fd, NULL, NULL, - keys, values, count, opts); + (void *)keys, (void *)values, count, opts); } int bpf_obj_pin(int fd, const char *pathname) diff --git a/src/bpf.h b/src/bpf.h index 00619f6..14e0d97 100644 --- a/src/bpf.h +++ b/src/bpf.h @@ -254,17 +254,128 @@ struct bpf_map_batch_opts { }; #define bpf_map_batch_opts__last_field flags -LIBBPF_API int bpf_map_delete_batch(int fd, void *keys, + +/** + * @brief **bpf_map_delete_batch()** allows for batch deletion of multiple + * elements in a BPF map. + * + * @param fd BPF map file descriptor + * @param keys pointer to an array of *count* keys + * @param count input and output parameter; on input **count** represents the + * number of elements in the map to delete in batch; + * on output if a non-EFAULT error is returned, **count** represents the number of deleted + * elements if the output **count** value is not equal to the input **count** value + * If EFAULT is returned, **count** should not be trusted to be correct. + * @param opts options for configuring the way the batch deletion works + * @return 0, on success; negative error code, otherwise (errno is also set to + * the error code) + */ +LIBBPF_API int bpf_map_delete_batch(int fd, const void *keys, __u32 *count, const struct bpf_map_batch_opts *opts); + +/** + * @brief **bpf_map_lookup_batch()** allows for batch lookup of BPF map elements. + * + * The parameter *in_batch* is the address of the first element in the batch to read. + * *out_batch* is an output parameter that should be passed as *in_batch* to subsequent + * calls to **bpf_map_lookup_batch()**. NULL can be passed for *in_batch* to indicate + * that the batched lookup starts from the beginning of the map. + * + * The *keys* and *values* are output parameters which must point to memory large enough to + * hold *count* items based on the key and value size of the map *map_fd*. The *keys* + * buffer must be of *key_size* * *count*. The *values* buffer must be of + * *value_size* * *count*. + * + * @param fd BPF map file descriptor + * @param in_batch address of the first element in batch to read, can pass NULL to + * indicate that the batched lookup starts from the beginning of the map. + * @param out_batch output parameter that should be passed to next call as *in_batch* + * @param keys pointer to an array large enough for *count* keys + * @param values pointer to an array large enough for *count* values + * @param count input and output parameter; on input it's the number of elements + * in the map to read in batch; on output it's the number of elements that were + * successfully read. + * If a non-EFAULT error is returned, count will be set as the number of elements + * that were read before the error occurred. + * If EFAULT is returned, **count** should not be trusted to be correct. + * @param opts options for configuring the way the batch lookup works + * @return 0, on success; negative error code, otherwise (errno is also set to + * the error code) + */ LIBBPF_API int bpf_map_lookup_batch(int fd, void *in_batch, void *out_batch, void *keys, void *values, __u32 *count, const struct bpf_map_batch_opts *opts); + +/** + * @brief **bpf_map_lookup_and_delete_batch()** allows for batch lookup and deletion + * of BPF map elements where each element is deleted after being retrieved. + * + * @param fd BPF map file descriptor + * @param in_batch address of the first element in batch to read, can pass NULL to + * get address of the first element in *out_batch* + * @param out_batch output parameter that should be passed to next call as *in_batch* + * @param keys pointer to an array of *count* keys + * @param values pointer to an array large enough for *count* values + * @param count input and output parameter; on input it's the number of elements + * in the map to read and delete in batch; on output it represents the number of + * elements that were successfully read and deleted + * If a non-**EFAULT** error code is returned and if the output **count** value + * is not equal to the input **count** value, up to **count** elements may + * have been deleted. + * if **EFAULT** is returned up to *count* elements may have been deleted without + * being returned via the *keys* and *values* output parameters. + * @param opts options for configuring the way the batch lookup and delete works + * @return 0, on success; negative error code, otherwise (errno is also set to + * the error code) + */ LIBBPF_API int bpf_map_lookup_and_delete_batch(int fd, void *in_batch, void *out_batch, void *keys, void *values, __u32 *count, const struct bpf_map_batch_opts *opts); -LIBBPF_API int bpf_map_update_batch(int fd, void *keys, void *values, + +/** + * @brief **bpf_map_update_batch()** updates multiple elements in a map + * by specifying keys and their corresponding values. + * + * The *keys* and *values* parameters must point to memory large enough + * to hold *count* items based on the key and value size of the map. + * + * The *opts* parameter can be used to control how *bpf_map_update_batch()* + * should handle keys that either do or do not already exist in the map. + * In particular the *flags* parameter of *bpf_map_batch_opts* can be + * one of the following: + * + * Note that *count* is an input and output parameter, where on output it + * represents how many elements were successfully updated. Also note that if + * **EFAULT** then *count* should not be trusted to be correct. + * + * **BPF_ANY** + * Create new elements or update existing. + * + * **BPF_NOEXIST** + * Create new elements only if they do not exist. + * + * **BPF_EXIST** + * Update existing elements. + * + * **BPF_F_LOCK** + * Update spin_lock-ed map elements. This must be + * specified if the map value contains a spinlock. + * + * @param fd BPF map file descriptor + * @param keys pointer to an array of *count* keys + * @param values pointer to an array of *count* values + * @param count input and output parameter; on input it's the number of elements + * in the map to update in batch; on output if a non-EFAULT error is returned, + * **count** represents the number of updated elements if the output **count** + * value is not equal to the input **count** value. + * If EFAULT is returned, **count** should not be trusted to be correct. + * @param opts options for configuring the way the batch update works + * @return 0, on success; negative error code, otherwise (errno is also set to + * the error code) + */ +LIBBPF_API int bpf_map_update_batch(int fd, const void *keys, const void *values, __u32 *count, const struct bpf_map_batch_opts *opts); From e99f34e14447866bcbc38944387531c36f6c85fe Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 6 Jan 2022 15:22:18 -0800 Subject: [PATCH 067/131] sync: latest libbpf changes from kernel Syncing latest libbpf commits from kernel repository. Baseline bpf-next commit: ecf45e60a62dfeb65658abac02f0bdb45b786911 Checkpoint bpf-next commit: 44bab87d8ca6f0544a9f8fc97bdf33aa5b3c899e Baseline bpf commit: 819d11507f6637731947836e6308f5966d64cf9d Checkpoint bpf commit: d6d86830705f173fca6087a3e67ceaf68db80523 Andrii Nakryiko (3): libbpf: Normalize PT_REGS_xxx() macro definitions libbpf: Use 100-character limit to make bpf_tracing.h easier to read libbpf: Improve LINUX_VERSION_CODE detection Christy Lee (3): libbpf: Deprecate bpf_perf_event_read_simple() API libbpf 1.0: Deprecate bpf_map__is_offload_neutral() libbpf 1.0: Deprecate bpf_object__find_map_by_offset() API Grant Seltzer (1): libbpf: Add documentation for bpf_map batch operations Qiang Wang (2): libbpf: Use probe_name for legacy kprobe libbpf: Support repeated legacy kprobes on same function src/bpf.c | 8 +- src/bpf.h | 115 ++++++++++- src/bpf_tracing.h | 431 +++++++++++++++++------------------------- src/libbpf.c | 56 ++++-- src/libbpf.h | 5 +- src/libbpf_internal.h | 2 + src/libbpf_probes.c | 16 -- 7 files changed, 342 insertions(+), 291 deletions(-) -- 2.30.2 --- BPF-CHECKPOINT-COMMIT | 2 +- CHECKPOINT-COMMIT | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BPF-CHECKPOINT-COMMIT b/BPF-CHECKPOINT-COMMIT index 417cd69..4854ec6 100644 --- a/BPF-CHECKPOINT-COMMIT +++ b/BPF-CHECKPOINT-COMMIT @@ -1 +1 @@ -819d11507f6637731947836e6308f5966d64cf9d +d6d86830705f173fca6087a3e67ceaf68db80523 diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index 4cc3273..3b9b714 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -ecf45e60a62dfeb65658abac02f0bdb45b786911 +44bab87d8ca6f0544a9f8fc97bdf33aa5b3c899e From 22411acc4b2c846868fd570b2d9f3b016d2af2cb Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Tue, 11 Jan 2022 22:52:29 +0530 Subject: [PATCH 068/131] ci: Add userfaultfd kernel config Add necessary kernel config values to run BPF mod race test in conntrack-bpf series. Signed-off-by: Kumar Kartikeya Dwivedi --- travis-ci/vmtest/configs/config-latest.x86_64 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/travis-ci/vmtest/configs/config-latest.x86_64 b/travis-ci/vmtest/configs/config-latest.x86_64 index adc49e4..d531f1a 100644 --- a/travis-ci/vmtest/configs/config-latest.x86_64 +++ b/travis-ci/vmtest/configs/config-latest.x86_64 @@ -223,6 +223,8 @@ CONFIG_SHMEM=y CONFIG_AIO=y CONFIG_IO_URING=y CONFIG_ADVISE_SYSCALLS=y +CONFIG_HAVE_ARCH_USERFAULTFD_WP=y +CONFIG_HAVE_ARCH_USERFAULTFD_MINOR=y CONFIG_MEMBARRIER=y CONFIG_KALLSYMS=y CONFIG_KALLSYMS_ALL=y @@ -236,7 +238,7 @@ CONFIG_BPF_JIT_DEFAULT_ON=y CONFIG_USERMODE_DRIVER=y CONFIG_BPF_PRELOAD=y CONFIG_BPF_PRELOAD_UMD=y -# CONFIG_USERFAULTFD is not set +CONFIG_USERFAULTFD=y CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y CONFIG_RSEQ=y # CONFIG_DEBUG_RSEQ is not set From 060c8a99c4781dafe1671a2371693e223738763d Mon Sep 17 00:00:00 2001 From: Adam Jensen Date: Wed, 8 May 2019 07:04:50 -0400 Subject: [PATCH 069/131] include: Include linux/stddef.h This fixes the build in environments such as Alpine Linux. See [0] for discussion. [0] https://github.com/libbpf/libbpf/pull/41 Signed-off-by: Adam Jensen --- include/linux/types.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/types.h b/include/linux/types.h index bae1ed8..b15252a 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -7,6 +7,8 @@ #include #include +#include + #include #include From cde5b418dd0e87aa278e516c62844a55bbf6f1fa Mon Sep 17 00:00:00 2001 From: Coco Li Date: Wed, 5 Jan 2022 02:48:38 -0800 Subject: [PATCH 070/131] gro: add ability to control gro max packet size Eric Dumazet suggested to allow users to modify max GRO packet size. We have seen GRO being disabled by users of appliances (such as wifi access points) because of claimed bufferbloat issues, or some work arounds in sch_cake, to split GRO/GSO packets. Instead of disabling GRO completely, one can chose to limit the maximum packet size of GRO packets, depending on their latency constraints. This patch adds a per device gro_max_size attribute that can be changed with ip link command. ip link set dev eth0 gro_max_size 16000 Suggested-by: Eric Dumazet Signed-off-by: Coco Li Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/uapi/linux/if_link.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index 4ac53b3..6218f93 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -347,6 +347,7 @@ enum { */ IFLA_PARENT_DEV_NAME, IFLA_PARENT_DEV_BUS_NAME, + IFLA_GRO_MAX_SIZE, __IFLA_MAX }; From d5daf275c7f740f800eac0b0a1043584dc6fed04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mauricio=20V=C3=A1squez?= Date: Fri, 7 Jan 2022 10:26:19 -0500 Subject: [PATCH 071/131] libbpf: Use IS_ERR_OR_NULL() in hashmap__free() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hashmap__new() uses ERR_PTR() to return an error so it's better to use IS_ERR_OR_NULL() in order to check the pointer before calling free(). This will prevent freeing an invalid pointer if somebody calls hashmap__free() with the result of a failed hashmap__new() call. Signed-off-by: Mauricio Vásquez Signed-off-by: Andrii Nakryiko Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20220107152620.192327-1-mauricio@kinvolk.io --- src/hashmap.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hashmap.c b/src/hashmap.c index 3c20b12..aeb09c2 100644 --- a/src/hashmap.c +++ b/src/hashmap.c @@ -75,7 +75,7 @@ void hashmap__clear(struct hashmap *map) void hashmap__free(struct hashmap *map) { - if (!map) + if (IS_ERR_OR_NULL(map)) return; hashmap__clear(map); @@ -238,4 +238,3 @@ bool hashmap__delete(struct hashmap *map, const void *key, return true; } - From e0de05d1b14dad05454289d4f351245f6fad64c3 Mon Sep 17 00:00:00 2001 From: Christy Lee Date: Fri, 7 Jan 2022 10:46:03 -0800 Subject: [PATCH 072/131] libbpf: Rename bpf_prog_attach_xattr() to bpf_prog_attach_opts() All xattr APIs are being dropped, let's converge to the convention used in high-level APIs and rename bpf_prog_attach_xattr to bpf_prog_attach_opts. Signed-off-by: Christy Lee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220107184604.3668544-2-christylee@fb.com --- src/bpf.c | 9 +++++++-- src/bpf.h | 4 ++++ src/libbpf.map | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/bpf.c b/src/bpf.c index 550b4cb..418b259 100644 --- a/src/bpf.c +++ b/src/bpf.c @@ -754,10 +754,10 @@ int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type, .flags = flags, ); - return bpf_prog_attach_xattr(prog_fd, target_fd, type, &opts); + return bpf_prog_attach_opts(prog_fd, target_fd, type, &opts); } -int bpf_prog_attach_xattr(int prog_fd, int target_fd, +int bpf_prog_attach_opts(int prog_fd, int target_fd, enum bpf_attach_type type, const struct bpf_prog_attach_opts *opts) { @@ -778,6 +778,11 @@ int bpf_prog_attach_xattr(int prog_fd, int target_fd, return libbpf_err_errno(ret); } +__attribute__((alias("bpf_prog_attach_opts"))) +int bpf_prog_attach_xattr(int prog_fd, int target_fd, + enum bpf_attach_type type, + const struct bpf_prog_attach_opts *opts); + int bpf_prog_detach(int target_fd, enum bpf_attach_type type) { union bpf_attr attr; diff --git a/src/bpf.h b/src/bpf.h index 14e0d97..c2e8327 100644 --- a/src/bpf.h +++ b/src/bpf.h @@ -391,6 +391,10 @@ struct bpf_prog_attach_opts { LIBBPF_API int bpf_prog_attach(int prog_fd, int attachable_fd, enum bpf_attach_type type, unsigned int flags); +LIBBPF_API int bpf_prog_attach_opts(int prog_fd, int attachable_fd, + enum bpf_attach_type type, + const struct bpf_prog_attach_opts *opts); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_prog_attach_opts() instead") LIBBPF_API int bpf_prog_attach_xattr(int prog_fd, int attachable_fd, enum bpf_attach_type type, const struct bpf_prog_attach_opts *opts); diff --git a/src/libbpf.map b/src/libbpf.map index 5297839..8262cfc 100644 --- a/src/libbpf.map +++ b/src/libbpf.map @@ -247,6 +247,7 @@ LIBBPF_0.0.8 { bpf_link_create; bpf_link_update; bpf_map__set_initial_value; + bpf_prog_attach_opts; bpf_program__attach_cgroup; bpf_program__attach_lsm; bpf_program__is_lsm; From 5b6dfd7f6b90e8a3842ade8af35e514d92756c0a Mon Sep 17 00:00:00 2001 From: Yafang Shao Date: Sat, 8 Jan 2022 13:47:39 +0000 Subject: [PATCH 073/131] libbpf: Fix possible NULL pointer dereference when destroying skeleton When I checked the code in skeleton header file generated with my own bpf prog, I found there may be possible NULL pointer dereference when destroying skeleton. Then I checked the in-tree bpf progs, finding that is a common issue. Let's take the generated samples/bpf/xdp_redirect_cpu.skel.h for example. Below is the generated code in xdp_redirect_cpu__create_skeleton(): xdp_redirect_cpu__create_skeleton struct bpf_object_skeleton *s; s = (struct bpf_object_skeleton *)calloc(1, sizeof(*s)); if (!s) goto error; ... error: bpf_object__destroy_skeleton(s); return -ENOMEM; After goto error, the NULL 's' will be deferenced in bpf_object__destroy_skeleton(). We can simply fix this issue by just adding a NULL check in bpf_object__destroy_skeleton(). Fixes: d66562fba1ce ("libbpf: Add BPF object skeleton support") Signed-off-by: Yafang Shao Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220108134739.32541-1-laoar.shao@gmail.com --- src/libbpf.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libbpf.c b/src/libbpf.c index 7f10dd5..fdb3536 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -11795,6 +11795,9 @@ void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) { + if (!s) + return; + if (s->progs) bpf_object__detach_skeleton(s); if (s->obj) From 7b8e97bffc1330eec5474531ea9dca354be8fd51 Mon Sep 17 00:00:00 2001 From: Christy Lee Date: Fri, 7 Jan 2022 16:42:18 -0800 Subject: [PATCH 074/131] libbpf: Deprecate bpf_map__def() API All fields accessed via bpf_map_def can now be accessed via appropirate getters and setters. Mark bpf_map__def() API as deprecated. Signed-off-by: Christy Lee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220108004218.355761-6-christylee@fb.com --- src/libbpf.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libbpf.h b/src/libbpf.h index 8b9bc5e..9728551 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -706,7 +706,8 @@ bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *map); LIBBPF_API int bpf_map__fd(const struct bpf_map *map); LIBBPF_API int bpf_map__reuse_fd(struct bpf_map *map, int fd); /* get map definition */ -LIBBPF_API const struct bpf_map_def *bpf_map__def(const struct bpf_map *map); +LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 8, "use appropriate getters or setters instead") +const struct bpf_map_def *bpf_map__def(const struct bpf_map *map); /* get map name */ LIBBPF_API const char *bpf_map__name(const struct bpf_map *map); /* get/set map type */ From be89b28f96be426e30a2b0c5312d13b30ee518c7 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 14 Jan 2022 18:43:00 -0800 Subject: [PATCH 075/131] sync: latest libbpf changes from kernel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Syncing latest libbpf commits from kernel repository. Baseline bpf-next commit: 44bab87d8ca6f0544a9f8fc97bdf33aa5b3c899e Checkpoint bpf-next commit: e80f2a0d194605553315de68284fc41969f81f62 Baseline bpf commit: d6d86830705f173fca6087a3e67ceaf68db80523 Checkpoint bpf commit: 343e53754b21ae45530623222aa079fecd3cf942 Christy Lee (2): libbpf: Rename bpf_prog_attach_xattr() to bpf_prog_attach_opts() libbpf: Deprecate bpf_map__def() API Coco Li (1): gro: add ability to control gro max packet size Mauricio Vásquez (1): libbpf: Use IS_ERR_OR_NULL() in hashmap__free() Yafang Shao (1): libbpf: Fix possible NULL pointer dereference when destroying skeleton include/uapi/linux/if_link.h | 1 + src/bpf.c | 9 +++++++-- src/bpf.h | 4 ++++ src/hashmap.c | 3 +-- src/libbpf.c | 3 +++ src/libbpf.h | 3 ++- src/libbpf.map | 1 + 7 files changed, 19 insertions(+), 5 deletions(-) -- 2.30.2 --- BPF-CHECKPOINT-COMMIT | 2 +- CHECKPOINT-COMMIT | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BPF-CHECKPOINT-COMMIT b/BPF-CHECKPOINT-COMMIT index 4854ec6..2faf619 100644 --- a/BPF-CHECKPOINT-COMMIT +++ b/BPF-CHECKPOINT-COMMIT @@ -1 +1 @@ -d6d86830705f173fca6087a3e67ceaf68db80523 +343e53754b21ae45530623222aa079fecd3cf942 diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index 3b9b714..3a3cc35 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -44bab87d8ca6f0544a9f8fc97bdf33aa5b3c899e +e80f2a0d194605553315de68284fc41969f81f62 From 8404d1396c0b239cb245819b42aa12daf38c9216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Tue, 18 Jan 2022 15:13:27 +0100 Subject: [PATCH 076/131] libbpf: Define BTF_KIND_* constants in btf.h to avoid compilation errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The btf.h header included with libbpf contains inline helper functions to check for various BTF kinds. These helpers directly reference the BTF_KIND_* constants defined in the kernel header, and because the header file is included in user applications, this happens in the user application compile units. This presents a problem if a user application is compiled on a system with older kernel headers because the constants are not available. To avoid this, add #defines of the constants directly in btf.h before using them. Since the kernel header moved to an enum for BTF_KIND_*, the #defines can shadow the enum values without any errors, so we only need #ifndef guards for the constants that predates the conversion to enum. We group these so there's only one guard for groups of values that were added together. [0] Closes: https://github.com/libbpf/libbpf/issues/436 Fixes: 223f903e9c83 ("bpf: Rename BTF_KIND_TAG to BTF_KIND_DECL_TAG") Fixes: 5b84bd10363e ("libbpf: Add support for BTF_KIND_TAG") Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Acked-by: Arnaldo Carvalho de Melo Link: https://lore.kernel.org/bpf/20220118141327.34231-1-toke@redhat.com --- src/btf.h | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/btf.h b/src/btf.h index 061839f..51862fd 100644 --- a/src/btf.h +++ b/src/btf.h @@ -375,8 +375,28 @@ btf_dump__dump_type_data(struct btf_dump *d, __u32 id, const struct btf_dump_type_data_opts *opts); /* - * A set of helpers for easier BTF types handling + * A set of helpers for easier BTF types handling. + * + * The inline functions below rely on constants from the kernel headers which + * may not be available for applications including this header file. To avoid + * compilation errors, we define all the constants here that were added after + * the initial introduction of the BTF_KIND* constants. */ +#ifndef BTF_KIND_FUNC +#define BTF_KIND_FUNC 12 /* Function */ +#define BTF_KIND_FUNC_PROTO 13 /* Function Proto */ +#endif +#ifndef BTF_KIND_VAR +#define BTF_KIND_VAR 14 /* Variable */ +#define BTF_KIND_DATASEC 15 /* Section */ +#endif +#ifndef BTF_KIND_FLOAT +#define BTF_KIND_FLOAT 16 /* Floating point */ +#endif +/* The kernel header switched to enums, so these two were never #defined */ +#define BTF_KIND_DECL_TAG 17 /* Decl Tag */ +#define BTF_KIND_TYPE_TAG 18 /* Type Tag */ + static inline __u16 btf_kind(const struct btf_type *t) { return BTF_INFO_KIND(t->info); From 0f99d97a9c8d8ba7a55f4df6239f242d8c1c36f7 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 19 Jan 2022 11:44:40 +0000 Subject: [PATCH 077/131] uapi/bpf: Add missing description and returns for helper documentation Both description and returns section will become mandatory for helpers and syscalls in a later commit to generate man pages. This commit also adds in the documentation that BPF_PROG_RUN is an alias for BPF_PROG_TEST_RUN for anyone searching for the syscall in the generated man pages. Signed-off-by: Usama Arif Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220119114442.1452088-1-usama.arif@bytedance.com --- include/uapi/linux/bpf.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index b0383d3..a9c96c2 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -330,6 +330,8 @@ union bpf_iter_link_info { * *ctx_out*, *data_in* and *data_out* must be NULL. * *repeat* must be zero. * + * BPF_PROG_RUN is an alias for BPF_PROG_TEST_RUN. + * * Return * Returns zero on success. On error, -1 is returned and *errno* * is set appropriately. @@ -1775,6 +1777,8 @@ union bpf_attr { * 0 on success, or a negative error in case of failure. * * u64 bpf_get_current_pid_tgid(void) + * Description + * Get the current pid and tgid. * Return * A 64-bit integer containing the current tgid and pid, and * created as such: @@ -1782,6 +1786,8 @@ union bpf_attr { * *current_task*\ **->pid**. * * u64 bpf_get_current_uid_gid(void) + * Description + * Get the current uid and gid. * Return * A 64-bit integer containing the current GID and UID, and * created as such: *current_gid* **<< 32 \|** *current_uid*. @@ -2256,6 +2262,8 @@ union bpf_attr { * The 32-bit hash. * * u64 bpf_get_current_task(void) + * Description + * Get the current task. * Return * A pointer to the current task struct. * @@ -2369,6 +2377,8 @@ union bpf_attr { * indicate that the hash is outdated and to trigger a * recalculation the next time the kernel tries to access this * hash or when the **bpf_get_hash_recalc**\ () helper is called. + * Return + * void. * * long bpf_get_numa_node_id(void) * Description @@ -2466,6 +2476,8 @@ union bpf_attr { * A 8-byte long unique number or 0 if *sk* is NULL. * * u32 bpf_get_socket_uid(struct sk_buff *skb) + * Description + * Get the owner UID of the socked associated to *skb*. * Return * The owner UID of the socket associated to *skb*. If the socket * is **NULL**, or if it is not a full socket (i.e. if it is a @@ -3240,6 +3252,9 @@ union bpf_attr { * The id is returned or 0 in case the id could not be retrieved. * * u64 bpf_get_current_cgroup_id(void) + * Description + * Get the current cgroup id based on the cgroup within which + * the current task is running. * Return * A 64-bit integer containing the current cgroup id based * on the cgroup within which the current task is running. From 75c7c722f5d80872cdf391de1f093109d05474c8 Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Wed, 19 Jan 2022 10:02:14 -0800 Subject: [PATCH 078/131] libbpf: Improve btf__add_btf() with an additional hashmap for strings. Add a hashmap to map the string offsets from a source btf to the string offsets from a target btf to reduce overheads. btf__add_btf() calls btf__add_str() to add strings from a source to a target btf. It causes many string comparisons, and it is a major hotspot when adding a big btf. btf__add_str() uses strcmp() to check if a hash entry is the right one. The extra hashmap here compares offsets of strings, that are much cheaper. It remembers the results of btf__add_str() for later uses to reduce the cost. We are parallelizing BTF encoding for pahole by creating separated btf instances for worker threads. These per-thread btf instances will be added to the btf instance of the main thread by calling btf__add_str() to deduplicate and write out. With this patch and -j4, the running time of pahole drops to about 6.0s from 6.6s. The following lines are the summary of 'perf stat' w/o the change. 6.668126396 seconds time elapsed 13.451054000 seconds user 0.715520000 seconds sys The following lines are the summary w/ the change. 5.986973919 seconds time elapsed 12.939903000 seconds user 0.724152000 seconds sys V4 fixes a bug of error checking against the pointer returned by hashmap__new(). [v3] https://lore.kernel.org/bpf/20220118232053.2113139-1-kuifeng@fb.com/ [v2] https://lore.kernel.org/bpf/20220114193713.461349-1-kuifeng@fb.com/ Signed-off-by: Kui-Feng Lee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220119180214.255634-1-kuifeng@fb.com --- src/btf.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/btf.c b/src/btf.c index 9aa19c8..1383e26 100644 --- a/src/btf.c +++ b/src/btf.c @@ -1620,20 +1620,37 @@ static int btf_commit_type(struct btf *btf, int data_sz) struct btf_pipe { const struct btf *src; struct btf *dst; + struct hashmap *str_off_map; /* map string offsets from src to dst */ }; static int btf_rewrite_str(__u32 *str_off, void *ctx) { struct btf_pipe *p = ctx; - int off; + void *mapped_off; + int off, err; if (!*str_off) /* nothing to do for empty strings */ return 0; + if (p->str_off_map && + hashmap__find(p->str_off_map, (void *)(long)*str_off, &mapped_off)) { + *str_off = (__u32)(long)mapped_off; + return 0; + } + off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off)); if (off < 0) return off; + /* Remember string mapping from src to dst. It avoids + * performing expensive string comparisons. + */ + if (p->str_off_map) { + err = hashmap__append(p->str_off_map, (void *)(long)*str_off, (void *)(long)off); + if (err) + return err; + } + *str_off = off; return 0; } @@ -1680,6 +1697,9 @@ static int btf_rewrite_type_ids(__u32 *type_id, void *ctx) return 0; } +static size_t btf_dedup_identity_hash_fn(const void *key, void *ctx); +static bool btf_dedup_equal_fn(const void *k1, const void *k2, void *ctx); + int btf__add_btf(struct btf *btf, const struct btf *src_btf) { struct btf_pipe p = { .src = src_btf, .dst = btf }; @@ -1713,6 +1733,11 @@ int btf__add_btf(struct btf *btf, const struct btf *src_btf) if (!off) return libbpf_err(-ENOMEM); + /* Map the string offsets from src_btf to the offsets from btf to improve performance */ + p.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL); + if (IS_ERR(p.str_off_map)) + return libbpf_err(-ENOMEM); + /* bulk copy types data for all types from src_btf */ memcpy(t, src_btf->types_data, data_sz); @@ -1754,6 +1779,8 @@ int btf__add_btf(struct btf *btf, const struct btf *src_btf) btf->hdr->str_off += data_sz; btf->nr_types += cnt; + hashmap__free(p.str_off_map); + /* return type ID of the first added BTF type */ return btf->start_id + btf->nr_types - cnt; err_out: @@ -1767,6 +1794,8 @@ err_out: * wasn't modified, so doesn't need restoring, see big comment above */ btf->hdr->str_len = old_strs_len; + hashmap__free(p.str_off_map); + return libbpf_err(err); } From ab022c8eb4f291516953a30532110894fb151124 Mon Sep 17 00:00:00 2001 From: YiFei Zhu Date: Thu, 16 Dec 2021 02:04:27 +0000 Subject: [PATCH 079/131] bpf: Add cgroup helpers bpf_{get,set}_retval to get/set syscall return value The helpers continue to use int for retval because all the hooks are int-returning rather than long-returning. The return value of bpf_set_retval is int for future-proofing, in case in the future there may be errors trying to set the retval. After the previous patch, if a program rejects a syscall by returning 0, an -EPERM will be generated no matter if the retval is already set to -err. This patch change it being forced only if retval is not -err. This is because we want to support, for example, invoking bpf_set_retval(-EINVAL) and return 0, and have the syscall return value be -EINVAL not -EPERM. For BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY, the prior behavior is that, if the return value is NET_XMIT_DROP, the packet is silently dropped. We preserve this behavior for backward compatibility reasons, so even if an errno is set, the errno does not return to caller. However, setting a non-err to retval cannot propagate so this is not allowed and we return a -EFAULT in that case. Signed-off-by: YiFei Zhu Reviewed-by: Stanislav Fomichev Link: https://lore.kernel.org/r/b4013fd5d16bed0b01977c1fafdeae12e1de61fb.1639619851.git.zhuyifei@google.com Signed-off-by: Alexei Starovoitov --- include/uapi/linux/bpf.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index a9c96c2..fe2272d 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -5033,6 +5033,22 @@ union bpf_attr { * * Return * The number of arguments of the traced function. + * + * int bpf_get_retval(void) + * Description + * Get the syscall's return value that will be returned to userspace. + * + * This helper is currently supported by cgroup programs only. + * Return + * The syscall's return value. + * + * int bpf_set_retval(int retval) + * Description + * Set the syscall's return value that will be returned to userspace. + * + * This helper is currently supported by cgroup programs only. + * Return + * 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5221,6 +5237,8 @@ union bpf_attr { FN(get_func_arg), \ FN(get_func_ret), \ FN(get_func_arg_cnt), \ + FN(get_retval), \ + FN(set_retval), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper From d788cd57b54c179a657f823cbe56c868f6974924 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 19 Jan 2022 22:05:28 -0800 Subject: [PATCH 080/131] libbpf: deprecate legacy BPF map definitions Enact deprecation of legacy BPF map definition in SEC("maps") ([0]). For the definitions themselves introduce LIBBPF_STRICT_MAP_DEFINITIONS flag for libbpf strict mode. If it is set, error out on any struct bpf_map_def-based map definition. If not set, libbpf will print out a warning for each legacy BPF map to raise awareness that it goes away. For any use of BPF_ANNOTATE_KV_PAIR() macro providing a legacy way to associate BTF key/value type information with legacy BPF map definition, warn through libbpf's pr_warn() error message (but don't fail BPF object open). BPF-side struct bpf_map_def is marked as deprecated. User-space struct bpf_map_def has to be used internally in libbpf, so it is left untouched. It should be enough for bpf_map__def() to be marked deprecated to raise awareness that it goes away. bpftool is an interesting case that utilizes libbpf to open BPF ELF object to generate skeleton. As such, even though bpftool itself uses full on strict libbpf mode (LIBBPF_STRICT_ALL), it has to relax it a bit for BPF map definition handling to minimize unnecessary disruptions. So opt-out of LIBBPF_STRICT_MAP_DEFINITIONS for bpftool. User's code that will later use generated skeleton will make its own decision whether to enforce LIBBPF_STRICT_MAP_DEFINITIONS or not. There are few tests in selftests/bpf that are consciously using legacy BPF map definitions to test libbpf functionality. For those, temporary opt out of LIBBPF_STRICT_MAP_DEFINITIONS mode for the duration of those tests. [0] Closes: https://github.com/libbpf/libbpf/issues/272 Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20220120060529.1890907-4-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- src/bpf_helpers.h | 2 +- src/libbpf.c | 8 ++++++++ src/libbpf_legacy.h | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/bpf_helpers.h b/src/bpf_helpers.h index 963b106..44df982 100644 --- a/src/bpf_helpers.h +++ b/src/bpf_helpers.h @@ -133,7 +133,7 @@ struct bpf_map_def { unsigned int value_size; unsigned int max_entries; unsigned int map_flags; -}; +} __attribute__((deprecated("use BTF-defined maps in .maps section"))); enum libbpf_pin_type { LIBBPF_PIN_NONE, diff --git a/src/libbpf.c b/src/libbpf.c index fdb3536..fc6d530 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -1937,6 +1937,11 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) if (obj->efile.maps_shndx < 0) return 0; + if (libbpf_mode & LIBBPF_STRICT_MAP_DEFINITIONS) { + pr_warn("legacy map definitions in SEC(\"maps\") are not supported\n"); + return -EOPNOTSUPP; + } + if (!symbols) return -EINVAL; @@ -1999,6 +2004,8 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) return -LIBBPF_ERRNO__FORMAT; } + pr_warn("map '%s' (legacy): legacy map definitions are deprecated, use BTF-defined maps instead\n", map_name); + if (ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { pr_warn("map '%s' (legacy): static maps are not supported\n", map_name); return -ENOTSUP; @@ -4190,6 +4197,7 @@ static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) return 0; if (!bpf_map__is_internal(map)) { + pr_warn("Use of BPF_ANNOTATE_KV_PAIR is deprecated, use BTF-defined maps in .maps section instead\n"); ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, def->value_size, &key_type_id, &value_type_id); diff --git a/src/libbpf_legacy.h b/src/libbpf_legacy.h index 79131f7..3c2b281 100644 --- a/src/libbpf_legacy.h +++ b/src/libbpf_legacy.h @@ -73,6 +73,11 @@ enum libbpf_strict_mode { * operation. */ LIBBPF_STRICT_AUTO_RLIMIT_MEMLOCK = 0x10, + /* + * Error out on any SEC("maps") map definition, which are deprecated + * in favor of BTF-defined map definitions in SEC(".maps"). + */ + LIBBPF_STRICT_MAP_DEFINITIONS = 0x20, __LIBBPF_STRICT_LAST, }; From 8fbe7eec3aacca51d81785f95da295d40e1cb965 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 19 Jan 2022 22:14:19 -0800 Subject: [PATCH 081/131] libbpf: streamline low-level XDP APIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce 4 new netlink-based XDP APIs for attaching, detaching, and querying XDP programs: - bpf_xdp_attach; - bpf_xdp_detach; - bpf_xdp_query; - bpf_xdp_query_id. These APIs replace bpf_set_link_xdp_fd, bpf_set_link_xdp_fd_opts, bpf_get_link_xdp_id, and bpf_get_link_xdp_info APIs ([0]). The latter don't follow a consistent naming pattern and some of them use non-extensible approaches (e.g., struct xdp_link_info which can't be modified without breaking libbpf ABI). The approach I took with these low-level XDP APIs is similar to what we did with low-level TC APIs. There is a nice duality of bpf_tc_attach vs bpf_xdp_attach, and so on. I left bpf_xdp_attach() to support detaching when -1 is specified for prog_fd for generality and convenience, but bpf_xdp_detach() is preferred due to clearer naming and associated semantics. Both bpf_xdp_attach() and bpf_xdp_detach() accept the same opts struct allowing to specify expected old_prog_fd. While doing the refactoring, I noticed that old APIs require users to specify opts with old_fd == -1 to declare "don't care about already attached XDP prog fd" condition. Otherwise, FD 0 is assumed, which is essentially never an intended behavior. So I made this behavior consistent with other kernel and libbpf APIs, in which zero FD means "no FD". This seems to be more in line with the latest thinking in BPF land and should cause less user confusion, hopefully. For querying, I left two APIs, both more generic bpf_xdp_query() allowing to query multiple IDs and attach mode, but also a specialization of it, bpf_xdp_query_id(), which returns only requested prog_id. Uses of prog_id returning bpf_get_link_xdp_id() were so prevalent across selftests and samples, that it seemed a very common use case and using bpf_xdp_query() for doing it felt very cumbersome with a highly branches if/else chain based on flags and attach mode. Old APIs are scheduled for deprecation in libbpf 0.8 release. [0] Closes: https://github.com/libbpf/libbpf/issues/309 Signed-off-by: Andrii Nakryiko Acked-by: Toke Høiland-Jørgensen Link: https://lore.kernel.org/r/20220120061422.2710637-2-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- src/libbpf.h | 29 +++++++++++ src/libbpf.map | 4 ++ src/netlink.c | 127 ++++++++++++++++++++++++++++++++++--------------- 3 files changed, 122 insertions(+), 38 deletions(-) diff --git a/src/libbpf.h b/src/libbpf.h index 9728551..9467006 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -833,13 +833,42 @@ struct bpf_xdp_set_link_opts { }; #define bpf_xdp_set_link_opts__last_field old_fd +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_attach() instead") LIBBPF_API int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_attach() instead") LIBBPF_API int bpf_set_link_xdp_fd_opts(int ifindex, int fd, __u32 flags, const struct bpf_xdp_set_link_opts *opts); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_query_id() instead") LIBBPF_API int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_query() instead") LIBBPF_API int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info, size_t info_size, __u32 flags); +struct bpf_xdp_attach_opts { + size_t sz; + int old_prog_fd; + size_t :0; +}; +#define bpf_xdp_attach_opts__last_field old_prog_fd + +struct bpf_xdp_query_opts { + size_t sz; + __u32 prog_id; /* output */ + __u32 drv_prog_id; /* output */ + __u32 hw_prog_id; /* output */ + __u32 skb_prog_id; /* output */ + __u8 attach_mode; /* output */ + size_t :0; +}; +#define bpf_xdp_query_opts__last_field attach_mode + +LIBBPF_API int bpf_xdp_attach(int ifindex, int prog_fd, __u32 flags, + const struct bpf_xdp_attach_opts *opts); +LIBBPF_API int bpf_xdp_detach(int ifindex, __u32 flags, + const struct bpf_xdp_attach_opts *opts); +LIBBPF_API int bpf_xdp_query(int ifindex, int flags, struct bpf_xdp_query_opts *opts); +LIBBPF_API int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id); + /* TC related API */ enum bpf_tc_attach_point { BPF_TC_INGRESS = 1 << 0, diff --git a/src/libbpf.map b/src/libbpf.map index 8262cfc..e10f082 100644 --- a/src/libbpf.map +++ b/src/libbpf.map @@ -428,6 +428,10 @@ LIBBPF_0.7.0 { bpf_program__log_level; bpf_program__set_log_buf; bpf_program__set_log_level; + bpf_xdp_attach; + bpf_xdp_detach; + bpf_xdp_query; + bpf_xdp_query_id; libbpf_probe_bpf_helper; libbpf_probe_bpf_map_type; libbpf_probe_bpf_prog_type; diff --git a/src/netlink.c b/src/netlink.c index 39f25e0..c39c37f 100644 --- a/src/netlink.c +++ b/src/netlink.c @@ -217,6 +217,28 @@ static int __bpf_set_link_xdp_fd_replace(int ifindex, int fd, int old_fd, return libbpf_netlink_send_recv(&req, NULL, NULL, NULL); } +int bpf_xdp_attach(int ifindex, int prog_fd, __u32 flags, const struct bpf_xdp_attach_opts *opts) +{ + int old_prog_fd, err; + + if (!OPTS_VALID(opts, bpf_xdp_attach_opts)) + return libbpf_err(-EINVAL); + + old_prog_fd = OPTS_GET(opts, old_prog_fd, 0); + if (old_prog_fd) + flags |= XDP_FLAGS_REPLACE; + else + old_prog_fd = -1; + + err = __bpf_set_link_xdp_fd_replace(ifindex, prog_fd, old_prog_fd, flags); + return libbpf_err(err); +} + +int bpf_xdp_detach(int ifindex, __u32 flags, const struct bpf_xdp_attach_opts *opts) +{ + return bpf_xdp_attach(ifindex, -1, flags, opts); +} + int bpf_set_link_xdp_fd_opts(int ifindex, int fd, __u32 flags, const struct bpf_xdp_set_link_opts *opts) { @@ -303,69 +325,98 @@ static int get_xdp_info(void *cookie, void *msg, struct nlattr **tb) return 0; } -int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info, - size_t info_size, __u32 flags) +int bpf_xdp_query(int ifindex, int xdp_flags, struct bpf_xdp_query_opts *opts) { - struct xdp_id_md xdp_id = {}; - __u32 mask; - int ret; struct libbpf_nla_req req = { .nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)), .nh.nlmsg_type = RTM_GETLINK, .nh.nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST, .ifinfo.ifi_family = AF_PACKET, }; + struct xdp_id_md xdp_id = {}; + int err; - if (flags & ~XDP_FLAGS_MASK || !info_size) + if (!OPTS_VALID(opts, bpf_xdp_query_opts)) + return libbpf_err(-EINVAL); + + if (xdp_flags & ~XDP_FLAGS_MASK) return libbpf_err(-EINVAL); /* Check whether the single {HW,DRV,SKB} mode is set */ - flags &= (XDP_FLAGS_SKB_MODE | XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE); - mask = flags - 1; - if (flags && flags & mask) + xdp_flags &= XDP_FLAGS_SKB_MODE | XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE; + if (xdp_flags & (xdp_flags - 1)) return libbpf_err(-EINVAL); xdp_id.ifindex = ifindex; - xdp_id.flags = flags; + xdp_id.flags = xdp_flags; - ret = libbpf_netlink_send_recv(&req, __dump_link_nlmsg, + err = libbpf_netlink_send_recv(&req, __dump_link_nlmsg, get_xdp_info, &xdp_id); - if (!ret) { - size_t sz = min(info_size, sizeof(xdp_id.info)); + if (err) + return libbpf_err(err); - memcpy(info, &xdp_id.info, sz); - memset((void *) info + sz, 0, info_size - sz); - } - - return libbpf_err(ret); -} - -static __u32 get_xdp_id(struct xdp_link_info *info, __u32 flags) -{ - flags &= XDP_FLAGS_MODES; - - if (info->attach_mode != XDP_ATTACHED_MULTI && !flags) - return info->prog_id; - if (flags & XDP_FLAGS_DRV_MODE) - return info->drv_prog_id; - if (flags & XDP_FLAGS_HW_MODE) - return info->hw_prog_id; - if (flags & XDP_FLAGS_SKB_MODE) - return info->skb_prog_id; + OPTS_SET(opts, prog_id, xdp_id.info.prog_id); + OPTS_SET(opts, drv_prog_id, xdp_id.info.drv_prog_id); + OPTS_SET(opts, hw_prog_id, xdp_id.info.hw_prog_id); + OPTS_SET(opts, skb_prog_id, xdp_id.info.skb_prog_id); + OPTS_SET(opts, attach_mode, xdp_id.info.attach_mode); return 0; } -int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags) +int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info, + size_t info_size, __u32 flags) { - struct xdp_link_info info; + LIBBPF_OPTS(bpf_xdp_query_opts, opts); + size_t sz; + int err; + + if (!info_size) + return libbpf_err(-EINVAL); + + err = bpf_xdp_query(ifindex, flags, &opts); + if (err) + return libbpf_err(err); + + /* struct xdp_link_info field layout matches struct bpf_xdp_query_opts + * layout after sz field + */ + sz = min(info_size, offsetofend(struct xdp_link_info, attach_mode)); + memcpy(info, &opts.prog_id, sz); + memset((void *)info + sz, 0, info_size - sz); + + return 0; +} + +int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id) +{ + LIBBPF_OPTS(bpf_xdp_query_opts, opts); int ret; - ret = bpf_get_link_xdp_info(ifindex, &info, sizeof(info), flags); - if (!ret) - *prog_id = get_xdp_id(&info, flags); + ret = bpf_xdp_query(ifindex, flags, &opts); + if (ret) + return libbpf_err(ret); - return libbpf_err(ret); + flags &= XDP_FLAGS_MODES; + + if (opts.attach_mode != XDP_ATTACHED_MULTI && !flags) + *prog_id = opts.prog_id; + else if (flags & XDP_FLAGS_DRV_MODE) + *prog_id = opts.drv_prog_id; + else if (flags & XDP_FLAGS_HW_MODE) + *prog_id = opts.hw_prog_id; + else if (flags & XDP_FLAGS_SKB_MODE) + *prog_id = opts.skb_prog_id; + else + *prog_id = 0; + + return 0; +} + + +int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags) +{ + return bpf_xdp_query_id(ifindex, flags, prog_id); } typedef int (*qdisc_config_t)(struct libbpf_nla_req *req); From d2ea0e2d03384eab3f89603f39d626c79410c6bc Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 21 Jan 2022 12:12:58 -0800 Subject: [PATCH 082/131] sync: auto-generate latest BPF helpers Latest changes to BPF helper definitions. --- src/bpf_helper_defs.h | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/bpf_helper_defs.h b/src/bpf_helper_defs.h index 1c70294..e2f9541 100644 --- a/src/bpf_helper_defs.h +++ b/src/bpf_helper_defs.h @@ -352,6 +352,7 @@ static long (*bpf_clone_redirect)(struct __sk_buff *skb, __u32 ifindex, __u64 fl /* * bpf_get_current_pid_tgid * + * Get the current pid and tgid. * * Returns * A 64-bit integer containing the current tgid and pid, and @@ -364,6 +365,7 @@ static __u64 (*bpf_get_current_pid_tgid)(void) = (void *) 14; /* * bpf_get_current_uid_gid * + * Get the current uid and gid. * * Returns * A 64-bit integer containing the current GID and UID, and @@ -919,6 +921,7 @@ static __u32 (*bpf_get_hash_recalc)(struct __sk_buff *skb) = (void *) 34; /* * bpf_get_current_task * + * Get the current task. * * Returns * A pointer to the current task struct. @@ -1057,6 +1060,8 @@ static __s64 (*bpf_csum_update)(struct __sk_buff *skb, __wsum csum) = (void *) 4 * recalculation the next time the kernel tries to access this * hash or when the **bpf_get_hash_recalc**\ () helper is called. * + * Returns + * void. */ static void (*bpf_set_hash_invalid)(struct __sk_buff *skb) = (void *) 41; @@ -1156,6 +1161,7 @@ static __u64 (*bpf_get_socket_cookie)(void *ctx) = (void *) 46; /* * bpf_get_socket_uid * + * Get the owner UID of the socked associated to *skb*. * * Returns * The owner UID of the socket associated to *skb*. If the socket @@ -2063,6 +2069,8 @@ static __u64 (*bpf_skb_cgroup_id)(struct __sk_buff *skb) = (void *) 79; /* * bpf_get_current_cgroup_id * + * Get the current cgroup id based on the cgroup within which + * the current task is running. * * Returns * A 64-bit integer containing the current cgroup id based @@ -4215,4 +4223,28 @@ static long (*bpf_get_func_ret)(void *ctx, __u64 *value) = (void *) 184; */ static long (*bpf_get_func_arg_cnt)(void *ctx) = (void *) 185; +/* + * bpf_get_retval + * + * Get the syscall's return value that will be returned to userspace. + * + * This helper is currently supported by cgroup programs only. + * + * Returns + * The syscall's return value. + */ +static int (*bpf_get_retval)(void) = (void *) 186; + +/* + * bpf_set_retval + * + * Set the syscall's return value that will be returned to userspace. + * + * This helper is currently supported by cgroup programs only. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static int (*bpf_set_retval)(int retval) = (void *) 187; + From 78e816a15d2889b91a72c6106efc45fa81ab5b57 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 21 Jan 2022 12:12:58 -0800 Subject: [PATCH 083/131] sync: latest libbpf changes from kernel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Syncing latest libbpf commits from kernel repository. Baseline bpf-next commit: e80f2a0d194605553315de68284fc41969f81f62 Checkpoint bpf-next commit: 820e6e227c4053b6b631ae65ef1f65d560cb392b Baseline bpf commit: 343e53754b21ae45530623222aa079fecd3cf942 Checkpoint bpf commit: baa59504c1cd0cca7d41954a45ee0b3dc78e41a0 Andrii Nakryiko (2): libbpf: deprecate legacy BPF map definitions libbpf: streamline low-level XDP APIs Kui-Feng Lee (1): libbpf: Improve btf__add_btf() with an additional hashmap for strings. Toke Høiland-Jørgensen (1): libbpf: Define BTF_KIND_* constants in btf.h to avoid compilation errors Usama Arif (1): uapi/bpf: Add missing description and returns for helper documentation YiFei Zhu (1): bpf: Add cgroup helpers bpf_{get,set}_retval to get/set syscall return value include/uapi/linux/bpf.h | 33 +++++++++++ src/bpf_helpers.h | 2 +- src/btf.c | 31 ++++++++++- src/btf.h | 22 +++++++- src/libbpf.c | 8 +++ src/libbpf.h | 29 ++++++++++ src/libbpf.map | 4 ++ src/libbpf_legacy.h | 5 ++ src/netlink.c | 117 ++++++++++++++++++++++++++++----------- 9 files changed, 215 insertions(+), 36 deletions(-) -- 2.30.2 --- BPF-CHECKPOINT-COMMIT | 2 +- CHECKPOINT-COMMIT | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BPF-CHECKPOINT-COMMIT b/BPF-CHECKPOINT-COMMIT index 2faf619..af72d6d 100644 --- a/BPF-CHECKPOINT-COMMIT +++ b/BPF-CHECKPOINT-COMMIT @@ -1 +1 @@ -343e53754b21ae45530623222aa079fecd3cf942 +baa59504c1cd0cca7d41954a45ee0b3dc78e41a0 diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index 3a3cc35..16a932e 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -e80f2a0d194605553315de68284fc41969f81f62 +820e6e227c4053b6b631ae65ef1f65d560cb392b From 5daafdccf91a0ae74c4f5ea00fdcf0e6e7cc9597 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 21 Jan 2022 15:07:05 -0800 Subject: [PATCH 084/131] ci: regenerate and check in latest vmlinux.h Add a bunch of new kernel types that are needed for successful selftest compilation. Signed-off-by: Andrii Nakryiko --- .github/actions/build-selftests/vmlinux.h | 17906 ++++++++++++-------- 1 file changed, 10421 insertions(+), 7485 deletions(-) diff --git a/.github/actions/build-selftests/vmlinux.h b/.github/actions/build-selftests/vmlinux.h index 545f3f4..8280fce 100644 --- a/.github/actions/build-selftests/vmlinux.h +++ b/.github/actions/build-selftests/vmlinux.h @@ -9,6 +9,8 @@ typedef signed char __s8; typedef unsigned char __u8; +typedef short int __s16; + typedef short unsigned int __u16; typedef int __s32; @@ -23,6 +25,8 @@ typedef __s8 s8; typedef __u8 u8; +typedef __s16 s16; + typedef __u16 u16; typedef __s32 s32; @@ -170,9 +174,16 @@ struct file_system_type { struct lock_class_key s_writers_key[3]; struct lock_class_key i_lock_key; struct lock_class_key i_mutex_key; + struct lock_class_key invalidate_lock_key; struct lock_class_key i_mutex_dir_key; }; +struct kernel_symbol { + int value_offset; + int name_offset; + int namespace_offset; +}; + struct taint_flag { char c_true; char c_false; @@ -291,6 +302,8 @@ struct kiocb; struct iov_iter; +struct io_comp_batch; + struct dir_context; struct poll_table_struct; @@ -314,7 +327,7 @@ struct file_operations { ssize_t (*write)(struct file *, const char *, size_t, loff_t *); ssize_t (*read_iter)(struct kiocb *, struct iov_iter *); ssize_t (*write_iter)(struct kiocb *, struct iov_iter *); - int (*iopoll)(struct kiocb *, bool); + int (*iopoll)(struct kiocb *, struct io_comp_batch *, unsigned int); int (*iterate)(struct file *, struct dir_context *); int (*iterate_shared)(struct file *, struct dir_context *); __poll_t (*poll)(struct file *, struct poll_table_struct *); @@ -361,11 +374,12 @@ struct static_call_key { enum system_states { SYSTEM_BOOTING = 0, SYSTEM_SCHEDULING = 1, - SYSTEM_RUNNING = 2, - SYSTEM_HALT = 3, - SYSTEM_POWER_OFF = 4, - SYSTEM_RESTART = 5, - SYSTEM_SUSPEND = 6, + SYSTEM_FREEING_INITMEM = 2, + SYSTEM_RUNNING = 3, + SYSTEM_HALT = 4, + SYSTEM_POWER_OFF = 5, + SYSTEM_RESTART = 6, + SYSTEM_SUSPEND = 7, }; struct jump_entry { @@ -393,6 +407,13 @@ struct static_key_false { struct static_key key; }; +struct bug_entry { + int bug_addr_disp; + int file_disp; + short unsigned int line; + short unsigned int flags; +}; + typedef __s64 time64_t; struct __kernel_timespec { @@ -460,6 +481,7 @@ struct thread_info { long unsigned int flags; long unsigned int syscall_work; u32 status; + u32 cpu; }; struct refcount_struct { @@ -493,36 +515,6 @@ struct rb_node { struct rb_node *rb_left; }; -struct sched_statistics { - u64 wait_start; - u64 wait_max; - u64 wait_count; - u64 wait_sum; - u64 iowait_count; - u64 iowait_sum; - u64 sleep_start; - u64 sleep_max; - s64 sum_sleep_runtime; - u64 block_start; - u64 block_max; - u64 exec_max; - u64 slice_max; - u64 nr_migrations_cold; - u64 nr_failed_migrations_affine; - u64 nr_failed_migrations_running; - u64 nr_failed_migrations_hot; - u64 nr_forced_migrations; - u64 nr_wakeups; - u64 nr_wakeups_sync; - u64 nr_wakeups_migrate; - u64 nr_wakeups_local; - u64 nr_wakeups_remote; - u64 nr_wakeups_affine; - u64 nr_wakeups_affine_attempts; - u64 nr_wakeups_passive; - u64 nr_wakeups_idle; -}; - struct util_est { unsigned int enqueued; unsigned int ewma; @@ -552,7 +544,6 @@ struct sched_entity { u64 vruntime; u64 prev_sum_exec_runtime; u64 nr_migrations; - struct sched_statistics statistics; int depth; struct sched_entity *parent; struct cfs_rq *cfs_rq; @@ -561,6 +552,9 @@ struct sched_entity { long: 64; long: 64; long: 64; + long: 64; + long: 64; + long: 64; struct sched_avg avg; }; @@ -618,6 +612,41 @@ struct sched_dl_entity { struct sched_dl_entity *pi_se; }; +struct sched_statistics { + u64 wait_start; + u64 wait_max; + u64 wait_count; + u64 wait_sum; + u64 iowait_count; + u64 iowait_sum; + u64 sleep_start; + u64 sleep_max; + s64 sum_sleep_runtime; + u64 block_start; + u64 block_max; + s64 sum_block_runtime; + u64 exec_max; + u64 slice_max; + u64 nr_migrations_cold; + u64 nr_failed_migrations_affine; + u64 nr_failed_migrations_running; + u64 nr_failed_migrations_hot; + u64 nr_forced_migrations; + u64 nr_wakeups; + u64 nr_wakeups_sync; + u64 nr_wakeups_migrate; + u64 nr_wakeups_local; + u64 nr_wakeups_remote; + u64 nr_wakeups_affine; + u64 nr_wakeups_affine_attempts; + u64 nr_wakeups_passive; + u64 nr_wakeups_idle; + long: 64; + long: 64; + long: 64; + long: 64; +}; + struct cpumask { long unsigned int bits[2]; }; @@ -797,7 +826,7 @@ struct optimistic_spin_queue { struct mutex { atomic_long_t owner; - spinlock_t wait_lock; + raw_spinlock_t wait_lock; struct optimistic_spin_queue osq; struct list_head wait_list; void *magic; @@ -842,6 +871,12 @@ struct desc_struct { u16 base2: 8; }; +struct fpu_state_perm { + u64 __state_perm; + unsigned int __state_size; + unsigned int __user_state_size; +}; + struct fregs_state { u32 cwd; u32 swd; @@ -923,16 +958,32 @@ union fpregs_state { u8 __padding[4096]; }; +struct fpstate { + unsigned int size; + unsigned int user_size; + u64 xfeatures; + u64 user_xfeatures; + u64 xfd; + unsigned int is_valloc: 1; + unsigned int is_guest: 1; + unsigned int is_confidential: 1; + unsigned int in_use: 1; + long: 60; + long: 64; + long: 64; + long: 64; + union fpregs_state regs; +}; + struct fpu { unsigned int last_cpu; long unsigned int avx512_timestamp; + struct fpstate *fpstate; + struct fpstate *__task_fpstate; + struct fpu_state_perm perm; long: 64; long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - union fpregs_state state; + struct fpstate __fpstate; }; struct perf_event; @@ -956,6 +1007,7 @@ struct thread_struct { long unsigned int error_code; struct io_bitmap *io_bitmap; long unsigned int iopl_emul; + unsigned int iopl_warn: 1; unsigned int sig_on_uaccess_err: 1; u32 pkru; long: 64; @@ -1057,7 +1109,6 @@ struct task_struct { unsigned int ptrace; int on_cpu; struct __call_single_node wake_entry; - unsigned int cpu; unsigned int wakee_flips; long unsigned int wakee_flip_decay_ts; struct task_struct *last_wakee; @@ -1068,22 +1119,21 @@ struct task_struct { int static_prio; int normal_prio; unsigned int rt_priority; - const struct sched_class *sched_class; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; struct sched_entity se; struct sched_rt_entity rt; struct sched_dl_entity dl; + const struct sched_class *sched_class; struct task_group *sched_task_group; + long: 64; + long: 64; + long: 64; + long: 64; + struct sched_statistics stats; unsigned int btrace_seq; unsigned int policy; int nr_cpus_allowed; const cpumask_t *cpus_ptr; + cpumask_t *user_cpus_ptr; cpumask_t cpus_mask; void *migration_pending; short unsigned int migration_disabled; @@ -1129,6 +1179,7 @@ struct task_struct { unsigned int no_cgroup_migration: 1; unsigned int frozen: 1; unsigned int use_memdelay: 1; + unsigned int in_eventfd_signal: 1; long unsigned int atomic_flags; struct restart_block restart_block; pid_t pid; @@ -1303,8 +1354,9 @@ struct task_struct { __u64 mce_whole_page: 1; __u64 __mce_reserved: 62; struct callback_head mce_kill_me; + int mce_count; struct llist_head kretprobe_instances; - long: 64; + struct callback_head l1d_flush_kill; long: 64; long: 64; long: 64; @@ -1713,7 +1765,11 @@ struct page { long unsigned int pp_magic; struct page_pool *pp; long unsigned int _pp_mapping_pad; - long unsigned int dma_addr[2]; + long unsigned int dma_addr; + union { + long unsigned int dma_addr_upper; + atomic_long_t pp_frag_count; + }; }; struct { union { @@ -1721,7 +1777,6 @@ struct page { struct { struct page *next; int pages; - int pobjects; }; }; struct kmem_cache *slab_cache; @@ -1793,6 +1848,16 @@ struct tracepoint { struct tracepoint_func *funcs; }; +typedef const int tracepoint_ptr_t; + +struct bpf_raw_event_map { + struct tracepoint *tp; + void *bpf_func; + u32 num_args; + u32 writable_size; + long: 64; +}; + struct idt_bits { u16 ist: 3; u16 zero: 5; @@ -1817,6 +1882,23 @@ struct desc_ptr { long unsigned int address; } __attribute__((packed)); +struct orc_entry { + s16 sp_offset; + s16 bp_offset; + unsigned int sp_reg: 4; + unsigned int bp_reg: 4; + unsigned int type: 2; + unsigned int end: 1; +} __attribute__((packed)); + +struct fpu_state_config { + unsigned int max_size; + unsigned int default_size; + u64 max_features; + u64 default_features; + u64 legacy_features; +}; + enum { UNAME26 = 131072, ADDR_NO_RANDOMIZE = 262144, @@ -1873,6 +1955,7 @@ struct cpuinfo_x86 { u16 cpu_die_id; u16 logical_die_id; u16 cpu_index; + bool smt_active; u32 microcode; u8 x86_cache_bits; unsigned int initialized: 1; @@ -2587,6 +2670,8 @@ struct device_dma_parameters; struct cma; +struct io_tlb_mem; + struct device_node; struct class; @@ -2611,6 +2696,7 @@ struct device { struct dev_pm_info power; struct dev_pm_domain *pm_domain; struct irq_domain *msi_domain; + raw_spinlock_t msi_lock; struct list_head msi_list; const struct dma_map_ops *dma_ops; u64 *dma_mask; @@ -2620,6 +2706,7 @@ struct device { struct device_dma_parameters *dma_parms; struct list_head dma_pools; struct cma *cma_area; + struct io_tlb_mem *dma_io_tlb_mem; struct dev_archdata archdata; struct device_node *of_node; struct fwnode_handle *fwnode; @@ -2671,46 +2758,6 @@ struct wait_queue_head { typedef struct wait_queue_head wait_queue_head_t; -struct ww_acquire_ctx; - -struct ww_class; - -struct ww_mutex { - struct mutex base; - struct ww_acquire_ctx *ctx; - struct ww_class *ww_class; -}; - -struct ww_acquire_ctx { - struct task_struct *task; - long unsigned int stamp; - unsigned int acquired; - short unsigned int wounded; - short unsigned int is_wait_die; - unsigned int done_acquire; - struct ww_class *ww_class; - struct ww_mutex *contending_lock; - struct lockdep_map dep_map; - unsigned int deadlock_inject_interval; - unsigned int deadlock_inject_countdown; -}; - -struct ww_class { - atomic_long_t stamp; - struct lock_class_key acquire_key; - struct lock_class_key mutex_key; - const char *acquire_name; - const char *mutex_name; - unsigned int is_wait_die; -}; - -struct mutex_waiter { - struct list_head list; - struct task_struct *task; - struct ww_acquire_ctx *ww_ctx; - void *magic; -}; - struct seqcount_raw_spinlock { seqcount_t seqcount; raw_spinlock_t *lock; @@ -2734,7 +2781,11 @@ enum node_states { NR_NODE_STATES = 6, }; -struct vm_userfaultfd_ctx {}; +struct userfaultfd_ctx; + +struct vm_userfaultfd_ctx { + struct userfaultfd_ctx *ctx; +}; struct anon_vma; @@ -2832,8 +2883,6 @@ struct work_struct { struct linux_binfmt; -struct core_state; - struct kioctx_table; struct user_namespace; @@ -2884,7 +2933,6 @@ struct mm_struct { struct linux_binfmt *binfmt; mm_context_t context; long unsigned int flags; - struct core_state *core_state; spinlock_t ioctx_lock; struct kioctx_table *ioctx_table; struct task_struct *owner; @@ -2902,6 +2950,92 @@ struct mm_struct { long unsigned int cpu_bitmap[0]; }; +struct timer_list { + struct hlist_node entry; + long unsigned int expires; + void (*function)(struct timer_list *); + u32 flags; + struct lockdep_map lockdep_map; +}; + +struct workqueue_struct; + +struct delayed_work { + struct work_struct work; + struct timer_list timer; + struct workqueue_struct *wq; + int cpu; +}; + +struct rcu_segcblist { + struct callback_head *head; + struct callback_head **tails[4]; + long unsigned int gp_seq[4]; + long int len; + long int seglen[4]; + u8 flags; +}; + +struct srcu_node; + +struct srcu_struct; + +struct srcu_data { + long unsigned int srcu_lock_count[2]; + long unsigned int srcu_unlock_count[2]; + long: 64; + long: 64; + long: 64; + long: 64; + spinlock_t lock; + struct rcu_segcblist srcu_cblist; + long unsigned int srcu_gp_seq_needed; + long unsigned int srcu_gp_seq_needed_exp; + bool srcu_cblist_invoking; + struct timer_list delay_work; + struct work_struct work; + struct callback_head srcu_barrier_head; + struct srcu_node *mynode; + long unsigned int grpmask; + int cpu; + struct srcu_struct *ssp; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; +}; + +struct srcu_node { + spinlock_t lock; + long unsigned int srcu_have_cbs[4]; + long unsigned int srcu_data_have_cbs[4]; + long unsigned int srcu_gp_seq_needed_exp; + struct srcu_node *srcu_parent; + int grplo; + int grphi; +}; + +struct srcu_struct { + struct srcu_node node[9]; + struct srcu_node *level[3]; + struct mutex srcu_cb_mutex; + spinlock_t lock; + struct mutex srcu_gp_mutex; + unsigned int srcu_idx; + long unsigned int srcu_gp_seq; + long unsigned int srcu_gp_seq_needed; + long unsigned int srcu_gp_seq_needed_exp; + long unsigned int srcu_last_gp_end; + struct srcu_data *sda; + long unsigned int srcu_barrier_seq; + struct mutex srcu_barrier_mutex; + struct completion srcu_barrier_completion; + atomic_t srcu_barrier_cpu_cnt; + struct delayed_work work; + struct lockdep_map dep_map; +}; + struct notifier_block; typedef int (*notifier_fn_t)(struct notifier_block *, long unsigned int, void *); @@ -2996,6 +3130,7 @@ struct address_space_operations; struct address_space { struct inode *host; struct xarray i_pages; + struct rw_semaphore invalidate_lock; gfp_t gfp_mask; atomic_t i_mmap_writable; struct rb_root_cached i_mmap; @@ -3149,17 +3284,6 @@ struct vm_operations_struct { struct page * (*find_special_page)(struct vm_area_struct *, long unsigned int); }; -struct core_thread { - struct task_struct *task; - struct core_thread *next; -}; - -struct core_state { - atomic_t nr_threads; - struct core_thread dumper; - struct completion startup; -}; - enum fault_flag { FAULT_FLAG_WRITE = 1, FAULT_FLAG_MKWRITE = 2, @@ -3282,13 +3406,14 @@ enum node_stat_item { NR_VMSCAN_IMMEDIATE = 30, NR_DIRTIED = 31, NR_WRITTEN = 32, - NR_KERNEL_MISC_RECLAIMABLE = 33, - NR_FOLL_PIN_ACQUIRED = 34, - NR_FOLL_PIN_RELEASED = 35, - NR_KERNEL_STACK_KB = 36, - NR_PAGETABLE = 37, - NR_SWAPCACHE = 38, - NR_VM_NODE_STAT_ITEMS = 39, + NR_THROTTLED_WRITTEN = 33, + NR_KERNEL_MISC_RECLAIMABLE = 34, + NR_FOLL_PIN_ACQUIRED = 35, + NR_FOLL_PIN_RELEASED = 36, + NR_KERNEL_STACK_KB = 37, + NR_PAGETABLE = 38, + NR_SWAPCACHE = 39, + NR_VM_NODE_STAT_ITEMS = 40, }; enum lru_list { @@ -3300,6 +3425,14 @@ enum lru_list { NR_LRU_LISTS = 5, }; +enum vmscan_throttle_state { + VMSCAN_THROTTLE_WRITEBACK = 0, + VMSCAN_THROTTLE_ISOLATED = 1, + VMSCAN_THROTTLE_NOPROGRESS = 2, + VMSCAN_THROTTLE_CONGESTED = 3, + NR_VMSCAN_THROTTLE = 4, +}; + struct pglist_data; struct lruvec { @@ -3395,6 +3528,9 @@ struct pglist_data { int node_id; wait_queue_head_t kswapd_wait; wait_queue_head_t pfmemalloc_wait; + wait_queue_head_t reclaim_wait[4]; + atomic_t nr_writeback_throttled; + long unsigned int nr_reclaim_start; struct task_struct *kswapd; int kswapd_order; enum zone_type kswapd_highest_zoneidx; @@ -3403,10 +3539,16 @@ struct pglist_data { enum zone_type kcompactd_highest_zoneidx; wait_queue_head_t kcompactd_wait; struct task_struct *kcompactd; + bool proactive_compact_trigger; long unsigned int totalreserve_pages; long unsigned int min_unmapped_pages; long unsigned int min_slab_pages; long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; struct zone_padding _pad1_; struct deferred_split deferred_split_queue; struct lruvec __lruvec; @@ -3416,7 +3558,14 @@ struct pglist_data { long: 64; struct zone_padding _pad2_; struct per_cpu_nodestat *per_cpu_nodestats; - atomic_long_t vm_stat[39]; + atomic_long_t vm_stat[40]; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; }; typedef unsigned int isolate_mode_t; @@ -3445,7 +3594,7 @@ struct per_cpu_zonestat { struct per_cpu_nodestat { s8 stat_threshold; - s8 vm_node_stat_diff[39]; + s8 vm_node_stat_diff[40]; }; enum { @@ -3844,6 +3993,8 @@ struct pacct_struct { long unsigned int ac_majflt; }; +struct core_state; + struct tty_struct; struct taskstats; @@ -3864,6 +4015,7 @@ struct signal_struct { struct task_struct *group_exit_task; int group_stop_count; unsigned int flags; + struct core_state *core_state; unsigned int is_child_subreaper: 1; unsigned int has_child_subreaper: 1; int posix_timer_id; @@ -4133,18 +4285,6 @@ enum rseq_event_mask_bits { RSEQ_EVENT_MIGRATE_BIT = 2, }; -struct movsl_mask { - int mask; - long: 32; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; -}; - enum fixed_addresses { VSYSCALL_PAGE = 511, FIX_DBGP_BASE = 512, @@ -4405,8 +4545,6 @@ struct xattr_handler; struct block_device; -struct workqueue_struct; - struct super_block { struct list_head s_list; dev_t s_dev; @@ -4452,7 +4590,7 @@ struct super_block { int cleancache_poolid; struct shrinker s_shrink; atomic_long_t s_remove_count; - atomic_long_t s_fsnotify_inode_refs; + atomic_long_t s_fsnotify_connectors; int s_readonly_remount; errseq_t s_wb_err; struct workqueue_struct *s_dio_done_wq; @@ -4553,6 +4691,24 @@ enum migrate_mode { MIGRATE_SYNC_NO_COPY = 3, }; +struct exception_table_entry { + int insn; + int fixup; + int type; +}; + +struct movsl_mask { + int mask; + long: 32; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; +}; + struct ctl_table; typedef int proc_handler(struct ctl_table *, int, void *, size_t *, loff_t *); @@ -4624,9 +4780,16 @@ struct key_restriction { struct key_type *keytype; }; +struct percpu_counter { + raw_spinlock_t lock; + s64 count; + struct list_head list; + s32 *counters; +}; + struct user_struct { refcount_t __count; - atomic_long_t epoll_watches; + struct percpu_counter epoll_watches; long unsigned int unix_inflight; atomic_long_t pipe_bufs; struct hlist_node uidhash_node; @@ -4641,6 +4804,17 @@ struct group_info { kgid_t gid[0]; }; +struct core_thread { + struct task_struct *task; + struct core_thread *next; +}; + +struct core_state { + atomic_t nr_threads; + struct core_thread dumper; + struct completion startup; +}; + struct delayed_call { void (*fn)(void *); void *arg; @@ -4681,15 +4855,12 @@ struct wait_page_queue; struct kiocb { struct file *ki_filp; loff_t ki_pos; - void (*ki_complete)(struct kiocb *, long int, long int); + void (*ki_complete)(struct kiocb *, long int); void *private; int ki_flags; u16 ki_hint; u16 ki_ioprio; - union { - unsigned int ki_cookie; - struct wait_page_queue *ki_waitq; - }; + struct wait_page_queue *ki_waitq; }; struct iattr { @@ -4704,13 +4875,6 @@ struct iattr { struct file *ia_file; }; -struct percpu_counter { - raw_spinlock_t lock; - s64 count; - struct list_head list; - s32 *counters; -}; - typedef __kernel_uid32_t projid_t; typedef struct { @@ -4876,6 +5040,150 @@ struct quotactl_ops { int (*rm_xquota)(struct super_block *, unsigned int); }; +enum module_state { + MODULE_STATE_LIVE = 0, + MODULE_STATE_COMING = 1, + MODULE_STATE_GOING = 2, + MODULE_STATE_UNFORMED = 3, +}; + +struct module_param_attrs; + +struct module_kobject { + struct kobject kobj; + struct module *mod; + struct kobject *drivers_dir; + struct module_param_attrs *mp; + struct completion *kobj_completion; +}; + +struct latch_tree_node { + struct rb_node node[2]; +}; + +struct mod_tree_node { + struct module *mod; + struct latch_tree_node node; +}; + +struct module_layout { + void *base; + unsigned int size; + unsigned int text_size; + unsigned int ro_size; + unsigned int ro_after_init_size; + struct mod_tree_node mtn; +}; + +struct mod_arch_specific { + unsigned int num_orcs; + int *orc_unwind_ip; + struct orc_entry *orc_unwind; +}; + +struct elf64_sym; + +typedef struct elf64_sym Elf64_Sym; + +struct mod_kallsyms { + Elf64_Sym *symtab; + unsigned int num_symtab; + char *strtab; + char *typetab; +}; + +struct module_attribute; + +struct kernel_param; + +struct module_sect_attrs; + +struct module_notes_attrs; + +struct trace_event_call; + +struct trace_eval_map; + +struct error_injection_entry; + +struct module { + enum module_state state; + struct list_head list; + char name[56]; + struct module_kobject mkobj; + struct module_attribute *modinfo_attrs; + const char *version; + const char *srcversion; + struct kobject *holders_dir; + const struct kernel_symbol *syms; + const s32 *crcs; + unsigned int num_syms; + struct mutex param_lock; + struct kernel_param *kp; + unsigned int num_kp; + unsigned int num_gpl_syms; + const struct kernel_symbol *gpl_syms; + const s32 *gpl_crcs; + bool using_gplonly_symbols; + bool async_probe_requested; + unsigned int num_exentries; + struct exception_table_entry *extable; + int (*init)(); + long: 64; + long: 64; + struct module_layout core_layout; + struct module_layout init_layout; + struct mod_arch_specific arch; + long unsigned int taints; + unsigned int num_bugs; + struct list_head bug_list; + struct bug_entry *bug_table; + struct mod_kallsyms *kallsyms; + struct mod_kallsyms core_kallsyms; + struct module_sect_attrs *sect_attrs; + struct module_notes_attrs *notes_attrs; + char *args; + void *percpu; + unsigned int percpu_size; + void *noinstr_text_start; + unsigned int noinstr_text_size; + unsigned int num_tracepoints; + tracepoint_ptr_t *tracepoints_ptrs; + unsigned int num_srcu_structs; + struct srcu_struct **srcu_struct_ptrs; + unsigned int num_bpf_raw_events; + struct bpf_raw_event_map *bpf_raw_events; + unsigned int btf_data_size; + void *btf_data; + struct jump_entry *jump_entries; + unsigned int num_jump_entries; + unsigned int num_trace_bprintk_fmt; + const char **trace_bprintk_fmt_start; + struct trace_event_call **trace_events; + unsigned int num_trace_events; + struct trace_eval_map **trace_evals; + unsigned int num_trace_evals; + unsigned int num_ftrace_callsites; + long unsigned int *ftrace_callsites; + void *kprobes_text_start; + unsigned int kprobes_text_size; + long unsigned int *kprobe_blacklist; + unsigned int num_kprobe_blacklist; + int num_static_call_sites; + struct static_call_site *static_call_sites; + struct list_head source_list; + struct list_head target_list; + void (*exit)(); + atomic_t refcnt; + struct error_injection_entry *ei_funcs; + unsigned int num_ei_funcs; + long: 32; + long: 64; + long: 64; + long: 64; + long: 64; +}; + struct writeback_control; struct readahead_control; @@ -4915,7 +5223,7 @@ struct inode_operations { struct dentry * (*lookup)(struct inode *, struct dentry *, unsigned int); const char * (*get_link)(struct dentry *, struct inode *, struct delayed_call *); int (*permission)(struct user_namespace *, struct inode *, int); - struct posix_acl * (*get_acl)(struct inode *, int); + struct posix_acl * (*get_acl)(struct inode *, int, bool); int (*readlink)(struct dentry *, char *, int); int (*create)(struct user_namespace *, struct inode *, struct dentry *, umode_t, bool); int (*link)(struct dentry *, struct inode *, struct dentry *); @@ -5156,84 +5464,86 @@ enum vm_event_item { PGREUSE = 24, PGSTEAL_KSWAPD = 25, PGSTEAL_DIRECT = 26, - PGSCAN_KSWAPD = 27, - PGSCAN_DIRECT = 28, - PGSCAN_DIRECT_THROTTLE = 29, - PGSCAN_ANON = 30, - PGSCAN_FILE = 31, - PGSTEAL_ANON = 32, - PGSTEAL_FILE = 33, - PGSCAN_ZONE_RECLAIM_FAILED = 34, - PGINODESTEAL = 35, - SLABS_SCANNED = 36, - KSWAPD_INODESTEAL = 37, - KSWAPD_LOW_WMARK_HIT_QUICKLY = 38, - KSWAPD_HIGH_WMARK_HIT_QUICKLY = 39, - PAGEOUTRUN = 40, - PGROTATED = 41, - DROP_PAGECACHE = 42, - DROP_SLAB = 43, - OOM_KILL = 44, - NUMA_PTE_UPDATES = 45, - NUMA_HUGE_PTE_UPDATES = 46, - NUMA_HINT_FAULTS = 47, - NUMA_HINT_FAULTS_LOCAL = 48, - NUMA_PAGE_MIGRATE = 49, - PGMIGRATE_SUCCESS = 50, - PGMIGRATE_FAIL = 51, - THP_MIGRATION_SUCCESS = 52, - THP_MIGRATION_FAIL = 53, - THP_MIGRATION_SPLIT = 54, - COMPACTMIGRATE_SCANNED = 55, - COMPACTFREE_SCANNED = 56, - COMPACTISOLATED = 57, - COMPACTSTALL = 58, - COMPACTFAIL = 59, - COMPACTSUCCESS = 60, - KCOMPACTD_WAKE = 61, - KCOMPACTD_MIGRATE_SCANNED = 62, - KCOMPACTD_FREE_SCANNED = 63, - HTLB_BUDDY_PGALLOC = 64, - HTLB_BUDDY_PGALLOC_FAIL = 65, - CMA_ALLOC_SUCCESS = 66, - CMA_ALLOC_FAIL = 67, - UNEVICTABLE_PGCULLED = 68, - UNEVICTABLE_PGSCANNED = 69, - UNEVICTABLE_PGRESCUED = 70, - UNEVICTABLE_PGMLOCKED = 71, - UNEVICTABLE_PGMUNLOCKED = 72, - UNEVICTABLE_PGCLEARED = 73, - UNEVICTABLE_PGSTRANDED = 74, - THP_FAULT_ALLOC = 75, - THP_FAULT_FALLBACK = 76, - THP_FAULT_FALLBACK_CHARGE = 77, - THP_COLLAPSE_ALLOC = 78, - THP_COLLAPSE_ALLOC_FAILED = 79, - THP_FILE_ALLOC = 80, - THP_FILE_FALLBACK = 81, - THP_FILE_FALLBACK_CHARGE = 82, - THP_FILE_MAPPED = 83, - THP_SPLIT_PAGE = 84, - THP_SPLIT_PAGE_FAILED = 85, - THP_DEFERRED_SPLIT_PAGE = 86, - THP_SPLIT_PMD = 87, - THP_SPLIT_PUD = 88, - THP_ZERO_PAGE_ALLOC = 89, - THP_ZERO_PAGE_ALLOC_FAILED = 90, - THP_SWPOUT = 91, - THP_SWPOUT_FALLBACK = 92, - BALLOON_INFLATE = 93, - BALLOON_DEFLATE = 94, - BALLOON_MIGRATE = 95, - SWAP_RA = 96, - SWAP_RA_HIT = 97, - DIRECT_MAP_LEVEL2_SPLIT = 98, - DIRECT_MAP_LEVEL3_SPLIT = 99, - NR_VM_EVENT_ITEMS = 100, + PGDEMOTE_KSWAPD = 27, + PGDEMOTE_DIRECT = 28, + PGSCAN_KSWAPD = 29, + PGSCAN_DIRECT = 30, + PGSCAN_DIRECT_THROTTLE = 31, + PGSCAN_ANON = 32, + PGSCAN_FILE = 33, + PGSTEAL_ANON = 34, + PGSTEAL_FILE = 35, + PGSCAN_ZONE_RECLAIM_FAILED = 36, + PGINODESTEAL = 37, + SLABS_SCANNED = 38, + KSWAPD_INODESTEAL = 39, + KSWAPD_LOW_WMARK_HIT_QUICKLY = 40, + KSWAPD_HIGH_WMARK_HIT_QUICKLY = 41, + PAGEOUTRUN = 42, + PGROTATED = 43, + DROP_PAGECACHE = 44, + DROP_SLAB = 45, + OOM_KILL = 46, + NUMA_PTE_UPDATES = 47, + NUMA_HUGE_PTE_UPDATES = 48, + NUMA_HINT_FAULTS = 49, + NUMA_HINT_FAULTS_LOCAL = 50, + NUMA_PAGE_MIGRATE = 51, + PGMIGRATE_SUCCESS = 52, + PGMIGRATE_FAIL = 53, + THP_MIGRATION_SUCCESS = 54, + THP_MIGRATION_FAIL = 55, + THP_MIGRATION_SPLIT = 56, + COMPACTMIGRATE_SCANNED = 57, + COMPACTFREE_SCANNED = 58, + COMPACTISOLATED = 59, + COMPACTSTALL = 60, + COMPACTFAIL = 61, + COMPACTSUCCESS = 62, + KCOMPACTD_WAKE = 63, + KCOMPACTD_MIGRATE_SCANNED = 64, + KCOMPACTD_FREE_SCANNED = 65, + HTLB_BUDDY_PGALLOC = 66, + HTLB_BUDDY_PGALLOC_FAIL = 67, + CMA_ALLOC_SUCCESS = 68, + CMA_ALLOC_FAIL = 69, + UNEVICTABLE_PGCULLED = 70, + UNEVICTABLE_PGSCANNED = 71, + UNEVICTABLE_PGRESCUED = 72, + UNEVICTABLE_PGMLOCKED = 73, + UNEVICTABLE_PGMUNLOCKED = 74, + UNEVICTABLE_PGCLEARED = 75, + UNEVICTABLE_PGSTRANDED = 76, + THP_FAULT_ALLOC = 77, + THP_FAULT_FALLBACK = 78, + THP_FAULT_FALLBACK_CHARGE = 79, + THP_COLLAPSE_ALLOC = 80, + THP_COLLAPSE_ALLOC_FAILED = 81, + THP_FILE_ALLOC = 82, + THP_FILE_FALLBACK = 83, + THP_FILE_FALLBACK_CHARGE = 84, + THP_FILE_MAPPED = 85, + THP_SPLIT_PAGE = 86, + THP_SPLIT_PAGE_FAILED = 87, + THP_DEFERRED_SPLIT_PAGE = 88, + THP_SPLIT_PMD = 89, + THP_SPLIT_PUD = 90, + THP_ZERO_PAGE_ALLOC = 91, + THP_ZERO_PAGE_ALLOC_FAILED = 92, + THP_SWPOUT = 93, + THP_SWPOUT_FALLBACK = 94, + BALLOON_INFLATE = 95, + BALLOON_DEFLATE = 96, + BALLOON_MIGRATE = 97, + SWAP_RA = 98, + SWAP_RA_HIT = 99, + DIRECT_MAP_LEVEL2_SPLIT = 100, + DIRECT_MAP_LEVEL3_SPLIT = 101, + NR_VM_EVENT_ITEMS = 102, }; struct vm_event_state { - long unsigned int event[100]; + long unsigned int event[102]; }; enum memblock_flags { @@ -5241,6 +5551,7 @@ enum memblock_flags { MEMBLOCK_HOTPLUG = 1, MEMBLOCK_MIRROR = 2, MEMBLOCK_NOMAP = 4, + MEMBLOCK_DRIVER_MANAGED = 8, }; struct memblock_region { @@ -5776,17 +6087,17 @@ struct debug_store_buffers { struct cea_exception_stacks { char DF_stack_guard[4096]; - char DF_stack[4096]; + char DF_stack[8192]; char NMI_stack_guard[4096]; - char NMI_stack[4096]; + char NMI_stack[8192]; char DB_stack_guard[4096]; - char DB_stack[4096]; + char DB_stack[8192]; char MCE_stack_guard[4096]; - char MCE_stack[4096]; + char MCE_stack[8192]; char VC_stack_guard[4096]; - char VC_stack[4096]; + char VC_stack[8192]; char VC2_stack_guard[4096]; - char VC2_stack[4096]; + char VC2_stack[8192]; char IST_top_guard[4096]; }; @@ -6308,7 +6619,7 @@ struct tlb_state { struct mm_struct *loaded_mm; union { struct mm_struct *last_user_mm; - long unsigned int last_user_mm_ibpb; + long unsigned int last_user_mm_spec; }; u16 loaded_mm_asid; u16 next_asid; @@ -6362,6 +6673,7 @@ struct kernfs_elem_dir { long unsigned int subdirs; struct rb_root children; struct kernfs_root *root; + long unsigned int rev; }; struct kernfs_syscall_ops; @@ -6430,7 +6742,6 @@ struct kernfs_ops { ssize_t (*write)(struct kernfs_open_file *, char *, size_t, loff_t); __poll_t (*poll)(struct kernfs_open_file *, struct poll_table_struct *); int (*mmap)(struct kernfs_open_file *, struct vm_area_struct *); - struct lock_class_key lockdep_key; }; struct kernfs_syscall_ops { @@ -6488,7 +6799,7 @@ struct bin_attribute { struct attribute attr; size_t size; void *private; - struct address_space *mapping; + struct address_space * (*f_mapping)(); ssize_t (*read)(struct file *, struct kobject *, struct bin_attribute *, char *, loff_t, size_t); ssize_t (*write)(struct file *, struct kobject *, struct bin_attribute *, char *, loff_t, size_t); int (*mmap)(struct file *, struct kobject *, struct bin_attribute *, struct vm_area_struct *); @@ -6617,7 +6928,7 @@ struct bus_type { int (*uevent)(struct device *, struct kobj_uevent_env *); int (*probe)(struct device *); void (*sync_state)(struct device *); - int (*remove)(struct device *); + void (*remove)(struct device *); void (*shutdown)(struct device *); int (*online)(struct device *); int (*offline)(struct device *); @@ -6695,6 +7006,96 @@ struct class { struct subsys_private *p; }; +struct va_alignment { + int flags; + long unsigned int mask; + long unsigned int bits; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; +}; + +typedef __u64 Elf64_Addr; + +typedef __u16 Elf64_Half; + +typedef __u32 Elf64_Word; + +typedef __u64 Elf64_Xword; + +typedef __s64 Elf64_Sxword; + +typedef struct { + Elf64_Sxword d_tag; + union { + Elf64_Xword d_val; + Elf64_Addr d_ptr; + } d_un; +} Elf64_Dyn; + +struct elf64_sym { + Elf64_Word st_name; + unsigned char st_info; + unsigned char st_other; + Elf64_Half st_shndx; + Elf64_Addr st_value; + Elf64_Xword st_size; +}; + +struct kernel_param_ops { + unsigned int flags; + int (*set)(const char *, const struct kernel_param *); + int (*get)(char *, const struct kernel_param *); + void (*free)(void *); +}; + +struct kparam_string; + +struct kparam_array; + +struct kernel_param { + const char *name; + struct module *mod; + const struct kernel_param_ops *ops; + const u16 perm; + s8 level; + u8 flags; + union { + void *arg; + const struct kparam_string *str; + const struct kparam_array *arr; + }; +}; + +struct kparam_string { + unsigned int maxlen; + char *string; +}; + +struct kparam_array { + unsigned int max; + unsigned int elemsize; + unsigned int *num; + const struct kernel_param_ops *ops; + void *elem; +}; + +struct error_injection_entry { + long unsigned int addr; + int etype; +}; + +struct module_attribute { + struct attribute attr; + ssize_t (*show)(struct module_attribute *, struct module_kobject *, char *); + ssize_t (*store)(struct module_attribute *, struct module_kobject *, const char *, size_t); + void (*setup)(struct module *, const char *); + int (*test)(struct module *); + void (*free)(struct module *); +}; + struct of_device_id { char name[32]; char type[32]; @@ -6802,21 +7203,15 @@ struct kobj_attribute___2; struct file_system_type___2; -typedef short int __s16; - -typedef __s16 s16; - typedef __u32 __le32; +typedef unsigned int uint; + typedef long unsigned int ulong; -typedef long unsigned int irq_hw_number_t; +typedef u16 uint16_t; -struct kernel_symbol { - int value_offset; - int name_offset; - int namespace_offset; -}; +typedef long unsigned int irq_hw_number_t; typedef int (*initcall_t)(); @@ -6847,32 +7242,6 @@ enum ftrace_dump_mode { DUMP_ORIG = 2, }; -struct bug_entry { - int bug_addr_disp; - int file_disp; - short unsigned int line; - short unsigned int flags; -}; - -typedef const int tracepoint_ptr_t; - -struct bpf_raw_event_map { - struct tracepoint *tp; - void *bpf_func; - u32 num_args; - u32 writable_size; - long: 64; -}; - -struct orc_entry { - s16 sp_offset; - s16 bp_offset; - unsigned int sp_reg: 4; - unsigned int bp_reg: 4; - unsigned int type: 2; - unsigned int end: 1; -} __attribute__((packed)); - enum perf_event_state { PERF_EVENT_STATE_DEAD = 4294967292, PERF_EVENT_STATE_EXIT = 4294967293, @@ -7043,6 +7412,7 @@ struct hw_perf_event { struct irq_work { struct __call_single_node node; void (*func)(struct irq_work *); + struct rcuwait irqwait; }; struct perf_addr_filters_head { @@ -7091,8 +7461,6 @@ struct perf_addr_filter_range; struct bpf_prog; -struct trace_event_call; - struct event_filter; struct perf_cgroup; @@ -7163,6 +7531,7 @@ struct perf_event { void *overflow_handler_context; perf_overflow_handler_t orig_overflow_handler; struct bpf_prog *prog; + u64 bpf_cookie; struct trace_event_call *tp_event; struct event_filter *filter; struct ftrace_ops ftrace_ops; @@ -7261,14 +7630,6 @@ struct pollfd { short int revents; }; -typedef void (*smp_call_func_t)(void *); - -struct __call_single_data { - struct __call_single_node node; - smp_call_func_t func; - void *info; -}; - struct wait_queue_entry; typedef int (*wait_queue_func_t)(struct wait_queue_entry *, unsigned int, int, void *); @@ -7282,94 +7643,32 @@ struct wait_queue_entry { typedef struct wait_queue_entry wait_queue_entry_t; -struct timer_list { - struct hlist_node entry; - long unsigned int expires; - void (*function)(struct timer_list *); - u32 flags; - struct lockdep_map lockdep_map; -}; - -struct delayed_work { - struct work_struct work; - struct timer_list timer; - struct workqueue_struct *wq; - int cpu; -}; - struct rcu_work { struct work_struct work; struct callback_head rcu; struct workqueue_struct *wq; }; -struct rcu_segcblist { - struct callback_head *head; - struct callback_head **tails[4]; - long unsigned int gp_seq[4]; - long int len; - long int seglen[4]; - u8 flags; +struct ldt_struct { + struct desc_struct *entries; + unsigned int nr_entries; + int slot; }; -struct srcu_node; - -struct srcu_struct; - -struct srcu_data { - long unsigned int srcu_lock_count[2]; - long unsigned int srcu_unlock_count[2]; - long: 64; - long: 64; - long: 64; - long: 64; - spinlock_t lock; - struct rcu_segcblist srcu_cblist; - long unsigned int srcu_gp_seq_needed; - long unsigned int srcu_gp_seq_needed_exp; - bool srcu_cblist_invoking; - struct timer_list delay_work; - struct work_struct work; - struct callback_head srcu_barrier_head; - struct srcu_node *mynode; - long unsigned int grpmask; - int cpu; - struct srcu_struct *ssp; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; -}; - -struct srcu_node { - spinlock_t lock; - long unsigned int srcu_have_cbs[4]; - long unsigned int srcu_data_have_cbs[4]; - long unsigned int srcu_gp_seq_needed_exp; - struct srcu_node *srcu_parent; - int grplo; - int grphi; -}; - -struct srcu_struct { - struct srcu_node node[9]; - struct srcu_node *level[3]; - struct mutex srcu_cb_mutex; - spinlock_t lock; - struct mutex srcu_gp_mutex; - unsigned int srcu_idx; - long unsigned int srcu_gp_seq; - long unsigned int srcu_gp_seq_needed; - long unsigned int srcu_gp_seq_needed_exp; - long unsigned int srcu_last_gp_end; - struct srcu_data *sda; - long unsigned int srcu_barrier_seq; - struct mutex srcu_barrier_mutex; - struct completion srcu_barrier_completion; - atomic_t srcu_barrier_cpu_cnt; - struct delayed_work work; - struct lockdep_map dep_map; +struct folio { + union { + struct { + long unsigned int flags; + struct list_head lru; + struct address_space *mapping; + long unsigned int index; + void *private; + atomic_t _mapcount; + atomic_t _refcount; + long unsigned int memcg_data; + }; + struct page page; + }; }; struct anon_vma { @@ -7455,52 +7754,6 @@ struct ctl_table_root { int (*permissions)(struct ctl_table_header *, struct ctl_table *); }; -enum umh_disable_depth { - UMH_ENABLED = 0, - UMH_FREEZING = 1, - UMH_DISABLED = 2, -}; - -struct va_alignment { - int flags; - long unsigned int mask; - long unsigned int bits; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; -}; - -typedef __u64 Elf64_Addr; - -typedef __u16 Elf64_Half; - -typedef __u32 Elf64_Word; - -typedef __u64 Elf64_Xword; - -typedef __s64 Elf64_Sxword; - -typedef struct { - Elf64_Sxword d_tag; - union { - Elf64_Xword d_val; - Elf64_Addr d_ptr; - } d_un; -} Elf64_Dyn; - -struct elf64_sym { - Elf64_Word st_name; - unsigned char st_info; - unsigned char st_other; - Elf64_Half st_shndx; - Elf64_Addr st_value; - Elf64_Xword st_size; -}; - -typedef struct elf64_sym Elf64_Sym; - struct seq_file { char *buf; size_t size; @@ -7531,204 +7784,6 @@ enum refcount_saturation_type { REFCOUNT_DEC_LEAK = 4, }; -struct kernel_param; - -struct kernel_param_ops { - unsigned int flags; - int (*set)(const char *, const struct kernel_param *); - int (*get)(char *, const struct kernel_param *); - void (*free)(void *); -}; - -struct kparam_string; - -struct kparam_array; - -struct kernel_param { - const char *name; - struct module *mod; - const struct kernel_param_ops *ops; - const u16 perm; - s8 level; - u8 flags; - union { - void *arg; - const struct kparam_string *str; - const struct kparam_array *arr; - }; -}; - -struct kparam_string { - unsigned int maxlen; - char *string; -}; - -struct kparam_array { - unsigned int max; - unsigned int elemsize; - unsigned int *num; - const struct kernel_param_ops *ops; - void *elem; -}; - -enum module_state { - MODULE_STATE_LIVE = 0, - MODULE_STATE_COMING = 1, - MODULE_STATE_GOING = 2, - MODULE_STATE_UNFORMED = 3, -}; - -struct module_param_attrs; - -struct module_kobject { - struct kobject kobj; - struct module *mod; - struct kobject *drivers_dir; - struct module_param_attrs *mp; - struct completion *kobj_completion; -}; - -struct latch_tree_node { - struct rb_node node[2]; -}; - -struct mod_tree_node { - struct module *mod; - struct latch_tree_node node; -}; - -struct module_layout { - void *base; - unsigned int size; - unsigned int text_size; - unsigned int ro_size; - unsigned int ro_after_init_size; - struct mod_tree_node mtn; -}; - -struct mod_arch_specific { - unsigned int num_orcs; - int *orc_unwind_ip; - struct orc_entry *orc_unwind; -}; - -struct mod_kallsyms { - Elf64_Sym *symtab; - unsigned int num_symtab; - char *strtab; - char *typetab; -}; - -struct module_attribute; - -struct exception_table_entry; - -struct module_sect_attrs; - -struct module_notes_attrs; - -struct trace_eval_map; - -struct error_injection_entry; - -struct module { - enum module_state state; - struct list_head list; - char name[56]; - struct module_kobject mkobj; - struct module_attribute *modinfo_attrs; - const char *version; - const char *srcversion; - struct kobject *holders_dir; - const struct kernel_symbol *syms; - const s32 *crcs; - unsigned int num_syms; - struct mutex param_lock; - struct kernel_param *kp; - unsigned int num_kp; - unsigned int num_gpl_syms; - const struct kernel_symbol *gpl_syms; - const s32 *gpl_crcs; - bool using_gplonly_symbols; - bool async_probe_requested; - unsigned int num_exentries; - struct exception_table_entry *extable; - int (*init)(); - long: 64; - long: 64; - struct module_layout core_layout; - struct module_layout init_layout; - struct mod_arch_specific arch; - long unsigned int taints; - unsigned int num_bugs; - struct list_head bug_list; - struct bug_entry *bug_table; - struct mod_kallsyms *kallsyms; - struct mod_kallsyms core_kallsyms; - struct module_sect_attrs *sect_attrs; - struct module_notes_attrs *notes_attrs; - char *args; - void *percpu; - unsigned int percpu_size; - void *noinstr_text_start; - unsigned int noinstr_text_size; - unsigned int num_tracepoints; - tracepoint_ptr_t *tracepoints_ptrs; - unsigned int num_srcu_structs; - struct srcu_struct **srcu_struct_ptrs; - unsigned int num_bpf_raw_events; - struct bpf_raw_event_map *bpf_raw_events; - unsigned int btf_data_size; - void *btf_data; - struct jump_entry *jump_entries; - unsigned int num_jump_entries; - unsigned int num_trace_bprintk_fmt; - const char **trace_bprintk_fmt_start; - struct trace_event_call **trace_events; - unsigned int num_trace_events; - struct trace_eval_map **trace_evals; - unsigned int num_trace_evals; - unsigned int num_ftrace_callsites; - long unsigned int *ftrace_callsites; - void *kprobes_text_start; - unsigned int kprobes_text_size; - long unsigned int *kprobe_blacklist; - unsigned int num_kprobe_blacklist; - int num_static_call_sites; - struct static_call_site *static_call_sites; - struct list_head source_list; - struct list_head target_list; - void (*exit)(); - atomic_t refcnt; - struct error_injection_entry *ei_funcs; - unsigned int num_ei_funcs; - long: 32; - long: 64; - long: 64; - long: 64; - long: 64; -}; - -struct error_injection_entry { - long unsigned int addr; - int etype; -}; - -struct module_attribute { - struct attribute attr; - ssize_t (*show)(struct module_attribute *, struct module_kobject *, char *); - ssize_t (*store)(struct module_attribute *, struct module_kobject *, const char *, size_t); - void (*setup)(struct module *, const char *); - int (*test)(struct module *); - void (*free)(struct module *); -}; - -struct exception_table_entry { - int insn; - int fixup; - int handler; -}; - struct trace_event_functions; struct trace_event { @@ -7752,7 +7807,10 @@ struct trace_event_call { struct trace_event event; char *print_fmt; struct event_filter *filter; - void *mod; + union { + void *module; + atomic_t refcnt; + }; void *data; int flags; int perf_refcount; @@ -7838,16 +7896,10 @@ struct memcg_padding { }; struct memcg_vmstats { - long int state[42]; - long unsigned int events[100]; - long int state_pending[42]; - long unsigned int events_pending[100]; -}; - -enum memcg_kmem_state { - KMEM_NONE = 0, - KMEM_ALLOCATED = 1, - KMEM_ONLINE = 2, + long int state[43]; + long unsigned int events[102]; + long int state_pending[43]; + long unsigned int events_pending[102]; }; struct fprop_global { @@ -7920,15 +7972,8 @@ struct mem_cgroup { bool tcpmem_active; int tcpmem_pressure; int kmemcg_id; - enum memcg_kmem_state kmem_state; struct obj_cgroup *objcg; struct list_head objcg_list; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; struct memcg_padding _pad2_; atomic_t moving_account; struct task_struct *move_lock_task; @@ -8006,12 +8051,17 @@ struct bio_list { struct bio *tail; }; +struct request; + struct blk_plug { - struct list_head mq_list; - struct list_head cb_list; + struct request *mq_list; + struct request *cached_rq; + short unsigned int nr_ios; short unsigned int rq_count; bool multiple_queues; + bool has_elevator; bool nowait; + struct list_head cb_list; }; struct reclaim_state { @@ -8045,6 +8095,7 @@ struct bdi_writeback { struct list_head b_more_io; struct list_head b_dirty_time; spinlock_t list_lock; + atomic_t writeback_inodes; struct percpu_counter stat[4]; long unsigned int congested; long unsigned int bw_time_stamp; @@ -8060,6 +8111,7 @@ struct bdi_writeback { spinlock_t work_lock; struct list_head work_list; struct delayed_work dwork; + struct delayed_work bw_dwork; long unsigned int dirty_sleep; struct list_head bdi_node; struct percpu_ref refcnt; @@ -8142,6 +8194,7 @@ struct perf_event_context { struct list_head flexible_active; int nr_events; int nr_active; + int nr_user; int is_active; int nr_stat; int nr_freq; @@ -8191,28 +8244,6 @@ struct blk_rq_stat { u64 batch; }; -struct sbitmap_word; - -struct sbitmap { - unsigned int depth; - unsigned int shift; - unsigned int map_nr; - bool round_robin; - struct sbitmap_word *map; - unsigned int *alloc_hint; -}; - -struct sbq_wait_state; - -struct sbitmap_queue { - struct sbitmap sb; - unsigned int wake_batch; - atomic_t wake_index; - struct sbq_wait_state *ws; - atomic_t ws_active; - unsigned int min_shallow_depth; -}; - enum blk_bounce { BLK_BOUNCE_NONE = 0, BLK_BOUNCE_HIGH = 1, @@ -8255,15 +8286,6 @@ struct queue_limits { enum blk_zoned_model zoned; }; -struct bsg_ops; - -struct bsg_class_device { - struct device *class_dev; - int minor; - struct request_queue *queue; - const struct bsg_ops *ops; -}; - typedef void *mempool_alloc_t(gfp_t, void *); typedef void mempool_free_t(void *, void *); @@ -8281,9 +8303,12 @@ struct mempool_s { typedef struct mempool_s mempool_t; +struct bio_alloc_cache; + struct bio_set { struct kmem_cache *bio_slab; unsigned int front_pad; + struct bio_alloc_cache *cache; mempool_t bio_pool; mempool_t bvec_pool; unsigned int back_pad; @@ -8291,10 +8316,9 @@ struct bio_set { struct bio_list rescue_list; struct work_struct rescue_work; struct workqueue_struct *rescue_workqueue; + struct hlist_node cpuhp_dead; }; -struct request; - struct elevator_queue; struct blk_queue_stats; @@ -8307,8 +8331,12 @@ struct blk_mq_ctx; struct blk_mq_hw_ctx; +struct gendisk; + struct blk_stat_callback; +struct blk_mq_tags; + struct blkcg_gq; struct blk_trace; @@ -8319,6 +8347,8 @@ struct throtl_data; struct blk_mq_tag_set; +struct blk_independent_access_ranges; + struct request_queue { struct request *last_merge; struct elevator_queue *elevator; @@ -8330,12 +8360,12 @@ struct request_queue { unsigned int queue_depth; struct blk_mq_hw_ctx **queue_hw_ctx; unsigned int nr_hw_queues; - struct backing_dev_info *backing_dev_info; void *queuedata; long unsigned int queue_flags; atomic_t pm_only; int id; spinlock_t queue_lock; + struct gendisk *disk; struct kobject kobj; struct kobject *mq_kobj; long unsigned int nr_requests; @@ -8347,17 +8377,14 @@ struct request_queue { struct blk_rq_stat poll_stat[16]; struct timer_list timeout; struct work_struct timeout_work; - atomic_t nr_active_requests_shared_sbitmap; - struct sbitmap_queue sched_bitmap_tags; - struct sbitmap_queue sched_breserved_tags; + atomic_t nr_active_requests_shared_tags; + struct blk_mq_tags *sched_shared_tags; struct list_head icq_list; long unsigned int blkcg_pols[1]; struct blkcg_gq *root_blkg; struct list_head blkg_list; struct queue_limits limits; unsigned int required_elevator_features; - unsigned int sg_timeout; - unsigned int sg_reserved_size; int node; struct mutex debugfs_mutex; struct blk_trace *blk_trace; @@ -8370,11 +8397,11 @@ struct request_queue { struct list_head unused_hctx_list; spinlock_t unused_hctx_lock; int mq_freeze_depth; - struct bsg_class_device bsg_dev; struct throtl_data *td; struct callback_head callback_head; wait_queue_head_t mq_freeze_wq; struct mutex mq_freeze_lock; + int quiesce_depth; struct blk_mq_tag_set *tag_set; struct list_head tag_set_list; struct bio_set bio_split; @@ -8382,12 +8409,10 @@ struct request_queue { struct dentry *sched_debugfs_dir; struct dentry *rqos_debugfs_dir; bool mq_sysfs_init_done; - size_t cmd_size; u64 write_hints[5]; + struct blk_independent_access_ranges *ia_ranges; }; -struct bpf_run_ctx {}; - struct cgroup_base_stat { struct task_cputime cputime; }; @@ -8395,9 +8420,9 @@ struct cgroup_base_stat { struct psi_group {}; struct cgroup_bpf { - struct bpf_prog_array *effective[41]; - struct list_head progs[41]; - u32 flags[41]; + struct bpf_prog_array *effective[23]; + struct list_head progs[23]; + u32 flags[23]; struct list_head storages; struct bpf_prog_array *inactive; struct percpu_ref refcnt; @@ -8507,7 +8532,7 @@ struct taskstats { }; struct wait_page_queue { - struct page *page; + struct folio *folio; int bit_nr; wait_queue_entry_t wait; }; @@ -8558,6 +8583,7 @@ struct bio_vec; struct iov_iter { u8 iter_type; + bool nofault; bool data_source; size_t iov_offset; size_t count; @@ -8632,12 +8658,11 @@ struct cdev { struct disk_stats; -struct gendisk; - struct partition_meta_info; struct block_device { sector_t bd_start_sect; + sector_t bd_nr_sectors; struct disk_stats *bd_stats; long unsigned int bd_stamp; bool bd_read_only; @@ -8650,18 +8675,23 @@ struct block_device { void *bd_holder; int bd_holders; bool bd_write_holder; - struct list_head bd_holder_disks; struct kobject *bd_holder_dir; u8 bd_partno; spinlock_t bd_size_lock; struct gendisk *bd_disk; - struct backing_dev_info *bd_bdi; + struct request_queue *bd_queue; int bd_fsfreeze_count; struct mutex bd_fsfreeze_mutex; struct super_block *bd_fsfreeze_sb; struct partition_meta_info *bd_meta_info; }; +struct io_comp_batch { + struct request *req_list; + bool need_ts; + void (*complete)(struct io_comp_batch *); +}; + struct fc_log; struct p_log { @@ -8729,7 +8759,9 @@ struct bvec_iter { unsigned int bi_size; unsigned int bi_idx; unsigned int bi_bvec_done; -}; +} __attribute__((packed)); + +typedef unsigned int blk_qc_t; typedef void bio_end_io_t(struct bio *); @@ -8753,6 +8785,7 @@ struct bio { blk_status_t bi_status; atomic_t __bi_remaining; struct bvec_iter bi_iter; + blk_qc_t bi_cookie; bio_end_io_t *bi_end_io; void *bi_private; struct blkcg_gq *bi_blkg; @@ -8840,169 +8873,174 @@ enum cpuhp_state { CPUHP_SLUB_DEAD = 11, CPUHP_DEBUG_OBJ_DEAD = 12, CPUHP_MM_WRITEBACK_DEAD = 13, - CPUHP_MM_VMSTAT_DEAD = 14, - CPUHP_SOFTIRQ_DEAD = 15, - CPUHP_NET_MVNETA_DEAD = 16, - CPUHP_CPUIDLE_DEAD = 17, - CPUHP_ARM64_FPSIMD_DEAD = 18, - CPUHP_ARM_OMAP_WAKE_DEAD = 19, - CPUHP_IRQ_POLL_DEAD = 20, - CPUHP_BLOCK_SOFTIRQ_DEAD = 21, - CPUHP_ACPI_CPUDRV_DEAD = 22, - CPUHP_S390_PFAULT_DEAD = 23, - CPUHP_BLK_MQ_DEAD = 24, - CPUHP_FS_BUFF_DEAD = 25, - CPUHP_PRINTK_DEAD = 26, - CPUHP_MM_MEMCQ_DEAD = 27, - CPUHP_PERCPU_CNT_DEAD = 28, - CPUHP_RADIX_DEAD = 29, - CPUHP_PAGE_ALLOC = 30, - CPUHP_NET_DEV_DEAD = 31, - CPUHP_PCI_XGENE_DEAD = 32, - CPUHP_IOMMU_IOVA_DEAD = 33, - CPUHP_LUSTRE_CFS_DEAD = 34, - CPUHP_AP_ARM_CACHE_B15_RAC_DEAD = 35, - CPUHP_PADATA_DEAD = 36, - CPUHP_WORKQUEUE_PREP = 37, - CPUHP_POWER_NUMA_PREPARE = 38, - CPUHP_HRTIMERS_PREPARE = 39, - CPUHP_PROFILE_PREPARE = 40, - CPUHP_X2APIC_PREPARE = 41, - CPUHP_SMPCFD_PREPARE = 42, - CPUHP_RELAY_PREPARE = 43, - CPUHP_SLAB_PREPARE = 44, - CPUHP_MD_RAID5_PREPARE = 45, - CPUHP_RCUTREE_PREP = 46, - CPUHP_CPUIDLE_COUPLED_PREPARE = 47, - CPUHP_POWERPC_PMAC_PREPARE = 48, - CPUHP_POWERPC_MMU_CTX_PREPARE = 49, - CPUHP_XEN_PREPARE = 50, - CPUHP_XEN_EVTCHN_PREPARE = 51, - CPUHP_ARM_SHMOBILE_SCU_PREPARE = 52, - CPUHP_SH_SH3X_PREPARE = 53, - CPUHP_NET_FLOW_PREPARE = 54, - CPUHP_TOPOLOGY_PREPARE = 55, - CPUHP_NET_IUCV_PREPARE = 56, - CPUHP_ARM_BL_PREPARE = 57, - CPUHP_TRACE_RB_PREPARE = 58, - CPUHP_MM_ZS_PREPARE = 59, - CPUHP_MM_ZSWP_MEM_PREPARE = 60, - CPUHP_MM_ZSWP_POOL_PREPARE = 61, - CPUHP_KVM_PPC_BOOK3S_PREPARE = 62, - CPUHP_ZCOMP_PREPARE = 63, - CPUHP_TIMERS_PREPARE = 64, - CPUHP_MIPS_SOC_PREPARE = 65, - CPUHP_BP_PREPARE_DYN = 66, - CPUHP_BP_PREPARE_DYN_END = 86, - CPUHP_BRINGUP_CPU = 87, - CPUHP_AP_IDLE_DEAD = 88, - CPUHP_AP_OFFLINE = 89, - CPUHP_AP_SCHED_STARTING = 90, - CPUHP_AP_RCUTREE_DYING = 91, - CPUHP_AP_CPU_PM_STARTING = 92, - CPUHP_AP_IRQ_GIC_STARTING = 93, - CPUHP_AP_IRQ_HIP04_STARTING = 94, - CPUHP_AP_IRQ_APPLE_AIC_STARTING = 95, - CPUHP_AP_IRQ_ARMADA_XP_STARTING = 96, - CPUHP_AP_IRQ_BCM2836_STARTING = 97, - CPUHP_AP_IRQ_MIPS_GIC_STARTING = 98, - CPUHP_AP_IRQ_RISCV_STARTING = 99, - CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING = 100, - CPUHP_AP_ARM_MVEBU_COHERENCY = 101, - CPUHP_AP_MICROCODE_LOADER = 102, - CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING = 103, - CPUHP_AP_PERF_X86_STARTING = 104, - CPUHP_AP_PERF_X86_AMD_IBS_STARTING = 105, - CPUHP_AP_PERF_X86_CQM_STARTING = 106, - CPUHP_AP_PERF_X86_CSTATE_STARTING = 107, - CPUHP_AP_PERF_XTENSA_STARTING = 108, - CPUHP_AP_MIPS_OP_LOONGSON3_STARTING = 109, - CPUHP_AP_ARM_SDEI_STARTING = 110, - CPUHP_AP_ARM_VFP_STARTING = 111, - CPUHP_AP_ARM64_DEBUG_MONITORS_STARTING = 112, - CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING = 113, - CPUHP_AP_PERF_ARM_ACPI_STARTING = 114, - CPUHP_AP_PERF_ARM_STARTING = 115, - CPUHP_AP_ARM_L2X0_STARTING = 116, - CPUHP_AP_EXYNOS4_MCT_TIMER_STARTING = 117, - CPUHP_AP_ARM_ARCH_TIMER_STARTING = 118, - CPUHP_AP_ARM_GLOBAL_TIMER_STARTING = 119, - CPUHP_AP_JCORE_TIMER_STARTING = 120, - CPUHP_AP_ARM_TWD_STARTING = 121, - CPUHP_AP_QCOM_TIMER_STARTING = 122, - CPUHP_AP_TEGRA_TIMER_STARTING = 123, - CPUHP_AP_ARMADA_TIMER_STARTING = 124, - CPUHP_AP_MARCO_TIMER_STARTING = 125, - CPUHP_AP_MIPS_GIC_TIMER_STARTING = 126, - CPUHP_AP_ARC_TIMER_STARTING = 127, - CPUHP_AP_RISCV_TIMER_STARTING = 128, - CPUHP_AP_CLINT_TIMER_STARTING = 129, - CPUHP_AP_CSKY_TIMER_STARTING = 130, - CPUHP_AP_TI_GP_TIMER_STARTING = 131, - CPUHP_AP_HYPERV_TIMER_STARTING = 132, - CPUHP_AP_KVM_STARTING = 133, - CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING = 134, - CPUHP_AP_KVM_ARM_VGIC_STARTING = 135, - CPUHP_AP_KVM_ARM_TIMER_STARTING = 136, - CPUHP_AP_DUMMY_TIMER_STARTING = 137, - CPUHP_AP_ARM_XEN_STARTING = 138, - CPUHP_AP_ARM_CORESIGHT_STARTING = 139, - CPUHP_AP_ARM_CORESIGHT_CTI_STARTING = 140, - CPUHP_AP_ARM64_ISNDEP_STARTING = 141, - CPUHP_AP_SMPCFD_DYING = 142, - CPUHP_AP_X86_TBOOT_DYING = 143, - CPUHP_AP_ARM_CACHE_B15_RAC_DYING = 144, - CPUHP_AP_ONLINE = 145, - CPUHP_TEARDOWN_CPU = 146, - CPUHP_AP_ONLINE_IDLE = 147, - CPUHP_AP_SCHED_WAIT_EMPTY = 148, - CPUHP_AP_SMPBOOT_THREADS = 149, - CPUHP_AP_X86_VDSO_VMA_ONLINE = 150, - CPUHP_AP_IRQ_AFFINITY_ONLINE = 151, - CPUHP_AP_BLK_MQ_ONLINE = 152, - CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS = 153, - CPUHP_AP_X86_INTEL_EPB_ONLINE = 154, - CPUHP_AP_PERF_ONLINE = 155, - CPUHP_AP_PERF_X86_ONLINE = 156, - CPUHP_AP_PERF_X86_UNCORE_ONLINE = 157, - CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE = 158, - CPUHP_AP_PERF_X86_AMD_POWER_ONLINE = 159, - CPUHP_AP_PERF_X86_RAPL_ONLINE = 160, - CPUHP_AP_PERF_X86_CQM_ONLINE = 161, - CPUHP_AP_PERF_X86_CSTATE_ONLINE = 162, - CPUHP_AP_PERF_X86_IDXD_ONLINE = 163, - CPUHP_AP_PERF_S390_CF_ONLINE = 164, - CPUHP_AP_PERF_S390_SF_ONLINE = 165, - CPUHP_AP_PERF_ARM_CCI_ONLINE = 166, - CPUHP_AP_PERF_ARM_CCN_ONLINE = 167, - CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE = 168, - CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE = 169, - CPUHP_AP_PERF_ARM_HISI_L3_ONLINE = 170, - CPUHP_AP_PERF_ARM_HISI_PA_ONLINE = 171, - CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE = 172, - CPUHP_AP_PERF_ARM_L2X0_ONLINE = 173, - CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE = 174, - CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE = 175, - CPUHP_AP_PERF_ARM_APM_XGENE_ONLINE = 176, - CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE = 177, - CPUHP_AP_PERF_POWERPC_NEST_IMC_ONLINE = 178, - CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE = 179, - CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE = 180, - CPUHP_AP_PERF_POWERPC_TRACE_IMC_ONLINE = 181, - CPUHP_AP_PERF_POWERPC_HV_24x7_ONLINE = 182, - CPUHP_AP_PERF_POWERPC_HV_GPCI_ONLINE = 183, - CPUHP_AP_PERF_CSKY_ONLINE = 184, - CPUHP_AP_WATCHDOG_ONLINE = 185, - CPUHP_AP_WORKQUEUE_ONLINE = 186, - CPUHP_AP_RCUTREE_ONLINE = 187, - CPUHP_AP_BASE_CACHEINFO_ONLINE = 188, - CPUHP_AP_ONLINE_DYN = 189, - CPUHP_AP_ONLINE_DYN_END = 219, - CPUHP_AP_X86_HPET_ONLINE = 220, - CPUHP_AP_X86_KVM_CLK_ONLINE = 221, - CPUHP_AP_DTPM_CPU_ONLINE = 222, - CPUHP_AP_ACTIVE = 223, - CPUHP_ONLINE = 224, + CPUHP_MM_DEMOTION_DEAD = 14, + CPUHP_MM_VMSTAT_DEAD = 15, + CPUHP_SOFTIRQ_DEAD = 16, + CPUHP_NET_MVNETA_DEAD = 17, + CPUHP_CPUIDLE_DEAD = 18, + CPUHP_ARM64_FPSIMD_DEAD = 19, + CPUHP_ARM_OMAP_WAKE_DEAD = 20, + CPUHP_IRQ_POLL_DEAD = 21, + CPUHP_BLOCK_SOFTIRQ_DEAD = 22, + CPUHP_BIO_DEAD = 23, + CPUHP_ACPI_CPUDRV_DEAD = 24, + CPUHP_S390_PFAULT_DEAD = 25, + CPUHP_BLK_MQ_DEAD = 26, + CPUHP_FS_BUFF_DEAD = 27, + CPUHP_PRINTK_DEAD = 28, + CPUHP_MM_MEMCQ_DEAD = 29, + CPUHP_XFS_DEAD = 30, + CPUHP_PERCPU_CNT_DEAD = 31, + CPUHP_RADIX_DEAD = 32, + CPUHP_PAGE_ALLOC = 33, + CPUHP_NET_DEV_DEAD = 34, + CPUHP_PCI_XGENE_DEAD = 35, + CPUHP_IOMMU_IOVA_DEAD = 36, + CPUHP_LUSTRE_CFS_DEAD = 37, + CPUHP_AP_ARM_CACHE_B15_RAC_DEAD = 38, + CPUHP_PADATA_DEAD = 39, + CPUHP_AP_DTPM_CPU_DEAD = 40, + CPUHP_WORKQUEUE_PREP = 41, + CPUHP_POWER_NUMA_PREPARE = 42, + CPUHP_HRTIMERS_PREPARE = 43, + CPUHP_PROFILE_PREPARE = 44, + CPUHP_X2APIC_PREPARE = 45, + CPUHP_SMPCFD_PREPARE = 46, + CPUHP_RELAY_PREPARE = 47, + CPUHP_SLAB_PREPARE = 48, + CPUHP_MD_RAID5_PREPARE = 49, + CPUHP_RCUTREE_PREP = 50, + CPUHP_CPUIDLE_COUPLED_PREPARE = 51, + CPUHP_POWERPC_PMAC_PREPARE = 52, + CPUHP_POWERPC_MMU_CTX_PREPARE = 53, + CPUHP_XEN_PREPARE = 54, + CPUHP_XEN_EVTCHN_PREPARE = 55, + CPUHP_ARM_SHMOBILE_SCU_PREPARE = 56, + CPUHP_SH_SH3X_PREPARE = 57, + CPUHP_NET_FLOW_PREPARE = 58, + CPUHP_TOPOLOGY_PREPARE = 59, + CPUHP_NET_IUCV_PREPARE = 60, + CPUHP_ARM_BL_PREPARE = 61, + CPUHP_TRACE_RB_PREPARE = 62, + CPUHP_MM_ZS_PREPARE = 63, + CPUHP_MM_ZSWP_MEM_PREPARE = 64, + CPUHP_MM_ZSWP_POOL_PREPARE = 65, + CPUHP_KVM_PPC_BOOK3S_PREPARE = 66, + CPUHP_ZCOMP_PREPARE = 67, + CPUHP_TIMERS_PREPARE = 68, + CPUHP_MIPS_SOC_PREPARE = 69, + CPUHP_BP_PREPARE_DYN = 70, + CPUHP_BP_PREPARE_DYN_END = 90, + CPUHP_BRINGUP_CPU = 91, + CPUHP_AP_IDLE_DEAD = 92, + CPUHP_AP_OFFLINE = 93, + CPUHP_AP_SCHED_STARTING = 94, + CPUHP_AP_RCUTREE_DYING = 95, + CPUHP_AP_CPU_PM_STARTING = 96, + CPUHP_AP_IRQ_GIC_STARTING = 97, + CPUHP_AP_IRQ_HIP04_STARTING = 98, + CPUHP_AP_IRQ_APPLE_AIC_STARTING = 99, + CPUHP_AP_IRQ_ARMADA_XP_STARTING = 100, + CPUHP_AP_IRQ_BCM2836_STARTING = 101, + CPUHP_AP_IRQ_MIPS_GIC_STARTING = 102, + CPUHP_AP_IRQ_RISCV_STARTING = 103, + CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING = 104, + CPUHP_AP_ARM_MVEBU_COHERENCY = 105, + CPUHP_AP_MICROCODE_LOADER = 106, + CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING = 107, + CPUHP_AP_PERF_X86_STARTING = 108, + CPUHP_AP_PERF_X86_AMD_IBS_STARTING = 109, + CPUHP_AP_PERF_X86_CQM_STARTING = 110, + CPUHP_AP_PERF_X86_CSTATE_STARTING = 111, + CPUHP_AP_PERF_XTENSA_STARTING = 112, + CPUHP_AP_MIPS_OP_LOONGSON3_STARTING = 113, + CPUHP_AP_ARM_SDEI_STARTING = 114, + CPUHP_AP_ARM_VFP_STARTING = 115, + CPUHP_AP_ARM64_DEBUG_MONITORS_STARTING = 116, + CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING = 117, + CPUHP_AP_PERF_ARM_ACPI_STARTING = 118, + CPUHP_AP_PERF_ARM_STARTING = 119, + CPUHP_AP_ARM_L2X0_STARTING = 120, + CPUHP_AP_EXYNOS4_MCT_TIMER_STARTING = 121, + CPUHP_AP_ARM_ARCH_TIMER_STARTING = 122, + CPUHP_AP_ARM_GLOBAL_TIMER_STARTING = 123, + CPUHP_AP_JCORE_TIMER_STARTING = 124, + CPUHP_AP_ARM_TWD_STARTING = 125, + CPUHP_AP_QCOM_TIMER_STARTING = 126, + CPUHP_AP_TEGRA_TIMER_STARTING = 127, + CPUHP_AP_ARMADA_TIMER_STARTING = 128, + CPUHP_AP_MARCO_TIMER_STARTING = 129, + CPUHP_AP_MIPS_GIC_TIMER_STARTING = 130, + CPUHP_AP_ARC_TIMER_STARTING = 131, + CPUHP_AP_RISCV_TIMER_STARTING = 132, + CPUHP_AP_CLINT_TIMER_STARTING = 133, + CPUHP_AP_CSKY_TIMER_STARTING = 134, + CPUHP_AP_TI_GP_TIMER_STARTING = 135, + CPUHP_AP_HYPERV_TIMER_STARTING = 136, + CPUHP_AP_KVM_STARTING = 137, + CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING = 138, + CPUHP_AP_KVM_ARM_VGIC_STARTING = 139, + CPUHP_AP_KVM_ARM_TIMER_STARTING = 140, + CPUHP_AP_DUMMY_TIMER_STARTING = 141, + CPUHP_AP_ARM_XEN_STARTING = 142, + CPUHP_AP_ARM_CORESIGHT_STARTING = 143, + CPUHP_AP_ARM_CORESIGHT_CTI_STARTING = 144, + CPUHP_AP_ARM64_ISNDEP_STARTING = 145, + CPUHP_AP_SMPCFD_DYING = 146, + CPUHP_AP_X86_TBOOT_DYING = 147, + CPUHP_AP_ARM_CACHE_B15_RAC_DYING = 148, + CPUHP_AP_ONLINE = 149, + CPUHP_TEARDOWN_CPU = 150, + CPUHP_AP_ONLINE_IDLE = 151, + CPUHP_AP_SCHED_WAIT_EMPTY = 152, + CPUHP_AP_SMPBOOT_THREADS = 153, + CPUHP_AP_X86_VDSO_VMA_ONLINE = 154, + CPUHP_AP_IRQ_AFFINITY_ONLINE = 155, + CPUHP_AP_BLK_MQ_ONLINE = 156, + CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS = 157, + CPUHP_AP_X86_INTEL_EPB_ONLINE = 158, + CPUHP_AP_PERF_ONLINE = 159, + CPUHP_AP_PERF_X86_ONLINE = 160, + CPUHP_AP_PERF_X86_UNCORE_ONLINE = 161, + CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE = 162, + CPUHP_AP_PERF_X86_AMD_POWER_ONLINE = 163, + CPUHP_AP_PERF_X86_RAPL_ONLINE = 164, + CPUHP_AP_PERF_X86_CQM_ONLINE = 165, + CPUHP_AP_PERF_X86_CSTATE_ONLINE = 166, + CPUHP_AP_PERF_X86_IDXD_ONLINE = 167, + CPUHP_AP_PERF_S390_CF_ONLINE = 168, + CPUHP_AP_PERF_S390_SF_ONLINE = 169, + CPUHP_AP_PERF_ARM_CCI_ONLINE = 170, + CPUHP_AP_PERF_ARM_CCN_ONLINE = 171, + CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE = 172, + CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE = 173, + CPUHP_AP_PERF_ARM_HISI_L3_ONLINE = 174, + CPUHP_AP_PERF_ARM_HISI_PA_ONLINE = 175, + CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE = 176, + CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE = 177, + CPUHP_AP_PERF_ARM_L2X0_ONLINE = 178, + CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE = 179, + CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE = 180, + CPUHP_AP_PERF_ARM_APM_XGENE_ONLINE = 181, + CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE = 182, + CPUHP_AP_PERF_POWERPC_NEST_IMC_ONLINE = 183, + CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE = 184, + CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE = 185, + CPUHP_AP_PERF_POWERPC_TRACE_IMC_ONLINE = 186, + CPUHP_AP_PERF_POWERPC_HV_24x7_ONLINE = 187, + CPUHP_AP_PERF_POWERPC_HV_GPCI_ONLINE = 188, + CPUHP_AP_PERF_CSKY_ONLINE = 189, + CPUHP_AP_WATCHDOG_ONLINE = 190, + CPUHP_AP_WORKQUEUE_ONLINE = 191, + CPUHP_AP_RCUTREE_ONLINE = 192, + CPUHP_AP_BASE_CACHEINFO_ONLINE = 193, + CPUHP_AP_ONLINE_DYN = 194, + CPUHP_AP_ONLINE_DYN_END = 224, + CPUHP_AP_MM_DEMOTION_ONLINE = 225, + CPUHP_AP_X86_HPET_ONLINE = 226, + CPUHP_AP_X86_KVM_CLK_ONLINE = 227, + CPUHP_AP_ACTIVE = 228, + CPUHP_ONLINE = 229, }; struct ring_buffer_event { @@ -9080,7 +9118,8 @@ union perf_mem_data_src { __u64 mem_remote: 1; __u64 mem_snoopx: 2; __u64 mem_blk: 3; - __u64 mem_rsvd: 21; + __u64 mem_hops: 3; + __u64 mem_rsvd: 18; }; }; @@ -9249,1046 +9288,6 @@ struct kernel_stat { struct u64_stats_sync {}; -struct bpf_insn { - __u8 code; - __u8 dst_reg: 4; - __u8 src_reg: 4; - __s16 off; - __s32 imm; -}; - -struct bpf_cgroup_storage_key { - __u64 cgroup_inode_id; - __u32 attach_type; -}; - -enum bpf_map_type { - BPF_MAP_TYPE_UNSPEC = 0, - BPF_MAP_TYPE_HASH = 1, - BPF_MAP_TYPE_ARRAY = 2, - BPF_MAP_TYPE_PROG_ARRAY = 3, - BPF_MAP_TYPE_PERF_EVENT_ARRAY = 4, - BPF_MAP_TYPE_PERCPU_HASH = 5, - BPF_MAP_TYPE_PERCPU_ARRAY = 6, - BPF_MAP_TYPE_STACK_TRACE = 7, - BPF_MAP_TYPE_CGROUP_ARRAY = 8, - BPF_MAP_TYPE_LRU_HASH = 9, - BPF_MAP_TYPE_LRU_PERCPU_HASH = 10, - BPF_MAP_TYPE_LPM_TRIE = 11, - BPF_MAP_TYPE_ARRAY_OF_MAPS = 12, - BPF_MAP_TYPE_HASH_OF_MAPS = 13, - BPF_MAP_TYPE_DEVMAP = 14, - BPF_MAP_TYPE_SOCKMAP = 15, - BPF_MAP_TYPE_CPUMAP = 16, - BPF_MAP_TYPE_XSKMAP = 17, - BPF_MAP_TYPE_SOCKHASH = 18, - BPF_MAP_TYPE_CGROUP_STORAGE = 19, - BPF_MAP_TYPE_REUSEPORT_SOCKARRAY = 20, - BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = 21, - BPF_MAP_TYPE_QUEUE = 22, - BPF_MAP_TYPE_STACK = 23, - BPF_MAP_TYPE_SK_STORAGE = 24, - BPF_MAP_TYPE_DEVMAP_HASH = 25, - BPF_MAP_TYPE_STRUCT_OPS = 26, - BPF_MAP_TYPE_RINGBUF = 27, - BPF_MAP_TYPE_INODE_STORAGE = 28, - BPF_MAP_TYPE_TASK_STORAGE = 29, -}; - -enum bpf_prog_type { - BPF_PROG_TYPE_UNSPEC = 0, - BPF_PROG_TYPE_SOCKET_FILTER = 1, - BPF_PROG_TYPE_KPROBE = 2, - BPF_PROG_TYPE_SCHED_CLS = 3, - BPF_PROG_TYPE_SCHED_ACT = 4, - BPF_PROG_TYPE_TRACEPOINT = 5, - BPF_PROG_TYPE_XDP = 6, - BPF_PROG_TYPE_PERF_EVENT = 7, - BPF_PROG_TYPE_CGROUP_SKB = 8, - BPF_PROG_TYPE_CGROUP_SOCK = 9, - BPF_PROG_TYPE_LWT_IN = 10, - BPF_PROG_TYPE_LWT_OUT = 11, - BPF_PROG_TYPE_LWT_XMIT = 12, - BPF_PROG_TYPE_SOCK_OPS = 13, - BPF_PROG_TYPE_SK_SKB = 14, - BPF_PROG_TYPE_CGROUP_DEVICE = 15, - BPF_PROG_TYPE_SK_MSG = 16, - BPF_PROG_TYPE_RAW_TRACEPOINT = 17, - BPF_PROG_TYPE_CGROUP_SOCK_ADDR = 18, - BPF_PROG_TYPE_LWT_SEG6LOCAL = 19, - BPF_PROG_TYPE_LIRC_MODE2 = 20, - BPF_PROG_TYPE_SK_REUSEPORT = 21, - BPF_PROG_TYPE_FLOW_DISSECTOR = 22, - BPF_PROG_TYPE_CGROUP_SYSCTL = 23, - BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE = 24, - BPF_PROG_TYPE_CGROUP_SOCKOPT = 25, - BPF_PROG_TYPE_TRACING = 26, - BPF_PROG_TYPE_STRUCT_OPS = 27, - BPF_PROG_TYPE_EXT = 28, - BPF_PROG_TYPE_LSM = 29, - BPF_PROG_TYPE_SK_LOOKUP = 30, - BPF_PROG_TYPE_SYSCALL = 31, -}; - -enum bpf_attach_type { - BPF_CGROUP_INET_INGRESS = 0, - BPF_CGROUP_INET_EGRESS = 1, - BPF_CGROUP_INET_SOCK_CREATE = 2, - BPF_CGROUP_SOCK_OPS = 3, - BPF_SK_SKB_STREAM_PARSER = 4, - BPF_SK_SKB_STREAM_VERDICT = 5, - BPF_CGROUP_DEVICE = 6, - BPF_SK_MSG_VERDICT = 7, - BPF_CGROUP_INET4_BIND = 8, - BPF_CGROUP_INET6_BIND = 9, - BPF_CGROUP_INET4_CONNECT = 10, - BPF_CGROUP_INET6_CONNECT = 11, - BPF_CGROUP_INET4_POST_BIND = 12, - BPF_CGROUP_INET6_POST_BIND = 13, - BPF_CGROUP_UDP4_SENDMSG = 14, - BPF_CGROUP_UDP6_SENDMSG = 15, - BPF_LIRC_MODE2 = 16, - BPF_FLOW_DISSECTOR = 17, - BPF_CGROUP_SYSCTL = 18, - BPF_CGROUP_UDP4_RECVMSG = 19, - BPF_CGROUP_UDP6_RECVMSG = 20, - BPF_CGROUP_GETSOCKOPT = 21, - BPF_CGROUP_SETSOCKOPT = 22, - BPF_TRACE_RAW_TP = 23, - BPF_TRACE_FENTRY = 24, - BPF_TRACE_FEXIT = 25, - BPF_MODIFY_RETURN = 26, - BPF_LSM_MAC = 27, - BPF_TRACE_ITER = 28, - BPF_CGROUP_INET4_GETPEERNAME = 29, - BPF_CGROUP_INET6_GETPEERNAME = 30, - BPF_CGROUP_INET4_GETSOCKNAME = 31, - BPF_CGROUP_INET6_GETSOCKNAME = 32, - BPF_XDP_DEVMAP = 33, - BPF_CGROUP_INET_SOCK_RELEASE = 34, - BPF_XDP_CPUMAP = 35, - BPF_SK_LOOKUP = 36, - BPF_XDP = 37, - BPF_SK_SKB_VERDICT = 38, - BPF_SK_REUSEPORT_SELECT = 39, - BPF_SK_REUSEPORT_SELECT_OR_MIGRATE = 40, - __MAX_BPF_ATTACH_TYPE = 41, -}; - -union bpf_attr { - struct { - __u32 map_type; - __u32 key_size; - __u32 value_size; - __u32 max_entries; - __u32 map_flags; - __u32 inner_map_fd; - __u32 numa_node; - char map_name[16]; - __u32 map_ifindex; - __u32 btf_fd; - __u32 btf_key_type_id; - __u32 btf_value_type_id; - __u32 btf_vmlinux_value_type_id; - }; - struct { - __u32 map_fd; - __u64 key; - union { - __u64 value; - __u64 next_key; - }; - __u64 flags; - }; - struct { - __u64 in_batch; - __u64 out_batch; - __u64 keys; - __u64 values; - __u32 count; - __u32 map_fd; - __u64 elem_flags; - __u64 flags; - } batch; - struct { - __u32 prog_type; - __u32 insn_cnt; - __u64 insns; - __u64 license; - __u32 log_level; - __u32 log_size; - __u64 log_buf; - __u32 kern_version; - __u32 prog_flags; - char prog_name[16]; - __u32 prog_ifindex; - __u32 expected_attach_type; - __u32 prog_btf_fd; - __u32 func_info_rec_size; - __u64 func_info; - __u32 func_info_cnt; - __u32 line_info_rec_size; - __u64 line_info; - __u32 line_info_cnt; - __u32 attach_btf_id; - union { - __u32 attach_prog_fd; - __u32 attach_btf_obj_fd; - }; - __u64 fd_array; - }; - struct { - __u64 pathname; - __u32 bpf_fd; - __u32 file_flags; - }; - struct { - __u32 target_fd; - __u32 attach_bpf_fd; - __u32 attach_type; - __u32 attach_flags; - __u32 replace_bpf_fd; - }; - struct { - __u32 prog_fd; - __u32 retval; - __u32 data_size_in; - __u32 data_size_out; - __u64 data_in; - __u64 data_out; - __u32 repeat; - __u32 duration; - __u32 ctx_size_in; - __u32 ctx_size_out; - __u64 ctx_in; - __u64 ctx_out; - __u32 flags; - __u32 cpu; - } test; - struct { - union { - __u32 start_id; - __u32 prog_id; - __u32 map_id; - __u32 btf_id; - __u32 link_id; - }; - __u32 next_id; - __u32 open_flags; - }; - struct { - __u32 bpf_fd; - __u32 info_len; - __u64 info; - } info; - struct { - __u32 target_fd; - __u32 attach_type; - __u32 query_flags; - __u32 attach_flags; - __u64 prog_ids; - __u32 prog_cnt; - } query; - struct { - __u64 name; - __u32 prog_fd; - } raw_tracepoint; - struct { - __u64 btf; - __u64 btf_log_buf; - __u32 btf_size; - __u32 btf_log_size; - __u32 btf_log_level; - }; - struct { - __u32 pid; - __u32 fd; - __u32 flags; - __u32 buf_len; - __u64 buf; - __u32 prog_id; - __u32 fd_type; - __u64 probe_offset; - __u64 probe_addr; - } task_fd_query; - struct { - __u32 prog_fd; - union { - __u32 target_fd; - __u32 target_ifindex; - }; - __u32 attach_type; - __u32 flags; - union { - __u32 target_btf_id; - struct { - __u64 iter_info; - __u32 iter_info_len; - }; - }; - } link_create; - struct { - __u32 link_fd; - __u32 new_prog_fd; - __u32 flags; - __u32 old_prog_fd; - } link_update; - struct { - __u32 link_fd; - } link_detach; - struct { - __u32 type; - } enable_stats; - struct { - __u32 link_fd; - __u32 flags; - } iter_create; - struct { - __u32 prog_fd; - __u32 map_fd; - __u32 flags; - } prog_bind_map; -}; - -enum bpf_func_id { - BPF_FUNC_unspec = 0, - BPF_FUNC_map_lookup_elem = 1, - BPF_FUNC_map_update_elem = 2, - BPF_FUNC_map_delete_elem = 3, - BPF_FUNC_probe_read = 4, - BPF_FUNC_ktime_get_ns = 5, - BPF_FUNC_trace_printk = 6, - BPF_FUNC_get_prandom_u32 = 7, - BPF_FUNC_get_smp_processor_id = 8, - BPF_FUNC_skb_store_bytes = 9, - BPF_FUNC_l3_csum_replace = 10, - BPF_FUNC_l4_csum_replace = 11, - BPF_FUNC_tail_call = 12, - BPF_FUNC_clone_redirect = 13, - BPF_FUNC_get_current_pid_tgid = 14, - BPF_FUNC_get_current_uid_gid = 15, - BPF_FUNC_get_current_comm = 16, - BPF_FUNC_get_cgroup_classid = 17, - BPF_FUNC_skb_vlan_push = 18, - BPF_FUNC_skb_vlan_pop = 19, - BPF_FUNC_skb_get_tunnel_key = 20, - BPF_FUNC_skb_set_tunnel_key = 21, - BPF_FUNC_perf_event_read = 22, - BPF_FUNC_redirect = 23, - BPF_FUNC_get_route_realm = 24, - BPF_FUNC_perf_event_output = 25, - BPF_FUNC_skb_load_bytes = 26, - BPF_FUNC_get_stackid = 27, - BPF_FUNC_csum_diff = 28, - BPF_FUNC_skb_get_tunnel_opt = 29, - BPF_FUNC_skb_set_tunnel_opt = 30, - BPF_FUNC_skb_change_proto = 31, - BPF_FUNC_skb_change_type = 32, - BPF_FUNC_skb_under_cgroup = 33, - BPF_FUNC_get_hash_recalc = 34, - BPF_FUNC_get_current_task = 35, - BPF_FUNC_probe_write_user = 36, - BPF_FUNC_current_task_under_cgroup = 37, - BPF_FUNC_skb_change_tail = 38, - BPF_FUNC_skb_pull_data = 39, - BPF_FUNC_csum_update = 40, - BPF_FUNC_set_hash_invalid = 41, - BPF_FUNC_get_numa_node_id = 42, - BPF_FUNC_skb_change_head = 43, - BPF_FUNC_xdp_adjust_head = 44, - BPF_FUNC_probe_read_str = 45, - BPF_FUNC_get_socket_cookie = 46, - BPF_FUNC_get_socket_uid = 47, - BPF_FUNC_set_hash = 48, - BPF_FUNC_setsockopt = 49, - BPF_FUNC_skb_adjust_room = 50, - BPF_FUNC_redirect_map = 51, - BPF_FUNC_sk_redirect_map = 52, - BPF_FUNC_sock_map_update = 53, - BPF_FUNC_xdp_adjust_meta = 54, - BPF_FUNC_perf_event_read_value = 55, - BPF_FUNC_perf_prog_read_value = 56, - BPF_FUNC_getsockopt = 57, - BPF_FUNC_override_return = 58, - BPF_FUNC_sock_ops_cb_flags_set = 59, - BPF_FUNC_msg_redirect_map = 60, - BPF_FUNC_msg_apply_bytes = 61, - BPF_FUNC_msg_cork_bytes = 62, - BPF_FUNC_msg_pull_data = 63, - BPF_FUNC_bind = 64, - BPF_FUNC_xdp_adjust_tail = 65, - BPF_FUNC_skb_get_xfrm_state = 66, - BPF_FUNC_get_stack = 67, - BPF_FUNC_skb_load_bytes_relative = 68, - BPF_FUNC_fib_lookup = 69, - BPF_FUNC_sock_hash_update = 70, - BPF_FUNC_msg_redirect_hash = 71, - BPF_FUNC_sk_redirect_hash = 72, - BPF_FUNC_lwt_push_encap = 73, - BPF_FUNC_lwt_seg6_store_bytes = 74, - BPF_FUNC_lwt_seg6_adjust_srh = 75, - BPF_FUNC_lwt_seg6_action = 76, - BPF_FUNC_rc_repeat = 77, - BPF_FUNC_rc_keydown = 78, - BPF_FUNC_skb_cgroup_id = 79, - BPF_FUNC_get_current_cgroup_id = 80, - BPF_FUNC_get_local_storage = 81, - BPF_FUNC_sk_select_reuseport = 82, - BPF_FUNC_skb_ancestor_cgroup_id = 83, - BPF_FUNC_sk_lookup_tcp = 84, - BPF_FUNC_sk_lookup_udp = 85, - BPF_FUNC_sk_release = 86, - BPF_FUNC_map_push_elem = 87, - BPF_FUNC_map_pop_elem = 88, - BPF_FUNC_map_peek_elem = 89, - BPF_FUNC_msg_push_data = 90, - BPF_FUNC_msg_pop_data = 91, - BPF_FUNC_rc_pointer_rel = 92, - BPF_FUNC_spin_lock = 93, - BPF_FUNC_spin_unlock = 94, - BPF_FUNC_sk_fullsock = 95, - BPF_FUNC_tcp_sock = 96, - BPF_FUNC_skb_ecn_set_ce = 97, - BPF_FUNC_get_listener_sock = 98, - BPF_FUNC_skc_lookup_tcp = 99, - BPF_FUNC_tcp_check_syncookie = 100, - BPF_FUNC_sysctl_get_name = 101, - BPF_FUNC_sysctl_get_current_value = 102, - BPF_FUNC_sysctl_get_new_value = 103, - BPF_FUNC_sysctl_set_new_value = 104, - BPF_FUNC_strtol = 105, - BPF_FUNC_strtoul = 106, - BPF_FUNC_sk_storage_get = 107, - BPF_FUNC_sk_storage_delete = 108, - BPF_FUNC_send_signal = 109, - BPF_FUNC_tcp_gen_syncookie = 110, - BPF_FUNC_skb_output = 111, - BPF_FUNC_probe_read_user = 112, - BPF_FUNC_probe_read_kernel = 113, - BPF_FUNC_probe_read_user_str = 114, - BPF_FUNC_probe_read_kernel_str = 115, - BPF_FUNC_tcp_send_ack = 116, - BPF_FUNC_send_signal_thread = 117, - BPF_FUNC_jiffies64 = 118, - BPF_FUNC_read_branch_records = 119, - BPF_FUNC_get_ns_current_pid_tgid = 120, - BPF_FUNC_xdp_output = 121, - BPF_FUNC_get_netns_cookie = 122, - BPF_FUNC_get_current_ancestor_cgroup_id = 123, - BPF_FUNC_sk_assign = 124, - BPF_FUNC_ktime_get_boot_ns = 125, - BPF_FUNC_seq_printf = 126, - BPF_FUNC_seq_write = 127, - BPF_FUNC_sk_cgroup_id = 128, - BPF_FUNC_sk_ancestor_cgroup_id = 129, - BPF_FUNC_ringbuf_output = 130, - BPF_FUNC_ringbuf_reserve = 131, - BPF_FUNC_ringbuf_submit = 132, - BPF_FUNC_ringbuf_discard = 133, - BPF_FUNC_ringbuf_query = 134, - BPF_FUNC_csum_level = 135, - BPF_FUNC_skc_to_tcp6_sock = 136, - BPF_FUNC_skc_to_tcp_sock = 137, - BPF_FUNC_skc_to_tcp_timewait_sock = 138, - BPF_FUNC_skc_to_tcp_request_sock = 139, - BPF_FUNC_skc_to_udp6_sock = 140, - BPF_FUNC_get_task_stack = 141, - BPF_FUNC_load_hdr_opt = 142, - BPF_FUNC_store_hdr_opt = 143, - BPF_FUNC_reserve_hdr_opt = 144, - BPF_FUNC_inode_storage_get = 145, - BPF_FUNC_inode_storage_delete = 146, - BPF_FUNC_d_path = 147, - BPF_FUNC_copy_from_user = 148, - BPF_FUNC_snprintf_btf = 149, - BPF_FUNC_seq_printf_btf = 150, - BPF_FUNC_skb_cgroup_classid = 151, - BPF_FUNC_redirect_neigh = 152, - BPF_FUNC_per_cpu_ptr = 153, - BPF_FUNC_this_cpu_ptr = 154, - BPF_FUNC_redirect_peer = 155, - BPF_FUNC_task_storage_get = 156, - BPF_FUNC_task_storage_delete = 157, - BPF_FUNC_get_current_task_btf = 158, - BPF_FUNC_bprm_opts_set = 159, - BPF_FUNC_ktime_get_coarse_ns = 160, - BPF_FUNC_ima_inode_hash = 161, - BPF_FUNC_sock_from_file = 162, - BPF_FUNC_check_mtu = 163, - BPF_FUNC_for_each_map_elem = 164, - BPF_FUNC_snprintf = 165, - BPF_FUNC_sys_bpf = 166, - BPF_FUNC_btf_find_by_name_kind = 167, - BPF_FUNC_sys_close = 168, - BPF_FUNC_timer_init = 169, - BPF_FUNC_timer_set_callback = 170, - BPF_FUNC_timer_start = 171, - BPF_FUNC_timer_cancel = 172, - BPF_FUNC_get_func_ip = 173, - __BPF_FUNC_MAX_ID = 174, -}; - -struct bpf_func_info { - __u32 insn_off; - __u32 type_id; -}; - -struct bpf_line_info { - __u32 insn_off; - __u32 file_name_off; - __u32 line_off; - __u32 line_col; -}; - -enum kmalloc_cache_type { - KMALLOC_NORMAL = 0, - KMALLOC_CGROUP = 1, - KMALLOC_RECLAIM = 2, - KMALLOC_DMA = 3, - NR_KMALLOC_TYPES = 4, -}; - -struct bpf_iter_aux_info; - -typedef int (*bpf_iter_init_seq_priv_t)(void *, struct bpf_iter_aux_info *); - -struct bpf_map; - -struct bpf_iter_aux_info { - struct bpf_map *map; -}; - -typedef void (*bpf_iter_fini_seq_priv_t)(void *); - -struct bpf_iter_seq_info { - const struct seq_operations *seq_ops; - bpf_iter_init_seq_priv_t init_seq_private; - bpf_iter_fini_seq_priv_t fini_seq_private; - u32 seq_priv_size; -}; - -struct btf; - -struct btf_type; - -struct bpf_prog_aux; - -struct bpf_local_storage_map; - -struct bpf_verifier_env; - -struct bpf_func_state; - -struct bpf_map_ops { - int (*map_alloc_check)(union bpf_attr *); - struct bpf_map * (*map_alloc)(union bpf_attr *); - void (*map_release)(struct bpf_map *, struct file *); - void (*map_free)(struct bpf_map *); - int (*map_get_next_key)(struct bpf_map *, void *, void *); - void (*map_release_uref)(struct bpf_map *); - void * (*map_lookup_elem_sys_only)(struct bpf_map *, void *); - int (*map_lookup_batch)(struct bpf_map *, const union bpf_attr *, union bpf_attr *); - int (*map_lookup_and_delete_elem)(struct bpf_map *, void *, void *, u64); - int (*map_lookup_and_delete_batch)(struct bpf_map *, const union bpf_attr *, union bpf_attr *); - int (*map_update_batch)(struct bpf_map *, const union bpf_attr *, union bpf_attr *); - int (*map_delete_batch)(struct bpf_map *, const union bpf_attr *, union bpf_attr *); - void * (*map_lookup_elem)(struct bpf_map *, void *); - int (*map_update_elem)(struct bpf_map *, void *, void *, u64); - int (*map_delete_elem)(struct bpf_map *, void *); - int (*map_push_elem)(struct bpf_map *, void *, u64); - int (*map_pop_elem)(struct bpf_map *, void *); - int (*map_peek_elem)(struct bpf_map *, void *); - void * (*map_fd_get_ptr)(struct bpf_map *, struct file *, int); - void (*map_fd_put_ptr)(void *); - int (*map_gen_lookup)(struct bpf_map *, struct bpf_insn *); - u32 (*map_fd_sys_lookup_elem)(void *); - void (*map_seq_show_elem)(struct bpf_map *, void *, struct seq_file *); - int (*map_check_btf)(const struct bpf_map *, const struct btf *, const struct btf_type *, const struct btf_type *); - int (*map_poke_track)(struct bpf_map *, struct bpf_prog_aux *); - void (*map_poke_untrack)(struct bpf_map *, struct bpf_prog_aux *); - void (*map_poke_run)(struct bpf_map *, u32, struct bpf_prog *, struct bpf_prog *); - int (*map_direct_value_addr)(const struct bpf_map *, u64 *, u32); - int (*map_direct_value_meta)(const struct bpf_map *, u64, u32 *); - int (*map_mmap)(struct bpf_map *, struct vm_area_struct *); - __poll_t (*map_poll)(struct bpf_map *, struct file *, struct poll_table_struct *); - int (*map_local_storage_charge)(struct bpf_local_storage_map *, void *, u32); - void (*map_local_storage_uncharge)(struct bpf_local_storage_map *, void *, u32); - struct bpf_local_storage ** (*map_owner_storage_ptr)(void *); - int (*map_redirect)(struct bpf_map *, u32, u64); - bool (*map_meta_equal)(const struct bpf_map *, const struct bpf_map *); - int (*map_set_for_each_callback_args)(struct bpf_verifier_env *, struct bpf_func_state *, struct bpf_func_state *); - int (*map_for_each_callback)(struct bpf_map *, void *, void *, u64); - const char * const map_btf_name; - int *map_btf_id; - const struct bpf_iter_seq_info *iter_seq_info; -}; - -struct bpf_map { - const struct bpf_map_ops *ops; - struct bpf_map *inner_map_meta; - void *security; - enum bpf_map_type map_type; - u32 key_size; - u32 value_size; - u32 max_entries; - u32 map_flags; - int spin_lock_off; - int timer_off; - u32 id; - int numa_node; - u32 btf_key_type_id; - u32 btf_value_type_id; - struct btf *btf; - struct mem_cgroup *memcg; - char name[16]; - u32 btf_vmlinux_value_type_id; - bool bypass_spec_v1; - bool frozen; - long: 16; - long: 64; - long: 64; - atomic64_t refcnt; - atomic64_t usercnt; - struct work_struct work; - struct mutex freeze_mutex; - u64 writecnt; - long: 64; - long: 64; -}; - -struct btf_header { - __u16 magic; - __u8 version; - __u8 flags; - __u32 hdr_len; - __u32 type_off; - __u32 type_len; - __u32 str_off; - __u32 str_len; -}; - -struct btf { - void *data; - struct btf_type **types; - u32 *resolved_ids; - u32 *resolved_sizes; - const char *strings; - void *nohdr_data; - struct btf_header hdr; - u32 nr_types; - u32 types_size; - u32 data_size; - refcount_t refcnt; - u32 id; - struct callback_head rcu; - struct btf *base_btf; - u32 start_id; - u32 start_str_off; - char name[56]; - bool kernel_btf; -}; - -struct btf_type { - __u32 name_off; - __u32 info; - union { - __u32 size; - __u32 type; - }; -}; - -struct bpf_ksym { - long unsigned int start; - long unsigned int end; - char name[128]; - struct list_head lnode; - struct latch_tree_node tnode; - bool prog; -}; - -struct bpf_ctx_arg_aux; - -struct bpf_trampoline; - -struct bpf_jit_poke_descriptor; - -struct bpf_kfunc_desc_tab; - -struct bpf_prog_ops; - -struct btf_mod_pair; - -struct bpf_prog_offload; - -struct bpf_func_info_aux; - -struct bpf_prog_aux { - atomic64_t refcnt; - u32 used_map_cnt; - u32 used_btf_cnt; - u32 max_ctx_offset; - u32 max_pkt_offset; - u32 max_tp_access; - u32 stack_depth; - u32 id; - u32 func_cnt; - u32 func_idx; - u32 attach_btf_id; - u32 ctx_arg_info_size; - u32 max_rdonly_access; - u32 max_rdwr_access; - struct btf *attach_btf; - const struct bpf_ctx_arg_aux *ctx_arg_info; - struct mutex dst_mutex; - struct bpf_prog *dst_prog; - struct bpf_trampoline *dst_trampoline; - enum bpf_prog_type saved_dst_prog_type; - enum bpf_attach_type saved_dst_attach_type; - bool verifier_zext; - bool offload_requested; - bool attach_btf_trace; - bool func_proto_unreliable; - bool sleepable; - bool tail_call_reachable; - struct hlist_node tramp_hlist; - const struct btf_type *attach_func_proto; - const char *attach_func_name; - struct bpf_prog **func; - void *jit_data; - struct bpf_jit_poke_descriptor *poke_tab; - struct bpf_kfunc_desc_tab *kfunc_tab; - u32 size_poke_tab; - struct bpf_ksym ksym; - const struct bpf_prog_ops *ops; - struct bpf_map **used_maps; - struct mutex used_maps_mutex; - struct btf_mod_pair *used_btfs; - struct bpf_prog *prog; - struct user_struct *user; - u64 load_time; - struct bpf_map *cgroup_storage[2]; - char name[16]; - void *security; - struct bpf_prog_offload *offload; - struct btf *btf; - struct bpf_func_info *func_info; - struct bpf_func_info_aux *func_info_aux; - struct bpf_line_info *linfo; - void **jited_linfo; - u32 func_info_cnt; - u32 nr_linfo; - u32 linfo_idx; - u32 num_exentries; - struct exception_table_entry *extable; - union { - struct work_struct work; - struct callback_head rcu; - }; -}; - -struct sock_filter { - __u16 code; - __u8 jt; - __u8 jf; - __u32 k; -}; - -struct bpf_prog_stats; - -struct sock_fprog_kern; - -struct bpf_prog { - u16 pages; - u16 jited: 1; - u16 jit_requested: 1; - u16 gpl_compatible: 1; - u16 cb_access: 1; - u16 dst_needed: 1; - u16 blinded: 1; - u16 is_func: 1; - u16 kprobe_override: 1; - u16 has_callchain_buf: 1; - u16 enforce_expected_attach_type: 1; - u16 call_get_stack: 1; - u16 call_get_func_ip: 1; - enum bpf_prog_type type; - enum bpf_attach_type expected_attach_type; - u32 len; - u32 jited_len; - u8 tag[8]; - struct bpf_prog_stats *stats; - int *active; - unsigned int (*bpf_func)(const void *, const struct bpf_insn *); - struct bpf_prog_aux *aux; - struct sock_fprog_kern *orig_prog; - struct sock_filter insns[0]; - struct bpf_insn insnsi[0]; -}; - -enum bpf_arg_type { - ARG_DONTCARE = 0, - ARG_CONST_MAP_PTR = 1, - ARG_PTR_TO_MAP_KEY = 2, - ARG_PTR_TO_MAP_VALUE = 3, - ARG_PTR_TO_UNINIT_MAP_VALUE = 4, - ARG_PTR_TO_MAP_VALUE_OR_NULL = 5, - ARG_PTR_TO_MEM = 6, - ARG_PTR_TO_MEM_OR_NULL = 7, - ARG_PTR_TO_UNINIT_MEM = 8, - ARG_CONST_SIZE = 9, - ARG_CONST_SIZE_OR_ZERO = 10, - ARG_PTR_TO_CTX = 11, - ARG_PTR_TO_CTX_OR_NULL = 12, - ARG_ANYTHING = 13, - ARG_PTR_TO_SPIN_LOCK = 14, - ARG_PTR_TO_SOCK_COMMON = 15, - ARG_PTR_TO_INT = 16, - ARG_PTR_TO_LONG = 17, - ARG_PTR_TO_SOCKET = 18, - ARG_PTR_TO_SOCKET_OR_NULL = 19, - ARG_PTR_TO_BTF_ID = 20, - ARG_PTR_TO_ALLOC_MEM = 21, - ARG_PTR_TO_ALLOC_MEM_OR_NULL = 22, - ARG_CONST_ALLOC_SIZE_OR_ZERO = 23, - ARG_PTR_TO_BTF_ID_SOCK_COMMON = 24, - ARG_PTR_TO_PERCPU_BTF_ID = 25, - ARG_PTR_TO_FUNC = 26, - ARG_PTR_TO_STACK_OR_NULL = 27, - ARG_PTR_TO_CONST_STR = 28, - ARG_PTR_TO_TIMER = 29, - __BPF_ARG_TYPE_MAX = 30, -}; - -enum bpf_return_type { - RET_INTEGER = 0, - RET_VOID = 1, - RET_PTR_TO_MAP_VALUE = 2, - RET_PTR_TO_MAP_VALUE_OR_NULL = 3, - RET_PTR_TO_SOCKET_OR_NULL = 4, - RET_PTR_TO_TCP_SOCK_OR_NULL = 5, - RET_PTR_TO_SOCK_COMMON_OR_NULL = 6, - RET_PTR_TO_ALLOC_MEM_OR_NULL = 7, - RET_PTR_TO_BTF_ID_OR_NULL = 8, - RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL = 9, - RET_PTR_TO_MEM_OR_BTF_ID = 10, - RET_PTR_TO_BTF_ID = 11, -}; - -struct bpf_func_proto { - u64 (*func)(u64, u64, u64, u64, u64); - bool gpl_only; - bool pkt_access; - enum bpf_return_type ret_type; - union { - struct { - enum bpf_arg_type arg1_type; - enum bpf_arg_type arg2_type; - enum bpf_arg_type arg3_type; - enum bpf_arg_type arg4_type; - enum bpf_arg_type arg5_type; - }; - enum bpf_arg_type arg_type[5]; - }; - union { - struct { - u32 *arg1_btf_id; - u32 *arg2_btf_id; - u32 *arg3_btf_id; - u32 *arg4_btf_id; - u32 *arg5_btf_id; - }; - u32 *arg_btf_id[5]; - }; - int *ret_btf_id; - bool (*allowed)(const struct bpf_prog *); -}; - -enum bpf_access_type { - BPF_READ = 1, - BPF_WRITE = 2, -}; - -enum bpf_reg_type { - NOT_INIT = 0, - SCALAR_VALUE = 1, - PTR_TO_CTX = 2, - CONST_PTR_TO_MAP = 3, - PTR_TO_MAP_VALUE = 4, - PTR_TO_MAP_VALUE_OR_NULL = 5, - PTR_TO_STACK = 6, - PTR_TO_PACKET_META = 7, - PTR_TO_PACKET = 8, - PTR_TO_PACKET_END = 9, - PTR_TO_FLOW_KEYS = 10, - PTR_TO_SOCKET = 11, - PTR_TO_SOCKET_OR_NULL = 12, - PTR_TO_SOCK_COMMON = 13, - PTR_TO_SOCK_COMMON_OR_NULL = 14, - PTR_TO_TCP_SOCK = 15, - PTR_TO_TCP_SOCK_OR_NULL = 16, - PTR_TO_TP_BUFFER = 17, - PTR_TO_XDP_SOCK = 18, - PTR_TO_BTF_ID = 19, - PTR_TO_BTF_ID_OR_NULL = 20, - PTR_TO_MEM = 21, - PTR_TO_MEM_OR_NULL = 22, - PTR_TO_RDONLY_BUF = 23, - PTR_TO_RDONLY_BUF_OR_NULL = 24, - PTR_TO_RDWR_BUF = 25, - PTR_TO_RDWR_BUF_OR_NULL = 26, - PTR_TO_PERCPU_BTF_ID = 27, - PTR_TO_FUNC = 28, - PTR_TO_MAP_KEY = 29, - __BPF_REG_TYPE_MAX = 30, -}; - -struct bpf_verifier_log; - -struct bpf_insn_access_aux { - enum bpf_reg_type reg_type; - union { - int ctx_field_size; - struct { - struct btf *btf; - u32 btf_id; - }; - }; - struct bpf_verifier_log *log; -}; - -struct bpf_prog_ops { - int (*test_run)(struct bpf_prog *, const union bpf_attr *, union bpf_attr *); -}; - -struct bpf_verifier_ops { - const struct bpf_func_proto * (*get_func_proto)(enum bpf_func_id, const struct bpf_prog *); - bool (*is_valid_access)(int, int, enum bpf_access_type, const struct bpf_prog *, struct bpf_insn_access_aux *); - int (*gen_prologue)(struct bpf_insn *, bool, const struct bpf_prog *); - int (*gen_ld_abs)(const struct bpf_insn *, struct bpf_insn *); - u32 (*convert_ctx_access)(enum bpf_access_type, const struct bpf_insn *, struct bpf_insn *, struct bpf_prog *, u32 *); - int (*btf_struct_access)(struct bpf_verifier_log *, const struct btf *, const struct btf_type *, int, int, enum bpf_access_type, u32 *); - bool (*check_kfunc_call)(u32); -}; - -struct net_device; - -struct bpf_offload_dev; - -struct bpf_prog_offload { - struct bpf_prog *prog; - struct net_device *netdev; - struct bpf_offload_dev *offdev; - void *dev_priv; - struct list_head offloads; - bool dev_state; - bool opt_failed; - void *jited_image; - u32 jited_len; -}; - -struct btf_func_model { - u8 ret_size; - u8 nr_args; - u8 arg_size[12]; -}; - -struct bpf_tramp_image { - void *image; - struct bpf_ksym ksym; - struct percpu_ref pcref; - void *ip_after_call; - void *ip_epilogue; - union { - struct callback_head rcu; - struct work_struct work; - }; -}; - -struct bpf_trampoline { - struct hlist_node hlist; - struct mutex mutex; - refcount_t refcnt; - u64 key; - struct { - struct btf_func_model model; - void *addr; - bool ftrace_managed; - } func; - struct bpf_prog *extension_prog; - struct hlist_head progs_hlist[3]; - int progs_cnt[3]; - struct bpf_tramp_image *cur_image; - u64 selector; - struct module *mod; -}; - -struct bpf_func_info_aux { - u16 linkage; - bool unreliable; -}; - -struct bpf_jit_poke_descriptor { - void *tailcall_target; - void *tailcall_bypass; - void *bypass_addr; - void *aux; - union { - struct { - struct bpf_map *map; - u32 key; - } tail_call; - }; - bool tailcall_target_stable; - u8 adj_off; - u16 reason; - u32 insn_idx; -}; - -struct bpf_ctx_arg_aux { - u32 offset; - enum bpf_reg_type reg_type; - u32 btf_id; -}; - -struct btf_mod_pair { - struct btf *btf; - struct module *module; -}; - -struct bpf_cgroup_storage; - -struct bpf_prog_array_item { - struct bpf_prog *prog; - struct bpf_cgroup_storage *cgroup_storage[2]; -}; - -struct bpf_storage_buffer; - -struct bpf_cgroup_storage_map; - -struct bpf_cgroup_storage { - union { - struct bpf_storage_buffer *buf; - void *percpu_buf; - }; - struct bpf_cgroup_storage_map *map; - struct bpf_cgroup_storage_key key; - struct list_head list_map; - struct list_head list_cg; - struct rb_node node; - struct callback_head rcu; -}; - -struct bpf_prog_array { - struct callback_head rcu; - struct bpf_prog_array_item items[0]; -}; - -struct bpf_storage_buffer { - struct callback_head rcu; - char data[0]; -}; - struct cgroup_taskset; struct cftype; @@ -10371,6 +9370,14 @@ struct cftype { struct lock_class_key lockdep_key; }; +enum kmalloc_cache_type { + KMALLOC_NORMAL = 0, + KMALLOC_CGROUP = 1, + KMALLOC_RECLAIM = 2, + KMALLOC_DMA = 3, + NR_KMALLOC_TYPES = 4, +}; + struct perf_callchain_entry { __u64 nr; __u64 ip[0]; @@ -10615,8 +9622,10 @@ enum { TRACE_EVENT_FL_NO_SET_FILTER_BIT = 2, TRACE_EVENT_FL_IGNORE_ENABLE_BIT = 3, TRACE_EVENT_FL_TRACEPOINT_BIT = 4, - TRACE_EVENT_FL_KPROBE_BIT = 5, - TRACE_EVENT_FL_UPROBE_BIT = 6, + TRACE_EVENT_FL_DYNAMIC_BIT = 5, + TRACE_EVENT_FL_KPROBE_BIT = 6, + TRACE_EVENT_FL_UPROBE_BIT = 7, + TRACE_EVENT_FL_EPROBE_BIT = 8, }; enum { @@ -10625,8 +9634,10 @@ enum { TRACE_EVENT_FL_NO_SET_FILTER = 4, TRACE_EVENT_FL_IGNORE_ENABLE = 8, TRACE_EVENT_FL_TRACEPOINT = 16, - TRACE_EVENT_FL_KPROBE = 32, - TRACE_EVENT_FL_UPROBE = 64, + TRACE_EVENT_FL_DYNAMIC = 32, + TRACE_EVENT_FL_KPROBE = 64, + TRACE_EVENT_FL_UPROBE = 128, + TRACE_EVENT_FL_EPROBE = 256, }; enum { @@ -10782,10 +9793,10 @@ enum acpi_irq_model_id { }; struct xbc_node { - u16 next; - u16 child; - u16 parent; - u16 data; + uint16_t next; + uint16_t child; + uint16_t parent; + uint16_t data; }; enum con_scroll { @@ -10997,6 +10008,8 @@ struct console { short int flags; short int index; int cflag; + uint ispeed; + uint ospeed; void *data; struct console *next; }; @@ -11106,6 +10119,7 @@ struct gendisk { long unsigned int state; struct mutex open_mutex; unsigned int open_partitions; + struct backing_dev_info *bdi; struct kobject *slave_dir; struct timer_rand_state *random; atomic_t sync_io; @@ -11113,6 +10127,7 @@ struct gendisk { int node_id; struct badblocks *bb; struct lockdep_map lockdep_map; + u64 diskseq; }; struct partition_meta_info { @@ -11120,71 +10135,6 @@ struct partition_meta_info { u8 volname[64]; }; -struct blkg_iostat { - u64 bytes[3]; - u64 ios[3]; -}; - -struct blkg_iostat_set { - struct u64_stats_sync sync; - struct blkg_iostat cur; - struct blkg_iostat last; -}; - -struct blkcg; - -struct blkg_policy_data; - -struct blkcg_gq { - struct request_queue *q; - struct list_head q_node; - struct hlist_node blkcg_node; - struct blkcg *blkcg; - struct blkcg_gq *parent; - struct percpu_ref refcnt; - bool online; - struct blkg_iostat_set *iostat_cpu; - struct blkg_iostat_set iostat; - struct blkg_policy_data *pd[5]; - spinlock_t async_bio_lock; - struct bio_list async_bios; - struct work_struct async_bio_work; - atomic_t use_delay; - atomic64_t delay_nsec; - atomic64_t delay_start; - u64 last_delay; - int last_use; - struct callback_head callback_head; -}; - -typedef unsigned int blk_qc_t; - -struct blk_zone; - -typedef int (*report_zones_cb)(struct blk_zone *, unsigned int, void *); - -struct hd_geometry; - -struct pr_ops; - -struct block_device_operations { - blk_qc_t (*submit_bio)(struct bio *); - int (*open)(struct block_device *, fmode_t); - void (*release)(struct gendisk *, fmode_t); - int (*rw_page)(struct block_device *, sector_t, struct page *, unsigned int); - int (*ioctl)(struct block_device *, fmode_t, unsigned int, long unsigned int); - int (*compat_ioctl)(struct block_device *, fmode_t, unsigned int, long unsigned int); - unsigned int (*check_events)(struct gendisk *, unsigned int); - void (*unlock_native_capacity)(struct gendisk *); - int (*getgeo)(struct block_device *, struct hd_geometry *); - int (*set_read_only)(struct block_device *, bool); - void (*swap_slot_free_notify)(struct block_device *, long unsigned int); - int (*report_zones)(struct gendisk *, sector_t, unsigned int, report_zones_cb, void *); - char * (*devnode)(struct gendisk *, umode_t *); - struct module *owner; - const struct pr_ops *pr_ops; -}; - struct iovec { void *iov_base; __kernel_size_t iov_len; @@ -11195,318 +10145,6 @@ struct kvec { size_t iov_len; }; -struct sg_io_v4 { - __s32 guard; - __u32 protocol; - __u32 subprotocol; - __u32 request_len; - __u64 request; - __u64 request_tag; - __u32 request_attr; - __u32 request_priority; - __u32 request_extra; - __u32 max_response_len; - __u64 response; - __u32 dout_iovec_count; - __u32 dout_xfer_len; - __u32 din_iovec_count; - __u32 din_xfer_len; - __u64 dout_xferp; - __u64 din_xferp; - __u32 timeout; - __u32 flags; - __u64 usr_ptr; - __u32 spare_in; - __u32 driver_status; - __u32 transport_status; - __u32 device_status; - __u32 retry_delay; - __u32 info; - __u32 duration; - __u32 response_len; - __s32 din_resid; - __s32 dout_resid; - __u64 generated_tag; - __u32 spare_out; - __u32 padding; -}; - -struct bsg_ops { - int (*check_proto)(struct sg_io_v4 *); - int (*fill_hdr)(struct request *, struct sg_io_v4 *, fmode_t); - int (*complete_rq)(struct request *, struct sg_io_v4 *); - void (*free_rq)(struct request *); -}; - -typedef __u32 req_flags_t; - -typedef void rq_end_io_fn(struct request *, blk_status_t); - -enum mq_rq_state { - MQ_RQ_IDLE = 0, - MQ_RQ_IN_FLIGHT = 1, - MQ_RQ_COMPLETE = 2, -}; - -struct request { - struct request_queue *q; - struct blk_mq_ctx *mq_ctx; - struct blk_mq_hw_ctx *mq_hctx; - unsigned int cmd_flags; - req_flags_t rq_flags; - int tag; - int internal_tag; - unsigned int __data_len; - sector_t __sector; - struct bio *bio; - struct bio *biotail; - struct list_head queuelist; - union { - struct hlist_node hash; - struct llist_node ipi_list; - }; - union { - struct rb_node rb_node; - struct bio_vec special_vec; - void *completion_data; - int error_count; - }; - union { - struct { - struct io_cq *icq; - void *priv[2]; - } elv; - struct { - unsigned int seq; - struct list_head list; - rq_end_io_fn *saved_end_io; - } flush; - }; - struct gendisk *rq_disk; - struct block_device *part; - u64 start_time_ns; - u64 io_start_time_ns; - short unsigned int stats_sectors; - short unsigned int nr_phys_segments; - short unsigned int write_hint; - short unsigned int ioprio; - enum mq_rq_state state; - refcount_t ref; - unsigned int timeout; - long unsigned int deadline; - union { - struct __call_single_data csd; - u64 fifo_time; - }; - rq_end_io_fn *end_io; - void *end_io_data; -}; - -struct blk_zone { - __u64 start; - __u64 len; - __u64 wp; - __u8 type; - __u8 cond; - __u8 non_seq; - __u8 reset; - __u8 resv[4]; - __u64 capacity; - __u8 reserved[24]; -}; - -struct sbitmap_word { - long unsigned int depth; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long unsigned int word; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long unsigned int cleared; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; -}; - -struct sbq_wait_state { - atomic_t wait_cnt; - wait_queue_head_t wait; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; -}; - -enum elv_merge { - ELEVATOR_NO_MERGE = 0, - ELEVATOR_FRONT_MERGE = 1, - ELEVATOR_BACK_MERGE = 2, - ELEVATOR_DISCARD_MERGE = 3, -}; - -struct elevator_type; - -struct blk_mq_alloc_data; - -struct elevator_mq_ops { - int (*init_sched)(struct request_queue *, struct elevator_type *); - void (*exit_sched)(struct elevator_queue *); - int (*init_hctx)(struct blk_mq_hw_ctx *, unsigned int); - void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); - void (*depth_updated)(struct blk_mq_hw_ctx *); - bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); - bool (*bio_merge)(struct request_queue *, struct bio *, unsigned int); - int (*request_merge)(struct request_queue *, struct request **, struct bio *); - void (*request_merged)(struct request_queue *, struct request *, enum elv_merge); - void (*requests_merged)(struct request_queue *, struct request *, struct request *); - void (*limit_depth)(unsigned int, struct blk_mq_alloc_data *); - void (*prepare_request)(struct request *); - void (*finish_request)(struct request *); - void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); - struct request * (*dispatch_request)(struct blk_mq_hw_ctx *); - bool (*has_work)(struct blk_mq_hw_ctx *); - void (*completed_request)(struct request *, u64); - void (*requeue_request)(struct request *); - struct request * (*former_request)(struct request_queue *, struct request *); - struct request * (*next_request)(struct request_queue *, struct request *); - void (*init_icq)(struct io_cq *); - void (*exit_icq)(struct io_cq *); -}; - -struct elv_fs_entry; - -struct blk_mq_debugfs_attr; - -struct elevator_type { - struct kmem_cache *icq_cache; - struct elevator_mq_ops ops; - size_t icq_size; - size_t icq_align; - struct elv_fs_entry *elevator_attrs; - const char *elevator_name; - const char *elevator_alias; - const unsigned int elevator_features; - struct module *elevator_owner; - const struct blk_mq_debugfs_attr *queue_debugfs_attrs; - const struct blk_mq_debugfs_attr *hctx_debugfs_attrs; - char icq_cache_name[22]; - struct list_head list; -}; - -struct elevator_queue { - struct elevator_type *type; - void *elevator_data; - struct kobject kobj; - struct mutex sysfs_lock; - unsigned int registered: 1; - struct hlist_head hash[64]; -}; - -struct elv_fs_entry { - struct attribute attr; - ssize_t (*show)(struct elevator_queue *, char *); - ssize_t (*store)(struct elevator_queue *, const char *, size_t); -}; - -struct blk_mq_debugfs_attr { - const char *name; - umode_t mode; - int (*show)(void *, struct seq_file *); - ssize_t (*write)(void *, const char *, size_t, loff_t *); - const struct seq_operations *seq_ops; -}; - -enum blk_eh_timer_return { - BLK_EH_DONE = 0, - BLK_EH_RESET_TIMER = 1, -}; - -struct blk_mq_queue_data; - -struct blk_mq_ops { - blk_status_t (*queue_rq)(struct blk_mq_hw_ctx *, const struct blk_mq_queue_data *); - void (*commit_rqs)(struct blk_mq_hw_ctx *); - int (*get_budget)(struct request_queue *); - void (*put_budget)(struct request_queue *, int); - void (*set_rq_budget_token)(struct request *, int); - int (*get_rq_budget_token)(struct request *); - enum blk_eh_timer_return (*timeout)(struct request *, bool); - int (*poll)(struct blk_mq_hw_ctx *); - void (*complete)(struct request *); - int (*init_hctx)(struct blk_mq_hw_ctx *, void *, unsigned int); - void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); - int (*init_request)(struct blk_mq_tag_set *, struct request *, unsigned int, unsigned int); - void (*exit_request)(struct blk_mq_tag_set *, struct request *, unsigned int); - void (*initialize_rq_fn)(struct request *); - void (*cleanup_rq)(struct request *); - bool (*busy)(struct request_queue *); - int (*map_queues)(struct blk_mq_tag_set *); - void (*show_rq)(struct seq_file *, struct request *); -}; - -enum pr_type { - PR_WRITE_EXCLUSIVE = 1, - PR_EXCLUSIVE_ACCESS = 2, - PR_WRITE_EXCLUSIVE_REG_ONLY = 3, - PR_EXCLUSIVE_ACCESS_REG_ONLY = 4, - PR_WRITE_EXCLUSIVE_ALL_REGS = 5, - PR_EXCLUSIVE_ACCESS_ALL_REGS = 6, -}; - -struct pr_ops { - int (*pr_register)(struct block_device *, u64, u64, u32); - int (*pr_reserve)(struct block_device *, u64, enum pr_type, u32); - int (*pr_release)(struct block_device *, u64, enum pr_type); - int (*pr_preempt)(struct block_device *, u64, u64, enum pr_type, bool); - int (*pr_clear)(struct block_device *, u64); -}; - -enum blkg_iostat_type { - BLKG_IOSTAT_READ = 0, - BLKG_IOSTAT_WRITE = 1, - BLKG_IOSTAT_DISCARD = 2, - BLKG_IOSTAT_NR = 3, -}; - -struct blkcg_policy_data; - -struct blkcg { - struct cgroup_subsys_state css; - spinlock_t lock; - refcount_t online_pin; - struct xarray blkg_tree; - struct blkcg_gq *blkg_hint; - struct hlist_head blkg_list; - struct blkcg_policy_data *cpd[5]; - struct list_head all_blkcgs_node; - struct list_head cgwb_list; -}; - -struct blkcg_policy_data { - struct blkcg *blkcg; - int plid; -}; - -struct blkg_policy_data { - struct blkcg_gq *blkg; - int plid; -}; - enum reboot_mode { REBOOT_UNDEFINED = 4294967295, REBOOT_COLD = 0, @@ -11710,10 +10348,10 @@ struct efi_runtime_work { }; enum memcg_stat_item { - MEMCG_SWAP = 39, - MEMCG_SOCK = 40, - MEMCG_PERCPU_B = 41, - MEMCG_NR_STAT = 42, + MEMCG_SWAP = 40, + MEMCG_SOCK = 41, + MEMCG_PERCPU_B = 42, + MEMCG_NR_STAT = 43, }; enum memcg_memory_event { @@ -11735,10 +10373,10 @@ enum mem_cgroup_events_target { }; struct memcg_vmstats_percpu { - long int state[42]; - long unsigned int events[100]; - long int state_prev[42]; - long unsigned int events_prev[100]; + long int state[43]; + long unsigned int events[102]; + long int state_prev[43]; + long unsigned int events_prev[102]; long unsigned int nr_page_events; long unsigned int targets[2]; }; @@ -11748,25 +10386,26 @@ struct mem_cgroup_reclaim_iter { unsigned int generation; }; -struct lruvec_stat { - long int count[39]; -}; - -struct batched_lruvec_stat { - s32 count[39]; -}; - struct shrinker_info { struct callback_head rcu; atomic_long_t *nr_deferred; long unsigned int *map; }; +struct lruvec_stats_percpu { + long int state[40]; + long int state_prev[40]; +}; + +struct lruvec_stats { + long int state[40]; + long int state_pending[40]; +}; + struct mem_cgroup_per_node { struct lruvec lruvec; - struct lruvec_stat *lruvec_stat_local; - struct batched_lruvec_stat *lruvec_stat_cpu; - atomic_long_t lruvec_stat[39]; + struct lruvec_stats_percpu *lruvec_stats_percpu; + struct lruvec_stats lruvec_stats; long unsigned int lru_zone_size[20]; struct mem_cgroup_reclaim_iter iter; struct shrinker_info *shrinker_info; @@ -11854,6 +10493,111 @@ struct fs_parse_result { }; }; +struct blk_zone; + +typedef int (*report_zones_cb)(struct blk_zone *, unsigned int, void *); + +enum blk_unique_id { + BLK_UID_T10 = 1, + BLK_UID_EUI64 = 2, + BLK_UID_NAA = 3, +}; + +struct hd_geometry; + +struct pr_ops; + +struct block_device_operations { + void (*submit_bio)(struct bio *); + int (*open)(struct block_device *, fmode_t); + void (*release)(struct gendisk *, fmode_t); + int (*rw_page)(struct block_device *, sector_t, struct page *, unsigned int); + int (*ioctl)(struct block_device *, fmode_t, unsigned int, long unsigned int); + int (*compat_ioctl)(struct block_device *, fmode_t, unsigned int, long unsigned int); + unsigned int (*check_events)(struct gendisk *, unsigned int); + void (*unlock_native_capacity)(struct gendisk *); + int (*getgeo)(struct block_device *, struct hd_geometry *); + int (*set_read_only)(struct block_device *, bool); + void (*swap_slot_free_notify)(struct block_device *, long unsigned int); + int (*report_zones)(struct gendisk *, sector_t, unsigned int, report_zones_cb, void *); + char * (*devnode)(struct gendisk *, umode_t *); + int (*get_unique_id)(struct gendisk *, u8 *, enum blk_unique_id); + struct module *owner; + const struct pr_ops *pr_ops; + int (*alternative_gpt_sector)(struct gendisk *, sector_t *); +}; + +struct blk_zone { + __u64 start; + __u64 len; + __u64 wp; + __u8 type; + __u8 cond; + __u8 non_seq; + __u8 reset; + __u8 resv[4]; + __u64 capacity; + __u8 reserved[24]; +}; + +struct blk_independent_access_range { + struct kobject kobj; + struct request_queue *queue; + sector_t sector; + sector_t nr_sectors; +}; + +struct blk_independent_access_ranges { + struct kobject kobj; + bool sysfs_registered; + unsigned int nr_ia_ranges; + struct blk_independent_access_range ia_range[0]; +}; + +enum blk_eh_timer_return { + BLK_EH_DONE = 0, + BLK_EH_RESET_TIMER = 1, +}; + +struct blk_mq_queue_data; + +struct blk_mq_ops { + blk_status_t (*queue_rq)(struct blk_mq_hw_ctx *, const struct blk_mq_queue_data *); + void (*commit_rqs)(struct blk_mq_hw_ctx *); + int (*get_budget)(struct request_queue *); + void (*put_budget)(struct request_queue *, int); + void (*set_rq_budget_token)(struct request *, int); + int (*get_rq_budget_token)(struct request *); + enum blk_eh_timer_return (*timeout)(struct request *, bool); + int (*poll)(struct blk_mq_hw_ctx *, struct io_comp_batch *); + void (*complete)(struct request *); + int (*init_hctx)(struct blk_mq_hw_ctx *, void *, unsigned int); + void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); + int (*init_request)(struct blk_mq_tag_set *, struct request *, unsigned int, unsigned int); + void (*exit_request)(struct blk_mq_tag_set *, struct request *, unsigned int); + void (*cleanup_rq)(struct request *); + bool (*busy)(struct request_queue *); + int (*map_queues)(struct blk_mq_tag_set *); + void (*show_rq)(struct seq_file *, struct request *); +}; + +enum pr_type { + PR_WRITE_EXCLUSIVE = 1, + PR_EXCLUSIVE_ACCESS = 2, + PR_WRITE_EXCLUSIVE_REG_ONLY = 3, + PR_EXCLUSIVE_ACCESS_REG_ONLY = 4, + PR_WRITE_EXCLUSIVE_ALL_REGS = 5, + PR_EXCLUSIVE_ACCESS_ALL_REGS = 6, +}; + +struct pr_ops { + int (*pr_register)(struct block_device *, u64, u64, u32); + int (*pr_reserve)(struct block_device *, u64, enum pr_type, u32); + int (*pr_release)(struct block_device *, u64, enum pr_type); + int (*pr_preempt)(struct block_device *, u64, u64, enum pr_type, bool); + int (*pr_clear)(struct block_device *, u64); +}; + struct trace_event_raw_initcall_level { struct trace_entry ent; u32 __data_loc_level; @@ -11926,6 +10670,14 @@ typedef u64 uint64_t; typedef unsigned int slab_flags_t; +typedef void (*smp_call_func_t)(void *); + +struct __call_single_data { + struct __call_single_node node; + smp_call_func_t func; + void *info; +}; + typedef struct __call_single_data call_single_data_t; struct raw_notifier_head { @@ -11941,6 +10693,8 @@ struct page_pool_params { enum dma_data_direction dma_dir; unsigned int max_len; unsigned int offset; + void (*init_callback)(struct page *, void *); + void *init_arg; }; struct pp_alloc_cache { @@ -11986,9 +10740,16 @@ struct page_pool { long unsigned int defer_start; long unsigned int defer_warn; u32 pages_state_hold_cnt; + unsigned int frag_offset; + struct page *frag_page; + long int frag_users; + u32 xdp_mem_id; long: 32; long: 64; long: 64; + long: 64; + long: 64; + long: 64; struct pp_alloc_cache alloc; long: 64; long: 64; @@ -12108,9 +10869,19 @@ typedef struct { struct sk_buff; -struct sk_buff_head { +struct sk_buff_list { struct sk_buff *next; struct sk_buff *prev; +}; + +struct sk_buff_head { + union { + struct { + struct sk_buff *next; + struct sk_buff *prev; + }; + struct sk_buff_list list; + }; __u32 qlen; spinlock_t lock; }; @@ -12118,38 +10889,33 @@ struct sk_buff_head { typedef u64 netdev_features_t; struct sock_cgroup_data { - union { - struct { - u8 is_data: 1; - u8 no_refcnt: 1; - u8 unused: 6; - u8 padding; - u16 prioidx; - u32 classid; - }; - u64 val; - }; + struct cgroup *cgroup; + u32 classid; }; +typedef struct {} netns_tracker; + +struct dst_entry; + struct sk_filter; struct socket_wq; struct xfrm_policy; -struct dst_entry; - struct socket; struct sock_reuseport; struct sock { struct sock_common __sk_common; + struct dst_entry *sk_rx_dst; + int sk_rx_dst_ifindex; + u32 sk_rx_dst_cookie; socket_lock_t sk_lock; atomic_t sk_drops; int sk_rcvlowat; struct sk_buff_head sk_error_queue; - struct sk_buff *sk_rx_skb_cache; struct sk_buff_head sk_receive_queue; struct { atomic_t rmem_alloc; @@ -12157,7 +10923,9 @@ struct sock { struct sk_buff *head; struct sk_buff *tail; } sk_backlog; + struct llist_head defer_list; int sk_forward_alloc; + u32 sk_reserved_mem; unsigned int sk_ll_usec; unsigned int sk_napi_id; int sk_rcvbuf; @@ -12167,7 +10935,6 @@ struct sock { struct socket_wq *sk_wq_raw; }; struct xfrm_policy *sk_policy[2]; - struct dst_entry *sk_rx_dst; struct dst_entry *sk_dst_cache; atomic_t sk_omem_alloc; int sk_sndbuf; @@ -12178,7 +10945,6 @@ struct sock { struct sk_buff *sk_send_head; struct rb_root tcp_rtx_queue; }; - struct sk_buff *sk_tx_skb_cache; struct sk_buff_head sk_write_queue; __s32 sk_peek_off; int sk_write_pending; @@ -12192,13 +10958,11 @@ struct sock { long unsigned int sk_max_pacing_rate; struct page_frag sk_frag; netdev_features_t sk_route_caps; - netdev_features_t sk_route_nocaps; - netdev_features_t sk_route_forced_caps; int sk_gso_type; unsigned int sk_gso_max_size; gfp_t sk_allocation; __u32 sk_txhash; - u8 sk_padding: 1; + u8 sk_gso_disabled: 1; u8 sk_kern_sock: 1; u8 sk_no_check_tx: 1; u8 sk_no_check_rx: 1; @@ -12217,12 +10981,13 @@ struct sock { kuid_t sk_uid; u8 sk_prefer_busy_poll; u16 sk_busy_poll_budget; + spinlock_t sk_peer_lock; + int sk_bind_phc; struct pid *sk_peer_pid; const struct cred *sk_peer_cred; long int sk_rcvtimeo; ktime_t sk_stamp; u16 sk_tsflags; - int sk_bind_phc; u8 sk_shutdown; u32 sk_tskey; atomic_t sk_zckey; @@ -12244,6 +11009,7 @@ struct sock { struct sock_reuseport *sk_reuseport_cb; struct bpf_local_storage *sk_bpf_storage; struct callback_head sk_rcu; + netns_tracker ns_tracker; }; struct rhash_head { @@ -12316,6 +11082,7 @@ struct pipe_inode_info { unsigned int files; unsigned int r_counter; unsigned int w_counter; + unsigned int poll_usage; struct page *tmp_page; struct fasync_struct *fasync_readers; struct fasync_struct *fasync_writers; @@ -12509,7 +11276,6 @@ struct tty_driver { struct tty_ldisc_ops { char *name; int num; - int flags; int (*open)(struct tty_struct *); void (*close)(struct tty_struct *); void (*flush_buffer)(struct tty_struct *); @@ -12519,7 +11285,7 @@ struct tty_ldisc_ops { int (*compat_ioctl)(struct tty_struct *, struct file *, unsigned int, long unsigned int); void (*set_termios)(struct tty_struct *, struct ktermios *); __poll_t (*poll)(struct tty_struct *, struct file *, struct poll_table_struct *); - int (*hangup)(struct tty_struct *); + void (*hangup)(struct tty_struct *); void (*receive_buf)(struct tty_struct *, const unsigned char *, const char *, int); void (*write_wakeup)(struct tty_struct *); void (*dcd_change)(struct tty_struct *, unsigned int); @@ -12545,12 +11311,13 @@ struct tty_port_client_operations { void (*write_wakeup)(struct tty_port *); }; +struct ref_tracker_dir {}; + struct prot_inuse; struct netns_core { struct ctl_table_header *sysctl_hdr; int sysctl_somaxconn; - int *sock_inuse; struct prot_inuse *prot_inuse; }; @@ -12683,6 +11450,8 @@ struct netns_ipv4 { u8 sysctl_icmp_errors_use_inbound_ifaddr; int sysctl_icmp_ratelimit; int sysctl_icmp_ratemask; + u32 ip_rt_min_pmtu; + int ip_rt_mtu_expires; struct local_ports ip_local_ports; u8 sysctl_tcp_ecn; u8 sysctl_tcp_ecn_fallback; @@ -12759,7 +11528,6 @@ struct netns_ipv4 { int sysctl_tcp_fastopen; const struct tcp_congestion_ops *tcp_congestion_control; struct tcp_fastopen_context *tcp_fastopen_ctx; - spinlock_t tcp_fastopen_ctx_lock; unsigned int sysctl_tcp_fastopen_blackhole_timeout; atomic_t tfo_active_disable_times; long unsigned int tfo_active_disable_stamp; @@ -12789,9 +11557,10 @@ struct netns_ipv4 { long: 64; long: 64; long: 64; - long: 64; }; +struct net_device; + struct neighbour; struct dst_ops { @@ -12856,6 +11625,8 @@ struct netns_sysctl_ipv6 { int max_dst_opts_len; int max_hbh_opts_len; int seg6_flowlabel; + u32 ioam6_id; + u64 ioam6_id_wide; bool skip_notify_on_dev_down; u8 fib_notify_on_flag_change; }; @@ -12872,6 +11643,8 @@ struct fib6_table; struct seg6_pernet_data; +struct ioam6_pernet_data; + struct netns_ipv6 { struct dst_ops ip6_dst_ops; struct netns_sysctl_ipv6 sysctl; @@ -12913,28 +11686,86 @@ struct netns_ipv6 { spinlock_t lock; u32 seq; } ip6addrlbl_table; + struct ioam6_pernet_data *ioam6_data; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; long: 64; long: 64; }; -struct nf_queue_handler; - struct nf_logger; struct nf_hook_entries; struct netns_nf { struct proc_dir_entry *proc_netfilter; - const struct nf_queue_handler *queue_handler; const struct nf_logger *nf_loggers[13]; struct ctl_table_header *nf_log_dir_header; struct nf_hook_entries *hooks_ipv4[5]; struct nf_hook_entries *hooks_ipv6[5]; + unsigned int defrag_ipv4_users; + unsigned int defrag_ipv6_users; }; -struct netns_xt { - bool notrack_deprecated_warning; - bool clusterip_deprecated_warning; +struct nf_ct_event_notifier; + +struct nf_generic_net { + unsigned int timeout; +}; + +struct nf_tcp_net { + unsigned int timeouts[14]; + u8 tcp_loose; + u8 tcp_be_liberal; + u8 tcp_max_retrans; + u8 tcp_ignore_invalid_rst; +}; + +struct nf_udp_net { + unsigned int timeouts[2]; +}; + +struct nf_icmp_net { + unsigned int timeout; +}; + +struct nf_dccp_net { + u8 dccp_loose; + unsigned int dccp_timeout[10]; +}; + +struct nf_sctp_net { + unsigned int timeouts[10]; +}; + +struct nf_ip_net { + struct nf_generic_net generic; + struct nf_tcp_net tcp; + struct nf_udp_net udp; + struct nf_icmp_net icmp; + struct nf_icmp_net icmpv6; + struct nf_dccp_net dccp; + struct nf_sctp_net sctp; +}; + +struct ct_pcpu; + +struct ip_conntrack_stat; + +struct netns_ct { + u8 sysctl_log_invalid; + u8 sysctl_events; + u8 sysctl_acct; + u8 sysctl_auto_assign_helper; + u8 sysctl_tstamp; + u8 sysctl_checksum; + struct ct_pcpu *pcpu_lists; + struct ip_conntrack_stat *stat; + struct nf_ct_event_notifier *nf_conntrack_event_cb; + struct nf_ip_net nf_ct_proto; }; struct netns_bpf { @@ -12985,6 +11816,7 @@ struct netns_xfrm { u32 sysctl_aevent_rseqth; int sysctl_larval_drop; u32 sysctl_acq_expires; + u8 policy_default; struct ctl_table_header *sysctl_hdr; long: 64; long: 64; @@ -12992,20 +11824,13 @@ struct netns_xfrm { long: 64; long: 64; long: 64; - long: 64; struct dst_ops xfrm4_dst_ops; struct dst_ops xfrm6_dst_ops; spinlock_t xfrm_state_lock; seqcount_spinlock_t xfrm_state_hash_generation; + seqcount_spinlock_t xfrm_policy_hash_generation; spinlock_t xfrm_policy_lock; struct mutex xfrm_cfg_mutex; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; }; struct mpls_route; @@ -13043,6 +11868,7 @@ struct net { struct ucounts *ucounts; struct idr netns_ids; struct ns_common ns; + struct ref_tracker_dir refcnt_tracker; struct list_head dev_base_head; struct proc_dir_entry *proc_net; struct proc_dir_entry *proc_net_stat; @@ -13067,10 +11893,11 @@ struct net { long: 64; long: 64; long: 64; + long: 64; struct netns_ipv4 ipv4; struct netns_ipv6 ipv6; struct netns_nf nf; - struct netns_xt xt; + struct netns_ct ct; struct net_generic *gen; struct netns_bpf bpf; long: 64; @@ -13095,443 +11922,6 @@ typedef struct { local64_t v; } u64_stats_t; -typedef struct { - union { - void *kernel; - void *user; - }; - bool is_kernel: 1; -} sockptr_t; - -struct bpf_offloaded_map; - -struct bpf_map_dev_ops { - int (*map_get_next_key)(struct bpf_offloaded_map *, void *, void *); - int (*map_lookup_elem)(struct bpf_offloaded_map *, void *, void *); - int (*map_update_elem)(struct bpf_offloaded_map *, void *, void *, u64); - int (*map_delete_elem)(struct bpf_offloaded_map *, void *); -}; - -struct bpf_offloaded_map { - struct bpf_map map; - struct net_device *netdev; - const struct bpf_map_dev_ops *dev_ops; - void *dev_priv; - struct list_head offloads; - long: 64; - long: 64; - long: 64; -}; - -struct net_device_stats { - long unsigned int rx_packets; - long unsigned int tx_packets; - long unsigned int rx_bytes; - long unsigned int tx_bytes; - long unsigned int rx_errors; - long unsigned int tx_errors; - long unsigned int rx_dropped; - long unsigned int tx_dropped; - long unsigned int multicast; - long unsigned int collisions; - long unsigned int rx_length_errors; - long unsigned int rx_over_errors; - long unsigned int rx_crc_errors; - long unsigned int rx_frame_errors; - long unsigned int rx_fifo_errors; - long unsigned int rx_missed_errors; - long unsigned int tx_aborted_errors; - long unsigned int tx_carrier_errors; - long unsigned int tx_fifo_errors; - long unsigned int tx_heartbeat_errors; - long unsigned int tx_window_errors; - long unsigned int rx_compressed; - long unsigned int tx_compressed; -}; - -struct netdev_hw_addr_list { - struct list_head list; - int count; -}; - -struct wireless_dev; - -enum rx_handler_result { - RX_HANDLER_CONSUMED = 0, - RX_HANDLER_ANOTHER = 1, - RX_HANDLER_EXACT = 2, - RX_HANDLER_PASS = 3, -}; - -typedef enum rx_handler_result rx_handler_result_t; - -typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **); - -enum netdev_ml_priv_type { - ML_PRIV_NONE = 0, - ML_PRIV_CAN = 1, -}; - -struct pcpu_dstats; - -struct netdev_tc_txq { - u16 count; - u16 offset; -}; - -struct sfp_bus; - -struct bpf_xdp_link; - -struct bpf_xdp_entity { - struct bpf_prog *prog; - struct bpf_xdp_link *link; -}; - -struct netdev_name_node; - -struct dev_ifalias; - -struct net_device_ops; - -struct ethtool_ops; - -struct ndisc_ops; - -struct header_ops; - -struct vlan_info; - -struct in_device; - -struct inet6_dev; - -struct wpan_dev; - -struct netdev_rx_queue; - -struct mini_Qdisc; - -struct netdev_queue; - -struct cpu_rmap; - -struct Qdisc; - -struct xdp_dev_bulk_queue; - -struct xps_dev_maps; - -struct pcpu_lstats; - -struct pcpu_sw_netstats; - -struct rtnl_link_ops; - -struct dcbnl_rtnl_ops; - -struct phy_device; - -struct udp_tunnel_nic_info; - -struct udp_tunnel_nic; - -struct net_device { - char name[16]; - struct netdev_name_node *name_node; - struct dev_ifalias *ifalias; - long unsigned int mem_end; - long unsigned int mem_start; - long unsigned int base_addr; - long unsigned int state; - struct list_head dev_list; - struct list_head napi_list; - struct list_head unreg_list; - struct list_head close_list; - struct list_head ptype_all; - struct list_head ptype_specific; - struct { - struct list_head upper; - struct list_head lower; - } adj_list; - unsigned int flags; - unsigned int priv_flags; - const struct net_device_ops *netdev_ops; - int ifindex; - short unsigned int gflags; - short unsigned int hard_header_len; - unsigned int mtu; - short unsigned int needed_headroom; - short unsigned int needed_tailroom; - netdev_features_t features; - netdev_features_t hw_features; - netdev_features_t wanted_features; - netdev_features_t vlan_features; - netdev_features_t hw_enc_features; - netdev_features_t mpls_features; - netdev_features_t gso_partial_features; - unsigned int min_mtu; - unsigned int max_mtu; - short unsigned int type; - unsigned char min_header_len; - unsigned char name_assign_type; - int group; - struct net_device_stats stats; - atomic_long_t rx_dropped; - atomic_long_t tx_dropped; - atomic_long_t rx_nohandler; - atomic_t carrier_up_count; - atomic_t carrier_down_count; - const struct ethtool_ops *ethtool_ops; - const struct ndisc_ops *ndisc_ops; - const struct header_ops *header_ops; - unsigned char operstate; - unsigned char link_mode; - unsigned char if_port; - unsigned char dma; - unsigned char perm_addr[32]; - unsigned char addr_assign_type; - unsigned char addr_len; - unsigned char upper_level; - unsigned char lower_level; - short unsigned int neigh_priv_len; - short unsigned int dev_id; - short unsigned int dev_port; - short unsigned int padded; - spinlock_t addr_list_lock; - int irq; - struct netdev_hw_addr_list uc; - struct netdev_hw_addr_list mc; - struct netdev_hw_addr_list dev_addrs; - struct kset *queues_kset; - struct list_head unlink_list; - unsigned int promiscuity; - unsigned int allmulti; - bool uc_promisc; - unsigned char nested_level; - struct vlan_info *vlan_info; - struct in_device *ip_ptr; - struct inet6_dev *ip6_ptr; - struct wireless_dev *ieee80211_ptr; - struct wpan_dev *ieee802154_ptr; - unsigned char *dev_addr; - struct netdev_rx_queue *_rx; - unsigned int num_rx_queues; - unsigned int real_num_rx_queues; - struct bpf_prog *xdp_prog; - long unsigned int gro_flush_timeout; - int napi_defer_hard_irqs; - rx_handler_func_t *rx_handler; - void *rx_handler_data; - struct mini_Qdisc *miniq_ingress; - struct netdev_queue *ingress_queue; - struct nf_hook_entries *nf_hooks_ingress; - unsigned char broadcast[32]; - struct cpu_rmap *rx_cpu_rmap; - struct hlist_node index_hlist; - long: 64; - struct netdev_queue *_tx; - unsigned int num_tx_queues; - unsigned int real_num_tx_queues; - struct Qdisc *qdisc; - unsigned int tx_queue_len; - spinlock_t tx_global_lock; - struct xdp_dev_bulk_queue *xdp_bulkq; - struct xps_dev_maps *xps_maps[2]; - struct mini_Qdisc *miniq_egress; - struct hlist_head qdisc_hash[16]; - struct timer_list watchdog_timer; - int watchdog_timeo; - u32 proto_down_reason; - struct list_head todo_list; - int *pcpu_refcnt; - struct list_head link_watch_list; - enum { - NETREG_UNINITIALIZED = 0, - NETREG_REGISTERED = 1, - NETREG_UNREGISTERING = 2, - NETREG_UNREGISTERED = 3, - NETREG_RELEASED = 4, - NETREG_DUMMY = 5, - } reg_state: 8; - bool dismantle; - enum { - RTNL_LINK_INITIALIZED = 0, - RTNL_LINK_INITIALIZING = 1, - } rtnl_link_state: 16; - bool needs_free_netdev; - void (*priv_destructor)(struct net_device *); - possible_net_t nd_net; - void *ml_priv; - enum netdev_ml_priv_type ml_priv_type; - union { - struct pcpu_lstats *lstats; - struct pcpu_sw_netstats *tstats; - struct pcpu_dstats *dstats; - }; - struct device dev; - const struct attribute_group *sysfs_groups[4]; - const struct attribute_group *sysfs_rx_queue_group; - const struct rtnl_link_ops *rtnl_link_ops; - unsigned int gso_max_size; - u16 gso_max_segs; - const struct dcbnl_rtnl_ops *dcbnl_ops; - s16 num_tc; - struct netdev_tc_txq tc_to_txq[16]; - u8 prio_tc_map[16]; - struct phy_device *phydev; - struct sfp_bus *sfp_bus; - struct lock_class_key *qdisc_tx_busylock; - struct lock_class_key *qdisc_running_key; - bool proto_down; - unsigned int wol_enabled: 1; - unsigned int threaded: 1; - struct list_head net_notifier_list; - const struct udp_tunnel_nic_info *udp_tunnel_nic_info; - struct udp_tunnel_nic *udp_tunnel_nic; - struct bpf_xdp_entity xdp_state[3]; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; -}; - -struct bpf_dispatcher_prog { - struct bpf_prog *prog; - refcount_t users; -}; - -struct bpf_dispatcher { - struct mutex mutex; - void *func; - struct bpf_dispatcher_prog progs[48]; - int num_progs; - void *image; - u32 image_off; - struct bpf_ksym ksym; -}; - -typedef unsigned int sk_buff_data_t; - -struct skb_ext; - -struct sk_buff { - union { - struct { - struct sk_buff *next; - struct sk_buff *prev; - union { - struct net_device *dev; - long unsigned int dev_scratch; - }; - }; - struct rb_node rbnode; - struct list_head list; - }; - union { - struct sock *sk; - int ip_defrag_offset; - }; - union { - ktime_t tstamp; - u64 skb_mstamp_ns; - }; - char cb[48]; - union { - struct { - long unsigned int _skb_refdst; - void (*destructor)(struct sk_buff *); - }; - struct list_head tcp_tsorted_anchor; - long unsigned int _sk_redir; - }; - unsigned int len; - unsigned int data_len; - __u16 mac_len; - __u16 hdr_len; - __u16 queue_mapping; - __u8 __cloned_offset[0]; - __u8 cloned: 1; - __u8 nohdr: 1; - __u8 fclone: 2; - __u8 peeked: 1; - __u8 head_frag: 1; - __u8 pfmemalloc: 1; - __u8 pp_recycle: 1; - __u8 active_extensions; - __u32 headers_start[0]; - __u8 __pkt_type_offset[0]; - __u8 pkt_type: 3; - __u8 ignore_df: 1; - __u8 nf_trace: 1; - __u8 ip_summed: 2; - __u8 ooo_okay: 1; - __u8 l4_hash: 1; - __u8 sw_hash: 1; - __u8 wifi_acked_valid: 1; - __u8 wifi_acked: 1; - __u8 no_fcs: 1; - __u8 encapsulation: 1; - __u8 encap_hdr_csum: 1; - __u8 csum_valid: 1; - __u8 __pkt_vlan_present_offset[0]; - __u8 vlan_present: 1; - __u8 csum_complete_sw: 1; - __u8 csum_level: 2; - __u8 csum_not_inet: 1; - __u8 dst_pending_confirm: 1; - __u8 ndisc_nodetype: 2; - __u8 ipvs_property: 1; - __u8 inner_protocol_type: 1; - __u8 remcsum_offload: 1; - __u8 tc_skip_classify: 1; - __u8 tc_at_ingress: 1; - __u8 redirected: 1; - __u16 tc_index; - union { - __wsum csum; - struct { - __u16 csum_start; - __u16 csum_offset; - }; - }; - __u32 priority; - int skb_iif; - __u32 hash; - __be16 vlan_proto; - __u16 vlan_tci; - union { - unsigned int napi_id; - unsigned int sender_cpu; - }; - __u32 secmark; - union { - __u32 mark; - __u32 reserved_tailroom; - }; - union { - __be16 inner_protocol; - __u8 inner_ipproto; - }; - __u16 inner_transport_header; - __u16 inner_network_header; - __u16 inner_mac_header; - __be16 protocol; - __u16 transport_header; - __u16 network_header; - __u16 mac_header; - __u32 headers_end[0]; - sk_buff_data_t tail; - sk_buff_data_t end; - unsigned char *head; - unsigned char *data; - unsigned int truesize; - refcount_t users; - struct skb_ext *extensions; -}; - enum suspend_stat_step { SUSPEND_FREEZE = 1, SUSPEND_PREPARE = 2, @@ -13811,10 +12201,11 @@ struct rpc_stat; struct rpc_program; struct rpc_clnt { - atomic_t cl_count; + refcount_t cl_count; unsigned int cl_clid; struct list_head cl_clients; struct list_head cl_tasks; + atomic_t cl_pid; spinlock_t cl_lock; struct rpc_xprt *cl_xprt; const struct rpc_procinfo *cl_procinfo; @@ -13847,6 +12238,7 @@ struct rpc_clnt { struct work_struct cl_work; }; const struct cred *cl_cred; + unsigned int cl_max_connect; }; struct svc_xprt; @@ -13945,6 +12337,195 @@ struct flow_dissector { short unsigned int offset[28]; }; +typedef unsigned int sk_buff_data_t; + +struct skb_ext; + +struct sk_buff { + union { + struct { + struct sk_buff *next; + struct sk_buff *prev; + union { + struct net_device *dev; + long unsigned int dev_scratch; + }; + }; + struct rb_node rbnode; + struct list_head list; + struct llist_node ll_node; + }; + union { + struct sock *sk; + int ip_defrag_offset; + }; + union { + ktime_t tstamp; + u64 skb_mstamp_ns; + }; + char cb[48]; + union { + struct { + long unsigned int _skb_refdst; + void (*destructor)(struct sk_buff *); + }; + struct list_head tcp_tsorted_anchor; + long unsigned int _sk_redir; + }; + long unsigned int _nfct; + unsigned int len; + unsigned int data_len; + __u16 mac_len; + __u16 hdr_len; + __u16 queue_mapping; + __u8 __cloned_offset[0]; + __u8 cloned: 1; + __u8 nohdr: 1; + __u8 fclone: 2; + __u8 peeked: 1; + __u8 head_frag: 1; + __u8 pfmemalloc: 1; + __u8 pp_recycle: 1; + __u8 active_extensions; + union { + struct { + __u8 __pkt_type_offset[0]; + __u8 pkt_type: 3; + __u8 ignore_df: 1; + __u8 nf_trace: 1; + __u8 ip_summed: 2; + __u8 ooo_okay: 1; + __u8 l4_hash: 1; + __u8 sw_hash: 1; + __u8 wifi_acked_valid: 1; + __u8 wifi_acked: 1; + __u8 no_fcs: 1; + __u8 encapsulation: 1; + __u8 encap_hdr_csum: 1; + __u8 csum_valid: 1; + __u8 __pkt_vlan_present_offset[0]; + __u8 vlan_present: 1; + __u8 csum_complete_sw: 1; + __u8 csum_level: 2; + __u8 csum_not_inet: 1; + __u8 dst_pending_confirm: 1; + __u8 ndisc_nodetype: 2; + __u8 ipvs_property: 1; + __u8 inner_protocol_type: 1; + __u8 remcsum_offload: 1; + __u8 tc_skip_classify: 1; + __u8 tc_at_ingress: 1; + __u8 redirected: 1; + __u8 nf_skip_egress: 1; + __u8 slow_gro: 1; + __u16 tc_index; + union { + __wsum csum; + struct { + __u16 csum_start; + __u16 csum_offset; + }; + }; + __u32 priority; + int skb_iif; + __u32 hash; + __be16 vlan_proto; + __u16 vlan_tci; + union { + unsigned int napi_id; + unsigned int sender_cpu; + }; + __u32 secmark; + union { + __u32 mark; + __u32 reserved_tailroom; + }; + union { + __be16 inner_protocol; + __u8 inner_ipproto; + }; + __u16 inner_transport_header; + __u16 inner_network_header; + __u16 inner_mac_header; + __be16 protocol; + __u16 transport_header; + __u16 network_header; + __u16 mac_header; + }; + struct { + __u8 __pkt_type_offset[0]; + __u8 pkt_type: 3; + __u8 ignore_df: 1; + __u8 nf_trace: 1; + __u8 ip_summed: 2; + __u8 ooo_okay: 1; + __u8 l4_hash: 1; + __u8 sw_hash: 1; + __u8 wifi_acked_valid: 1; + __u8 wifi_acked: 1; + __u8 no_fcs: 1; + __u8 encapsulation: 1; + __u8 encap_hdr_csum: 1; + __u8 csum_valid: 1; + __u8 __pkt_vlan_present_offset[0]; + __u8 vlan_present: 1; + __u8 csum_complete_sw: 1; + __u8 csum_level: 2; + __u8 csum_not_inet: 1; + __u8 dst_pending_confirm: 1; + __u8 ndisc_nodetype: 2; + __u8 ipvs_property: 1; + __u8 inner_protocol_type: 1; + __u8 remcsum_offload: 1; + __u8 tc_skip_classify: 1; + __u8 tc_at_ingress: 1; + __u8 redirected: 1; + __u8 nf_skip_egress: 1; + __u8 slow_gro: 1; + __u16 tc_index; + union { + __wsum csum; + struct { + __u16 csum_start; + __u16 csum_offset; + }; + }; + __u32 priority; + int skb_iif; + __u32 hash; + __be16 vlan_proto; + __u16 vlan_tci; + union { + unsigned int napi_id; + unsigned int sender_cpu; + }; + __u32 secmark; + union { + __u32 mark; + __u32 reserved_tailroom; + }; + union { + __be16 inner_protocol; + __u8 inner_ipproto; + }; + __u16 inner_transport_header; + __u16 inner_network_header; + __u16 inner_mac_header; + __be16 protocol; + __u16 transport_header; + __u16 network_header; + __u16 mac_header; + } headers; + }; + sk_buff_data_t tail; + sk_buff_data_t end; + unsigned char *head; + unsigned char *data; + unsigned int truesize; + refcount_t users; + struct skb_ext *extensions; +}; + struct flowi_tunnel { __be64 tun_id; }; @@ -14014,6 +12595,11 @@ struct flowi { } u; }; +struct prot_inuse { + int all; + int val[64]; +}; + struct ipstats_mib { u64 mibs[37]; struct u64_stats_sync syncp; @@ -14149,8 +12735,6 @@ struct nlattr; struct netlink_ext_ack; -struct nla_policy; - struct fib_rules_ops { int family; struct list_head list; @@ -14160,7 +12744,7 @@ struct fib_rules_ops { int nr_goto_rules; unsigned int fib_rules_seq; int (*action)(struct fib_rule *, struct flowi *, int, struct fib_lookup_arg *); - bool (*suppress)(struct fib_rule *, struct fib_lookup_arg *); + bool (*suppress)(struct fib_rule *, int, struct fib_lookup_arg *); int (*match)(struct fib_rule *, struct flowi *, int); int (*configure)(struct fib_rule *, struct sk_buff *, struct fib_rule_hdr *, struct nlattr **, struct netlink_ext_ack *); int (*delete)(struct fib_rule *); @@ -14169,7 +12753,6 @@ struct fib_rules_ops { size_t (*nlmsg_payload)(struct fib_rule *); void (*flush_cache)(struct fib_rules_ops *); int nlgroup; - const struct nla_policy *policy; struct list_head rules_list; struct module *owner; struct net *fro_net; @@ -14226,6 +12809,8 @@ struct fib_notifier_ops { struct callback_head rcu; }; +typedef struct {} netdevice_tracker; + struct xfrm_state; struct lwtunnel_state; @@ -14250,6 +12835,287 @@ struct dst_entry { short int error; short int __pad; __u32 tclassid; + netdevice_tracker dev_tracker; +}; + +struct net_device_stats { + long unsigned int rx_packets; + long unsigned int tx_packets; + long unsigned int rx_bytes; + long unsigned int tx_bytes; + long unsigned int rx_errors; + long unsigned int tx_errors; + long unsigned int rx_dropped; + long unsigned int tx_dropped; + long unsigned int multicast; + long unsigned int collisions; + long unsigned int rx_length_errors; + long unsigned int rx_over_errors; + long unsigned int rx_crc_errors; + long unsigned int rx_frame_errors; + long unsigned int rx_fifo_errors; + long unsigned int rx_missed_errors; + long unsigned int tx_aborted_errors; + long unsigned int tx_carrier_errors; + long unsigned int tx_fifo_errors; + long unsigned int tx_heartbeat_errors; + long unsigned int tx_window_errors; + long unsigned int rx_compressed; + long unsigned int tx_compressed; +}; + +struct netdev_hw_addr_list { + struct list_head list; + int count; + struct rb_root tree; +}; + +struct wireless_dev; + +enum rx_handler_result { + RX_HANDLER_CONSUMED = 0, + RX_HANDLER_ANOTHER = 1, + RX_HANDLER_EXACT = 2, + RX_HANDLER_PASS = 3, +}; + +typedef enum rx_handler_result rx_handler_result_t; + +typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **); + +enum netdev_ml_priv_type { + ML_PRIV_NONE = 0, + ML_PRIV_CAN = 1, +}; + +struct pcpu_dstats; + +struct netdev_tc_txq { + u16 count; + u16 offset; +}; + +struct sfp_bus; + +struct bpf_xdp_link; + +struct bpf_xdp_entity { + struct bpf_prog *prog; + struct bpf_xdp_link *link; +}; + +struct netdev_name_node; + +struct dev_ifalias; + +struct net_device_ops; + +struct ethtool_ops; + +struct ndisc_ops; + +struct header_ops; + +struct vlan_info; + +struct in_device; + +struct inet6_dev; + +struct wpan_dev; + +struct netdev_rx_queue; + +struct mini_Qdisc; + +struct netdev_queue; + +struct cpu_rmap; + +struct Qdisc; + +struct xdp_dev_bulk_queue; + +struct xps_dev_maps; + +struct pcpu_lstats; + +struct pcpu_sw_netstats; + +struct rtnl_link_ops; + +struct dcbnl_rtnl_ops; + +struct phy_device; + +struct udp_tunnel_nic_info; + +struct udp_tunnel_nic; + +struct net_device { + char name[16]; + struct netdev_name_node *name_node; + struct dev_ifalias *ifalias; + long unsigned int mem_end; + long unsigned int mem_start; + long unsigned int base_addr; + long unsigned int state; + struct list_head dev_list; + struct list_head napi_list; + struct list_head unreg_list; + struct list_head close_list; + struct list_head ptype_all; + struct list_head ptype_specific; + struct { + struct list_head upper; + struct list_head lower; + } adj_list; + unsigned int flags; + long long unsigned int priv_flags; + const struct net_device_ops *netdev_ops; + int ifindex; + short unsigned int gflags; + short unsigned int hard_header_len; + unsigned int mtu; + short unsigned int needed_headroom; + short unsigned int needed_tailroom; + netdev_features_t features; + netdev_features_t hw_features; + netdev_features_t wanted_features; + netdev_features_t vlan_features; + netdev_features_t hw_enc_features; + netdev_features_t mpls_features; + netdev_features_t gso_partial_features; + unsigned int min_mtu; + unsigned int max_mtu; + short unsigned int type; + unsigned char min_header_len; + unsigned char name_assign_type; + int group; + struct net_device_stats stats; + atomic_long_t rx_dropped; + atomic_long_t tx_dropped; + atomic_long_t rx_nohandler; + atomic_t carrier_up_count; + atomic_t carrier_down_count; + const struct ethtool_ops *ethtool_ops; + const struct ndisc_ops *ndisc_ops; + const struct header_ops *header_ops; + unsigned char operstate; + unsigned char link_mode; + unsigned char if_port; + unsigned char dma; + unsigned char perm_addr[32]; + unsigned char addr_assign_type; + unsigned char addr_len; + unsigned char upper_level; + unsigned char lower_level; + short unsigned int neigh_priv_len; + short unsigned int dev_id; + short unsigned int dev_port; + short unsigned int padded; + spinlock_t addr_list_lock; + int irq; + struct netdev_hw_addr_list uc; + struct netdev_hw_addr_list mc; + struct netdev_hw_addr_list dev_addrs; + struct kset *queues_kset; + struct list_head unlink_list; + unsigned int promiscuity; + unsigned int allmulti; + bool uc_promisc; + unsigned char nested_level; + struct vlan_info *vlan_info; + struct in_device *ip_ptr; + struct inet6_dev *ip6_ptr; + struct wireless_dev *ieee80211_ptr; + struct wpan_dev *ieee802154_ptr; + const unsigned char *dev_addr; + struct netdev_rx_queue *_rx; + unsigned int num_rx_queues; + unsigned int real_num_rx_queues; + struct bpf_prog *xdp_prog; + long unsigned int gro_flush_timeout; + int napi_defer_hard_irqs; + unsigned int gro_max_size; + rx_handler_func_t *rx_handler; + void *rx_handler_data; + struct mini_Qdisc *miniq_ingress; + struct netdev_queue *ingress_queue; + struct nf_hook_entries *nf_hooks_ingress; + unsigned char broadcast[32]; + struct cpu_rmap *rx_cpu_rmap; + struct hlist_node index_hlist; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + struct netdev_queue *_tx; + unsigned int num_tx_queues; + unsigned int real_num_tx_queues; + struct Qdisc *qdisc; + unsigned int tx_queue_len; + spinlock_t tx_global_lock; + struct xdp_dev_bulk_queue *xdp_bulkq; + struct xps_dev_maps *xps_maps[2]; + struct mini_Qdisc *miniq_egress; + struct nf_hook_entries *nf_hooks_egress; + struct hlist_head qdisc_hash[16]; + struct timer_list watchdog_timer; + int watchdog_timeo; + u32 proto_down_reason; + struct list_head todo_list; + int *pcpu_refcnt; + struct ref_tracker_dir refcnt_tracker; + struct list_head link_watch_list; + enum { + NETREG_UNINITIALIZED = 0, + NETREG_REGISTERED = 1, + NETREG_UNREGISTERING = 2, + NETREG_UNREGISTERED = 3, + NETREG_RELEASED = 4, + NETREG_DUMMY = 5, + } reg_state: 8; + bool dismantle; + enum { + RTNL_LINK_INITIALIZED = 0, + RTNL_LINK_INITIALIZING = 1, + } rtnl_link_state: 16; + bool needs_free_netdev; + void (*priv_destructor)(struct net_device *); + possible_net_t nd_net; + void *ml_priv; + enum netdev_ml_priv_type ml_priv_type; + union { + struct pcpu_lstats *lstats; + struct pcpu_sw_netstats *tstats; + struct pcpu_dstats *dstats; + }; + struct device dev; + const struct attribute_group *sysfs_groups[4]; + const struct attribute_group *sysfs_rx_queue_group; + const struct rtnl_link_ops *rtnl_link_ops; + unsigned int gso_max_size; + u16 gso_max_segs; + const struct dcbnl_rtnl_ops *dcbnl_ops; + s16 num_tc; + struct netdev_tc_txq tc_to_txq[16]; + u8 prio_tc_map[16]; + struct phy_device *phydev; + struct sfp_bus *sfp_bus; + struct lock_class_key *qdisc_tx_busylock; + bool proto_down; + unsigned int wol_enabled: 1; + unsigned int threaded: 1; + struct list_head net_notifier_list; + const struct udp_tunnel_nic_info *udp_tunnel_nic_info; + struct udp_tunnel_nic *udp_tunnel_nic; + struct bpf_xdp_entity xdp_state[3]; + u8 dev_addr_shadow[32]; + netdevice_tracker linkwatch_dev_tracker; + netdevice_tracker watchdog_dev_tracker; + long: 64; }; struct hh_cache { @@ -14277,19 +13143,21 @@ struct neighbour { struct timer_list timer; long unsigned int used; atomic_t probes; - __u8 flags; - __u8 nud_state; - __u8 type; - __u8 dead; + u8 nud_state; + u8 type; + u8 dead; u8 protocol; + u32 flags; seqlock_t ha_lock; unsigned char ha[32]; struct hh_cache hh; int (*output)(struct neighbour *, struct sk_buff *); const struct neigh_ops *ops; struct list_head gc_list; + struct list_head managed_list; struct callback_head rcu; struct net_device *dev; + netdevice_tracker dev_tracker; u8 primary_key[0]; }; @@ -14348,16 +13216,13 @@ struct ipv6_devconf { __s32 disable_policy; __s32 ndisc_tclass; __s32 rpl_seg_enabled; + __u32 ioam6_id; + __u32 ioam6_id_wide; + __u8 ioam6_enabled; + __u8 ndisc_evict_nocarrier; struct ctl_table_header *sysctl_header; }; -struct nf_queue_entry; - -struct nf_queue_handler { - int (*outfn)(struct nf_queue_entry *, unsigned int); - void (*nf_hook_drop)(struct net *); -}; - enum nf_log_type { NF_LOG_TYPE_LOG = 0, NF_LOG_TYPE_ULOG = 1, @@ -14377,6 +13242,40 @@ struct nf_logger { struct module *me; }; +struct hlist_nulls_head { + struct hlist_nulls_node *first; +}; + +struct ip_conntrack_stat { + unsigned int found; + unsigned int invalid; + unsigned int insert; + unsigned int insert_failed; + unsigned int clash_resolve; + unsigned int drop; + unsigned int early_drop; + unsigned int error; + unsigned int expect_new; + unsigned int expect_create; + unsigned int expect_delete; + unsigned int search_restart; + unsigned int chaintoolong; +}; + +struct ct_pcpu { + spinlock_t lock; + struct hlist_nulls_head unconfirmed; + struct hlist_nulls_head dying; +}; + +typedef struct { + union { + void *kernel; + void *user; + }; + bool is_kernel: 1; +} sockptr_t; + typedef enum { SS_FREE = 0, SS_UNCONNECTED = 1, @@ -14613,6 +13512,7 @@ struct rpc_xprt_switch { unsigned int xps_id; unsigned int xps_nxprts; unsigned int xps_nactive; + unsigned int xps_nunique_destaddr_xprts; atomic_long_t xps_queuelen; struct list_head xps_xprt_list; struct net *xps_net; @@ -14825,6 +13725,20 @@ struct xdp_rxq_info { long: 64; }; +struct xdp_txq_info { + struct net_device *dev; +}; + +struct xdp_buff { + void *data; + void *data_end; + void *data_meta; + void *data_hard_start; + struct xdp_rxq_info *rxq; + struct xdp_txq_info *txq; + u32 frame_sz; +}; + struct xdp_frame { void *data; u16 len; @@ -14848,6 +13762,8 @@ struct nlattr { __u16 nla_type; }; +struct nla_policy; + struct netlink_ext_ack { const char *_msg; const struct nlattr *bad_attr; @@ -15063,28 +13979,6 @@ struct ifla_vf_info { __be16 vlan_proto; }; -struct tc_stats { - __u64 bytes; - __u32 packets; - __u32 drops; - __u32 overlimits; - __u32 bps; - __u32 pps; - __u32 qlen; - __u32 backlog; -}; - -struct tc_sizespec { - unsigned char cell_log; - unsigned char size_log; - short int cell_align; - int overhead; - unsigned int linklayer; - unsigned int mpu; - unsigned int mtu; - unsigned int tsize; -}; - enum netdev_tx { __NETDEV_TX_MIN = 2147483648, NETDEV_TX_OK = 0, @@ -15130,12 +14024,13 @@ struct xsk_buff_pool; struct netdev_queue { struct net_device *dev; + netdevice_tracker dev_tracker; struct Qdisc *qdisc; struct Qdisc *qdisc_sleeping; struct kobject kobj; int numa_node; long unsigned int tx_maxrate; - long unsigned int trans_timeout; + atomic_long_t trans_timeout; struct net_device *sb_dev; struct xsk_buff_pool *pool; spinlock_t _xmit_lock; @@ -15150,75 +14045,6 @@ struct netdev_queue { struct dql dql; }; -struct qdisc_skb_head { - struct sk_buff *head; - struct sk_buff *tail; - __u32 qlen; - spinlock_t lock; -}; - -struct gnet_stats_basic_packed { - __u64 bytes; - __u64 packets; -}; - -struct gnet_stats_queue { - __u32 qlen; - __u32 backlog; - __u32 drops; - __u32 requeues; - __u32 overlimits; -}; - -struct Qdisc_ops; - -struct qdisc_size_table; - -struct net_rate_estimator; - -struct gnet_stats_basic_cpu; - -struct Qdisc { - int (*enqueue)(struct sk_buff *, struct Qdisc *, struct sk_buff **); - struct sk_buff * (*dequeue)(struct Qdisc *); - unsigned int flags; - u32 limit; - const struct Qdisc_ops *ops; - struct qdisc_size_table *stab; - struct hlist_node hash; - u32 handle; - u32 parent; - struct netdev_queue *dev_queue; - struct net_rate_estimator *rate_est; - struct gnet_stats_basic_cpu *cpu_bstats; - struct gnet_stats_queue *cpu_qstats; - int pad; - refcount_t refcnt; - long: 64; - long: 64; - long: 64; - struct sk_buff_head gso_skb; - struct qdisc_skb_head q; - struct gnet_stats_basic_packed bstats; - seqcount_t running; - struct gnet_stats_queue qstats; - long unsigned int state; - struct Qdisc *next_sched; - struct sk_buff_head skb_bad_txq; - long: 64; - long: 64; - spinlock_t busylock; - spinlock_t seqlock; - struct callback_head rcu; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long int privdata[0]; -}; - struct rps_map { unsigned int len; struct callback_head rcu; @@ -15251,24 +14077,17 @@ struct rps_sock_flow_table { }; struct netdev_rx_queue { + struct xdp_rxq_info xdp_rxq; struct rps_map *rps_map; struct rps_dev_flow_table *rps_flow_table; struct kobject kobj; struct net_device *dev; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - struct xdp_rxq_info xdp_rxq; + netdevice_tracker dev_tracker; struct xsk_buff_pool *pool; long: 64; long: 64; long: 64; long: 64; - long: 64; - long: 64; - long: 64; }; struct xps_map { @@ -15354,6 +14173,7 @@ enum tc_setup_type { TC_SETUP_QDISC_TBF = 16, TC_SETUP_QDISC_FIFO = 17, TC_SETUP_QDISC_HTB = 18, + TC_SETUP_ACT = 19, }; enum bpf_netdev_command { @@ -15364,6 +14184,8 @@ enum bpf_netdev_command { XDP_SETUP_XSK_POOL = 4, }; +struct bpf_offloaded_map; + struct netdev_bpf { enum bpf_netdev_command command; union { @@ -15411,6 +14233,10 @@ struct net_device_ops { int (*ndo_set_mac_address)(struct net_device *, void *); int (*ndo_validate_addr)(struct net_device *); int (*ndo_do_ioctl)(struct net_device *, struct ifreq *, int); + int (*ndo_eth_ioctl)(struct net_device *, struct ifreq *, int); + int (*ndo_siocbond)(struct net_device *, struct ifreq *, int); + int (*ndo_siocwandev)(struct net_device *, struct if_settings *); + int (*ndo_siocdevprivate)(struct net_device *, struct ifreq *, void *, int); int (*ndo_set_config)(struct net_device *, struct ifmap *); int (*ndo_change_mtu)(struct net_device *, int); int (*ndo_neigh_setup)(struct net_device *, struct neigh_parms *); @@ -15459,11 +14285,11 @@ struct net_device_ops { void (*ndo_dfwd_del_station)(struct net_device *, void *); int (*ndo_set_tx_maxrate)(struct net_device *, int, u32); int (*ndo_get_iflink)(const struct net_device *); - int (*ndo_change_proto_down)(struct net_device *, bool); int (*ndo_fill_metadata_dst)(struct net_device *, struct sk_buff *); void (*ndo_set_rx_headroom)(struct net_device *, int); int (*ndo_bpf)(struct net_device *, struct netdev_bpf *); int (*ndo_xdp_xmit)(struct net_device *, int, struct xdp_frame **, u32); + struct net_device * (*ndo_xdp_get_xmit_slave)(struct net_device *, struct xdp_buff *); int (*ndo_xsk_wakeup)(struct net_device *, u32, u32); struct devlink_port * (*ndo_get_devlink_port)(struct net_device *); int (*ndo_tunnel_ctl)(struct net_device *, struct ip_tunnel_parm *, int); @@ -15474,6 +14300,7 @@ struct net_device_ops { struct neigh_parms { possible_net_t net; struct net_device *dev; + netdevice_tracker dev_tracker; struct list_head list; int (*neigh_setup)(struct neighbour *); struct neigh_table *tbl; @@ -15519,8 +14346,12 @@ struct ethtool_eeprom; struct ethtool_coalesce; +struct kernel_ethtool_coalesce; + struct ethtool_ringparam; +struct kernel_ethtool_ringparam; + struct ethtool_pause_stats; struct ethtool_pauseparam; @@ -15563,9 +14394,12 @@ struct ethtool_rmon_stats; struct ethtool_rmon_hist_range; +struct ethtool_module_power_mode_params; + struct ethtool_ops { u32 cap_link_lanes_supported: 1; u32 supported_coalesce_params; + u32 supported_ring_params; void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *); int (*get_regs_len)(struct net_device *); void (*get_regs)(struct net_device *, struct ethtool_regs *, void *); @@ -15579,10 +14413,10 @@ struct ethtool_ops { int (*get_eeprom_len)(struct net_device *); int (*get_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *); int (*set_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *); - int (*get_coalesce)(struct net_device *, struct ethtool_coalesce *); - int (*set_coalesce)(struct net_device *, struct ethtool_coalesce *); - void (*get_ringparam)(struct net_device *, struct ethtool_ringparam *); - int (*set_ringparam)(struct net_device *, struct ethtool_ringparam *); + int (*get_coalesce)(struct net_device *, struct ethtool_coalesce *, struct kernel_ethtool_coalesce *, struct netlink_ext_ack *); + int (*set_coalesce)(struct net_device *, struct ethtool_coalesce *, struct kernel_ethtool_coalesce *, struct netlink_ext_ack *); + void (*get_ringparam)(struct net_device *, struct ethtool_ringparam *, struct kernel_ethtool_ringparam *, struct netlink_ext_ack *); + int (*set_ringparam)(struct net_device *, struct ethtool_ringparam *, struct kernel_ethtool_ringparam *, struct netlink_ext_ack *); void (*get_pause_stats)(struct net_device *, struct ethtool_pause_stats *); void (*get_pauseparam)(struct net_device *, struct ethtool_pauseparam *); int (*set_pauseparam)(struct net_device *, struct ethtool_pauseparam *); @@ -15632,6 +14466,8 @@ struct ethtool_ops { void (*get_eth_mac_stats)(struct net_device *, struct ethtool_eth_mac_stats *); void (*get_eth_ctrl_stats)(struct net_device *, struct ethtool_eth_ctrl_stats *); void (*get_rmon_stats)(struct net_device *, struct ethtool_rmon_stats *, const struct ethtool_rmon_hist_range **); + int (*get_module_power_mode)(struct net_device *, struct ethtool_module_power_mode_params *, struct netlink_ext_ack *); + int (*set_module_power_mode)(struct net_device *, const struct ethtool_module_power_mode_params *, struct netlink_ext_ack *); }; struct nd_opt_hdr; @@ -15662,6 +14498,7 @@ struct ifacaddr6; struct inet6_dev { struct net_device *dev; + netdevice_tracker dev_tracker; struct list_head addr_list; struct ifmcaddr6 *mc_list; struct ifmcaddr6 *mc_tomb; @@ -15699,18 +14536,7 @@ struct inet6_dev { __u8 rs_probes; long unsigned int tstamp; struct callback_head rcu; -}; - -struct tcf_proto; - -struct tcf_block; - -struct mini_Qdisc { - struct tcf_proto *filter_list; - struct tcf_block *block; - struct gnet_stats_basic_cpu *cpu_bstats; - struct gnet_stats_queue *cpu_qstats; - struct callback_head rcu; + unsigned int ra_mtu; }; struct rtnl_link_ops { @@ -15832,33 +14658,6 @@ enum { __RTAX_MAX = 18, }; -struct tcmsg { - unsigned char tcm_family; - unsigned char tcm__pad1; - short unsigned int tcm__pad2; - int tcm_ifindex; - __u32 tcm_handle; - __u32 tcm_parent; - __u32 tcm_info; -}; - -struct gnet_stats_basic_cpu { - struct gnet_stats_basic_packed bstats; - struct u64_stats_sync syncp; -}; - -struct gnet_dump { - spinlock_t *lock; - struct sk_buff *skb; - struct nlattr *tail; - int compat_tc_stats; - int compat_xstats; - int padattr; - void *xstats; - int xstats_len; - struct tc_stats tc_stats; -}; - struct netlink_range_validation { u64 min; u64 max; @@ -15877,212 +14676,6 @@ struct nl_info { u8 skip_notify_kernel: 1; }; -enum flow_action_hw_stats_bit { - FLOW_ACTION_HW_STATS_IMMEDIATE_BIT = 0, - FLOW_ACTION_HW_STATS_DELAYED_BIT = 1, - FLOW_ACTION_HW_STATS_DISABLED_BIT = 2, - FLOW_ACTION_HW_STATS_NUM_BITS = 3, -}; - -struct flow_block { - struct list_head cb_list; -}; - -typedef int flow_setup_cb_t(enum tc_setup_type, void *, void *); - -struct qdisc_size_table { - struct callback_head rcu; - struct list_head list; - struct tc_sizespec szopts; - int refcnt; - u16 data[0]; -}; - -struct Qdisc_class_ops; - -struct Qdisc_ops { - struct Qdisc_ops *next; - const struct Qdisc_class_ops *cl_ops; - char id[16]; - int priv_size; - unsigned int static_flags; - int (*enqueue)(struct sk_buff *, struct Qdisc *, struct sk_buff **); - struct sk_buff * (*dequeue)(struct Qdisc *); - struct sk_buff * (*peek)(struct Qdisc *); - int (*init)(struct Qdisc *, struct nlattr *, struct netlink_ext_ack *); - void (*reset)(struct Qdisc *); - void (*destroy)(struct Qdisc *); - int (*change)(struct Qdisc *, struct nlattr *, struct netlink_ext_ack *); - void (*attach)(struct Qdisc *); - int (*change_tx_queue_len)(struct Qdisc *, unsigned int); - int (*dump)(struct Qdisc *, struct sk_buff *); - int (*dump_stats)(struct Qdisc *, struct gnet_dump *); - void (*ingress_block_set)(struct Qdisc *, u32); - void (*egress_block_set)(struct Qdisc *, u32); - u32 (*ingress_block_get)(struct Qdisc *); - u32 (*egress_block_get)(struct Qdisc *); - struct module *owner; -}; - -struct qdisc_walker; - -struct Qdisc_class_ops { - unsigned int flags; - struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); - int (*graft)(struct Qdisc *, long unsigned int, struct Qdisc *, struct Qdisc **, struct netlink_ext_ack *); - struct Qdisc * (*leaf)(struct Qdisc *, long unsigned int); - void (*qlen_notify)(struct Qdisc *, long unsigned int); - long unsigned int (*find)(struct Qdisc *, u32); - int (*change)(struct Qdisc *, u32, u32, struct nlattr **, long unsigned int *, struct netlink_ext_ack *); - int (*delete)(struct Qdisc *, long unsigned int, struct netlink_ext_ack *); - void (*walk)(struct Qdisc *, struct qdisc_walker *); - struct tcf_block * (*tcf_block)(struct Qdisc *, long unsigned int, struct netlink_ext_ack *); - long unsigned int (*bind_tcf)(struct Qdisc *, long unsigned int, u32); - void (*unbind_tcf)(struct Qdisc *, long unsigned int); - int (*dump)(struct Qdisc *, long unsigned int, struct sk_buff *, struct tcmsg *); - int (*dump_stats)(struct Qdisc *, long unsigned int, struct gnet_dump *); -}; - -struct tcf_chain; - -struct tcf_block { - struct mutex lock; - struct list_head chain_list; - u32 index; - u32 classid; - refcount_t refcnt; - struct net *net; - struct Qdisc *q; - struct rw_semaphore cb_lock; - struct flow_block flow_block; - struct list_head owner_list; - bool keep_dst; - atomic_t offloadcnt; - unsigned int nooffloaddevcnt; - unsigned int lockeddevcnt; - struct { - struct tcf_chain *chain; - struct list_head filter_chain_list; - } chain0; - struct callback_head rcu; - struct hlist_head proto_destroy_ht[128]; - struct mutex proto_destroy_lock; -}; - -struct tcf_result; - -struct tcf_proto_ops; - -struct tcf_proto { - struct tcf_proto *next; - void *root; - int (*classify)(struct sk_buff *, const struct tcf_proto *, struct tcf_result *); - __be16 protocol; - u32 prio; - void *data; - const struct tcf_proto_ops *ops; - struct tcf_chain *chain; - spinlock_t lock; - bool deleting; - refcount_t refcnt; - struct callback_head rcu; - struct hlist_node destroy_ht_node; -}; - -struct tcf_result { - union { - struct { - long unsigned int class; - u32 classid; - }; - const struct tcf_proto *goto_tp; - struct { - bool ingress; - struct gnet_stats_queue *qstats; - }; - }; -}; - -struct tcf_walker; - -struct tcf_proto_ops { - struct list_head head; - char kind[16]; - int (*classify)(struct sk_buff *, const struct tcf_proto *, struct tcf_result *); - int (*init)(struct tcf_proto *); - void (*destroy)(struct tcf_proto *, bool, struct netlink_ext_ack *); - void * (*get)(struct tcf_proto *, u32); - void (*put)(struct tcf_proto *, void *); - int (*change)(struct net *, struct sk_buff *, struct tcf_proto *, long unsigned int, u32, struct nlattr **, void **, bool, bool, struct netlink_ext_ack *); - int (*delete)(struct tcf_proto *, void *, bool *, bool, struct netlink_ext_ack *); - bool (*delete_empty)(struct tcf_proto *); - void (*walk)(struct tcf_proto *, struct tcf_walker *, bool); - int (*reoffload)(struct tcf_proto *, bool, flow_setup_cb_t *, void *, struct netlink_ext_ack *); - void (*hw_add)(struct tcf_proto *, void *); - void (*hw_del)(struct tcf_proto *, void *); - void (*bind_class)(void *, u32, long unsigned int, void *, long unsigned int); - void * (*tmplt_create)(struct net *, struct tcf_chain *, struct nlattr **, struct netlink_ext_ack *); - void (*tmplt_destroy)(void *); - int (*dump)(struct net *, struct tcf_proto *, void *, struct sk_buff *, struct tcmsg *, bool); - int (*terse_dump)(struct net *, struct tcf_proto *, void *, struct sk_buff *, struct tcmsg *, bool); - int (*tmplt_dump)(struct sk_buff *, struct net *, void *); - struct module *owner; - int flags; -}; - -struct tcf_chain { - struct mutex filter_chain_lock; - struct tcf_proto *filter_chain; - struct list_head list; - struct tcf_block *block; - u32 index; - unsigned int refcnt; - unsigned int action_refcnt; - bool explicitly_created; - bool flushing; - const struct tcf_proto_ops *tmplt_ops; - void *tmplt_priv; - struct callback_head rcu; -}; - -struct sock_fprog_kern { - u16 len; - struct sock_filter *filter; -}; - -struct bpf_prog_stats { - u64 cnt; - u64 nsecs; - u64 misses; - struct u64_stats_sync syncp; - long: 64; -}; - -struct sk_filter { - refcount_t refcnt; - struct callback_head rcu; - struct bpf_prog *prog; -}; - -struct bpf_nh_params { - u32 nh_family; - union { - u32 ipv4_nh; - struct in6_addr ipv6_nh; - }; -}; - -struct bpf_redirect_info { - u32 flags; - u32 tgt_index; - void *tgt_value; - struct bpf_map *map; - u32 map_id; - enum bpf_map_type map_type; - u32 kern_flags; - struct bpf_nh_params nh; -}; - enum { NEIGH_VAR_MCAST_PROBES = 0, NEIGH_VAR_UCAST_PROBES = 1, @@ -16135,11 +14728,13 @@ struct neigh_table { int gc_thresh3; long unsigned int last_flush; struct delayed_work gc_work; + struct delayed_work managed_work; struct timer_list proxy_timer; struct sk_buff_head proxy_queue; atomic_t entries; atomic_t gc_entries; struct list_head gc_list; + struct list_head managed_list; rwlock_t lock; long unsigned int last_rand; struct neigh_statistics *stats; @@ -16174,7 +14769,8 @@ struct pneigh_entry { struct pneigh_entry *next; possible_net_t net; struct net_device *dev; - u8 flags; + netdevice_tracker dev_tracker; + u32 flags; u8 protocol; u8 key[0]; }; @@ -16303,10 +14899,11 @@ struct proto { void (*unhash)(struct sock *); void (*rehash)(struct sock *); int (*get_port)(struct sock *, short unsigned int); + void (*put_port)(struct sock *); int (*psock_update_sk_prot)(struct sock *, struct sk_psock *, bool); unsigned int inuse_idx; bool (*stream_memory_free)(const struct sock *, int); - bool (*stream_memory_read)(const struct sock *); + bool (*sock_is_readable)(struct sock *); void (*enter_memory_pressure)(struct sock *); void (*leave_memory_pressure)(struct sock *); atomic_long_t *memory_allocated; @@ -16324,7 +14921,7 @@ struct proto { slab_flags_t slab_flags; unsigned int useroffset; unsigned int usersize; - struct percpu_counter *orphan_count; + unsigned int *orphan_count; struct request_sock_ops *rsk_prot; struct timewait_sock_ops *twsk_prot; union { @@ -16459,6 +15056,7 @@ struct ipv6_stub { u32 (*ip6_mtu_from_fib6)(const struct fib6_result *, const struct in6_addr *, const struct in6_addr *); int (*fib6_nh_init)(struct net *, struct fib6_nh *, struct fib6_config *, gfp_t, struct netlink_ext_ack *); void (*fib6_nh_release)(struct fib6_nh *); + void (*fib6_nh_release_dsts)(struct fib6_nh *); void (*fib6_update_sernum)(struct net *, struct fib6_info *); int (*ip6_del_rt)(struct net *, struct fib6_info *, bool); void (*fib6_rt_update)(struct net *, struct fib6_info *, struct nl_info *); @@ -16907,6 +15505,7 @@ struct nfs_client { struct nfs_subversion *cl_nfs_mod; u32 cl_minorversion; unsigned int cl_nconnect; + unsigned int cl_max_connect; const char *cl_principal; char cl_ipaddr[48]; struct net *cl_net; @@ -17016,7 +15615,6 @@ struct nfs_entry { int eof; struct nfs_fh *fh; struct nfs_fattr *fattr; - struct nfs4_label *label; unsigned char d_type; struct nfs_server *server; }; @@ -17178,10 +15776,10 @@ struct nfs_rpc_ops { int (*getroot)(struct nfs_server *, struct nfs_fh *, struct nfs_fsinfo *); int (*submount)(struct fs_context *, struct nfs_server *); int (*try_get_tree)(struct fs_context *); - int (*getattr)(struct nfs_server *, struct nfs_fh *, struct nfs_fattr *, struct nfs4_label *, struct inode *); + int (*getattr)(struct nfs_server *, struct nfs_fh *, struct nfs_fattr *, struct inode *); int (*setattr)(struct dentry *, struct nfs_fattr *, struct iattr *); - int (*lookup)(struct inode *, struct dentry *, struct nfs_fh *, struct nfs_fattr *, struct nfs4_label *); - int (*lookupp)(struct inode *, struct nfs_fh *, struct nfs_fattr *, struct nfs4_label *); + int (*lookup)(struct inode *, struct dentry *, struct nfs_fh *, struct nfs_fattr *); + int (*lookupp)(struct inode *, struct nfs_fh *, struct nfs_fattr *); int (*access)(struct inode *, struct nfs_access_entry *); int (*readlink)(struct inode *, struct page *, unsigned int, unsigned int); int (*create)(struct inode *, struct dentry *, struct iattr *, int); @@ -17368,9 +15966,13 @@ struct __va_list_tag { void *reg_save_area; }; -typedef __builtin_va_list __gnuc_va_list; +typedef __builtin_va_list va_list; -typedef __gnuc_va_list va_list; +enum umh_disable_depth { + UMH_ENABLED = 0, + UMH_FREEZING = 1, + UMH_DISABLED = 2, +}; typedef u64 async_cookie_t; @@ -17834,6 +16436,64 @@ enum { NFPROTO_NUMPROTO = 13, }; +enum tcp_conntrack { + TCP_CONNTRACK_NONE = 0, + TCP_CONNTRACK_SYN_SENT = 1, + TCP_CONNTRACK_SYN_RECV = 2, + TCP_CONNTRACK_ESTABLISHED = 3, + TCP_CONNTRACK_FIN_WAIT = 4, + TCP_CONNTRACK_CLOSE_WAIT = 5, + TCP_CONNTRACK_LAST_ACK = 6, + TCP_CONNTRACK_TIME_WAIT = 7, + TCP_CONNTRACK_CLOSE = 8, + TCP_CONNTRACK_LISTEN = 9, + TCP_CONNTRACK_MAX = 10, + TCP_CONNTRACK_IGNORE = 11, + TCP_CONNTRACK_RETRANS = 12, + TCP_CONNTRACK_UNACK = 13, + TCP_CONNTRACK_TIMEOUT_MAX = 14, +}; + +enum ct_dccp_states { + CT_DCCP_NONE = 0, + CT_DCCP_REQUEST = 1, + CT_DCCP_RESPOND = 2, + CT_DCCP_PARTOPEN = 3, + CT_DCCP_OPEN = 4, + CT_DCCP_CLOSEREQ = 5, + CT_DCCP_CLOSING = 6, + CT_DCCP_TIMEWAIT = 7, + CT_DCCP_IGNORE = 8, + CT_DCCP_INVALID = 9, + __CT_DCCP_MAX = 10, +}; + +enum ip_conntrack_dir { + IP_CT_DIR_ORIGINAL = 0, + IP_CT_DIR_REPLY = 1, + IP_CT_DIR_MAX = 2, +}; + +enum sctp_conntrack { + SCTP_CONNTRACK_NONE = 0, + SCTP_CONNTRACK_CLOSED = 1, + SCTP_CONNTRACK_COOKIE_WAIT = 2, + SCTP_CONNTRACK_COOKIE_ECHOED = 3, + SCTP_CONNTRACK_ESTABLISHED = 4, + SCTP_CONNTRACK_SHUTDOWN_SENT = 5, + SCTP_CONNTRACK_SHUTDOWN_RECD = 6, + SCTP_CONNTRACK_SHUTDOWN_ACK_SENT = 7, + SCTP_CONNTRACK_HEARTBEAT_SENT = 8, + SCTP_CONNTRACK_HEARTBEAT_ACKED = 9, + SCTP_CONNTRACK_MAX = 10, +}; + +enum udp_conntrack { + UDP_CT_UNREPLIED = 0, + UDP_CT_REPLIED = 1, + UDP_CT_MAX = 2, +}; + enum { XFRM_POLICY_IN = 0, XFRM_POLICY_OUT = 1, @@ -17882,6 +16542,67 @@ enum { EI_ETYPE_TRUE = 4, }; +struct blkg_iostat { + u64 bytes[3]; + u64 ios[3]; +}; + +struct blkg_iostat_set { + struct u64_stats_sync sync; + struct blkg_iostat cur; + struct blkg_iostat last; +}; + +struct blkcg; + +struct blkg_policy_data; + +struct blkcg_gq { + struct request_queue *q; + struct list_head q_node; + struct hlist_node blkcg_node; + struct blkcg *blkcg; + struct blkcg_gq *parent; + struct percpu_ref refcnt; + bool online; + struct blkg_iostat_set *iostat_cpu; + struct blkg_iostat_set iostat; + struct blkg_policy_data *pd[6]; + spinlock_t async_bio_lock; + struct bio_list async_bios; + struct work_struct async_bio_work; + atomic_t use_delay; + atomic64_t delay_nsec; + atomic64_t delay_start; + u64 last_delay; + int last_use; + struct callback_head callback_head; +}; + +struct blkcg_policy_data; + +struct blkcg { + struct cgroup_subsys_state css; + spinlock_t lock; + refcount_t online_pin; + struct xarray blkg_tree; + struct blkcg_gq *blkg_hint; + struct hlist_head blkg_list; + struct blkcg_policy_data *cpd[6]; + struct list_head all_blkcgs_node; + struct list_head cgwb_list; +}; + +struct blkcg_policy_data { + struct blkcg *blkcg; + int plid; +}; + +struct blkg_policy_data { + struct blkcg_gq *blkg; + int plid; +}; + struct syscall_metadata { const char *name; int syscall_nr; @@ -18017,7 +16738,17 @@ struct ms_hyperv_info { u32 max_vp_index; u32 max_lp_index; u32 isolation_config_a; - u32 isolation_config_b; + union { + u32 isolation_config_b; + struct { + u32 cvm_type: 4; + u32 reserved1: 1; + u32 shared_gpa_boundary_active: 1; + u32 shared_gpa_boundary_bits: 6; + u32 reserved2: 20; + }; + }; + u64 shared_gpa_boundary; }; enum { @@ -18041,18 +16772,32 @@ enum cpu_usage_stat { NR_STATS = 10, }; -enum bpf_cgroup_storage_type { - BPF_CGROUP_STORAGE_SHARED = 0, - BPF_CGROUP_STORAGE_PERCPU = 1, - __BPF_CGROUP_STORAGE_MAX = 2, -}; - -enum bpf_tramp_prog_type { - BPF_TRAMP_FENTRY = 0, - BPF_TRAMP_FEXIT = 1, - BPF_TRAMP_MODIFY_RETURN = 2, - BPF_TRAMP_MAX = 3, - BPF_TRAMP_REPLACE = 4, +enum cgroup_bpf_attach_type { + CGROUP_BPF_ATTACH_TYPE_INVALID = 4294967295, + CGROUP_INET_INGRESS = 0, + CGROUP_INET_EGRESS = 1, + CGROUP_INET_SOCK_CREATE = 2, + CGROUP_SOCK_OPS = 3, + CGROUP_DEVICE = 4, + CGROUP_INET4_BIND = 5, + CGROUP_INET6_BIND = 6, + CGROUP_INET4_CONNECT = 7, + CGROUP_INET6_CONNECT = 8, + CGROUP_INET4_POST_BIND = 9, + CGROUP_INET6_POST_BIND = 10, + CGROUP_UDP4_SENDMSG = 11, + CGROUP_UDP6_SENDMSG = 12, + CGROUP_SYSCTL = 13, + CGROUP_UDP4_RECVMSG = 14, + CGROUP_UDP6_RECVMSG = 15, + CGROUP_GETSOCKOPT = 16, + CGROUP_SETSOCKOPT = 17, + CGROUP_INET4_GETPEERNAME = 18, + CGROUP_INET6_GETPEERNAME = 19, + CGROUP_INET4_GETSOCKNAME = 20, + CGROUP_INET6_GETSOCKNAME = 21, + CGROUP_INET_SOCK_RELEASE = 22, + MAX_CGROUP_BPF_ATTACH_TYPE = 23, }; enum cgroup_subsys_id { @@ -18084,6 +16829,13 @@ enum syscall_work_bit { SYSCALL_WORK_BIT_SYSCALL_EXIT_TRAP = 6, }; +enum blkg_iostat_type { + BLKG_IOSTAT_READ = 0, + BLKG_IOSTAT_WRITE = 1, + BLKG_IOSTAT_DISCARD = 2, + BLKG_IOSTAT_NR = 3, +}; + enum x86_pf_error_code { X86_PF_PROT = 1, X86_PF_WRITE = 2, @@ -18250,12 +17002,6 @@ struct perf_event_mmap_page { __u64 aux_size; }; -struct ldt_struct { - struct desc_struct *entries; - unsigned int nr_entries; - int slot; -}; - enum apic_delivery_modes { APIC_DELIVERY_MODE_FIXED = 0, APIC_DELIVERY_MODE_LOWESTPRIO = 1, @@ -18458,6 +17204,7 @@ struct unwind_state { long unsigned int stack_mask; struct task_struct *task; int graph_idx; + struct llist_node *kr_cur; bool error; bool signal; bool full_regs; @@ -18650,6 +17397,9 @@ struct x86_hybrid_pmu { struct event_constraint *event_constraints; struct event_constraint *pebs_constraints; struct extra_reg *extra_regs; + unsigned int late_ack: 1; + unsigned int mid_ack: 1; + unsigned int enabled_ack: 1; }; enum hybrid_pmu_type { @@ -18666,6 +17416,7 @@ struct x86_pmu { void (*enable_all)(int); void (*enable)(struct perf_event *); void (*disable)(struct perf_event *); + void (*assign)(struct perf_event *, int); void (*add)(struct perf_event *); void (*del)(struct perf_event *); void (*read)(struct perf_event *); @@ -18698,6 +17449,7 @@ struct x86_pmu { int perfctr_second_write; u64 (*limit_period)(struct perf_event *, u64); unsigned int late_ack: 1; + unsigned int mid_ack: 1; unsigned int enabled_ack: 1; int attr_rdpmc_broken; int attr_rdpmc; @@ -18803,25 +17555,18 @@ struct perf_msr { u64 mask; }; -struct amd_uncore { - int id; - int refcnt; - int cpu; - int num_counters; - int rdpmc_base; - u32 msr_base; - cpumask_t *active_mask; - struct pmu *pmu; - struct perf_event *events[6]; - struct hlist_node node; -}; - typedef int pci_power_t; typedef unsigned int pci_channel_state_t; typedef short unsigned int pci_dev_flags_t; +struct pci_vpd { + struct mutex lock; + unsigned int len; + u8 cap; +}; + struct pci_bus; struct pci_slot; @@ -18832,8 +17577,6 @@ struct pci_driver; struct pcie_link_state; -struct pci_vpd; - struct pci_sriov; struct pci_dev { @@ -18853,6 +17596,7 @@ struct pci_dev { u8 hdr_type; struct rcec_ea *rcec_ea; struct pci_dev *rcec; + u32 devcap; u8 pcie_cap; u8 msi_cap; u8 msix_cap; @@ -18887,6 +17631,7 @@ struct pci_dev { struct pcie_link_state *link_state; unsigned int ltr_path: 1; u16 l1ss; + unsigned int pasid_no_tlp: 1; unsigned int eetlp_prefix_path: 1; pci_channel_state_t error_state; struct device dev; @@ -18916,7 +17661,6 @@ struct pci_dev { unsigned int state_saved: 1; unsigned int is_physfn: 1; unsigned int is_virtfn: 1; - unsigned int reset_fn: 1; unsigned int is_hotplug_bridge: 1; unsigned int shpc_managed: 1; unsigned int is_thunderbolt: 1; @@ -18938,7 +17682,7 @@ struct pci_dev { struct bin_attribute *res_attr[17]; struct bin_attribute *res_attr_wc[17]; const struct attribute_group **msi_irq_groups; - struct pci_vpd *vpd; + struct pci_vpd vpd; union { struct pci_sriov *sriov; struct pci_dev *physfn; @@ -18950,6 +17694,7 @@ struct pci_dev { size_t romlen; char *driver_override; long unsigned int priv_flags; + u8 reset_methods[7]; }; struct pci_device_id { @@ -18960,6 +17705,7 @@ struct pci_device_id { __u32 class; __u32 class_mask; kernel_ulong_t driver_data; + __u32 override_only; }; struct hotplug_slot; @@ -19070,6 +17816,50 @@ enum pcie_bus_config_types { PCIE_BUS_PEER2PEER = 4, }; +union ibs_fetch_ctl { + __u64 val; + struct { + __u64 fetch_maxcnt: 16; + __u64 fetch_cnt: 16; + __u64 fetch_lat: 16; + __u64 fetch_en: 1; + __u64 fetch_val: 1; + __u64 fetch_comp: 1; + __u64 ic_miss: 1; + __u64 phy_addr_valid: 1; + __u64 l1tlb_pgsz: 2; + __u64 l1tlb_miss: 1; + __u64 l2tlb_miss: 1; + __u64 rand_en: 1; + __u64 fetch_l2_miss: 1; + __u64 reserved: 5; + }; +}; + +union ibs_op_ctl { + __u64 val; + struct { + __u64 opmaxcnt: 16; + __u64 reserved0: 1; + __u64 op_en: 1; + __u64 op_val: 1; + __u64 cnt_ctl: 1; + __u64 opmaxcnt_ext: 7; + __u64 reserved1: 5; + __u64 opcurcnt: 27; + __u64 reserved2: 5; + }; +}; + +struct perf_ibs_data { + u32 size; + union { + u32 data[0]; + u32 caps; + }; + u64 regs[8]; +}; + enum ibs_states { IBS_ENABLED = 0, IBS_STARTED = 1, @@ -19094,6 +17884,7 @@ struct perf_ibs { long unsigned int offset_mask[1]; int offset_max; unsigned int fetch_count_reset_broken: 1; + unsigned int fetch_ignore_if_zero_rip: 1; struct cpu_perf_ibs *pcpu; struct attribute **format_attrs; struct attribute_group format_group; @@ -19101,13 +17892,19 @@ struct perf_ibs { u64 (*get_count)(u64); }; -struct perf_ibs_data { - u32 size; - union { - u32 data[0]; - u32 caps; - }; - u64 regs[8]; +typedef void (*exitcall_t)(); + +struct amd_uncore { + int id; + int refcnt; + int cpu; + int num_counters; + int rdpmc_base; + u32 msr_base; + cpumask_t *active_mask; + struct pmu *pmu; + struct perf_event *events[6]; + struct hlist_node node; }; enum perf_msr_id { @@ -19169,6 +17966,8 @@ struct perf_pmu_format_hybrid_attr { u64 pmu_type; }; +typedef int perf_snapshot_branch_stack_t(struct perf_branch_entry *, unsigned int); + enum { LBR_FORMAT_32 = 0, LBR_FORMAT_LIP = 1, @@ -19228,6 +18027,7 @@ enum pageflags { PG_hwpoison = 23, PG_arch_2 = 24, __NR_PAGEFLAGS = 25, + PG_readahead = 18, PG_checked = 10, PG_swapcache = 10, PG_fscache = 14, @@ -19237,6 +18037,7 @@ enum pageflags { PG_xen_remapped = 10, PG_slob_free = 13, PG_double_map = 6, + PG_has_hwpoisoned = 17, PG_isolated = 18, PG_reported = 2, }; @@ -20555,7 +19356,10 @@ enum xfeature { XFEATURE_RSRVD_COMP_13 = 13, XFEATURE_RSRVD_COMP_14 = 14, XFEATURE_LBR = 15, - XFEATURE_MAX = 16, + XFEATURE_RSRVD_COMP_16 = 16, + XFEATURE_XTILE_CFG = 17, + XFEATURE_XTILE_DATA = 18, + XFEATURE_MAX = 19, }; struct arch_lbr_state { @@ -21096,8 +19900,6 @@ struct topa_page { struct topa topa; }; -typedef void (*exitcall_t)(); - struct x86_cpu_id { __u16 vendor; __u16 family; @@ -21279,6 +20081,7 @@ struct intel_uncore_init_fun { void (*cpu_init)(); int (*pci_init)(); void (*mmio_init)(); + bool use_discovery; }; enum { @@ -21317,6 +20120,13 @@ enum perf_tgl_uncore_imc_freerunning_types { TGL_MMIO_UNCORE_IMC_FREERUNNING_TYPE_MAX = 3, }; +enum uncore_access_type { + UNCORE_ACCESS_MSR = 0, + UNCORE_ACCESS_MMIO = 1, + UNCORE_ACCESS_PCI = 2, + UNCORE_ACCESS_MAX = 3, +}; + enum { SNBEP_PCI_QPI_PORT0_FILTER = 0, SNBEP_PCI_QPI_PORT1_FILTER = 1, @@ -21435,11 +20245,17 @@ enum perf_uncore_icx_imc_freerunning_type_id { ICX_IMC_FREERUNNING_TYPE_MAX = 3, }; -enum uncore_access_type { - UNCORE_ACCESS_MSR = 0, - UNCORE_ACCESS_MMIO = 1, - UNCORE_ACCESS_PCI = 2, - UNCORE_ACCESS_MAX = 3, +enum perf_uncore_spr_iio_freerunning_type_id { + SPR_IIO_MSR_IOCLK = 0, + SPR_IIO_MSR_BW_IN = 1, + SPR_IIO_MSR_BW_OUT = 2, + SPR_IIO_FREERUNNING_TYPE_MAX = 3, +}; + +enum perf_uncore_spr_imc_freerunning_type_id { + SPR_IMC_DCLK = 0, + SPR_IMC_PQ_CYCLES = 1, + SPR_IMC_FREERUNNING_TYPE_MAX = 2, }; struct uncore_global_discovery { @@ -21503,6 +20319,14 @@ struct intel_uncore_discovery_type { unsigned int *box_offset; }; +enum cc_attr { + CC_ATTR_MEM_ENCRYPT = 0, + CC_ATTR_HOST_MEM_ENCRYPT = 1, + CC_ATTR_GUEST_MEM_ENCRYPT = 2, + CC_ATTR_GUEST_STATE_ENCRYPT = 3, + CC_ATTR_GUEST_UNROLL_STRING_IO = 4, +}; + struct trampoline_header { u64 start; u64 efer; @@ -21626,18 +20450,6 @@ struct mce { typedef long unsigned int mce_banks_t[1]; -struct smca_hwid { - unsigned int bank_type; - u32 hwid_mcatype; - u8 count; -}; - -struct smca_bank { - struct smca_hwid *hwid; - u32 id; - u8 sysfs_id; -}; - struct kernel_vm86_regs { struct pt_regs pt; short unsigned int es; @@ -21862,8 +20674,6 @@ struct irq_chip { int (*irq_set_wake)(struct irq_data *, unsigned int); void (*irq_bus_lock)(struct irq_data *); void (*irq_bus_sync_unlock)(struct irq_data *); - void (*irq_cpu_online)(struct irq_data *); - void (*irq_cpu_offline)(struct irq_data *); void (*irq_suspend)(struct irq_data *); void (*irq_resume)(struct irq_data *); void (*irq_pm_shutdown)(struct irq_data *); @@ -22233,12 +21043,15 @@ struct msi_desc { void *write_msi_msg_data; union { struct { - u32 masked; + union { + u32 msi_mask; + u32 msix_ctrl; + }; struct { u8 is_msix: 1; u8 multiple: 3; u8 multi_cap: 3; - u8 maskbit: 1; + u8 can_mask: 1; u8 is_64: 1; u8 is_virtual: 1; u16 entry_nr; @@ -22344,15 +21157,16 @@ enum lockdown_reason { LOCKDOWN_MMIOTRACE = 13, LOCKDOWN_DEBUGFS = 14, LOCKDOWN_XMON_WR = 15, - LOCKDOWN_INTEGRITY_MAX = 16, - LOCKDOWN_KCORE = 17, - LOCKDOWN_KPROBES = 18, - LOCKDOWN_BPF_READ = 19, - LOCKDOWN_PERF = 20, - LOCKDOWN_TRACEFS = 21, - LOCKDOWN_XMON_RW = 22, - LOCKDOWN_XFRM_SECRET = 23, - LOCKDOWN_CONFIDENTIALITY_MAX = 24, + LOCKDOWN_BPF_WRITE_USER = 16, + LOCKDOWN_INTEGRITY_MAX = 17, + LOCKDOWN_KCORE = 18, + LOCKDOWN_KPROBES = 19, + LOCKDOWN_BPF_READ_KERNEL = 20, + LOCKDOWN_PERF = 21, + LOCKDOWN_TRACEFS = 22, + LOCKDOWN_XMON_RW = 23, + LOCKDOWN_XFRM_SECRET = 24, + LOCKDOWN_CONFIDENTIALITY_MAX = 25, }; enum lockdep_ok { @@ -22476,19 +21290,6 @@ struct hvm_start_info { uint32_t reserved; }; -struct balloon_stats { - long unsigned int current_pages; - long unsigned int target_pages; - long unsigned int target_unpopulated; - long unsigned int balloon_low; - long unsigned int balloon_high; - long unsigned int total_pages; - long unsigned int schedule_delay; - long unsigned int max_schedule_delay; - long unsigned int retry_count; - long unsigned int max_retry_count; -}; - struct setup_data { __u64 next; __u32 type; @@ -22508,6 +21309,22 @@ struct atomic_notifier_head { struct notifier_block *head; }; +struct io_tlb_slot; + +struct io_tlb_mem { + phys_addr_t start; + phys_addr_t end; + long unsigned int nslabs; + long unsigned int used; + unsigned int index; + spinlock_t lock; + struct dentry *debugfs; + bool late_alloc; + bool force_bounce; + bool for_alloc; + struct io_tlb_slot *slots; +}; + struct scatterlist { long unsigned int page_link; unsigned int offset; @@ -22534,18 +21351,6 @@ struct io_tlb_slot { unsigned int list; }; -struct io_tlb_mem { - phys_addr_t start; - phys_addr_t end; - long unsigned int nslabs; - long unsigned int used; - unsigned int index; - spinlock_t lock; - struct dentry *debugfs; - bool late_alloc; - struct io_tlb_slot slots[0]; -}; - enum efi_secureboot_mode { efi_secureboot_mode_unset = 0, efi_secureboot_mode_unknown = 1, @@ -22553,16 +21358,12 @@ enum efi_secureboot_mode { efi_secureboot_mode_enabled = 3, }; -struct acpi_table_ibft { - struct acpi_table_header header; - u8 reserved[12]; -}; - struct hstate { struct mutex resize_lock; int next_nid_to_alloc; int next_nid_to_free; unsigned int order; + unsigned int demote_order; long unsigned int mask; long unsigned int max_huge_pages; long unsigned int nr_huge_pages; @@ -22572,6 +21373,7 @@ struct hstate { long unsigned int nr_overcommit_huge_pages; struct list_head hugepage_activelist; struct list_head hugepage_freelists[64]; + unsigned int max_huge_pages_node[64]; unsigned int nr_huge_pages_node[64]; unsigned int free_huge_pages_node[64]; unsigned int surplus_huge_pages_node[64]; @@ -22625,6 +21427,11 @@ enum { JL_STATE_UPDATE = 2, }; +enum align_flags { + ALIGN_VA_32 = 1, + ALIGN_VA_64 = 2, +}; + struct vm_unmapped_area_info { long unsigned int flags; long unsigned int length; @@ -22634,11 +21441,6 @@ struct vm_unmapped_area_info { long unsigned int align_offset; }; -enum align_flags { - ALIGN_VA_32 = 1, - ALIGN_VA_64 = 2, -}; - enum { MEMREMAP_WB = 1, MEMREMAP_WT = 2, @@ -22803,18 +21605,17 @@ struct system_counterval_t { enum { WORK_STRUCT_PENDING_BIT = 0, - WORK_STRUCT_DELAYED_BIT = 1, + WORK_STRUCT_INACTIVE_BIT = 1, WORK_STRUCT_PWQ_BIT = 2, WORK_STRUCT_LINKED_BIT = 3, WORK_STRUCT_COLOR_SHIFT = 4, WORK_STRUCT_COLOR_BITS = 4, WORK_STRUCT_PENDING = 1, - WORK_STRUCT_DELAYED = 2, + WORK_STRUCT_INACTIVE = 2, WORK_STRUCT_PWQ = 4, WORK_STRUCT_LINKED = 8, WORK_STRUCT_STATIC = 0, - WORK_NR_COLORS = 15, - WORK_NO_COLOR = 15, + WORK_NR_COLORS = 16, WORK_CPU_UNBOUND = 128, WORK_STRUCT_FLAG_BITS = 8, WORK_OFFQ_FLAG_BASE = 4, @@ -22966,6 +21767,7 @@ struct cpufreq_policy { bool fast_switch_possible; bool fast_switch_enabled; bool strict_target; + bool efficiencies_available; unsigned int transition_delay_us; bool dvfs_possible_from_any_cpu; unsigned int cached_target_freq; @@ -23325,12 +22127,6 @@ struct user32_fxsr_struct { int padding[56]; }; -enum xstate_copy_mode { - XSTATE_COPY_FP = 0, - XSTATE_COPY_FX = 1, - XSTATE_COPY_XSAVE = 2, -}; - struct membuf { void *p; size_t left; @@ -23358,6 +22154,12 @@ struct user_regset { unsigned int core_note_type; }; +enum xstate_copy_mode { + XSTATE_COPY_FP = 0, + XSTATE_COPY_FX = 1, + XSTATE_COPY_XSAVE = 2, +}; + struct _fpx_sw_bytes { __u32 magic1; __u32 extended_size; @@ -23722,6 +22524,11 @@ enum srbds_mitigations { SRBDS_MITIGATION_HYPERVISOR = 4, }; +enum l1d_flush_mitigations { + L1D_FLUSH_OFF = 0, + L1D_FLUSH_ON = 1, +}; + enum spectre_v1_mitigation { SPECTRE_V1_MITIGATION_NONE = 0, SPECTRE_V1_MITIGATION_AUTO = 1, @@ -23853,6 +22660,7 @@ enum mcp_flags { MCP_TIMESTAMP = 1, MCP_UC = 2, MCP_DONTLOG = 4, + MCP_QUEUE_LOG = 8, }; enum severity_level { @@ -23873,22 +22681,22 @@ struct mce_evt_llist { }; struct mca_config { - bool dont_log_ce; - bool cmci_disabled; - bool ignore_ce; - bool print_all; __u64 lmce_disabled: 1; __u64 disabled: 1; __u64 ser: 1; __u64 recovery: 1; __u64 bios_cmci_threshold: 1; - int: 27; - __u64 __reserved: 59; - s8 bootlog; + __u64 initialized: 1; + __u64 __reserved: 58; + bool dont_log_ce; + bool cmci_disabled; + bool ignore_ce; + bool print_all; int tolerant; int monarch_timeout; int panic_timeout; u32 rip_msr; + s8 bootlog; }; struct mce_vendor_flags { @@ -23896,14 +22704,17 @@ struct mce_vendor_flags { __u64 succor: 1; __u64 smca: 1; __u64 amd_threshold: 1; - __u64 __reserved_0: 60; + __u64 p5: 1; + __u64 winchip: 1; + __u64 snb_ifu_quirk: 1; + __u64 __reserved_0: 57; }; -struct mca_msr_regs { - u32 (*ctl)(int); - u32 (*status)(int); - u32 (*addr)(int); - u32 (*misc)(int); +enum mca_msr { + MCA_CTL = 0, + MCA_STATUS = 1, + MCA_ADDR = 2, + MCA_MISC = 3, }; struct trace_event_raw_mce_record { @@ -23943,13 +22754,6 @@ struct mce_bank_dev { u8 bank; }; -enum handler_type { - EX_HANDLER_NONE = 0, - EX_HANDLER_FAULT = 1, - EX_HANDLER_UACCESS = 2, - EX_HANDLER_OTHER = 3, -}; - enum context { IN_KERNEL = 1, IN_USER = 2, @@ -24034,13 +22838,31 @@ enum smca_bank_types { SMCA_SMU = 17, SMCA_SMU_V2 = 18, SMCA_MP5 = 19, - SMCA_NBIO = 20, - SMCA_PCIE = 21, - SMCA_PCIE_V2 = 22, - SMCA_XGMI_PCS = 23, - SMCA_XGMI_PHY = 24, - SMCA_WAFL_PHY = 25, - N_SMCA_BANK_TYPES = 26, + SMCA_MPDMA = 20, + SMCA_NBIO = 21, + SMCA_PCIE = 22, + SMCA_PCIE_V2 = 23, + SMCA_XGMI_PCS = 24, + SMCA_NBIF = 25, + SMCA_SHUB = 26, + SMCA_SATA = 27, + SMCA_USB = 28, + SMCA_GMI_PCS = 29, + SMCA_XGMI_PHY = 30, + SMCA_WAFL_PHY = 31, + SMCA_GMI_PHY = 32, + N_SMCA_BANK_TYPES = 33, +}; + +struct smca_hwid { + unsigned int bank_type; + u32 hwid_mcatype; +}; + +struct smca_bank { + const struct smca_hwid *hwid; + u32 id; + u8 sysfs_id; }; struct smca_bank_name { @@ -24958,7 +23780,7 @@ struct sched_domain { unsigned int balance_interval; unsigned int nr_balance_failed; u64 max_newidle_lb_cost; - long unsigned int next_decay_max_lb_cost; + long unsigned int last_decay_max_lb_cost; u64 avg_scan_cost; unsigned int lb_count[3]; unsigned int lb_failed[3]; @@ -25241,6 +24063,7 @@ enum { IRQCHIP_SUPPORTS_LEVEL_MSI = 128, IRQCHIP_SUPPORTS_NMI = 256, IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND = 512, + IRQCHIP_AFFINITY_PRE_STARTUP = 1024, }; union IO_APIC_reg_00 { @@ -25402,6 +24225,34 @@ enum { MSI_FLAG_LEVEL_CAPABLE = 64, }; +enum { + TRACE_FTRACE_BIT = 0, + TRACE_FTRACE_NMI_BIT = 1, + TRACE_FTRACE_IRQ_BIT = 2, + TRACE_FTRACE_SIRQ_BIT = 3, + TRACE_FTRACE_TRANSITION_BIT = 4, + TRACE_INTERNAL_BIT = 5, + TRACE_INTERNAL_NMI_BIT = 6, + TRACE_INTERNAL_IRQ_BIT = 7, + TRACE_INTERNAL_SIRQ_BIT = 8, + TRACE_INTERNAL_TRANSITION_BIT = 9, + TRACE_BRANCH_BIT = 10, + TRACE_IRQ_BIT = 11, + TRACE_GRAPH_BIT = 12, + TRACE_GRAPH_DEPTH_START_BIT = 13, + TRACE_GRAPH_DEPTH_END_BIT = 14, + TRACE_GRAPH_NOTRACE_BIT = 15, + TRACE_RECORD_RECURSION_BIT = 16, +}; + +enum { + TRACE_CTX_NMI = 0, + TRACE_CTX_IRQ = 1, + TRACE_CTX_SOFTIRQ = 2, + TRACE_CTX_NORMAL = 3, + TRACE_CTX_TRANSITION = 4, +}; + struct dyn_arch_ftrace {}; enum { @@ -25538,32 +24389,6 @@ struct optimized_kprobe { struct arch_optimized_insn optinsn; }; -enum { - TRACE_FTRACE_BIT = 0, - TRACE_FTRACE_NMI_BIT = 1, - TRACE_FTRACE_IRQ_BIT = 2, - TRACE_FTRACE_SIRQ_BIT = 3, - TRACE_INTERNAL_BIT = 4, - TRACE_INTERNAL_NMI_BIT = 5, - TRACE_INTERNAL_IRQ_BIT = 6, - TRACE_INTERNAL_SIRQ_BIT = 7, - TRACE_BRANCH_BIT = 8, - TRACE_IRQ_BIT = 9, - TRACE_GRAPH_BIT = 10, - TRACE_GRAPH_DEPTH_START_BIT = 11, - TRACE_GRAPH_DEPTH_END_BIT = 12, - TRACE_GRAPH_NOTRACE_BIT = 13, - TRACE_TRANSITION_BIT = 14, - TRACE_RECORD_RECURSION_BIT = 15, -}; - -enum { - TRACE_CTX_NMI = 0, - TRACE_CTX_IRQ = 1, - TRACE_CTX_SOFTIRQ = 2, - TRACE_CTX_NORMAL = 3, -}; - typedef __u64 Elf64_Off; struct elf64_rela { @@ -25739,397 +24564,6 @@ enum rp_check { RP_CHECK_RET = 2, }; -struct fb_fix_screeninfo { - char id[16]; - long unsigned int smem_start; - __u32 smem_len; - __u32 type; - __u32 type_aux; - __u32 visual; - __u16 xpanstep; - __u16 ypanstep; - __u16 ywrapstep; - __u32 line_length; - long unsigned int mmio_start; - __u32 mmio_len; - __u32 accel; - __u16 capabilities; - __u16 reserved[2]; -}; - -struct fb_bitfield { - __u32 offset; - __u32 length; - __u32 msb_right; -}; - -struct fb_var_screeninfo { - __u32 xres; - __u32 yres; - __u32 xres_virtual; - __u32 yres_virtual; - __u32 xoffset; - __u32 yoffset; - __u32 bits_per_pixel; - __u32 grayscale; - struct fb_bitfield red; - struct fb_bitfield green; - struct fb_bitfield blue; - struct fb_bitfield transp; - __u32 nonstd; - __u32 activate; - __u32 height; - __u32 width; - __u32 accel_flags; - __u32 pixclock; - __u32 left_margin; - __u32 right_margin; - __u32 upper_margin; - __u32 lower_margin; - __u32 hsync_len; - __u32 vsync_len; - __u32 sync; - __u32 vmode; - __u32 rotate; - __u32 colorspace; - __u32 reserved[4]; -}; - -struct fb_cmap { - __u32 start; - __u32 len; - __u16 *red; - __u16 *green; - __u16 *blue; - __u16 *transp; -}; - -struct fb_copyarea { - __u32 dx; - __u32 dy; - __u32 width; - __u32 height; - __u32 sx; - __u32 sy; -}; - -struct fb_fillrect { - __u32 dx; - __u32 dy; - __u32 width; - __u32 height; - __u32 color; - __u32 rop; -}; - -struct fb_image { - __u32 dx; - __u32 dy; - __u32 width; - __u32 height; - __u32 fg_color; - __u32 bg_color; - __u8 depth; - const char *data; - struct fb_cmap cmap; -}; - -struct fbcurpos { - __u16 x; - __u16 y; -}; - -struct fb_cursor { - __u16 set; - __u16 enable; - __u16 rop; - const char *mask; - struct fbcurpos hot; - struct fb_image image; -}; - -struct fb_chroma { - __u32 redx; - __u32 greenx; - __u32 bluex; - __u32 whitex; - __u32 redy; - __u32 greeny; - __u32 bluey; - __u32 whitey; -}; - -struct fb_videomode; - -struct fb_monspecs { - struct fb_chroma chroma; - struct fb_videomode *modedb; - __u8 manufacturer[4]; - __u8 monitor[14]; - __u8 serial_no[14]; - __u8 ascii[14]; - __u32 modedb_len; - __u32 model; - __u32 serial; - __u32 year; - __u32 week; - __u32 hfmin; - __u32 hfmax; - __u32 dclkmin; - __u32 dclkmax; - __u16 input; - __u16 dpms; - __u16 signal; - __u16 vfmin; - __u16 vfmax; - __u16 gamma; - __u16 gtf: 1; - __u16 misc; - __u8 version; - __u8 revision; - __u8 max_x; - __u8 max_y; -}; - -struct fb_info; - -struct fb_pixmap { - u8 *addr; - u32 size; - u32 offset; - u32 buf_align; - u32 scan_align; - u32 access_align; - u32 flags; - u32 blit_x; - u32 blit_y; - void (*writeio)(struct fb_info *, void *, void *, unsigned int); - void (*readio)(struct fb_info *, void *, void *, unsigned int); -}; - -struct fb_ops; - -struct fb_tile_ops; - -struct apertures_struct; - -struct fb_info { - atomic_t count; - int node; - int flags; - int fbcon_rotate_hint; - struct mutex lock; - struct mutex mm_lock; - struct fb_var_screeninfo var; - struct fb_fix_screeninfo fix; - struct fb_monspecs monspecs; - struct work_struct queue; - struct fb_pixmap pixmap; - struct fb_pixmap sprite; - struct fb_cmap cmap; - struct list_head modelist; - struct fb_videomode *mode; - const struct fb_ops *fbops; - struct device *device; - struct device *dev; - int class_flag; - struct fb_tile_ops *tileops; - union { - char *screen_base; - char *screen_buffer; - }; - long unsigned int screen_size; - void *pseudo_palette; - u32 state; - void *fbcon_par; - void *par; - struct apertures_struct *apertures; - bool skip_vt_switch; -}; - -struct fb_videomode { - const char *name; - u32 refresh; - u32 xres; - u32 yres; - u32 pixclock; - u32 left_margin; - u32 right_margin; - u32 upper_margin; - u32 lower_margin; - u32 hsync_len; - u32 vsync_len; - u32 sync; - u32 vmode; - u32 flag; -}; - -struct fb_blit_caps { - u32 x; - u32 y; - u32 len; - u32 flags; -}; - -struct fb_ops { - struct module *owner; - int (*fb_open)(struct fb_info *, int); - int (*fb_release)(struct fb_info *, int); - ssize_t (*fb_read)(struct fb_info *, char *, size_t, loff_t *); - ssize_t (*fb_write)(struct fb_info *, const char *, size_t, loff_t *); - int (*fb_check_var)(struct fb_var_screeninfo *, struct fb_info *); - int (*fb_set_par)(struct fb_info *); - int (*fb_setcolreg)(unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, struct fb_info *); - int (*fb_setcmap)(struct fb_cmap *, struct fb_info *); - int (*fb_blank)(int, struct fb_info *); - int (*fb_pan_display)(struct fb_var_screeninfo *, struct fb_info *); - void (*fb_fillrect)(struct fb_info *, const struct fb_fillrect *); - void (*fb_copyarea)(struct fb_info *, const struct fb_copyarea *); - void (*fb_imageblit)(struct fb_info *, const struct fb_image *); - int (*fb_cursor)(struct fb_info *, struct fb_cursor *); - int (*fb_sync)(struct fb_info *); - int (*fb_ioctl)(struct fb_info *, unsigned int, long unsigned int); - int (*fb_compat_ioctl)(struct fb_info *, unsigned int, long unsigned int); - int (*fb_mmap)(struct fb_info *, struct vm_area_struct *); - void (*fb_get_caps)(struct fb_info *, struct fb_blit_caps *, struct fb_var_screeninfo *); - void (*fb_destroy)(struct fb_info *); - int (*fb_debug_enter)(struct fb_info *); - int (*fb_debug_leave)(struct fb_info *); -}; - -struct fb_tilemap { - __u32 width; - __u32 height; - __u32 depth; - __u32 length; - const __u8 *data; -}; - -struct fb_tilerect { - __u32 sx; - __u32 sy; - __u32 width; - __u32 height; - __u32 index; - __u32 fg; - __u32 bg; - __u32 rop; -}; - -struct fb_tilearea { - __u32 sx; - __u32 sy; - __u32 dx; - __u32 dy; - __u32 width; - __u32 height; -}; - -struct fb_tileblit { - __u32 sx; - __u32 sy; - __u32 width; - __u32 height; - __u32 fg; - __u32 bg; - __u32 length; - __u32 *indices; -}; - -struct fb_tilecursor { - __u32 sx; - __u32 sy; - __u32 mode; - __u32 shape; - __u32 fg; - __u32 bg; -}; - -struct fb_tile_ops { - void (*fb_settile)(struct fb_info *, struct fb_tilemap *); - void (*fb_tilecopy)(struct fb_info *, struct fb_tilearea *); - void (*fb_tilefill)(struct fb_info *, struct fb_tilerect *); - void (*fb_tileblit)(struct fb_info *, struct fb_tileblit *); - void (*fb_tilecursor)(struct fb_info *, struct fb_tilecursor *); - int (*fb_get_tilemax)(struct fb_info *); -}; - -struct aperture { - resource_size_t base; - resource_size_t size; -}; - -struct apertures_struct { - unsigned int count; - struct aperture ranges[0]; -}; - -struct dmt_videomode { - u32 dmt_id; - u32 std_2byte_code; - u32 cvt_3byte_code; - const struct fb_videomode *mode; -}; - -struct simplefb_platform_data { - u32 width; - u32 height; - u32 stride; - const char *format; -}; - -struct efifb_dmi_info { - char *optname; - long unsigned int base; - int stride; - int width; - int height; - int flags; -}; - -enum { - M_I17 = 0, - M_I20 = 1, - M_I20_SR = 2, - M_I24 = 3, - M_I24_8_1 = 4, - M_I24_10_1 = 5, - M_I27_11_1 = 6, - M_MINI = 7, - M_MINI_3_1 = 8, - M_MINI_4_1 = 9, - M_MB = 10, - M_MB_2 = 11, - M_MB_3 = 12, - M_MB_5_1 = 13, - M_MB_6_1 = 14, - M_MB_7_1 = 15, - M_MB_SR = 16, - M_MBA = 17, - M_MBA_3 = 18, - M_MBP = 19, - M_MBP_2 = 20, - M_MBP_2_2 = 21, - M_MBP_SR = 22, - M_MBP_4 = 23, - M_MBP_5_1 = 24, - M_MBP_5_2 = 25, - M_MBP_5_3 = 26, - M_MBP_6_1 = 27, - M_MBP_6_2 = 28, - M_MBP_7_1 = 29, - M_MBP_8_2 = 30, - M_UNKNOWN = 31, -}; - -enum { - OVERRIDE_NONE = 0, - OVERRIDE_BASE = 1, - OVERRIDE_STRIDE = 2, - OVERRIDE_HEIGHT = 4, - OVERRIDE_WIDTH = 8, -}; - enum perf_sample_regs_abi { PERF_SAMPLE_REGS_ABI_NONE = 0, PERF_SAMPLE_REGS_ABI_32 = 1, @@ -26141,6 +24575,17 @@ struct va_format { va_list *va; }; +enum auditsc_class_t { + AUDITSC_NATIVE = 0, + AUDITSC_COMPAT = 1, + AUDITSC_OPEN = 2, + AUDITSC_OPENAT = 3, + AUDITSC_SOCKETCALL = 4, + AUDITSC_EXECVE = 5, + AUDITSC_OPENAT2 = 6, + AUDITSC_NVALS = 7, +}; + enum chipset_type { NOT_SUPPORTED = 0, SUPPORTED = 1, @@ -26176,8 +24621,6 @@ struct pci_hostbridge_probe { typedef u8 uint8_t; -typedef u16 uint16_t; - enum pg_level { PG_LEVEL_NONE = 0, PG_LEVEL_4K = 1, @@ -26262,8 +24705,6 @@ struct ioremap_desc { unsigned int flags; }; -typedef bool (*ex_handler_t)(const struct exception_table_entry *, struct pt_regs *, int, long unsigned int, long unsigned int); - struct hugepage_subpool { spinlock_t lock; long int count; @@ -26297,13 +24738,13 @@ struct flush_tlb_info { struct exception_stacks { char DF_stack_guard[0]; - char DF_stack[4096]; + char DF_stack[8192]; char NMI_stack_guard[0]; - char NMI_stack[4096]; + char NMI_stack[8192]; char DB_stack_guard[0]; - char DB_stack[4096]; + char DB_stack[8192]; char MCE_stack_guard[0]; - char MCE_stack[4096]; + char MCE_stack[8192]; char VC_stack_guard[0]; char VC_stack[0]; char VC2_stack_guard[0]; @@ -26521,6 +24962,353 @@ typedef union { efi_system_table_32_t mixed_mode; } efi_system_table_t; +struct bpf_run_ctx {}; + +struct bpf_cgroup_storage; + +struct bpf_prog_array_item { + struct bpf_prog *prog; + union { + struct bpf_cgroup_storage *cgroup_storage[2]; + u64 bpf_cookie; + }; +}; + +struct bpf_prog_array { + struct callback_head rcu; + struct bpf_prog_array_item items[0]; +}; + +enum bpf_prog_type { + BPF_PROG_TYPE_UNSPEC = 0, + BPF_PROG_TYPE_SOCKET_FILTER = 1, + BPF_PROG_TYPE_KPROBE = 2, + BPF_PROG_TYPE_SCHED_CLS = 3, + BPF_PROG_TYPE_SCHED_ACT = 4, + BPF_PROG_TYPE_TRACEPOINT = 5, + BPF_PROG_TYPE_XDP = 6, + BPF_PROG_TYPE_PERF_EVENT = 7, + BPF_PROG_TYPE_CGROUP_SKB = 8, + BPF_PROG_TYPE_CGROUP_SOCK = 9, + BPF_PROG_TYPE_LWT_IN = 10, + BPF_PROG_TYPE_LWT_OUT = 11, + BPF_PROG_TYPE_LWT_XMIT = 12, + BPF_PROG_TYPE_SOCK_OPS = 13, + BPF_PROG_TYPE_SK_SKB = 14, + BPF_PROG_TYPE_CGROUP_DEVICE = 15, + BPF_PROG_TYPE_SK_MSG = 16, + BPF_PROG_TYPE_RAW_TRACEPOINT = 17, + BPF_PROG_TYPE_CGROUP_SOCK_ADDR = 18, + BPF_PROG_TYPE_LWT_SEG6LOCAL = 19, + BPF_PROG_TYPE_LIRC_MODE2 = 20, + BPF_PROG_TYPE_SK_REUSEPORT = 21, + BPF_PROG_TYPE_FLOW_DISSECTOR = 22, + BPF_PROG_TYPE_CGROUP_SYSCTL = 23, + BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE = 24, + BPF_PROG_TYPE_CGROUP_SOCKOPT = 25, + BPF_PROG_TYPE_TRACING = 26, + BPF_PROG_TYPE_STRUCT_OPS = 27, + BPF_PROG_TYPE_EXT = 28, + BPF_PROG_TYPE_LSM = 29, + BPF_PROG_TYPE_SK_LOOKUP = 30, + BPF_PROG_TYPE_SYSCALL = 31, +}; + +enum bpf_attach_type { + BPF_CGROUP_INET_INGRESS = 0, + BPF_CGROUP_INET_EGRESS = 1, + BPF_CGROUP_INET_SOCK_CREATE = 2, + BPF_CGROUP_SOCK_OPS = 3, + BPF_SK_SKB_STREAM_PARSER = 4, + BPF_SK_SKB_STREAM_VERDICT = 5, + BPF_CGROUP_DEVICE = 6, + BPF_SK_MSG_VERDICT = 7, + BPF_CGROUP_INET4_BIND = 8, + BPF_CGROUP_INET6_BIND = 9, + BPF_CGROUP_INET4_CONNECT = 10, + BPF_CGROUP_INET6_CONNECT = 11, + BPF_CGROUP_INET4_POST_BIND = 12, + BPF_CGROUP_INET6_POST_BIND = 13, + BPF_CGROUP_UDP4_SENDMSG = 14, + BPF_CGROUP_UDP6_SENDMSG = 15, + BPF_LIRC_MODE2 = 16, + BPF_FLOW_DISSECTOR = 17, + BPF_CGROUP_SYSCTL = 18, + BPF_CGROUP_UDP4_RECVMSG = 19, + BPF_CGROUP_UDP6_RECVMSG = 20, + BPF_CGROUP_GETSOCKOPT = 21, + BPF_CGROUP_SETSOCKOPT = 22, + BPF_TRACE_RAW_TP = 23, + BPF_TRACE_FENTRY = 24, + BPF_TRACE_FEXIT = 25, + BPF_MODIFY_RETURN = 26, + BPF_LSM_MAC = 27, + BPF_TRACE_ITER = 28, + BPF_CGROUP_INET4_GETPEERNAME = 29, + BPF_CGROUP_INET6_GETPEERNAME = 30, + BPF_CGROUP_INET4_GETSOCKNAME = 31, + BPF_CGROUP_INET6_GETSOCKNAME = 32, + BPF_XDP_DEVMAP = 33, + BPF_CGROUP_INET_SOCK_RELEASE = 34, + BPF_XDP_CPUMAP = 35, + BPF_SK_LOOKUP = 36, + BPF_XDP = 37, + BPF_SK_SKB_VERDICT = 38, + BPF_SK_REUSEPORT_SELECT = 39, + BPF_SK_REUSEPORT_SELECT_OR_MIGRATE = 40, + BPF_PERF_EVENT = 41, + __MAX_BPF_ATTACH_TYPE = 42, +}; + +struct sock_filter { + __u16 code; + __u8 jt; + __u8 jf; + __u32 k; +}; + +struct bpf_insn { + __u8 code; + __u8 dst_reg: 4; + __u8 src_reg: 4; + __s16 off; + __s32 imm; +}; + +struct bpf_prog_stats; + +struct bpf_prog_aux; + +struct sock_fprog_kern; + +struct bpf_prog { + u16 pages; + u16 jited: 1; + u16 jit_requested: 1; + u16 gpl_compatible: 1; + u16 cb_access: 1; + u16 dst_needed: 1; + u16 blinded: 1; + u16 is_func: 1; + u16 kprobe_override: 1; + u16 has_callchain_buf: 1; + u16 enforce_expected_attach_type: 1; + u16 call_get_stack: 1; + u16 call_get_func_ip: 1; + enum bpf_prog_type type; + enum bpf_attach_type expected_attach_type; + u32 len; + u32 jited_len; + u8 tag[8]; + struct bpf_prog_stats *stats; + int *active; + unsigned int (*bpf_func)(const void *, const struct bpf_insn *); + struct bpf_prog_aux *aux; + struct sock_fprog_kern *orig_prog; + union { + struct { + struct { } __empty_insns; + struct sock_filter insns[0]; + }; + struct { + struct { } __empty_insnsi; + struct bpf_insn insnsi[0]; + }; + }; +}; + +struct tc_stats { + __u64 bytes; + __u32 packets; + __u32 drops; + __u32 overlimits; + __u32 bps; + __u32 pps; + __u32 qlen; + __u32 backlog; +}; + +struct tc_sizespec { + unsigned char cell_log; + unsigned char size_log; + short int cell_align; + int overhead; + unsigned int linklayer; + unsigned int mpu; + unsigned int mtu; + unsigned int tsize; +}; + +struct net_rate_estimator; + +struct qdisc_skb_head { + struct sk_buff *head; + struct sk_buff *tail; + __u32 qlen; + spinlock_t lock; +}; + +struct gnet_stats_basic_sync { + u64_stats_t bytes; + u64_stats_t packets; + struct u64_stats_sync syncp; +}; + +struct gnet_stats_queue { + __u32 qlen; + __u32 backlog; + __u32 drops; + __u32 requeues; + __u32 overlimits; +}; + +struct Qdisc_ops; + +struct qdisc_size_table; + +struct Qdisc { + int (*enqueue)(struct sk_buff *, struct Qdisc *, struct sk_buff **); + struct sk_buff * (*dequeue)(struct Qdisc *); + unsigned int flags; + u32 limit; + const struct Qdisc_ops *ops; + struct qdisc_size_table *stab; + struct hlist_node hash; + u32 handle; + u32 parent; + struct netdev_queue *dev_queue; + struct net_rate_estimator *rate_est; + struct gnet_stats_basic_sync *cpu_bstats; + struct gnet_stats_queue *cpu_qstats; + int pad; + refcount_t refcnt; + long: 64; + long: 64; + long: 64; + struct sk_buff_head gso_skb; + struct qdisc_skb_head q; + struct gnet_stats_basic_sync bstats; + struct gnet_stats_queue qstats; + long unsigned int state; + long unsigned int state2; + struct Qdisc *next_sched; + struct sk_buff_head skb_bad_txq; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + spinlock_t busylock; + spinlock_t seqlock; + struct callback_head rcu; + netdevice_tracker dev_tracker; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long int privdata[0]; +}; + +enum bpf_map_type { + BPF_MAP_TYPE_UNSPEC = 0, + BPF_MAP_TYPE_HASH = 1, + BPF_MAP_TYPE_ARRAY = 2, + BPF_MAP_TYPE_PROG_ARRAY = 3, + BPF_MAP_TYPE_PERF_EVENT_ARRAY = 4, + BPF_MAP_TYPE_PERCPU_HASH = 5, + BPF_MAP_TYPE_PERCPU_ARRAY = 6, + BPF_MAP_TYPE_STACK_TRACE = 7, + BPF_MAP_TYPE_CGROUP_ARRAY = 8, + BPF_MAP_TYPE_LRU_HASH = 9, + BPF_MAP_TYPE_LRU_PERCPU_HASH = 10, + BPF_MAP_TYPE_LPM_TRIE = 11, + BPF_MAP_TYPE_ARRAY_OF_MAPS = 12, + BPF_MAP_TYPE_HASH_OF_MAPS = 13, + BPF_MAP_TYPE_DEVMAP = 14, + BPF_MAP_TYPE_SOCKMAP = 15, + BPF_MAP_TYPE_CPUMAP = 16, + BPF_MAP_TYPE_XSKMAP = 17, + BPF_MAP_TYPE_SOCKHASH = 18, + BPF_MAP_TYPE_CGROUP_STORAGE = 19, + BPF_MAP_TYPE_REUSEPORT_SOCKARRAY = 20, + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = 21, + BPF_MAP_TYPE_QUEUE = 22, + BPF_MAP_TYPE_STACK = 23, + BPF_MAP_TYPE_SK_STORAGE = 24, + BPF_MAP_TYPE_DEVMAP_HASH = 25, + BPF_MAP_TYPE_STRUCT_OPS = 26, + BPF_MAP_TYPE_RINGBUF = 27, + BPF_MAP_TYPE_INODE_STORAGE = 28, + BPF_MAP_TYPE_TASK_STORAGE = 29, + BPF_MAP_TYPE_BLOOM_FILTER = 30, +}; + +struct bpf_map_ops; + +struct btf; + +struct bpf_map { + const struct bpf_map_ops *ops; + struct bpf_map *inner_map_meta; + void *security; + enum bpf_map_type map_type; + u32 key_size; + u32 value_size; + u32 max_entries; + u64 map_extra; + u32 map_flags; + int spin_lock_off; + int timer_off; + u32 id; + int numa_node; + u32 btf_key_type_id; + u32 btf_value_type_id; + u32 btf_vmlinux_value_type_id; + struct btf *btf; + struct mem_cgroup *memcg; + char name[16]; + bool bypass_spec_v1; + bool frozen; + long: 48; + long: 64; + atomic64_t refcnt; + atomic64_t usercnt; + struct work_struct work; + struct mutex freeze_mutex; + atomic64_t writecnt; + long: 64; + long: 64; +}; + +struct bpf_map_dev_ops; + +struct bpf_offloaded_map { + struct bpf_map map; + struct net_device *netdev; + const struct bpf_map_dev_ops *dev_ops; + void *dev_priv; + struct list_head offloads; + long: 64; + long: 64; + long: 64; +}; + +struct tcf_proto; + +struct tcf_block; + +struct mini_Qdisc { + struct tcf_proto *filter_list; + struct tcf_block *block; + struct gnet_stats_basic_sync *cpu_bstats; + struct gnet_stats_queue *cpu_qstats; + long unsigned int rcu_state; +}; + enum { BPF_REG_0 = 0, BPF_REG_1 = 1, @@ -26536,18 +25324,876 @@ enum { __MAX_BPF_REG = 11, }; +union bpf_attr { + struct { + __u32 map_type; + __u32 key_size; + __u32 value_size; + __u32 max_entries; + __u32 map_flags; + __u32 inner_map_fd; + __u32 numa_node; + char map_name[16]; + __u32 map_ifindex; + __u32 btf_fd; + __u32 btf_key_type_id; + __u32 btf_value_type_id; + __u32 btf_vmlinux_value_type_id; + __u64 map_extra; + }; + struct { + __u32 map_fd; + __u64 key; + union { + __u64 value; + __u64 next_key; + }; + __u64 flags; + }; + struct { + __u64 in_batch; + __u64 out_batch; + __u64 keys; + __u64 values; + __u32 count; + __u32 map_fd; + __u64 elem_flags; + __u64 flags; + } batch; + struct { + __u32 prog_type; + __u32 insn_cnt; + __u64 insns; + __u64 license; + __u32 log_level; + __u32 log_size; + __u64 log_buf; + __u32 kern_version; + __u32 prog_flags; + char prog_name[16]; + __u32 prog_ifindex; + __u32 expected_attach_type; + __u32 prog_btf_fd; + __u32 func_info_rec_size; + __u64 func_info; + __u32 func_info_cnt; + __u32 line_info_rec_size; + __u64 line_info; + __u32 line_info_cnt; + __u32 attach_btf_id; + union { + __u32 attach_prog_fd; + __u32 attach_btf_obj_fd; + }; + __u32 core_relo_cnt; + __u64 fd_array; + __u64 core_relos; + __u32 core_relo_rec_size; + }; + struct { + __u64 pathname; + __u32 bpf_fd; + __u32 file_flags; + }; + struct { + __u32 target_fd; + __u32 attach_bpf_fd; + __u32 attach_type; + __u32 attach_flags; + __u32 replace_bpf_fd; + }; + struct { + __u32 prog_fd; + __u32 retval; + __u32 data_size_in; + __u32 data_size_out; + __u64 data_in; + __u64 data_out; + __u32 repeat; + __u32 duration; + __u32 ctx_size_in; + __u32 ctx_size_out; + __u64 ctx_in; + __u64 ctx_out; + __u32 flags; + __u32 cpu; + } test; + struct { + union { + __u32 start_id; + __u32 prog_id; + __u32 map_id; + __u32 btf_id; + __u32 link_id; + }; + __u32 next_id; + __u32 open_flags; + }; + struct { + __u32 bpf_fd; + __u32 info_len; + __u64 info; + } info; + struct { + __u32 target_fd; + __u32 attach_type; + __u32 query_flags; + __u32 attach_flags; + __u64 prog_ids; + __u32 prog_cnt; + } query; + struct { + __u64 name; + __u32 prog_fd; + } raw_tracepoint; + struct { + __u64 btf; + __u64 btf_log_buf; + __u32 btf_size; + __u32 btf_log_size; + __u32 btf_log_level; + }; + struct { + __u32 pid; + __u32 fd; + __u32 flags; + __u32 buf_len; + __u64 buf; + __u32 prog_id; + __u32 fd_type; + __u64 probe_offset; + __u64 probe_addr; + } task_fd_query; + struct { + __u32 prog_fd; + union { + __u32 target_fd; + __u32 target_ifindex; + }; + __u32 attach_type; + __u32 flags; + union { + __u32 target_btf_id; + struct { + __u64 iter_info; + __u32 iter_info_len; + }; + struct { + __u64 bpf_cookie; + } perf_event; + }; + } link_create; + struct { + __u32 link_fd; + __u32 new_prog_fd; + __u32 flags; + __u32 old_prog_fd; + } link_update; + struct { + __u32 link_fd; + } link_detach; + struct { + __u32 type; + } enable_stats; + struct { + __u32 link_fd; + __u32 flags; + } iter_create; + struct { + __u32 prog_fd; + __u32 map_fd; + __u32 flags; + } prog_bind_map; +}; + +enum bpf_func_id { + BPF_FUNC_unspec = 0, + BPF_FUNC_map_lookup_elem = 1, + BPF_FUNC_map_update_elem = 2, + BPF_FUNC_map_delete_elem = 3, + BPF_FUNC_probe_read = 4, + BPF_FUNC_ktime_get_ns = 5, + BPF_FUNC_trace_printk = 6, + BPF_FUNC_get_prandom_u32 = 7, + BPF_FUNC_get_smp_processor_id = 8, + BPF_FUNC_skb_store_bytes = 9, + BPF_FUNC_l3_csum_replace = 10, + BPF_FUNC_l4_csum_replace = 11, + BPF_FUNC_tail_call = 12, + BPF_FUNC_clone_redirect = 13, + BPF_FUNC_get_current_pid_tgid = 14, + BPF_FUNC_get_current_uid_gid = 15, + BPF_FUNC_get_current_comm = 16, + BPF_FUNC_get_cgroup_classid = 17, + BPF_FUNC_skb_vlan_push = 18, + BPF_FUNC_skb_vlan_pop = 19, + BPF_FUNC_skb_get_tunnel_key = 20, + BPF_FUNC_skb_set_tunnel_key = 21, + BPF_FUNC_perf_event_read = 22, + BPF_FUNC_redirect = 23, + BPF_FUNC_get_route_realm = 24, + BPF_FUNC_perf_event_output = 25, + BPF_FUNC_skb_load_bytes = 26, + BPF_FUNC_get_stackid = 27, + BPF_FUNC_csum_diff = 28, + BPF_FUNC_skb_get_tunnel_opt = 29, + BPF_FUNC_skb_set_tunnel_opt = 30, + BPF_FUNC_skb_change_proto = 31, + BPF_FUNC_skb_change_type = 32, + BPF_FUNC_skb_under_cgroup = 33, + BPF_FUNC_get_hash_recalc = 34, + BPF_FUNC_get_current_task = 35, + BPF_FUNC_probe_write_user = 36, + BPF_FUNC_current_task_under_cgroup = 37, + BPF_FUNC_skb_change_tail = 38, + BPF_FUNC_skb_pull_data = 39, + BPF_FUNC_csum_update = 40, + BPF_FUNC_set_hash_invalid = 41, + BPF_FUNC_get_numa_node_id = 42, + BPF_FUNC_skb_change_head = 43, + BPF_FUNC_xdp_adjust_head = 44, + BPF_FUNC_probe_read_str = 45, + BPF_FUNC_get_socket_cookie = 46, + BPF_FUNC_get_socket_uid = 47, + BPF_FUNC_set_hash = 48, + BPF_FUNC_setsockopt = 49, + BPF_FUNC_skb_adjust_room = 50, + BPF_FUNC_redirect_map = 51, + BPF_FUNC_sk_redirect_map = 52, + BPF_FUNC_sock_map_update = 53, + BPF_FUNC_xdp_adjust_meta = 54, + BPF_FUNC_perf_event_read_value = 55, + BPF_FUNC_perf_prog_read_value = 56, + BPF_FUNC_getsockopt = 57, + BPF_FUNC_override_return = 58, + BPF_FUNC_sock_ops_cb_flags_set = 59, + BPF_FUNC_msg_redirect_map = 60, + BPF_FUNC_msg_apply_bytes = 61, + BPF_FUNC_msg_cork_bytes = 62, + BPF_FUNC_msg_pull_data = 63, + BPF_FUNC_bind = 64, + BPF_FUNC_xdp_adjust_tail = 65, + BPF_FUNC_skb_get_xfrm_state = 66, + BPF_FUNC_get_stack = 67, + BPF_FUNC_skb_load_bytes_relative = 68, + BPF_FUNC_fib_lookup = 69, + BPF_FUNC_sock_hash_update = 70, + BPF_FUNC_msg_redirect_hash = 71, + BPF_FUNC_sk_redirect_hash = 72, + BPF_FUNC_lwt_push_encap = 73, + BPF_FUNC_lwt_seg6_store_bytes = 74, + BPF_FUNC_lwt_seg6_adjust_srh = 75, + BPF_FUNC_lwt_seg6_action = 76, + BPF_FUNC_rc_repeat = 77, + BPF_FUNC_rc_keydown = 78, + BPF_FUNC_skb_cgroup_id = 79, + BPF_FUNC_get_current_cgroup_id = 80, + BPF_FUNC_get_local_storage = 81, + BPF_FUNC_sk_select_reuseport = 82, + BPF_FUNC_skb_ancestor_cgroup_id = 83, + BPF_FUNC_sk_lookup_tcp = 84, + BPF_FUNC_sk_lookup_udp = 85, + BPF_FUNC_sk_release = 86, + BPF_FUNC_map_push_elem = 87, + BPF_FUNC_map_pop_elem = 88, + BPF_FUNC_map_peek_elem = 89, + BPF_FUNC_msg_push_data = 90, + BPF_FUNC_msg_pop_data = 91, + BPF_FUNC_rc_pointer_rel = 92, + BPF_FUNC_spin_lock = 93, + BPF_FUNC_spin_unlock = 94, + BPF_FUNC_sk_fullsock = 95, + BPF_FUNC_tcp_sock = 96, + BPF_FUNC_skb_ecn_set_ce = 97, + BPF_FUNC_get_listener_sock = 98, + BPF_FUNC_skc_lookup_tcp = 99, + BPF_FUNC_tcp_check_syncookie = 100, + BPF_FUNC_sysctl_get_name = 101, + BPF_FUNC_sysctl_get_current_value = 102, + BPF_FUNC_sysctl_get_new_value = 103, + BPF_FUNC_sysctl_set_new_value = 104, + BPF_FUNC_strtol = 105, + BPF_FUNC_strtoul = 106, + BPF_FUNC_sk_storage_get = 107, + BPF_FUNC_sk_storage_delete = 108, + BPF_FUNC_send_signal = 109, + BPF_FUNC_tcp_gen_syncookie = 110, + BPF_FUNC_skb_output = 111, + BPF_FUNC_probe_read_user = 112, + BPF_FUNC_probe_read_kernel = 113, + BPF_FUNC_probe_read_user_str = 114, + BPF_FUNC_probe_read_kernel_str = 115, + BPF_FUNC_tcp_send_ack = 116, + BPF_FUNC_send_signal_thread = 117, + BPF_FUNC_jiffies64 = 118, + BPF_FUNC_read_branch_records = 119, + BPF_FUNC_get_ns_current_pid_tgid = 120, + BPF_FUNC_xdp_output = 121, + BPF_FUNC_get_netns_cookie = 122, + BPF_FUNC_get_current_ancestor_cgroup_id = 123, + BPF_FUNC_sk_assign = 124, + BPF_FUNC_ktime_get_boot_ns = 125, + BPF_FUNC_seq_printf = 126, + BPF_FUNC_seq_write = 127, + BPF_FUNC_sk_cgroup_id = 128, + BPF_FUNC_sk_ancestor_cgroup_id = 129, + BPF_FUNC_ringbuf_output = 130, + BPF_FUNC_ringbuf_reserve = 131, + BPF_FUNC_ringbuf_submit = 132, + BPF_FUNC_ringbuf_discard = 133, + BPF_FUNC_ringbuf_query = 134, + BPF_FUNC_csum_level = 135, + BPF_FUNC_skc_to_tcp6_sock = 136, + BPF_FUNC_skc_to_tcp_sock = 137, + BPF_FUNC_skc_to_tcp_timewait_sock = 138, + BPF_FUNC_skc_to_tcp_request_sock = 139, + BPF_FUNC_skc_to_udp6_sock = 140, + BPF_FUNC_get_task_stack = 141, + BPF_FUNC_load_hdr_opt = 142, + BPF_FUNC_store_hdr_opt = 143, + BPF_FUNC_reserve_hdr_opt = 144, + BPF_FUNC_inode_storage_get = 145, + BPF_FUNC_inode_storage_delete = 146, + BPF_FUNC_d_path = 147, + BPF_FUNC_copy_from_user = 148, + BPF_FUNC_snprintf_btf = 149, + BPF_FUNC_seq_printf_btf = 150, + BPF_FUNC_skb_cgroup_classid = 151, + BPF_FUNC_redirect_neigh = 152, + BPF_FUNC_per_cpu_ptr = 153, + BPF_FUNC_this_cpu_ptr = 154, + BPF_FUNC_redirect_peer = 155, + BPF_FUNC_task_storage_get = 156, + BPF_FUNC_task_storage_delete = 157, + BPF_FUNC_get_current_task_btf = 158, + BPF_FUNC_bprm_opts_set = 159, + BPF_FUNC_ktime_get_coarse_ns = 160, + BPF_FUNC_ima_inode_hash = 161, + BPF_FUNC_sock_from_file = 162, + BPF_FUNC_check_mtu = 163, + BPF_FUNC_for_each_map_elem = 164, + BPF_FUNC_snprintf = 165, + BPF_FUNC_sys_bpf = 166, + BPF_FUNC_btf_find_by_name_kind = 167, + BPF_FUNC_sys_close = 168, + BPF_FUNC_timer_init = 169, + BPF_FUNC_timer_set_callback = 170, + BPF_FUNC_timer_start = 171, + BPF_FUNC_timer_cancel = 172, + BPF_FUNC_get_func_ip = 173, + BPF_FUNC_get_attach_cookie = 174, + BPF_FUNC_task_pt_regs = 175, + BPF_FUNC_get_branch_snapshot = 176, + BPF_FUNC_trace_vprintk = 177, + BPF_FUNC_skc_to_unix_sock = 178, + BPF_FUNC_kallsyms_lookup_name = 179, + BPF_FUNC_find_vma = 180, + BPF_FUNC_loop = 181, + BPF_FUNC_strncmp = 182, + BPF_FUNC_get_func_arg = 183, + BPF_FUNC_get_func_ret = 184, + BPF_FUNC_get_func_arg_cnt = 185, + BPF_FUNC_get_retval = 186, + BPF_FUNC_set_retval = 187, + __BPF_FUNC_MAX_ID = 188, +}; + +struct bpf_func_info { + __u32 insn_off; + __u32 type_id; +}; + +struct bpf_line_info { + __u32 insn_off; + __u32 file_name_off; + __u32 line_off; + __u32 line_col; +}; + +typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); + +struct bpf_iter_aux_info; + +typedef int (*bpf_iter_init_seq_priv_t)(void *, struct bpf_iter_aux_info *); + +struct bpf_iter_aux_info { + struct bpf_map *map; +}; + +typedef void (*bpf_iter_fini_seq_priv_t)(void *); + +struct bpf_iter_seq_info { + const struct seq_operations *seq_ops; + bpf_iter_init_seq_priv_t init_seq_private; + bpf_iter_fini_seq_priv_t fini_seq_private; + u32 seq_priv_size; +}; + +struct btf_type; + +struct bpf_local_storage_map; + +struct bpf_verifier_env; + +struct bpf_func_state; + +struct bpf_map_ops { + int (*map_alloc_check)(union bpf_attr *); + struct bpf_map * (*map_alloc)(union bpf_attr *); + void (*map_release)(struct bpf_map *, struct file *); + void (*map_free)(struct bpf_map *); + int (*map_get_next_key)(struct bpf_map *, void *, void *); + void (*map_release_uref)(struct bpf_map *); + void * (*map_lookup_elem_sys_only)(struct bpf_map *, void *); + int (*map_lookup_batch)(struct bpf_map *, const union bpf_attr *, union bpf_attr *); + int (*map_lookup_and_delete_elem)(struct bpf_map *, void *, void *, u64); + int (*map_lookup_and_delete_batch)(struct bpf_map *, const union bpf_attr *, union bpf_attr *); + int (*map_update_batch)(struct bpf_map *, const union bpf_attr *, union bpf_attr *); + int (*map_delete_batch)(struct bpf_map *, const union bpf_attr *, union bpf_attr *); + void * (*map_lookup_elem)(struct bpf_map *, void *); + int (*map_update_elem)(struct bpf_map *, void *, void *, u64); + int (*map_delete_elem)(struct bpf_map *, void *); + int (*map_push_elem)(struct bpf_map *, void *, u64); + int (*map_pop_elem)(struct bpf_map *, void *); + int (*map_peek_elem)(struct bpf_map *, void *); + void * (*map_fd_get_ptr)(struct bpf_map *, struct file *, int); + void (*map_fd_put_ptr)(void *); + int (*map_gen_lookup)(struct bpf_map *, struct bpf_insn *); + u32 (*map_fd_sys_lookup_elem)(void *); + void (*map_seq_show_elem)(struct bpf_map *, void *, struct seq_file *); + int (*map_check_btf)(const struct bpf_map *, const struct btf *, const struct btf_type *, const struct btf_type *); + int (*map_poke_track)(struct bpf_map *, struct bpf_prog_aux *); + void (*map_poke_untrack)(struct bpf_map *, struct bpf_prog_aux *); + void (*map_poke_run)(struct bpf_map *, u32, struct bpf_prog *, struct bpf_prog *); + int (*map_direct_value_addr)(const struct bpf_map *, u64 *, u32); + int (*map_direct_value_meta)(const struct bpf_map *, u64, u32 *); + int (*map_mmap)(struct bpf_map *, struct vm_area_struct *); + __poll_t (*map_poll)(struct bpf_map *, struct file *, struct poll_table_struct *); + int (*map_local_storage_charge)(struct bpf_local_storage_map *, void *, u32); + void (*map_local_storage_uncharge)(struct bpf_local_storage_map *, void *, u32); + struct bpf_local_storage ** (*map_owner_storage_ptr)(void *); + int (*map_redirect)(struct bpf_map *, u32, u64); + bool (*map_meta_equal)(const struct bpf_map *, const struct bpf_map *); + int (*map_set_for_each_callback_args)(struct bpf_verifier_env *, struct bpf_func_state *, struct bpf_func_state *); + int (*map_for_each_callback)(struct bpf_map *, bpf_callback_t, void *, u64); + const char * const map_btf_name; + int *map_btf_id; + const struct bpf_iter_seq_info *iter_seq_info; +}; + +struct btf_header { + __u16 magic; + __u8 version; + __u8 flags; + __u32 hdr_len; + __u32 type_off; + __u32 type_len; + __u32 str_off; + __u32 str_len; +}; + +struct btf_kfunc_set_tab; + +struct btf { + void *data; + struct btf_type **types; + u32 *resolved_ids; + u32 *resolved_sizes; + const char *strings; + void *nohdr_data; + struct btf_header hdr; + u32 nr_types; + u32 types_size; + u32 data_size; + refcount_t refcnt; + u32 id; + struct callback_head rcu; + struct btf_kfunc_set_tab *kfunc_set_tab; + struct btf *base_btf; + u32 start_id; + u32 start_str_off; + char name[56]; + bool kernel_btf; +}; + +struct btf_type { + __u32 name_off; + __u32 info; + union { + __u32 size; + __u32 type; + }; +}; + +struct bpf_ksym { + long unsigned int start; + long unsigned int end; + char name[128]; + struct list_head lnode; + struct latch_tree_node tnode; + bool prog; +}; + +struct bpf_ctx_arg_aux; + +struct bpf_trampoline; + +struct bpf_jit_poke_descriptor; + +struct bpf_kfunc_desc_tab; + +struct bpf_kfunc_btf_tab; + +struct bpf_prog_ops; + +struct btf_mod_pair; + +struct bpf_prog_offload; + +struct bpf_func_info_aux; + +struct bpf_prog_aux { + atomic64_t refcnt; + u32 used_map_cnt; + u32 used_btf_cnt; + u32 max_ctx_offset; + u32 max_pkt_offset; + u32 max_tp_access; + u32 stack_depth; + u32 id; + u32 func_cnt; + u32 func_idx; + u32 attach_btf_id; + u32 ctx_arg_info_size; + u32 max_rdonly_access; + u32 max_rdwr_access; + struct btf *attach_btf; + const struct bpf_ctx_arg_aux *ctx_arg_info; + struct mutex dst_mutex; + struct bpf_prog *dst_prog; + struct bpf_trampoline *dst_trampoline; + enum bpf_prog_type saved_dst_prog_type; + enum bpf_attach_type saved_dst_attach_type; + bool verifier_zext; + bool offload_requested; + bool attach_btf_trace; + bool func_proto_unreliable; + bool sleepable; + bool tail_call_reachable; + struct hlist_node tramp_hlist; + const struct btf_type *attach_func_proto; + const char *attach_func_name; + struct bpf_prog **func; + void *jit_data; + struct bpf_jit_poke_descriptor *poke_tab; + struct bpf_kfunc_desc_tab *kfunc_tab; + struct bpf_kfunc_btf_tab *kfunc_btf_tab; + u32 size_poke_tab; + struct bpf_ksym ksym; + const struct bpf_prog_ops *ops; + struct bpf_map **used_maps; + struct mutex used_maps_mutex; + struct btf_mod_pair *used_btfs; + struct bpf_prog *prog; + struct user_struct *user; + u64 load_time; + u32 verified_insns; + struct bpf_map *cgroup_storage[2]; + char name[16]; + void *security; + struct bpf_prog_offload *offload; + struct btf *btf; + struct bpf_func_info *func_info; + struct bpf_func_info_aux *func_info_aux; + struct bpf_line_info *linfo; + void **jited_linfo; + u32 func_info_cnt; + u32 nr_linfo; + u32 linfo_idx; + u32 num_exentries; + struct exception_table_entry *extable; + union { + struct work_struct work; + struct callback_head rcu; + }; +}; + +struct bpf_map_dev_ops { + int (*map_get_next_key)(struct bpf_offloaded_map *, void *, void *); + int (*map_lookup_elem)(struct bpf_offloaded_map *, void *, void *); + int (*map_update_elem)(struct bpf_offloaded_map *, void *, void *, u64); + int (*map_delete_elem)(struct bpf_offloaded_map *, void *); +}; + +enum bpf_arg_type { + ARG_DONTCARE = 0, + ARG_CONST_MAP_PTR = 1, + ARG_PTR_TO_MAP_KEY = 2, + ARG_PTR_TO_MAP_VALUE = 3, + ARG_PTR_TO_UNINIT_MAP_VALUE = 4, + ARG_PTR_TO_MEM = 5, + ARG_PTR_TO_UNINIT_MEM = 6, + ARG_CONST_SIZE = 7, + ARG_CONST_SIZE_OR_ZERO = 8, + ARG_PTR_TO_CTX = 9, + ARG_ANYTHING = 10, + ARG_PTR_TO_SPIN_LOCK = 11, + ARG_PTR_TO_SOCK_COMMON = 12, + ARG_PTR_TO_INT = 13, + ARG_PTR_TO_LONG = 14, + ARG_PTR_TO_SOCKET = 15, + ARG_PTR_TO_BTF_ID = 16, + ARG_PTR_TO_ALLOC_MEM = 17, + ARG_CONST_ALLOC_SIZE_OR_ZERO = 18, + ARG_PTR_TO_BTF_ID_SOCK_COMMON = 19, + ARG_PTR_TO_PERCPU_BTF_ID = 20, + ARG_PTR_TO_FUNC = 21, + ARG_PTR_TO_STACK = 22, + ARG_PTR_TO_CONST_STR = 23, + ARG_PTR_TO_TIMER = 24, + __BPF_ARG_TYPE_MAX = 25, + ARG_PTR_TO_MAP_VALUE_OR_NULL = 259, + ARG_PTR_TO_MEM_OR_NULL = 261, + ARG_PTR_TO_CTX_OR_NULL = 265, + ARG_PTR_TO_SOCKET_OR_NULL = 271, + ARG_PTR_TO_ALLOC_MEM_OR_NULL = 273, + ARG_PTR_TO_STACK_OR_NULL = 278, + __BPF_ARG_TYPE_LIMIT = 1023, +}; + +enum bpf_return_type { + RET_INTEGER = 0, + RET_VOID = 1, + RET_PTR_TO_MAP_VALUE = 2, + RET_PTR_TO_SOCKET = 3, + RET_PTR_TO_TCP_SOCK = 4, + RET_PTR_TO_SOCK_COMMON = 5, + RET_PTR_TO_ALLOC_MEM = 6, + RET_PTR_TO_MEM_OR_BTF_ID = 7, + RET_PTR_TO_BTF_ID = 8, + __BPF_RET_TYPE_MAX = 9, + RET_PTR_TO_MAP_VALUE_OR_NULL = 258, + RET_PTR_TO_SOCKET_OR_NULL = 259, + RET_PTR_TO_TCP_SOCK_OR_NULL = 260, + RET_PTR_TO_SOCK_COMMON_OR_NULL = 261, + RET_PTR_TO_ALLOC_MEM_OR_NULL = 262, + RET_PTR_TO_BTF_ID_OR_NULL = 264, + __BPF_RET_TYPE_LIMIT = 1023, +}; + +struct bpf_func_proto { + u64 (*func)(u64, u64, u64, u64, u64); + bool gpl_only; + bool pkt_access; + enum bpf_return_type ret_type; + union { + struct { + enum bpf_arg_type arg1_type; + enum bpf_arg_type arg2_type; + enum bpf_arg_type arg3_type; + enum bpf_arg_type arg4_type; + enum bpf_arg_type arg5_type; + }; + enum bpf_arg_type arg_type[5]; + }; + union { + struct { + u32 *arg1_btf_id; + u32 *arg2_btf_id; + u32 *arg3_btf_id; + u32 *arg4_btf_id; + u32 *arg5_btf_id; + }; + u32 *arg_btf_id[5]; + }; + int *ret_btf_id; + bool (*allowed)(const struct bpf_prog *); +}; + +enum bpf_access_type { + BPF_READ = 1, + BPF_WRITE = 2, +}; + +enum bpf_reg_type { + NOT_INIT = 0, + SCALAR_VALUE = 1, + PTR_TO_CTX = 2, + CONST_PTR_TO_MAP = 3, + PTR_TO_MAP_VALUE = 4, + PTR_TO_MAP_KEY = 5, + PTR_TO_STACK = 6, + PTR_TO_PACKET_META = 7, + PTR_TO_PACKET = 8, + PTR_TO_PACKET_END = 9, + PTR_TO_FLOW_KEYS = 10, + PTR_TO_SOCKET = 11, + PTR_TO_SOCK_COMMON = 12, + PTR_TO_TCP_SOCK = 13, + PTR_TO_TP_BUFFER = 14, + PTR_TO_XDP_SOCK = 15, + PTR_TO_BTF_ID = 16, + PTR_TO_MEM = 17, + PTR_TO_BUF = 18, + PTR_TO_PERCPU_BTF_ID = 19, + PTR_TO_FUNC = 20, + __BPF_REG_TYPE_MAX = 21, + PTR_TO_MAP_VALUE_OR_NULL = 260, + PTR_TO_SOCKET_OR_NULL = 267, + PTR_TO_SOCK_COMMON_OR_NULL = 268, + PTR_TO_TCP_SOCK_OR_NULL = 269, + PTR_TO_BTF_ID_OR_NULL = 272, + __BPF_REG_TYPE_LIMIT = 1023, +}; + +struct bpf_verifier_log; + +struct bpf_insn_access_aux { + enum bpf_reg_type reg_type; + union { + int ctx_field_size; + struct { + struct btf *btf; + u32 btf_id; + }; + }; + struct bpf_verifier_log *log; +}; + +struct bpf_prog_ops { + int (*test_run)(struct bpf_prog *, const union bpf_attr *, union bpf_attr *); +}; + +struct bpf_verifier_ops { + const struct bpf_func_proto * (*get_func_proto)(enum bpf_func_id, const struct bpf_prog *); + bool (*is_valid_access)(int, int, enum bpf_access_type, const struct bpf_prog *, struct bpf_insn_access_aux *); + int (*gen_prologue)(struct bpf_insn *, bool, const struct bpf_prog *); + int (*gen_ld_abs)(const struct bpf_insn *, struct bpf_insn *); + u32 (*convert_ctx_access)(enum bpf_access_type, const struct bpf_insn *, struct bpf_insn *, struct bpf_prog *, u32 *); + int (*btf_struct_access)(struct bpf_verifier_log *, const struct btf *, const struct btf_type *, int, int, enum bpf_access_type, u32 *); +}; + +struct bpf_offload_dev; + +struct bpf_prog_offload { + struct bpf_prog *prog; + struct net_device *netdev; + struct bpf_offload_dev *offdev; + void *dev_priv; + struct list_head offloads; + bool dev_state; + bool opt_failed; + void *jited_image; + u32 jited_len; +}; + +struct btf_func_model { + u8 ret_size; + u8 nr_args; + u8 arg_size[12]; +}; + struct bpf_tramp_progs { struct bpf_prog *progs[38]; int nr_progs; }; +enum bpf_tramp_prog_type { + BPF_TRAMP_FENTRY = 0, + BPF_TRAMP_FEXIT = 1, + BPF_TRAMP_MODIFY_RETURN = 2, + BPF_TRAMP_MAX = 3, + BPF_TRAMP_REPLACE = 4, +}; + +struct bpf_tramp_image { + void *image; + struct bpf_ksym ksym; + struct percpu_ref pcref; + void *ip_after_call; + void *ip_epilogue; + union { + struct callback_head rcu; + struct work_struct work; + }; +}; + +struct bpf_trampoline { + struct hlist_node hlist; + struct mutex mutex; + refcount_t refcnt; + u64 key; + struct { + struct btf_func_model model; + void *addr; + bool ftrace_managed; + } func; + struct bpf_prog *extension_prog; + struct hlist_head progs_hlist[3]; + int progs_cnt[3]; + struct bpf_tramp_image *cur_image; + u64 selector; + struct module *mod; +}; + +struct bpf_dispatcher_prog { + struct bpf_prog *prog; + refcount_t users; +}; + +struct bpf_dispatcher { + struct mutex mutex; + void *func; + struct bpf_dispatcher_prog progs[48]; + int num_progs; + void *image; + u32 image_off; + struct bpf_ksym ksym; +}; + +struct bpf_func_info_aux { + u16 linkage; + bool unreliable; +}; + enum bpf_jit_poke_reason { BPF_POKE_REASON_TAIL_CALL = 0, }; +struct bpf_jit_poke_descriptor { + void *tailcall_target; + void *tailcall_bypass; + void *bypass_addr; + void *aux; + union { + struct { + struct bpf_map *map; + u32 key; + } tail_call; + }; + bool tailcall_target_stable; + u8 adj_off; + u16 reason; + u32 insn_idx; +}; + +struct bpf_ctx_arg_aux { + u32 offset; + enum bpf_reg_type reg_type; + u32 btf_id; +}; + +struct btf_mod_pair { + struct btf *btf; + struct module *module; +}; + struct bpf_array_aux { - enum bpf_prog_type type; - bool jited; + struct { + spinlock_t lock; + enum bpf_prog_type type; + bool jited; + } owner; struct list_head poke_progs; struct bpf_map *map; struct mutex poke_mutex; @@ -26577,14 +26223,239 @@ enum bpf_text_poke_type { BPF_MOD_JUMP = 1, }; +struct tcmsg { + unsigned char tcm_family; + unsigned char tcm__pad1; + short unsigned int tcm__pad2; + int tcm_ifindex; + __u32 tcm_handle; + __u32 tcm_parent; + __u32 tcm_info; +}; + +struct gnet_dump { + spinlock_t *lock; + struct sk_buff *skb; + struct nlattr *tail; + int compat_tc_stats; + int compat_xstats; + int padattr; + void *xstats; + int xstats_len; + struct tc_stats tc_stats; +}; + +enum flow_action_hw_stats_bit { + FLOW_ACTION_HW_STATS_IMMEDIATE_BIT = 0, + FLOW_ACTION_HW_STATS_DELAYED_BIT = 1, + FLOW_ACTION_HW_STATS_DISABLED_BIT = 2, + FLOW_ACTION_HW_STATS_NUM_BITS = 3, +}; + +struct flow_block { + struct list_head cb_list; +}; + +typedef int flow_setup_cb_t(enum tc_setup_type, void *, void *); + +struct qdisc_size_table { + struct callback_head rcu; + struct list_head list; + struct tc_sizespec szopts; + int refcnt; + u16 data[0]; +}; + +struct Qdisc_class_ops; + +struct Qdisc_ops { + struct Qdisc_ops *next; + const struct Qdisc_class_ops *cl_ops; + char id[16]; + int priv_size; + unsigned int static_flags; + int (*enqueue)(struct sk_buff *, struct Qdisc *, struct sk_buff **); + struct sk_buff * (*dequeue)(struct Qdisc *); + struct sk_buff * (*peek)(struct Qdisc *); + int (*init)(struct Qdisc *, struct nlattr *, struct netlink_ext_ack *); + void (*reset)(struct Qdisc *); + void (*destroy)(struct Qdisc *); + int (*change)(struct Qdisc *, struct nlattr *, struct netlink_ext_ack *); + void (*attach)(struct Qdisc *); + int (*change_tx_queue_len)(struct Qdisc *, unsigned int); + void (*change_real_num_tx)(struct Qdisc *, unsigned int); + int (*dump)(struct Qdisc *, struct sk_buff *); + int (*dump_stats)(struct Qdisc *, struct gnet_dump *); + void (*ingress_block_set)(struct Qdisc *, u32); + void (*egress_block_set)(struct Qdisc *, u32); + u32 (*ingress_block_get)(struct Qdisc *); + u32 (*egress_block_get)(struct Qdisc *); + struct module *owner; +}; + +struct qdisc_walker; + +struct Qdisc_class_ops { + unsigned int flags; + struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); + int (*graft)(struct Qdisc *, long unsigned int, struct Qdisc *, struct Qdisc **, struct netlink_ext_ack *); + struct Qdisc * (*leaf)(struct Qdisc *, long unsigned int); + void (*qlen_notify)(struct Qdisc *, long unsigned int); + long unsigned int (*find)(struct Qdisc *, u32); + int (*change)(struct Qdisc *, u32, u32, struct nlattr **, long unsigned int *, struct netlink_ext_ack *); + int (*delete)(struct Qdisc *, long unsigned int, struct netlink_ext_ack *); + void (*walk)(struct Qdisc *, struct qdisc_walker *); + struct tcf_block * (*tcf_block)(struct Qdisc *, long unsigned int, struct netlink_ext_ack *); + long unsigned int (*bind_tcf)(struct Qdisc *, long unsigned int, u32); + void (*unbind_tcf)(struct Qdisc *, long unsigned int); + int (*dump)(struct Qdisc *, long unsigned int, struct sk_buff *, struct tcmsg *); + int (*dump_stats)(struct Qdisc *, long unsigned int, struct gnet_dump *); +}; + +struct tcf_chain; + +struct tcf_block { + struct mutex lock; + struct list_head chain_list; + u32 index; + u32 classid; + refcount_t refcnt; + struct net *net; + struct Qdisc *q; + struct rw_semaphore cb_lock; + struct flow_block flow_block; + struct list_head owner_list; + bool keep_dst; + atomic_t offloadcnt; + unsigned int nooffloaddevcnt; + unsigned int lockeddevcnt; + struct { + struct tcf_chain *chain; + struct list_head filter_chain_list; + } chain0; + struct callback_head rcu; + struct hlist_head proto_destroy_ht[128]; + struct mutex proto_destroy_lock; +}; + +struct tcf_result; + +struct tcf_proto_ops; + +struct tcf_proto { + struct tcf_proto *next; + void *root; + int (*classify)(struct sk_buff *, const struct tcf_proto *, struct tcf_result *); + __be16 protocol; + u32 prio; + void *data; + const struct tcf_proto_ops *ops; + struct tcf_chain *chain; + spinlock_t lock; + bool deleting; + refcount_t refcnt; + struct callback_head rcu; + struct hlist_node destroy_ht_node; +}; + +struct tcf_result { + union { + struct { + long unsigned int class; + u32 classid; + }; + const struct tcf_proto *goto_tp; + struct { + bool ingress; + struct gnet_stats_queue *qstats; + }; + }; +}; + +struct tcf_walker; + +struct tcf_proto_ops { + struct list_head head; + char kind[16]; + int (*classify)(struct sk_buff *, const struct tcf_proto *, struct tcf_result *); + int (*init)(struct tcf_proto *); + void (*destroy)(struct tcf_proto *, bool, struct netlink_ext_ack *); + void * (*get)(struct tcf_proto *, u32); + void (*put)(struct tcf_proto *, void *); + int (*change)(struct net *, struct sk_buff *, struct tcf_proto *, long unsigned int, u32, struct nlattr **, void **, u32, struct netlink_ext_ack *); + int (*delete)(struct tcf_proto *, void *, bool *, bool, struct netlink_ext_ack *); + bool (*delete_empty)(struct tcf_proto *); + void (*walk)(struct tcf_proto *, struct tcf_walker *, bool); + int (*reoffload)(struct tcf_proto *, bool, flow_setup_cb_t *, void *, struct netlink_ext_ack *); + void (*hw_add)(struct tcf_proto *, void *); + void (*hw_del)(struct tcf_proto *, void *); + void (*bind_class)(void *, u32, long unsigned int, void *, long unsigned int); + void * (*tmplt_create)(struct net *, struct tcf_chain *, struct nlattr **, struct netlink_ext_ack *); + void (*tmplt_destroy)(void *); + int (*dump)(struct net *, struct tcf_proto *, void *, struct sk_buff *, struct tcmsg *, bool); + int (*terse_dump)(struct net *, struct tcf_proto *, void *, struct sk_buff *, struct tcmsg *, bool); + int (*tmplt_dump)(struct sk_buff *, struct net *, void *); + struct module *owner; + int flags; +}; + +struct tcf_chain { + struct mutex filter_chain_lock; + struct tcf_proto *filter_chain; + struct list_head list; + struct tcf_block *block; + u32 index; + unsigned int refcnt; + unsigned int action_refcnt; + bool explicitly_created; + bool flushing; + const struct tcf_proto_ops *tmplt_ops; + void *tmplt_priv; + struct callback_head rcu; +}; + +struct sock_fprog_kern { + u16 len; + struct sock_filter *filter; +}; + struct bpf_binary_header { u32 pages; int: 32; u8 image[0]; }; +struct bpf_prog_stats { + u64_stats_t cnt; + u64_stats_t nsecs; + u64_stats_t misses; + struct u64_stats_sync syncp; + long: 64; +}; + +struct bpf_nh_params { + u32 nh_family; + union { + u32 ipv4_nh; + struct in6_addr ipv6_nh; + }; +}; + +struct bpf_redirect_info { + u32 flags; + u32 tgt_index; + void *tgt_value; + struct bpf_map *map; + u32 map_id; + enum bpf_map_type map_type; + u32 kern_flags; + struct bpf_nh_params nh; +}; + struct jit_context { int cleanup_addr; + int tail_call_direct_label; + int tail_call_indirect_label; }; struct x64_jit_data { @@ -26718,6 +26589,18 @@ struct proc_fs_info { enum proc_pidonly pidonly; }; +enum bpf_type_flag { + PTR_MAYBE_NULL = 256, + MEM_RDONLY = 512, + __BPF_TYPE_LAST_FLAG = 512, +}; + +enum bpf_cgroup_storage_type { + BPF_CGROUP_STORAGE_SHARED = 0, + BPF_CGROUP_STORAGE_PERCPU = 1, + __BPF_CGROUP_STORAGE_MAX = 2, +}; + struct trace_event_raw_task_newtask { struct trace_entry ent; pid_t pid; @@ -27010,6 +26893,53 @@ struct region_devres { resource_size_t n; }; +enum { + IPPROTO_IP = 0, + IPPROTO_ICMP = 1, + IPPROTO_IGMP = 2, + IPPROTO_IPIP = 4, + IPPROTO_TCP = 6, + IPPROTO_EGP = 8, + IPPROTO_PUP = 12, + IPPROTO_UDP = 17, + IPPROTO_IDP = 22, + IPPROTO_TP = 29, + IPPROTO_DCCP = 33, + IPPROTO_IPV6 = 41, + IPPROTO_RSVP = 46, + IPPROTO_GRE = 47, + IPPROTO_ESP = 50, + IPPROTO_AH = 51, + IPPROTO_MTP = 92, + IPPROTO_BEETPH = 94, + IPPROTO_ENCAP = 98, + IPPROTO_PIM = 103, + IPPROTO_COMP = 108, + IPPROTO_SCTP = 132, + IPPROTO_UDPLITE = 136, + IPPROTO_MPLS = 137, + IPPROTO_ETHERNET = 143, + IPPROTO_RAW = 255, + IPPROTO_MPTCP = 262, + IPPROTO_MAX = 263, +}; + +struct sk_filter { + refcount_t refcnt; + struct callback_head rcu; + struct bpf_prog *prog; +}; + +enum rpc_display_format_t { + RPC_DISPLAY_ADDR = 0, + RPC_DISPLAY_PORT = 1, + RPC_DISPLAY_PROTO = 2, + RPC_DISPLAY_HEX_ADDR = 3, + RPC_DISPLAY_HEX_PORT = 4, + RPC_DISPLAY_NETID = 5, + RPC_DISPLAY_MAX = 6, +}; + enum sysctl_writes_mode { SYSCTL_WRITES_LEGACY = 4294967295, SYSCTL_WRITES_WARN = 0, @@ -27098,7 +27028,7 @@ enum siginfo_layout { SIL_FAULT_MCEERR = 5, SIL_FAULT_BNDERR = 6, SIL_FAULT_PKUERR = 7, - SIL_PERF_EVENT = 8, + SIL_FAULT_PERF_EVENT = 8, SIL_CHLD = 9, SIL_RT = 10, SIL_SYS = 11, @@ -27147,6 +27077,12 @@ typedef void (*btf_trace_signal_generate)(void *, int, struct kernel_siginfo *, typedef void (*btf_trace_signal_deliver)(void *, int, struct kernel_siginfo *, struct k_sigaction *); +enum sig_handler { + HANDLER_CURRENT = 0, + HANDLER_SIG_DFL = 1, + HANDLER_EXIT = 2, +}; + struct sysinfo { __kernel_long_t uptime; __kernel_ulong_t loads[3]; @@ -27345,6 +27281,7 @@ struct worker { struct work_struct *current_work; work_func_t current_func; struct pool_workqueue *current_pwq; + unsigned int current_color; struct list_head scheduled; struct task_struct *task; struct worker_pool *pool; @@ -27364,17 +27301,16 @@ struct pool_workqueue { int work_color; int flush_color; int refcnt; - int nr_in_flight[15]; + int nr_in_flight[16]; int nr_active; int max_active; - struct list_head delayed_works; + struct list_head inactive_works; struct list_head pwqs_node; struct list_head mayday_node; struct work_struct unbound_release_work; struct callback_head rcu; long: 64; long: 64; - long: 64; }; struct worker_pool { @@ -27737,6 +27673,8 @@ enum { CSD_FLAG_TYPE_MASK = 240, }; +typedef int (*task_call_f)(struct task_struct *, void *); + struct dl_bw { raw_spinlock_t lock; u64 bw; @@ -27793,6 +27731,7 @@ struct cfs_rq { struct load_weight load; unsigned int nr_running; unsigned int h_nr_running; + unsigned int idle_nr_running; unsigned int idle_h_nr_running; u64 exec_clock; u64 min_vruntime; @@ -27828,6 +27767,7 @@ struct cfs_rq { int on_list; struct list_head leaf_cfs_rq_list; struct task_group *tg; + int idle; int runtime_enabled; s64 runtime_remaining; u64 throttled_clock; @@ -27849,6 +27789,7 @@ struct cfs_bandwidth { u64 quota; u64 runtime; u64 burst; + u64 runtime_snap; s64 hierarchical_quota; u8 idle; u8 period_active; @@ -27858,7 +27799,9 @@ struct cfs_bandwidth { struct list_head throttled_cfs_rq; int nr_periods; int nr_throttled; + int nr_burst; u64 throttled_time; + u64 burst_time; }; struct task_group { @@ -27866,7 +27809,8 @@ struct task_group { struct sched_entity **se; struct cfs_rq **cfs_rq; long unsigned int shares; - long: 64; + int idle; + long: 32; long: 64; atomic_long_t load_avg; struct callback_head rcu; @@ -27876,9 +27820,6 @@ struct task_group { struct list_head children; struct cfs_bandwidth cfs_bandwidth; long: 64; - long: 64; - long: 64; - long: 64; }; struct update_util_data { @@ -27891,6 +27832,7 @@ struct sched_group { unsigned int group_weight; struct sched_group_capacity *sgc; int asym_prefer_cpu; + int flags; long unsigned int cpumask[0]; }; @@ -27909,12 +27851,13 @@ struct em_perf_state { long unsigned int frequency; long unsigned int power; long unsigned int cost; + long unsigned int flags; }; struct em_perf_domain { struct em_perf_state *table; int nr_perf_states; - int milliwatts; + long unsigned int flags; long unsigned int cpus[0]; }; @@ -27925,17 +27868,6 @@ enum ctx_state { CONTEXT_GUEST = 2, }; -enum { - MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = 1, - MEMBARRIER_STATE_PRIVATE_EXPEDITED = 2, - MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY = 4, - MEMBARRIER_STATE_GLOBAL_EXPEDITED = 8, - MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY = 16, - MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE = 32, - MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY = 64, - MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ = 128, -}; - enum { CFTYPE_ONLY_ON_ROOT = 1, CFTYPE_NOT_ON_ROOT = 2, @@ -28369,6 +28301,17 @@ typedef void (*btf_trace_sched_util_est_se_tp)(void *, struct sched_entity *); typedef void (*btf_trace_sched_update_nr_running_tp)(void *, struct rq *, int); +enum { + MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = 1, + MEMBARRIER_STATE_PRIVATE_EXPEDITED = 2, + MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY = 4, + MEMBARRIER_STATE_GLOBAL_EXPEDITED = 8, + MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY = 16, + MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE = 32, + MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY = 64, + MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ = 128, +}; + struct wake_q_head { struct wake_q_node *first; struct wake_q_node **lastp; @@ -28427,6 +28370,11 @@ enum numa_topology_type { NUMA_BACKPLANE = 2, }; +struct sched_entity_stats { + struct sched_entity se; + struct sched_statistics stats; +}; + enum { __SCHED_FEAT_GENTLE_FAIR_SLEEPERS = 0, __SCHED_FEAT_START_DEBIT = 1, @@ -28485,6 +28433,7 @@ struct migration_swap_arg { }; enum { + preempt_dynamic_undefined = 4294967295, preempt_dynamic_none = 0, preempt_dynamic_voluntary = 1, preempt_dynamic_full = 2, @@ -28533,7 +28482,6 @@ struct numa_group { struct callback_head rcu; long unsigned int total_faults; long unsigned int max_faults_cpu; - long unsigned int *faults_cpu; long unsigned int faults[0]; }; @@ -28773,6 +28721,44 @@ enum membarrier_cmd_flag { MEMBARRIER_CMD_FLAG_CPU = 1, }; +struct ww_class { + atomic_long_t stamp; + struct lock_class_key acquire_key; + struct lock_class_key mutex_key; + const char *acquire_name; + const char *mutex_name; + unsigned int is_wait_die; +}; + +struct ww_acquire_ctx; + +struct ww_mutex { + struct mutex base; + struct ww_acquire_ctx *ctx; + struct ww_class *ww_class; +}; + +struct ww_acquire_ctx { + struct task_struct *task; + long unsigned int stamp; + unsigned int acquired; + short unsigned int wounded; + short unsigned int is_wait_die; + unsigned int done_acquire; + struct ww_class *ww_class; + void *contending_lock; + struct lockdep_map dep_map; + unsigned int deadlock_inject_interval; + unsigned int deadlock_inject_countdown; +}; + +struct mutex_waiter { + struct list_head list; + struct task_struct *task; + struct ww_acquire_ctx *ww_ctx; + void *magic; +}; + struct semaphore { raw_spinlock_t lock; unsigned int count; @@ -28795,6 +28781,7 @@ struct rwsem_waiter { struct task_struct *task; enum rwsem_waiter_type type; long unsigned int timeout; + bool handoff_set; }; enum rwsem_wake_type { @@ -28803,12 +28790,6 @@ enum rwsem_wake_type { RWSEM_WAKE_READ_OWNED = 2, }; -enum writer_wait_state { - WRITER_NOT_FIRST = 0, - WRITER_FIRST = 1, - WRITER_HANDOFF = 2, -}; - enum owner_state { OWNER_NULL = 1, OWNER_WRITER = 2, @@ -28931,27 +28912,36 @@ struct qnode { struct mcs_spinlock mcs; }; +struct rt_mutex_base { + raw_spinlock_t wait_lock; + struct rb_root_cached waiters; + struct task_struct *owner; +}; + +struct rt_mutex { + struct rt_mutex_base rtmutex; + struct lockdep_map dep_map; +}; + struct hrtimer_sleeper { struct hrtimer timer; struct task_struct *task; }; -struct rt_mutex; - struct rt_mutex_waiter { struct rb_node tree_entry; struct rb_node pi_tree_entry; struct task_struct *task; - struct rt_mutex *lock; + struct rt_mutex_base *lock; + unsigned int wake_state; int prio; u64 deadline; + struct ww_acquire_ctx *ww_ctx; }; -struct rt_mutex { - raw_spinlock_t wait_lock; - struct rb_root_cached waiters; - struct task_struct *owner; - struct lockdep_map dep_map; +struct rt_wake_q_head { + struct wake_q_head head; + struct task_struct *rtlock_task; }; enum rtmutex_chainwalk { @@ -29081,6 +29071,11 @@ struct console_cmdline { char *options; }; +enum printk_info_flags { + LOG_NEWLINE = 2, + LOG_CONT = 8, +}; + enum devkmsg_log_bits { __DEVKMSG_LOG_BIT_ON = 0, __DEVKMSG_LOG_BIT_OFF = 1, @@ -29098,11 +29093,6 @@ enum con_msg_format_flags { MSG_FORMAT_SYSLOG = 1, }; -enum log_flags { - LOG_NEWLINE = 2, - LOG_CONT = 8, -}; - struct latched_seq { seqcount_latch_t latch; u64 val[2]; @@ -29118,13 +29108,6 @@ struct devkmsg_user { struct printk_record record; }; -struct printk_safe_seq_buf { - atomic_t len; - atomic_t message_lost; - struct irq_work work; - unsigned char buffer[8160]; -}; - struct prb_data_block { long unsigned int id; char data[0]; @@ -29920,6 +29903,7 @@ struct timer_base { unsigned int cpu; bool next_expiry_recalc; bool is_idle; + bool timers_pending; long unsigned int pending_map[9]; struct hlist_head vectors[576]; long: 64; @@ -30166,6 +30150,17 @@ struct rtc_wkalrm { struct rtc_time time; }; +struct rtc_param { + __u64 param; + union { + __u64 uvalue; + __s64 svalue; + __u64 ptr; + }; + __u32 index; + __u32 __pad; +}; + struct rtc_class_ops { int (*ioctl)(struct device *, unsigned int, long unsigned int); int (*read_time)(struct device *, struct rtc_time *); @@ -30176,6 +30171,8 @@ struct rtc_class_ops { int (*alarm_irq_enable)(struct device *, unsigned int); int (*read_offset)(struct device *, long int *); int (*set_offset)(struct device *, long int); + int (*param_get)(struct device *, struct rtc_param *); + int (*param_set)(struct device *, struct rtc_param *); }; struct rtc_device; @@ -30281,8 +30278,6 @@ struct sigevent { typedef struct sigevent sigevent_t; -typedef unsigned int uint; - struct posix_clock; struct posix_clock_operations { @@ -30349,23 +30344,12 @@ union futex_key { struct futex_pi_state { struct list_head list; - struct rt_mutex pi_mutex; + struct rt_mutex_base pi_mutex; struct task_struct *owner; refcount_t refcount; union futex_key key; }; -struct futex_q { - struct plist_node list; - struct task_struct *task; - spinlock_t *lock_ptr; - union futex_key key; - struct futex_pi_state *pi_state; - struct rt_mutex_waiter *rt_waiter; - union futex_key *requeue_pi_key; - u32 bitset; -}; - struct futex_hash_bucket { atomic_t waiters; spinlock_t lock; @@ -30377,11 +30361,44 @@ struct futex_hash_bucket { long: 64; }; +struct futex_q { + struct plist_node list; + struct task_struct *task; + spinlock_t *lock_ptr; + union futex_key key; + struct futex_pi_state *pi_state; + struct rt_mutex_waiter *rt_waiter; + union futex_key *requeue_pi_key; + u32 bitset; + atomic_t requeue_state; +}; + enum futex_access { FUTEX_READ = 0, FUTEX_WRITE = 1, }; +struct futex_waitv { + __u64 val; + __u64 uaddr; + __u32 flags; + __u32 __reserved; +}; + +struct futex_vector { + struct futex_waitv w; + struct futex_q q; +}; + +enum { + Q_REQUEUE_PI_NONE = 0, + Q_REQUEUE_PI_IGNORE = 1, + Q_REQUEUE_PI_IN_PROGRESS = 2, + Q_REQUEUE_PI_WAIT = 3, + Q_REQUEUE_PI_DONE = 4, + Q_REQUEUE_PI_LOCKED = 5, +}; + struct dma_chan { int lock; const char *device_id; @@ -30745,80 +30762,6 @@ struct kernfs_fs_context { bool new_sb_created; }; -enum bpf_link_type { - BPF_LINK_TYPE_UNSPEC = 0, - BPF_LINK_TYPE_RAW_TRACEPOINT = 1, - BPF_LINK_TYPE_TRACING = 2, - BPF_LINK_TYPE_CGROUP = 3, - BPF_LINK_TYPE_ITER = 4, - BPF_LINK_TYPE_NETNS = 5, - BPF_LINK_TYPE_XDP = 6, - MAX_BPF_LINK_TYPE = 7, -}; - -struct bpf_link_info { - __u32 type; - __u32 id; - __u32 prog_id; - union { - struct { - __u64 tp_name; - __u32 tp_name_len; - } raw_tracepoint; - struct { - __u32 attach_type; - __u32 target_obj_id; - __u32 target_btf_id; - } tracing; - struct { - __u64 cgroup_id; - __u32 attach_type; - } cgroup; - struct { - __u64 target_name; - __u32 target_name_len; - union { - struct { - __u32 map_id; - } map; - }; - } iter; - struct { - __u32 netns_ino; - __u32 attach_type; - } netns; - struct { - __u32 ifindex; - } xdp; - }; -}; - -struct bpf_link_ops; - -struct bpf_link { - atomic64_t refcnt; - u32 id; - enum bpf_link_type type; - const struct bpf_link_ops *ops; - struct bpf_prog *prog; - struct work_struct work; -}; - -struct bpf_link_ops { - void (*release)(struct bpf_link *); - void (*dealloc)(struct bpf_link *); - int (*detach)(struct bpf_link *); - int (*update_prog)(struct bpf_link *, struct bpf_prog *, struct bpf_prog *); - void (*show_fdinfo)(const struct bpf_link *, struct seq_file *); - int (*fill_link_info)(const struct bpf_link *, struct bpf_link_info *); -}; - -struct bpf_cgroup_link { - struct bpf_link link; - struct cgroup *cgroup; - enum bpf_attach_type type; -}; - enum { CGRP_NOTIFY_ON_RELEASE = 0, CGRP_CPUSET_CLONE_CHILDREN = 1, @@ -30874,6 +30817,22 @@ struct cgroup_fs_context { char *release_agent; }; +struct cgroup_pidlist; + +struct cgroup_file_ctx { + struct cgroup_namespace *ns; + struct { + void *trigger; + } psi; + struct { + bool started; + struct css_task_iter iter; + } procs; + struct { + struct cgroup_pidlist *pidlist; + } procs1; +}; + struct cgrp_cset_link { struct cgroup *cgrp; struct css_set *cset; @@ -30888,6 +30847,33 @@ struct cgroup_mgctx { u16 ss_mask; }; +struct bpf_cgroup_storage_key { + __u64 cgroup_inode_id; + __u32 attach_type; +}; + +struct bpf_storage_buffer; + +struct bpf_cgroup_storage_map; + +struct bpf_cgroup_storage { + union { + struct bpf_storage_buffer *buf; + void *percpu_buf; + }; + struct bpf_cgroup_storage_map *map; + struct bpf_cgroup_storage_key key; + struct list_head list_map; + struct list_head list_cg; + struct rb_node node; + struct callback_head rcu; +}; + +struct bpf_storage_buffer { + struct callback_head rcu; + char data[0]; +}; + struct trace_event_raw_cgroup_root { struct trace_entry ent; int root; @@ -31053,6 +31039,7 @@ struct cpuset { int partition_root_state; int use_parent_ecpus; int child_ecpus_count; + struct cgroup_file partition_file; }; struct tmpmasks { @@ -31236,6 +31223,12 @@ struct mq_attr { __kernel_long_t __reserved[4]; }; +struct open_how { + __u64 flags; + __u64 mode; + __u64 resolve; +}; + struct audit_proctitle { int len; char *value; @@ -31247,11 +31240,16 @@ struct audit_tree_refs; struct audit_context { int dummy; - int in_syscall; + enum { + AUDIT_CTX_UNUSED = 0, + AUDIT_CTX_SYSCALL = 1, + AUDIT_CTX_URING = 2, + } context; enum audit_state state; enum audit_state current_state; unsigned int serial; int major; + int uring_op; struct timespec64 ctime; long unsigned int argv[4]; long int return_code; @@ -31332,6 +31330,7 @@ struct audit_context { int fd; int flags; } mmap; + struct open_how openat2; struct { int argc; } execve; @@ -31384,6 +31383,16 @@ struct audit_sig_info { char ctx[0]; }; +enum skb_drop_reason { + SKB_DROP_REASON_NOT_SPECIFIED = 0, + SKB_DROP_REASON_NO_SOCKET = 1, + SKB_DROP_REASON_PKT_TOO_SMALL = 2, + SKB_DROP_REASON_TCP_CSUM = 3, + SKB_DROP_REASON_TCP_FILTER = 4, + SKB_DROP_REASON_UDP_CSUM = 5, + SKB_DROP_REASON_MAX = 6, +}; + struct net_generic { union { struct { @@ -31670,7 +31679,7 @@ struct fsnotify_ops { int (*handle_inode_event)(struct fsnotify_mark *, u32, struct inode *, struct inode *, const struct qstr *, u32); void (*free_group_priv)(struct fsnotify_group *); void (*freeing_mark)(struct fsnotify_mark *, struct fsnotify_group *); - void (*free_event)(struct fsnotify_event *); + void (*free_event)(struct fsnotify_group *, struct fsnotify_event *); void (*free_mark)(struct fsnotify_mark *); }; @@ -31751,7 +31760,7 @@ struct audit_tree { char pathname[0]; }; -struct node___2 { +struct audit_node { struct list_head list; struct audit_tree *owner; unsigned int index; @@ -31765,7 +31774,7 @@ struct audit_chunk___2 { int count; atomic_long_t refs; struct callback_head head; - struct node___2 owners[0]; + struct audit_node owners[0]; }; struct audit_tree_mark { @@ -31864,7 +31873,7 @@ struct seccomp_notif_addfd { }; struct action_cache { - long unsigned int allow_native[7]; + long unsigned int allow_native[8]; }; struct notification; @@ -31890,6 +31899,8 @@ struct sock_fprog { struct sock_filter *filter; }; +typedef unsigned int (*bpf_dispatcher_fn)(const void *, const struct bpf_insn *, unsigned int (*)(const void *, const struct bpf_insn *)); + enum notify_state { SECCOMP_NOTIFY_INIT = 0, SECCOMP_NOTIFY_SENT = 1, @@ -31958,7 +31969,6 @@ struct rchan_buf { long: 64; long: 64; long: 64; - long: 64; }; struct rchan_callbacks; @@ -32167,6 +32177,25 @@ struct tp_module { struct module *mod; }; +enum tp_func_state { + TP_FUNC_0 = 0, + TP_FUNC_1 = 1, + TP_FUNC_2 = 2, + TP_FUNC_N = 3, +}; + +enum tp_transition_sync { + TP_TRANSITION_SYNC_1_0_1 = 0, + TP_TRANSITION_SYNC_N_2_1 = 1, + _NR_TP_TRANSITION_SYNC = 2, +}; + +struct tp_transition_snapshot { + long unsigned int rcu; + long unsigned int srcu; + bool ongoing; +}; + struct tp_probes { struct callback_head rcu; struct tracepoint_func probes[0]; @@ -32238,8 +32267,8 @@ struct trace_array { int buffer_disabled; int sys_refcount_enter; int sys_refcount_exit; - struct trace_event_file *enter_syscall_files[448]; - struct trace_event_file *exit_syscall_files[448]; + struct trace_event_file *enter_syscall_files[450]; + struct trace_event_file *exit_syscall_files[450]; int stop_count; int clock_id; int nr_topts; @@ -32313,6 +32342,26 @@ struct trace_subsystem_dir { int nr_events; }; +union lower_chunk { + union lower_chunk *next; + long unsigned int data[256]; +}; + +union upper_chunk { + union upper_chunk *next; + union lower_chunk *data[256]; +}; + +struct trace_pid_list { + raw_spinlock_t lock; + struct irq_work refill_irqwork; + union upper_chunk *upper[256]; + union upper_chunk *upper_list; + union lower_chunk *lower_list; + int free_upper_chunks; + int free_lower_chunks; +}; + struct trace_array_cpu { atomic_t disabled; void *buffer_page; @@ -32349,11 +32398,6 @@ struct trace_option_dentry { struct dentry *entry; }; -struct trace_pid_list { - int pid_max; - long unsigned int *pids; -}; - enum { TRACE_PIDS = 1, TRACE_NO_PIDS = 2, @@ -32740,6 +32784,7 @@ enum event_trigger_type { ETT_EVENT_ENABLE = 8, ETT_EVENT_HIST = 16, ETT_HIST_ENABLE = 32, + ETT_EVENT_EPROBE = 64, }; enum trace_type { @@ -33093,12 +33138,145 @@ enum req_flag_bits { __REQ_NOWAIT = 21, __REQ_CGROUP_PUNT = 22, __REQ_NOUNMAP = 23, - __REQ_HIPRI = 24, + __REQ_POLLED = 24, __REQ_DRV = 25, __REQ_SWAP = 26, __REQ_NR_BITS = 27, }; +struct sbitmap_word { + long unsigned int depth; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long unsigned int word; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long unsigned int cleared; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; +}; + +struct sbitmap { + unsigned int depth; + unsigned int shift; + unsigned int map_nr; + bool round_robin; + struct sbitmap_word *map; + unsigned int *alloc_hint; +}; + +struct sbq_wait_state { + atomic_t wait_cnt; + wait_queue_head_t wait; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; +}; + +struct sbitmap_queue { + struct sbitmap sb; + unsigned int wake_batch; + atomic_t wake_index; + struct sbq_wait_state *ws; + atomic_t ws_active; + unsigned int min_shallow_depth; +}; + +typedef __u32 req_flags_t; + +enum mq_rq_state { + MQ_RQ_IDLE = 0, + MQ_RQ_IN_FLIGHT = 1, + MQ_RQ_COMPLETE = 2, +}; + +typedef void rq_end_io_fn(struct request *, blk_status_t); + +struct request { + struct request_queue *q; + struct blk_mq_ctx *mq_ctx; + struct blk_mq_hw_ctx *mq_hctx; + unsigned int cmd_flags; + req_flags_t rq_flags; + int tag; + int internal_tag; + unsigned int timeout; + unsigned int __data_len; + sector_t __sector; + struct bio *bio; + struct bio *biotail; + union { + struct list_head queuelist; + struct request *rq_next; + }; + struct gendisk *rq_disk; + struct block_device *part; + u64 start_time_ns; + u64 io_start_time_ns; + short unsigned int stats_sectors; + short unsigned int nr_phys_segments; + short unsigned int write_hint; + short unsigned int ioprio; + enum mq_rq_state state; + refcount_t ref; + long unsigned int deadline; + union { + struct hlist_node hash; + struct llist_node ipi_list; + }; + union { + struct rb_node rb_node; + struct bio_vec special_vec; + void *completion_data; + int error_count; + }; + union { + struct { + struct io_cq *icq; + void *priv[2]; + } elv; + struct { + unsigned int seq; + struct list_head list; + rq_end_io_fn *saved_end_io; + } flush; + }; + union { + struct __call_single_data csd; + u64 fifo_time; + }; + rq_end_io_fn *end_io; + void *end_io_data; +}; + +struct elevator_type; + +struct elevator_queue { + struct elevator_type *type; + void *elevator_data; + struct kobject kobj; + struct mutex sysfs_lock; + unsigned int registered: 1; + struct hlist_head hash[64]; +}; + struct blk_mq_ctxs; struct blk_mq_ctx { @@ -33111,20 +33289,12 @@ struct blk_mq_ctx { unsigned int cpu; short unsigned int index_hw[3]; struct blk_mq_hw_ctx *hctxs[3]; - long unsigned int rq_dispatched[2]; - long unsigned int rq_merged; - long unsigned int rq_completed[2]; struct request_queue *queue; struct blk_mq_ctxs *ctxs; struct kobject kobj; long: 64; - long: 64; - long: 64; - long: 64; }; -struct blk_mq_tags; - struct blk_mq_hw_ctx { struct { spinlock_t lock; @@ -33158,35 +33328,16 @@ struct blk_mq_hw_ctx { struct blk_mq_tags *sched_tags; long unsigned int queued; long unsigned int run; - long unsigned int dispatched[7]; unsigned int numa_node; unsigned int queue_num; atomic_t nr_active; struct hlist_node cpuhp_online; struct hlist_node cpuhp_dead; struct kobject kobj; - long unsigned int poll_considered; - long unsigned int poll_invoked; - long unsigned int poll_success; struct dentry *debugfs_dir; struct dentry *sched_debugfs_dir; struct list_head hctx_list; struct srcu_struct srcu[0]; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; -}; - -struct blk_mq_alloc_data { - struct request_queue *q; - blk_mq_req_flags_t flags; - unsigned int shallow_depth; - unsigned int cmd_flags; - struct blk_mq_ctx *ctx; - struct blk_mq_hw_ctx *hctx; }; struct blk_stat_callback { @@ -33201,6 +33352,18 @@ struct blk_stat_callback { struct callback_head rcu; }; +struct blk_mq_tags { + unsigned int nr_tags; + unsigned int nr_reserved_tags; + atomic_t active_queues; + struct sbitmap_queue bitmap_tags; + struct sbitmap_queue breserved_tags; + struct request **rqs; + struct request **static_rqs; + struct list_head page_list; + spinlock_t lock; +}; + struct blk_trace { int trace_state; struct rchan *rchan; @@ -33245,14 +33408,17 @@ struct blk_mq_tag_set { unsigned int timeout; unsigned int flags; void *driver_data; - atomic_t active_queues_shared_sbitmap; - struct sbitmap_queue __bitmap_tags; - struct sbitmap_queue __breserved_tags; struct blk_mq_tags **tags; + struct blk_mq_tags *shared_tags; struct mutex tag_list_lock; struct list_head tag_list; }; +struct blk_mq_queue_data { + struct request *rq; + bool last; +}; + enum blktrace_cat { BLK_TC_READ = 1, BLK_TC_WRITE = 2, @@ -33338,25 +33504,6 @@ struct blk_user_trace_setup { __u32 pid; }; -struct blk_mq_tags { - unsigned int nr_tags; - unsigned int nr_reserved_tags; - atomic_t active_queues; - struct sbitmap_queue *bitmap_tags; - struct sbitmap_queue *breserved_tags; - struct sbitmap_queue __bitmap_tags; - struct sbitmap_queue __breserved_tags; - struct request **rqs; - struct request **static_rqs; - struct list_head page_list; - spinlock_t lock; -}; - -struct blk_mq_queue_data { - struct request *rq; - bool last; -}; - struct blk_crypto_mode { const char *cipher_str; unsigned int keysize; @@ -33368,6 +33515,84 @@ struct blk_mq_ctxs { struct blk_mq_ctx *queue_ctx; }; +struct blk_mq_alloc_data { + struct request_queue *q; + blk_mq_req_flags_t flags; + unsigned int shallow_depth; + unsigned int cmd_flags; + req_flags_t rq_flags; + unsigned int nr_tags; + struct request **cached_rq; + struct blk_mq_ctx *ctx; + struct blk_mq_hw_ctx *hctx; +}; + +enum elv_merge { + ELEVATOR_NO_MERGE = 0, + ELEVATOR_FRONT_MERGE = 1, + ELEVATOR_BACK_MERGE = 2, + ELEVATOR_DISCARD_MERGE = 3, +}; + +struct elevator_mq_ops { + int (*init_sched)(struct request_queue *, struct elevator_type *); + void (*exit_sched)(struct elevator_queue *); + int (*init_hctx)(struct blk_mq_hw_ctx *, unsigned int); + void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); + void (*depth_updated)(struct blk_mq_hw_ctx *); + bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); + bool (*bio_merge)(struct request_queue *, struct bio *, unsigned int); + int (*request_merge)(struct request_queue *, struct request **, struct bio *); + void (*request_merged)(struct request_queue *, struct request *, enum elv_merge); + void (*requests_merged)(struct request_queue *, struct request *, struct request *); + void (*limit_depth)(unsigned int, struct blk_mq_alloc_data *); + void (*prepare_request)(struct request *); + void (*finish_request)(struct request *); + void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); + struct request * (*dispatch_request)(struct blk_mq_hw_ctx *); + bool (*has_work)(struct blk_mq_hw_ctx *); + void (*completed_request)(struct request *, u64); + void (*requeue_request)(struct request *); + struct request * (*former_request)(struct request_queue *, struct request *); + struct request * (*next_request)(struct request_queue *, struct request *); + void (*init_icq)(struct io_cq *); + void (*exit_icq)(struct io_cq *); +}; + +struct elv_fs_entry; + +struct blk_mq_debugfs_attr; + +struct elevator_type { + struct kmem_cache *icq_cache; + struct elevator_mq_ops ops; + size_t icq_size; + size_t icq_align; + struct elv_fs_entry *elevator_attrs; + const char *elevator_name; + const char *elevator_alias; + const unsigned int elevator_features; + struct module *elevator_owner; + const struct blk_mq_debugfs_attr *queue_debugfs_attrs; + const struct blk_mq_debugfs_attr *hctx_debugfs_attrs; + char icq_cache_name[22]; + struct list_head list; +}; + +struct elv_fs_entry { + struct attribute attr; + ssize_t (*show)(struct elevator_queue *, char *); + ssize_t (*store)(struct elevator_queue *, const char *, size_t); +}; + +struct blk_mq_debugfs_attr { + const char *name; + umode_t mode; + int (*show)(void *, struct seq_file *); + ssize_t (*write)(void *, const char *, size_t, loff_t *); + const struct seq_operations *seq_ops; +}; + typedef void blk_log_action_t(struct trace_iterator *, const char *, bool); struct ftrace_event_field { @@ -33417,7 +33642,7 @@ struct syscall_tp_t___2 { long unsigned int args[6]; }; -typedef long unsigned int perf_trace_t[256]; +typedef long unsigned int perf_trace_t[1024]; struct filter_pred; @@ -33514,6 +33739,10 @@ struct function_filter_data { int first_notrace; }; +enum { + EVENT_TRIGGER_FL_PROBE = 1, +}; + struct event_trigger_ops; struct event_command; @@ -33521,6 +33750,7 @@ struct event_command; struct event_trigger_data { long unsigned int count; int ref; + int flags; struct event_trigger_ops *ops; struct event_command *cmd_ops; struct event_filter *filter; @@ -33565,6 +33795,191 @@ enum event_command_flags { EVENT_CMD_FL_NEEDS_REC = 2, }; +struct eprobe_trace_entry_head { + struct trace_entry ent; + unsigned int type; +}; + +struct dyn_event; + +struct dyn_event_operations { + struct list_head list; + int (*create)(const char *); + int (*show)(struct seq_file *, struct dyn_event *); + bool (*is_busy)(struct dyn_event *); + int (*free)(struct dyn_event *); + bool (*match)(const char *, const char *, int, const char **, struct dyn_event *); +}; + +struct dyn_event { + struct list_head list; + struct dyn_event_operations *ops; +}; + +typedef int (*print_type_func_t)(struct trace_seq *, void *, void *); + +enum fetch_op { + FETCH_OP_NOP = 0, + FETCH_OP_REG = 1, + FETCH_OP_STACK = 2, + FETCH_OP_STACKP = 3, + FETCH_OP_RETVAL = 4, + FETCH_OP_IMM = 5, + FETCH_OP_COMM = 6, + FETCH_OP_ARG = 7, + FETCH_OP_FOFFS = 8, + FETCH_OP_DATA = 9, + FETCH_OP_DEREF = 10, + FETCH_OP_UDEREF = 11, + FETCH_OP_ST_RAW = 12, + FETCH_OP_ST_MEM = 13, + FETCH_OP_ST_UMEM = 14, + FETCH_OP_ST_STRING = 15, + FETCH_OP_ST_USTRING = 16, + FETCH_OP_MOD_BF = 17, + FETCH_OP_LP_ARRAY = 18, + FETCH_OP_TP_ARG = 19, + FETCH_OP_END = 20, + FETCH_NOP_SYMBOL = 21, +}; + +struct fetch_insn { + enum fetch_op op; + union { + unsigned int param; + struct { + unsigned int size; + int offset; + }; + struct { + unsigned char basesize; + unsigned char lshift; + unsigned char rshift; + }; + long unsigned int immediate; + void *data; + }; +}; + +struct fetch_type { + const char *name; + size_t size; + int is_signed; + print_type_func_t print; + const char *fmt; + const char *fmttype; +}; + +struct probe_arg { + struct fetch_insn *code; + bool dynamic; + unsigned int offset; + unsigned int count; + const char *name; + const char *comm; + char *fmt; + const struct fetch_type *type; +}; + +struct trace_uprobe_filter { + rwlock_t rwlock; + int nr_systemwide; + struct list_head perf_events; +}; + +struct trace_probe_event { + unsigned int flags; + struct trace_event_class class; + struct trace_event_call call; + struct list_head files; + struct list_head probes; + struct trace_uprobe_filter filter[0]; +}; + +struct trace_probe { + struct list_head list; + struct trace_probe_event *event; + ssize_t size; + unsigned int nr_args; + struct probe_arg args[0]; +}; + +enum probe_print_type { + PROBE_PRINT_NORMAL = 0, + PROBE_PRINT_RETURN = 1, + PROBE_PRINT_EVENT = 2, +}; + +enum { + TP_ERR_FILE_NOT_FOUND = 0, + TP_ERR_NO_REGULAR_FILE = 1, + TP_ERR_BAD_REFCNT = 2, + TP_ERR_REFCNT_OPEN_BRACE = 3, + TP_ERR_BAD_REFCNT_SUFFIX = 4, + TP_ERR_BAD_UPROBE_OFFS = 5, + TP_ERR_MAXACT_NO_KPROBE = 6, + TP_ERR_BAD_MAXACT = 7, + TP_ERR_MAXACT_TOO_BIG = 8, + TP_ERR_BAD_PROBE_ADDR = 9, + TP_ERR_BAD_RETPROBE = 10, + TP_ERR_BAD_ADDR_SUFFIX = 11, + TP_ERR_NO_GROUP_NAME = 12, + TP_ERR_GROUP_TOO_LONG = 13, + TP_ERR_BAD_GROUP_NAME = 14, + TP_ERR_NO_EVENT_NAME = 15, + TP_ERR_EVENT_TOO_LONG = 16, + TP_ERR_BAD_EVENT_NAME = 17, + TP_ERR_EVENT_EXIST = 18, + TP_ERR_RETVAL_ON_PROBE = 19, + TP_ERR_BAD_STACK_NUM = 20, + TP_ERR_BAD_ARG_NUM = 21, + TP_ERR_BAD_VAR = 22, + TP_ERR_BAD_REG_NAME = 23, + TP_ERR_BAD_MEM_ADDR = 24, + TP_ERR_BAD_IMM = 25, + TP_ERR_IMMSTR_NO_CLOSE = 26, + TP_ERR_FILE_ON_KPROBE = 27, + TP_ERR_BAD_FILE_OFFS = 28, + TP_ERR_SYM_ON_UPROBE = 29, + TP_ERR_TOO_MANY_OPS = 30, + TP_ERR_DEREF_NEED_BRACE = 31, + TP_ERR_BAD_DEREF_OFFS = 32, + TP_ERR_DEREF_OPEN_BRACE = 33, + TP_ERR_COMM_CANT_DEREF = 34, + TP_ERR_BAD_FETCH_ARG = 35, + TP_ERR_ARRAY_NO_CLOSE = 36, + TP_ERR_BAD_ARRAY_SUFFIX = 37, + TP_ERR_BAD_ARRAY_NUM = 38, + TP_ERR_ARRAY_TOO_BIG = 39, + TP_ERR_BAD_TYPE = 40, + TP_ERR_BAD_STRING = 41, + TP_ERR_BAD_BITFIELD = 42, + TP_ERR_ARG_NAME_TOO_LONG = 43, + TP_ERR_NO_ARG_NAME = 44, + TP_ERR_BAD_ARG_NAME = 45, + TP_ERR_USED_ARG_NAME = 46, + TP_ERR_ARG_TOO_LONG = 47, + TP_ERR_NO_ARG_BODY = 48, + TP_ERR_BAD_INSN_BNDRY = 49, + TP_ERR_FAIL_REG_PROBE = 50, + TP_ERR_DIFF_PROBE_TYPE = 51, + TP_ERR_DIFF_ARG_TYPE = 52, + TP_ERR_SAME_PROBE = 53, +}; + +struct trace_eprobe { + const char *event_system; + const char *event_name; + struct trace_event_call *event; + struct dyn_event devent; + struct trace_probe tp; +}; + +struct eprobe_data { + struct trace_event_file *file; + struct trace_eprobe *ep; +}; + enum { BPF_F_INDEX_MASK = 4294967295, BPF_F_CURRENT_CPU = 4294967295, @@ -33642,11 +34057,13 @@ struct bpf_event_entry { typedef long unsigned int (*bpf_ctx_copy_t)(void *, const void *, long unsigned int, long unsigned int); -struct bpf_cg_run_ctx { +struct bpf_trace_run_ctx { struct bpf_run_ctx run_ctx; - struct bpf_prog_array_item *prog_item; + u64 bpf_cookie; }; +typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *, const void *); + typedef struct pt_regs bpf_user_pt_regs_t; struct bpf_perf_event_data { @@ -33655,6 +34072,11 @@ struct bpf_perf_event_data { __u64 addr; }; +struct btf_id_set { + u32 cnt; + u32 ids[0]; +}; + struct perf_event_query_bpf { __u32 ids_len; __u32 prog_cnt; @@ -33667,9 +34089,11 @@ struct bpf_perf_event_data_kern { struct perf_event *event; }; -struct btf_id_set { - u32 cnt; - u32 ids[0]; +enum { + BTF_TRACING_TYPE_TASK = 0, + BTF_TRACING_TYPE_FILE = 1, + BTF_TRACING_TYPE_VMA = 2, + MAX_BTF_TRACING_TYPE = 3, }; struct security_hook_heads { @@ -33883,6 +34307,8 @@ struct security_hook_heads { struct hlist_head perf_event_free; struct hlist_head perf_event_read; struct hlist_head perf_event_write; + struct hlist_head uring_override_creds; + struct hlist_head uring_sqpoll; }; struct lsm_blob_sizes { @@ -33954,6 +34380,8 @@ typedef u64 (*btf_bpf_probe_write_user)(void *, const void *, u32); typedef u64 (*btf_bpf_trace_printk)(char *, u32, u64, u64, u64); +typedef u64 (*btf_bpf_trace_vprintk)(char *, u32, const void *, u32); + typedef u64 (*btf_bpf_seq_printf)(struct seq_file *, char *, u32, const void *, u32); typedef u64 (*btf_bpf_seq_write)(struct seq_file *, const void *, u32); @@ -33978,6 +34406,8 @@ typedef u64 (*btf_bpf_get_current_task)(); typedef u64 (*btf_bpf_get_current_task_btf)(); +typedef u64 (*btf_bpf_task_pt_regs)(struct task_struct *); + typedef u64 (*btf_bpf_current_task_under_cgroup)(struct bpf_map *, u32); struct send_signal_irq_work { @@ -33999,6 +34429,18 @@ typedef u64 (*btf_bpf_get_func_ip_tracing)(void *); typedef u64 (*btf_bpf_get_func_ip_kprobe)(struct pt_regs *); +typedef u64 (*btf_bpf_get_attach_cookie_trace)(void *); + +typedef u64 (*btf_bpf_get_attach_cookie_pe)(struct bpf_perf_event_data_kern *); + +typedef u64 (*btf_bpf_get_branch_snapshot)(void *, u32, u64); + +typedef u64 (*btf_get_func_arg)(void *, u32, u64 *); + +typedef u64 (*btf_get_func_ret)(void *, u64 *); + +typedef u64 (*btf_get_func_arg_cnt)(void *); + typedef u64 (*btf_bpf_perf_event_output_tp)(void *, struct bpf_map *, u64, void *, u64); typedef u64 (*btf_bpf_get_stackid_tp)(void *, struct bpf_map *, u64); @@ -34049,175 +34491,16 @@ struct kretprobe_trace_entry_head { long unsigned int ret_ip; }; -struct dyn_event; - -struct dyn_event_operations { - struct list_head list; - int (*create)(const char *); - int (*show)(struct seq_file *, struct dyn_event *); - bool (*is_busy)(struct dyn_event *); - int (*free)(struct dyn_event *); - bool (*match)(const char *, const char *, int, const char **, struct dyn_event *); -}; - -struct dyn_event { - struct list_head list; - struct dyn_event_operations *ops; -}; - struct dynevent_arg { const char *str; char separator; }; -typedef int (*print_type_func_t)(struct trace_seq *, void *, void *); - -enum fetch_op { - FETCH_OP_NOP = 0, - FETCH_OP_REG = 1, - FETCH_OP_STACK = 2, - FETCH_OP_STACKP = 3, - FETCH_OP_RETVAL = 4, - FETCH_OP_IMM = 5, - FETCH_OP_COMM = 6, - FETCH_OP_ARG = 7, - FETCH_OP_FOFFS = 8, - FETCH_OP_DATA = 9, - FETCH_OP_DEREF = 10, - FETCH_OP_UDEREF = 11, - FETCH_OP_ST_RAW = 12, - FETCH_OP_ST_MEM = 13, - FETCH_OP_ST_UMEM = 14, - FETCH_OP_ST_STRING = 15, - FETCH_OP_ST_USTRING = 16, - FETCH_OP_MOD_BF = 17, - FETCH_OP_LP_ARRAY = 18, - FETCH_OP_END = 19, - FETCH_NOP_SYMBOL = 20, -}; - -struct fetch_insn { - enum fetch_op op; - union { - unsigned int param; - struct { - unsigned int size; - int offset; - }; - struct { - unsigned char basesize; - unsigned char lshift; - unsigned char rshift; - }; - long unsigned int immediate; - void *data; - }; -}; - -struct fetch_type { - const char *name; - size_t size; - int is_signed; - print_type_func_t print; - const char *fmt; - const char *fmttype; -}; - -struct probe_arg { - struct fetch_insn *code; - bool dynamic; - unsigned int offset; - unsigned int count; - const char *name; - const char *comm; - char *fmt; - const struct fetch_type *type; -}; - -struct trace_uprobe_filter { - rwlock_t rwlock; - int nr_systemwide; - struct list_head perf_events; -}; - -struct trace_probe_event { - unsigned int flags; - struct trace_event_class class; - struct trace_event_call call; - struct list_head files; - struct list_head probes; - struct trace_uprobe_filter filter[0]; -}; - -struct trace_probe { - struct list_head list; - struct trace_probe_event *event; - ssize_t size; - unsigned int nr_args; - struct probe_arg args[0]; -}; - struct event_file_link { struct trace_event_file *file; struct list_head list; }; -enum { - TP_ERR_FILE_NOT_FOUND = 0, - TP_ERR_NO_REGULAR_FILE = 1, - TP_ERR_BAD_REFCNT = 2, - TP_ERR_REFCNT_OPEN_BRACE = 3, - TP_ERR_BAD_REFCNT_SUFFIX = 4, - TP_ERR_BAD_UPROBE_OFFS = 5, - TP_ERR_MAXACT_NO_KPROBE = 6, - TP_ERR_BAD_MAXACT = 7, - TP_ERR_MAXACT_TOO_BIG = 8, - TP_ERR_BAD_PROBE_ADDR = 9, - TP_ERR_BAD_RETPROBE = 10, - TP_ERR_BAD_ADDR_SUFFIX = 11, - TP_ERR_NO_GROUP_NAME = 12, - TP_ERR_GROUP_TOO_LONG = 13, - TP_ERR_BAD_GROUP_NAME = 14, - TP_ERR_NO_EVENT_NAME = 15, - TP_ERR_EVENT_TOO_LONG = 16, - TP_ERR_BAD_EVENT_NAME = 17, - TP_ERR_RETVAL_ON_PROBE = 18, - TP_ERR_BAD_STACK_NUM = 19, - TP_ERR_BAD_ARG_NUM = 20, - TP_ERR_BAD_VAR = 21, - TP_ERR_BAD_REG_NAME = 22, - TP_ERR_BAD_MEM_ADDR = 23, - TP_ERR_BAD_IMM = 24, - TP_ERR_IMMSTR_NO_CLOSE = 25, - TP_ERR_FILE_ON_KPROBE = 26, - TP_ERR_BAD_FILE_OFFS = 27, - TP_ERR_SYM_ON_UPROBE = 28, - TP_ERR_TOO_MANY_OPS = 29, - TP_ERR_DEREF_NEED_BRACE = 30, - TP_ERR_BAD_DEREF_OFFS = 31, - TP_ERR_DEREF_OPEN_BRACE = 32, - TP_ERR_COMM_CANT_DEREF = 33, - TP_ERR_BAD_FETCH_ARG = 34, - TP_ERR_ARRAY_NO_CLOSE = 35, - TP_ERR_BAD_ARRAY_SUFFIX = 36, - TP_ERR_BAD_ARRAY_NUM = 37, - TP_ERR_ARRAY_TOO_BIG = 38, - TP_ERR_BAD_TYPE = 39, - TP_ERR_BAD_STRING = 40, - TP_ERR_BAD_BITFIELD = 41, - TP_ERR_ARG_NAME_TOO_LONG = 42, - TP_ERR_NO_ARG_NAME = 43, - TP_ERR_BAD_ARG_NAME = 44, - TP_ERR_USED_ARG_NAME = 45, - TP_ERR_ARG_TOO_LONG = 46, - TP_ERR_NO_ARG_BODY = 47, - TP_ERR_BAD_INSN_BNDRY = 48, - TP_ERR_FAIL_REG_PROBE = 49, - TP_ERR_DIFF_PROBE_TYPE = 50, - TP_ERR_DIFF_ARG_TYPE = 51, - TP_ERR_SAME_PROBE = 52, -}; - struct trace_kprobe { struct dyn_event devent; struct kretprobe rp; @@ -34496,6 +34779,14 @@ struct uprobe_cpu_buffer { typedef bool (*filter_func_t)(struct uprobe_consumer *, enum uprobe_filter_ctx, struct mm_struct *); +enum xdp_action { + XDP_ABORTED = 0, + XDP_DROP = 1, + XDP_PASS = 2, + XDP_TX = 3, + XDP_REDIRECT = 4, +}; + struct rhash_lock_head; struct bucket_table { @@ -34513,6 +34804,163 @@ struct bucket_table { struct rhash_lock_head *buckets[0]; }; +typedef sockptr_t bpfptr_t; + +struct bpf_verifier_log { + u32 level; + char kbuf[1024]; + char *ubuf; + u32 len_used; + u32 len_total; +}; + +struct bpf_subprog_info { + u32 start; + u32 linfo_idx; + u16 stack_depth; + bool has_tail_call; + bool tail_call_reachable; + bool has_ld_abs; + bool is_async_cb; +}; + +struct bpf_id_pair { + u32 old; + u32 cur; +}; + +struct bpf_verifier_stack_elem; + +struct bpf_verifier_state; + +struct bpf_verifier_state_list; + +struct bpf_insn_aux_data; + +struct bpf_verifier_env { + u32 insn_idx; + u32 prev_insn_idx; + struct bpf_prog *prog; + const struct bpf_verifier_ops *ops; + struct bpf_verifier_stack_elem *head; + int stack_size; + bool strict_alignment; + bool test_state_freq; + struct bpf_verifier_state *cur_state; + struct bpf_verifier_state_list **explored_states; + struct bpf_verifier_state_list *free_list; + struct bpf_map *used_maps[64]; + struct btf_mod_pair used_btfs[64]; + u32 used_map_cnt; + u32 used_btf_cnt; + u32 id_gen; + bool explore_alu_limits; + bool allow_ptr_leaks; + bool allow_uninit_stack; + bool allow_ptr_to_map_access; + bool bpf_capable; + bool bypass_spec_v1; + bool bypass_spec_v4; + bool seen_direct_write; + struct bpf_insn_aux_data *insn_aux_data; + const struct bpf_line_info *prev_linfo; + struct bpf_verifier_log log; + struct bpf_subprog_info subprog_info[257]; + struct bpf_id_pair idmap_scratch[75]; + struct { + int *insn_state; + int *insn_stack; + int cur_stack; + } cfg; + u32 pass_cnt; + u32 subprog_cnt; + u32 prev_insn_processed; + u32 insn_processed; + u32 prev_jmps_processed; + u32 jmps_processed; + u64 verification_time; + u32 max_states_per_insn; + u32 total_states; + u32 peak_states; + u32 longest_mark_read_walk; + bpfptr_t fd_array; + u32 scratched_regs; + u64 scratched_stack_slots; + u32 prev_log_len; + u32 prev_insn_print_len; + char type_str_buf[64]; +}; + +struct tnum { + u64 value; + u64 mask; +}; + +enum bpf_reg_liveness { + REG_LIVE_NONE = 0, + REG_LIVE_READ32 = 1, + REG_LIVE_READ64 = 2, + REG_LIVE_READ = 3, + REG_LIVE_WRITTEN = 4, + REG_LIVE_DONE = 8, +}; + +struct bpf_reg_state { + enum bpf_reg_type type; + s32 off; + union { + int range; + struct { + struct bpf_map *map_ptr; + u32 map_uid; + }; + struct { + struct btf *btf; + u32 btf_id; + }; + u32 mem_size; + struct { + long unsigned int raw1; + long unsigned int raw2; + } raw; + u32 subprogno; + }; + u32 id; + u32 ref_obj_id; + struct tnum var_off; + s64 smin_value; + s64 smax_value; + u64 umin_value; + u64 umax_value; + s32 s32_min_value; + s32 s32_max_value; + u32 u32_min_value; + u32 u32_max_value; + struct bpf_reg_state *parent; + u32 frameno; + s32 subreg_def; + enum bpf_reg_liveness live; + bool precise; +}; + +struct bpf_reference_state; + +struct bpf_stack_state; + +struct bpf_func_state { + struct bpf_reg_state regs[11]; + int callsite; + u32 frameno; + u32 subprogno; + u32 async_entry_cnt; + bool in_callback_fn; + bool in_async_callback_fn; + int acquired_refs; + struct bpf_reference_state *refs; + int allocated_stack; + struct bpf_stack_state *stack; +}; + struct rnd_state { __u32 s1; __u32 s2; @@ -34520,14 +34968,6 @@ struct rnd_state { __u32 s4; }; -enum xdp_action { - XDP_ABORTED = 0, - XDP_DROP = 1, - XDP_PASS = 2, - XDP_TX = 3, - XDP_REDIRECT = 4, -}; - enum xdp_mem_type { MEM_TYPE_PAGE_SHARED = 0, MEM_TYPE_PAGE_ORDER0 = 1, @@ -34544,6 +34984,81 @@ struct xdp_cpumap_stats { typedef void (*bpf_jit_fill_hole_t)(void *, unsigned int); +enum btf_kfunc_type { + BTF_KFUNC_TYPE_CHECK = 0, + BTF_KFUNC_TYPE_ACQUIRE = 1, + BTF_KFUNC_TYPE_RELEASE = 2, + BTF_KFUNC_TYPE_RET_NULL = 3, + BTF_KFUNC_TYPE_MAX = 4, +}; + +struct bpf_stack_state { + struct bpf_reg_state spilled_ptr; + u8 slot_type[8]; +}; + +struct bpf_reference_state { + int id; + int insn_idx; +}; + +struct bpf_idx_pair { + u32 prev_idx; + u32 idx; +}; + +struct bpf_verifier_state { + struct bpf_func_state *frame[8]; + struct bpf_verifier_state *parent; + u32 branches; + u32 insn_idx; + u32 curframe; + u32 active_spin_lock; + bool speculative; + u32 first_insn_idx; + u32 last_insn_idx; + struct bpf_idx_pair *jmp_history; + u32 jmp_history_cnt; +}; + +struct bpf_verifier_state_list { + struct bpf_verifier_state state; + struct bpf_verifier_state_list *next; + int miss_cnt; + int hit_cnt; +}; + +struct bpf_insn_aux_data { + union { + enum bpf_reg_type ptr_type; + long unsigned int map_ptr_state; + s32 call_imm; + u32 alu_limit; + struct { + u32 map_index; + u32 map_off; + }; + struct { + enum bpf_reg_type reg_type; + union { + struct { + struct btf *btf; + u32 btf_id; + }; + u32 mem_size; + }; + } btf_var; + }; + u64 map_key_state; + int ctx_field_size; + u32 seen; + bool sanitize_stack_spill; + bool zext_dst; + u8 alu_state; + unsigned int orig_idx; + bool prune_point; +}; + struct bpf_prog_dummy { struct bpf_prog prog; }; @@ -34558,14 +35073,11 @@ struct _bpf_dtab_netdev { struct rhash_lock_head {}; -struct zero_copy_allocator; - struct xdp_mem_allocator { struct xdp_mem_info mem; union { void *allocator; struct page_pool *page_pool; - struct zero_copy_allocator *zc_alloc; }; struct rhash_head node; struct callback_head rcu; @@ -34747,6 +35259,18 @@ enum bpf_cmd { BPF_PROG_BIND_MAP = 35, }; +enum bpf_link_type { + BPF_LINK_TYPE_UNSPEC = 0, + BPF_LINK_TYPE_RAW_TRACEPOINT = 1, + BPF_LINK_TYPE_TRACING = 2, + BPF_LINK_TYPE_CGROUP = 3, + BPF_LINK_TYPE_ITER = 4, + BPF_LINK_TYPE_NETNS = 5, + BPF_LINK_TYPE_XDP = 6, + BPF_LINK_TYPE_PERF_EVENT = 7, + MAX_BPF_LINK_TYPE = 8, +}; + enum { BPF_ANY = 0, BPF_NOEXIST = 1, @@ -34810,6 +35334,7 @@ struct bpf_prog_info { __u64 run_time_ns; __u64 run_cnt; __u64 recursion_misses; + __u32 verified_insns; }; struct bpf_map_info { @@ -34827,6 +35352,7 @@ struct bpf_map_info { __u32 btf_id; __u32 btf_key_type_id; __u32 btf_value_type_id; + __u64 map_extra; }; struct bpf_btf_info { @@ -34838,6 +35364,43 @@ struct bpf_btf_info { __u32 kernel_btf; }; +struct bpf_link_info { + __u32 type; + __u32 id; + __u32 prog_id; + union { + struct { + __u64 tp_name; + __u32 tp_name_len; + } raw_tracepoint; + struct { + __u32 attach_type; + __u32 target_obj_id; + __u32 target_btf_id; + } tracing; + struct { + __u64 cgroup_id; + __u32 attach_type; + } cgroup; + struct { + __u64 target_name; + __u32 target_name_len; + union { + struct { + __u32 map_id; + } map; + }; + } iter; + struct { + __u32 netns_ino; + __u32 attach_type; + } netns; + struct { + __u32 ifindex; + } xdp; + }; +}; + struct bpf_spin_lock { __u32 val; }; @@ -34846,157 +35409,6 @@ struct bpf_timer { long: 64; long: 64;}; -typedef sockptr_t bpfptr_t; - -struct bpf_verifier_log { - u32 level; - char kbuf[1024]; - char *ubuf; - u32 len_used; - u32 len_total; -}; - -struct bpf_subprog_info { - u32 start; - u32 linfo_idx; - u16 stack_depth; - bool has_tail_call; - bool tail_call_reachable; - bool has_ld_abs; - bool is_async_cb; -}; - -struct bpf_id_pair { - u32 old; - u32 cur; -}; - -struct bpf_verifier_stack_elem; - -struct bpf_verifier_state; - -struct bpf_verifier_state_list; - -struct bpf_insn_aux_data; - -struct bpf_verifier_env { - u32 insn_idx; - u32 prev_insn_idx; - struct bpf_prog *prog; - const struct bpf_verifier_ops *ops; - struct bpf_verifier_stack_elem *head; - int stack_size; - bool strict_alignment; - bool test_state_freq; - struct bpf_verifier_state *cur_state; - struct bpf_verifier_state_list **explored_states; - struct bpf_verifier_state_list *free_list; - struct bpf_map *used_maps[64]; - struct btf_mod_pair used_btfs[64]; - u32 used_map_cnt; - u32 used_btf_cnt; - u32 id_gen; - bool allow_ptr_leaks; - bool allow_uninit_stack; - bool allow_ptr_to_map_access; - bool bpf_capable; - bool bypass_spec_v1; - bool bypass_spec_v4; - bool seen_direct_write; - struct bpf_insn_aux_data *insn_aux_data; - const struct bpf_line_info *prev_linfo; - struct bpf_verifier_log log; - struct bpf_subprog_info subprog_info[257]; - struct bpf_id_pair idmap_scratch[75]; - struct { - int *insn_state; - int *insn_stack; - int cur_stack; - } cfg; - u32 pass_cnt; - u32 subprog_cnt; - u32 prev_insn_processed; - u32 insn_processed; - u32 prev_jmps_processed; - u32 jmps_processed; - u64 verification_time; - u32 max_states_per_insn; - u32 total_states; - u32 peak_states; - u32 longest_mark_read_walk; - bpfptr_t fd_array; -}; - -struct tnum { - u64 value; - u64 mask; -}; - -enum bpf_reg_liveness { - REG_LIVE_NONE = 0, - REG_LIVE_READ32 = 1, - REG_LIVE_READ64 = 2, - REG_LIVE_READ = 3, - REG_LIVE_WRITTEN = 4, - REG_LIVE_DONE = 8, -}; - -struct bpf_reg_state { - enum bpf_reg_type type; - s32 off; - union { - int range; - struct { - struct bpf_map *map_ptr; - u32 map_uid; - }; - struct { - struct btf *btf; - u32 btf_id; - }; - u32 mem_size; - struct { - long unsigned int raw1; - long unsigned int raw2; - } raw; - u32 subprogno; - }; - u32 id; - u32 ref_obj_id; - struct tnum var_off; - s64 smin_value; - s64 smax_value; - u64 umin_value; - u64 umax_value; - s32 s32_min_value; - s32 s32_max_value; - u32 u32_min_value; - u32 u32_max_value; - struct bpf_reg_state *parent; - u32 frameno; - s32 subreg_def; - enum bpf_reg_liveness live; - bool precise; -}; - -struct bpf_reference_state; - -struct bpf_stack_state; - -struct bpf_func_state { - struct bpf_reg_state regs[11]; - int callsite; - u32 frameno; - u32 subprogno; - u32 async_entry_cnt; - bool in_callback_fn; - bool in_async_callback_fn; - int acquired_refs; - struct bpf_reference_state *refs; - int allocated_stack; - struct bpf_stack_state *stack; -}; - struct bpf_attach_target_info { struct btf_func_model fmodel; long int tgt_addr; @@ -35004,6 +35416,26 @@ struct bpf_attach_target_info { const struct btf_type *tgt_type; }; +struct bpf_link_ops; + +struct bpf_link { + atomic64_t refcnt; + u32 id; + enum bpf_link_type type; + const struct bpf_link_ops *ops; + struct bpf_prog *prog; + struct work_struct work; +}; + +struct bpf_link_ops { + void (*release)(struct bpf_link *); + void (*dealloc)(struct bpf_link *); + int (*detach)(struct bpf_link *); + int (*update_prog)(struct bpf_link *, struct bpf_prog *, struct bpf_prog *); + void (*show_fdinfo)(const struct bpf_link *, struct seq_file *); + int (*fill_link_info)(const struct bpf_link *, struct bpf_link_info *); +}; + struct bpf_link_primer { struct bpf_link *link; struct file *file; @@ -35011,73 +35443,6 @@ struct bpf_link_primer { u32 id; }; -struct bpf_stack_state { - struct bpf_reg_state spilled_ptr; - u8 slot_type[8]; -}; - -struct bpf_reference_state { - int id; - int insn_idx; -}; - -struct bpf_idx_pair { - u32 prev_idx; - u32 idx; -}; - -struct bpf_verifier_state { - struct bpf_func_state *frame[8]; - struct bpf_verifier_state *parent; - u32 branches; - u32 insn_idx; - u32 curframe; - u32 active_spin_lock; - bool speculative; - u32 first_insn_idx; - u32 last_insn_idx; - struct bpf_idx_pair *jmp_history; - u32 jmp_history_cnt; -}; - -struct bpf_verifier_state_list { - struct bpf_verifier_state state; - struct bpf_verifier_state_list *next; - int miss_cnt; - int hit_cnt; -}; - -struct bpf_insn_aux_data { - union { - enum bpf_reg_type ptr_type; - long unsigned int map_ptr_state; - s32 call_imm; - u32 alu_limit; - struct { - u32 map_index; - u32 map_off; - }; - struct { - enum bpf_reg_type reg_type; - union { - struct { - struct btf *btf; - u32 btf_id; - }; - u32 mem_size; - }; - } btf_var; - }; - u64 map_key_state; - int ctx_field_size; - int sanitize_stack_off; - u32 seen; - bool zext_dst; - u8 alu_state; - unsigned int orig_idx; - bool prune_point; -}; - enum perf_bpf_event_type { PERF_BPF_EVENT_UNKNOWN = 0, PERF_BPF_EVENT_PROG_LOAD = 1, @@ -35091,6 +35456,12 @@ enum bpf_audit { BPF_AUDIT_MAX = 2, }; +struct bpf_prog_kstats { + u64 nsecs; + u64 cnt; + u64 misses; +}; + struct bpf_tracing_link { struct bpf_link link; enum bpf_attach_type attach_type; @@ -35103,10 +35474,41 @@ struct bpf_raw_tp_link { struct bpf_raw_event_map *btp; }; +struct bpf_perf_link { + struct bpf_link link; + struct file *perf_file; +}; + typedef u64 (*btf_bpf_sys_bpf)(int, void *, u32); typedef u64 (*btf_bpf_sys_close)(u32); +typedef u64 (*btf_bpf_kallsyms_lookup_name)(const char *, int, int, u64 *); + +enum { + BTF_KIND_UNKN = 0, + BTF_KIND_INT = 1, + BTF_KIND_PTR = 2, + BTF_KIND_ARRAY = 3, + BTF_KIND_STRUCT = 4, + BTF_KIND_UNION = 5, + BTF_KIND_ENUM = 6, + BTF_KIND_FWD = 7, + BTF_KIND_TYPEDEF = 8, + BTF_KIND_VOLATILE = 9, + BTF_KIND_CONST = 10, + BTF_KIND_RESTRICT = 11, + BTF_KIND_FUNC = 12, + BTF_KIND_FUNC_PROTO = 13, + BTF_KIND_VAR = 14, + BTF_KIND_DATASEC = 15, + BTF_KIND_FLOAT = 16, + BTF_KIND_DECL_TAG = 17, + BTF_KIND_TYPE_TAG = 18, + NR_BTF_KINDS = 19, + BTF_KIND_MAX = 18, +}; + struct btf_member { __u32 name_off; __u32 type; @@ -35135,10 +35537,33 @@ enum sk_action { SK_PASS = 1, }; +enum bpf_core_relo_kind { + BPF_CORE_FIELD_BYTE_OFFSET = 0, + BPF_CORE_FIELD_BYTE_SIZE = 1, + BPF_CORE_FIELD_EXISTS = 2, + BPF_CORE_FIELD_SIGNED = 3, + BPF_CORE_FIELD_LSHIFT_U64 = 4, + BPF_CORE_FIELD_RSHIFT_U64 = 5, + BPF_CORE_TYPE_ID_LOCAL = 6, + BPF_CORE_TYPE_ID_TARGET = 7, + BPF_CORE_TYPE_EXISTS = 8, + BPF_CORE_TYPE_SIZE = 9, + BPF_CORE_ENUMVAL_EXISTS = 10, + BPF_CORE_ENUMVAL_VALUE = 11, +}; + +struct bpf_core_relo { + __u32 insn_off; + __u32 type_id; + __u32 access_str_off; + enum bpf_core_relo_kind kind; +}; + struct bpf_kfunc_desc { struct btf_func_model func_model; u32 func_id; s32 imm; + u16 offset; }; struct bpf_kfunc_desc_tab { @@ -35146,6 +35571,17 @@ struct bpf_kfunc_desc_tab { u32 nr_descs; }; +struct bpf_kfunc_btf { + struct btf *btf; + struct module *module; + u16 offset; +}; + +struct bpf_kfunc_btf_tab { + struct bpf_kfunc_btf descs[256]; + u32 nr_descs; +}; + struct bpf_struct_ops { const struct bpf_verifier_ops *verifier_ops; int (*init)(struct btf *); @@ -35163,6 +35599,11 @@ struct bpf_struct_ops { typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type, const struct bpf_insn *, struct bpf_insn *, struct bpf_prog *, u32 *); +struct bpf_core_ctx { + struct bpf_verifier_log *log; + const struct btf *btf; +}; + enum bpf_stack_slot_type { STACK_INVALID = 0, STACK_SPILL = 1, @@ -35192,7 +35633,8 @@ enum { BTF_SOCK_TYPE_TCP6 = 10, BTF_SOCK_TYPE_UDP = 11, BTF_SOCK_TYPE_UDP6 = 12, - MAX_BTF_SOCK_TYPE = 13, + BTF_SOCK_TYPE_UNIX = 13, + MAX_BTF_SOCK_TYPE = 14, }; typedef void (*bpf_insn_print_t)(void *, const char *, ...); @@ -35317,6 +35759,12 @@ struct bpf_pidns_info { __u32 tgid; }; +struct bpf_cg_run_ctx { + struct bpf_run_ctx run_ctx; + const struct bpf_prog_array_item *prog_item; + int retval; +}; + typedef u64 (*btf_bpf_map_lookup_elem)(struct bpf_map *, void *); typedef u64 (*btf_bpf_map_update_elem)(struct bpf_map *, void *, void *, u64); @@ -35361,6 +35809,8 @@ typedef u64 (*btf_bpf_strtol)(const char *, size_t, u64, long int *); typedef u64 (*btf_bpf_strtoul)(const char *, size_t, u64, long unsigned int *); +typedef u64 (*btf_bpf_strncmp)(const char *, u32, const char *); + typedef u64 (*btf_bpf_get_ns_current_pid_tgid)(u64, u64, struct bpf_pidns_info *, u32); typedef u64 (*btf_bpf_event_output_data)(void *, struct bpf_map *, u64, void *, u64); @@ -35412,6 +35862,8 @@ typedef void (*bpf_iter_show_fdinfo_t)(const struct bpf_iter_aux_info *, struct typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *, struct bpf_link_info *); +typedef const struct bpf_func_proto * (*bpf_iter_get_func_proto_t)(enum bpf_func_id, const struct bpf_prog *); + enum bpf_iter_feature { BPF_ITER_RESCHED = 1, }; @@ -35422,6 +35874,7 @@ struct bpf_iter_reg { bpf_iter_detach_target_t detach_target; bpf_iter_show_fdinfo_t show_fdinfo; bpf_iter_fill_link_info_t fill_link_info; + bpf_iter_get_func_proto_t get_func_proto; u32 ctx_arg_info_size; u32 feature; struct bpf_ctx_arg_aux ctx_arg_info[2]; @@ -35461,6 +35914,8 @@ struct bpf_iter_priv_data { typedef u64 (*btf_bpf_for_each_map_elem)(struct bpf_map *, void *, void *, u64); +typedef u64 (*btf_bpf_loop)(u32, void *, void *, u64); + struct bpf_iter_seq_map_info { u32 map_id; }; @@ -35474,6 +35929,11 @@ struct bpf_iter__bpf_map { }; }; +struct mmap_unlock_irq_work { + struct irq_work irq_work; + struct mm_struct *mm; +}; + struct bpf_iter_seq_task_common { struct pid_namespace *ns; }; @@ -35539,6 +35999,8 @@ struct bpf_iter__task_vma { }; }; +typedef u64 (*btf_bpf_find_vma)(struct task_struct *, u64, bpf_callback_t, void *, u64); + struct bpf_iter_seq_prog_info { u32 prog_id; }; @@ -35567,10 +36029,6 @@ struct bpf_iter__bpf_map_elem { }; }; -struct hlist_nulls_head { - struct hlist_nulls_node *first; -}; - struct pcpu_freelist_node; struct pcpu_freelist_head { @@ -35748,6 +36206,21 @@ struct lpm_trie { long: 64; }; +struct bpf_bloom_filter { + struct bpf_map map; + u32 bitset_mask; + u32 hash_seed; + u32 aligned_u32_count; + u32 nr_hash_funcs; + long unsigned int bitset[0]; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; +}; + struct bpf_cgroup_storage_map { struct bpf_map map; spinlock_t lock; @@ -35800,6 +36273,14 @@ struct bpf_ringbuf { u64 mask; struct page **pages; int nr_pages; + long: 32; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; spinlock_t spinlock; long: 64; long: 64; @@ -36281,14 +36762,6 @@ struct bpf_ringbuf { long: 64; long: 64; long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; long unsigned int consumer_pos; long: 64; long: 64; @@ -37402,6 +37875,10 @@ struct btf_var { __u32 linkage; }; +struct btf_decl_tag { + __s32 component_idx; +}; + struct bpf_flow_keys { __u16 nhoff; __u16 thoff; @@ -37481,6 +37958,7 @@ struct __sk_buff { struct bpf_sock *sk; }; __u32 gso_size; + __u64 hwtstamp; }; struct xdp_md { @@ -37642,6 +38120,7 @@ struct bpf_sk_lookup { __u32 local_ip4; __u32 local_ip6[4]; __u32 local_port; + __u32 ingress_ifindex; }; struct sk_reuseport_kern { @@ -37655,6 +38134,19 @@ struct sk_reuseport_kern { bool bind_inany; }; +struct btf_kfunc_id_set { + struct module *owner; + union { + struct { + struct btf_id_set *check_set; + struct btf_id_set *acquire_set; + struct btf_id_set *release_set; + struct btf_id_set *ret_null_set; + }; + struct btf_id_set *sets[4]; + }; +}; + struct bpf_flow_dissector { struct bpf_flow_keys *flow_keys; const struct sk_buff *skb; @@ -37724,20 +38216,6 @@ struct in_addr { __be32 s_addr; }; -struct xdp_txq_info { - struct net_device *dev; -}; - -struct xdp_buff { - void *data; - void *data_end; - void *data_meta; - void *data_hard_start; - struct xdp_rxq_info *rxq; - struct xdp_txq_info *txq; - u32 frame_sz; -}; - struct bpf_sock_addr_kern { struct sock *sk; struct sockaddr *uaddr; @@ -37781,7 +38259,8 @@ struct bpf_sockopt_kern { s32 level; s32 optname; s32 optlen; - s32 retval; + struct task_struct *current_task; + u64 tmp_reg; }; struct bpf_sk_lookup_kern { @@ -37798,6 +38277,7 @@ struct bpf_sk_lookup_kern { const struct in6_addr *daddr; } v6; struct sock *selected_sk; + u32 ingress_ifindex; bool no_reuseport; }; @@ -37956,6 +38436,7 @@ struct tcp_md5sig_key { u8 keylen; u8 family; u8 prefixlen; + u8 flags; union tcp_md5_addr addr; int l3index; u8 key[80]; @@ -37987,7 +38468,9 @@ struct ack_sample { struct rate_sample { u64 prior_mstamp; u32 prior_delivered; + u32 prior_delivered_ce; s32 delivered; + s32 delivered_ce; long int interval_us; u32 snd_interval_us; u32 rcv_interval_us; @@ -38023,6 +38506,48 @@ struct sk_msg { struct list_head list; }; +struct bpf_core_cand { + const struct btf *btf; + __u32 id; +}; + +struct bpf_core_cand_list { + struct bpf_core_cand *cands; + int len; +}; + +struct bpf_core_accessor { + __u32 type_id; + __u32 idx; + const char *name; +}; + +struct bpf_core_spec { + const struct btf *btf; + struct bpf_core_accessor spec[64]; + __u32 root_type_id; + enum bpf_core_relo_kind relo_kind; + int len; + int raw_spec[64]; + int raw_len; + __u32 bit_offset; +}; + +enum btf_kfunc_hook { + BTF_KFUNC_HOOK_XDP = 0, + BTF_KFUNC_HOOK_TC = 1, + BTF_KFUNC_HOOK_STRUCT_OPS = 2, + BTF_KFUNC_HOOK_MAX = 3, +}; + +enum { + BTF_KFUNC_SET_MAX_CNT = 32, +}; + +struct btf_kfunc_set_tab { + struct btf_id_set *sets[12]; +}; + enum verifier_phase { CHECK_META = 0, CHECK_TYPE = 1, @@ -38146,6 +38671,8 @@ struct bpf_ctx_convert { struct bpf_sysctl_kern BPF_PROG_TYPE_CGROUP_SYSCTL_kern; struct bpf_sockopt BPF_PROG_TYPE_CGROUP_SOCKOPT_prog; struct bpf_sockopt_kern BPF_PROG_TYPE_CGROUP_SOCKOPT_kern; + __u32 BPF_PROG_TYPE_LIRC_MODE2_prog; + u32 BPF_PROG_TYPE_LIRC_MODE2_kern; struct sk_reuseport_md BPF_PROG_TYPE_SK_REUSEPORT_prog; struct sk_reuseport_kern BPF_PROG_TYPE_SK_REUSEPORT_kern; struct bpf_sk_lookup BPF_PROG_TYPE_SK_LOOKUP_prog; @@ -38185,13 +38712,14 @@ enum { __ctx_convertBPF_PROG_TYPE_CGROUP_DEVICE = 21, __ctx_convertBPF_PROG_TYPE_CGROUP_SYSCTL = 22, __ctx_convertBPF_PROG_TYPE_CGROUP_SOCKOPT = 23, - __ctx_convertBPF_PROG_TYPE_SK_REUSEPORT = 24, - __ctx_convertBPF_PROG_TYPE_SK_LOOKUP = 25, - __ctx_convertBPF_PROG_TYPE_STRUCT_OPS = 26, - __ctx_convertBPF_PROG_TYPE_EXT = 27, - __ctx_convertBPF_PROG_TYPE_LSM = 28, - __ctx_convertBPF_PROG_TYPE_SYSCALL = 29, - __ctx_convert_unused = 30, + __ctx_convertBPF_PROG_TYPE_LIRC_MODE2 = 24, + __ctx_convertBPF_PROG_TYPE_SK_REUSEPORT = 25, + __ctx_convertBPF_PROG_TYPE_SK_LOOKUP = 26, + __ctx_convertBPF_PROG_TYPE_STRUCT_OPS = 27, + __ctx_convertBPF_PROG_TYPE_EXT = 28, + __ctx_convertBPF_PROG_TYPE_LSM = 29, + __ctx_convertBPF_PROG_TYPE_SYSCALL = 30, + __ctx_convert_unused = 31, }; enum bpf_struct_walk_result { @@ -38206,15 +38734,31 @@ struct btf_show_snprintf { int len; }; +enum { + BTF_MODULE_F_LIVE = 1, +}; + struct btf_module { struct list_head list; struct module *module; struct btf *btf; struct bin_attribute *sysfs_attr; + int flags; }; typedef u64 (*btf_bpf_btf_find_by_name_kind)(char *, int, u32, int); +struct bpf_cand_cache { + const char *name; + u32 name_len; + u16 kind; + u16 cnt; + struct { + const struct btf *btf; + u32 id; + } cands[0]; +}; + enum { BPF_F_BROADCAST = 8, BPF_F_EXCLUDE_INGRESS = 16, @@ -38250,6 +38794,42 @@ enum net_device_flags { IFF_ECHO = 262144, }; +enum netdev_priv_flags { + IFF_802_1Q_VLAN = 1, + IFF_EBRIDGE = 2, + IFF_BONDING = 4, + IFF_ISATAP = 8, + IFF_WAN_HDLC = 16, + IFF_XMIT_DST_RELEASE = 32, + IFF_DONT_BRIDGE = 64, + IFF_DISABLE_NETPOLL = 128, + IFF_MACVLAN_PORT = 256, + IFF_BRIDGE_PORT = 512, + IFF_OVS_DATAPATH = 1024, + IFF_TX_SKB_SHARING = 2048, + IFF_UNICAST_FLT = 4096, + IFF_TEAM_PORT = 8192, + IFF_SUPP_NOFCS = 16384, + IFF_LIVE_ADDR_CHANGE = 32768, + IFF_MACVLAN = 65536, + IFF_XMIT_DST_RELEASE_PERM = 131072, + IFF_L3MDEV_MASTER = 262144, + IFF_NO_QUEUE = 524288, + IFF_OPENVSWITCH = 1048576, + IFF_L3MDEV_SLAVE = 2097152, + IFF_TEAM = 4194304, + IFF_RXFH_CONFIGURED = 8388608, + IFF_PHONY_HEADROOM = 16777216, + IFF_MACSEC = 33554432, + IFF_NO_RX_HANDLER = 67108864, + IFF_FAILOVER = 134217728, + IFF_FAILOVER_SLAVE = 268435456, + IFF_L3MDEV_RX_HANDLER = 536870912, + IFF_LIVE_RENAME_OK = 1073741824, + IFF_TX_SKB_NO_LINEAR = 2147483648, + IFF_CHANGE_PROTO_DOWN = 0, +}; + struct xdp_dev_bulk_queue { struct xdp_frame *q[16]; struct list_head flush_node; @@ -38471,11 +39051,6 @@ struct bpf_stack_map { long: 64; }; -struct stack_map_irq_work { - struct irq_work irq_work; - struct mm_struct *mm; -}; - typedef u64 (*btf_bpf_get_stackid)(struct pt_regs *, struct bpf_map *, u64); typedef u64 (*btf_bpf_get_stackid_pe)(struct bpf_perf_event_data_kern *, struct bpf_map *, u64); @@ -38490,13 +39065,6 @@ enum { BPF_F_SYSCTL_BASE_NAME = 1, }; -struct bpf_prog_list { - struct list_head node; - struct bpf_prog *prog; - struct bpf_cgroup_link *link; - struct bpf_cgroup_storage *storage[2]; -}; - struct qdisc_skb_cb { struct { unsigned int pkt_len; @@ -38504,8 +39072,6 @@ struct qdisc_skb_cb { u16 tc_classid; }; unsigned char data[20]; - u16 mru; - bool post_ct; }; struct bpf_skb_data_end { @@ -38518,6 +39084,19 @@ struct bpf_sockopt_buf { u8 data[32]; }; +struct bpf_cgroup_link { + struct bpf_link link; + struct cgroup *cgroup; + enum bpf_attach_type type; +}; + +struct bpf_prog_list { + struct list_head node; + struct bpf_prog *prog; + struct bpf_cgroup_link *link; + struct bpf_cgroup_storage *storage[2]; +}; + enum { TCPF_ESTABLISHED = 2, TCPF_SYN_SENT = 4, @@ -38533,6 +39112,10 @@ enum { TCPF_NEW_SYN_RECV = 4096, }; +typedef u64 (*btf_bpf_get_retval)(); + +typedef u64 (*btf_bpf_set_retval)(int); + typedef u64 (*btf_bpf_sysctl_get_name)(struct bpf_sysctl_kern *, char *, size_t, u64); typedef u64 (*btf_bpf_sysctl_get_current_value)(struct bpf_sysctl_kern *, char *, size_t); @@ -38541,6 +39124,8 @@ typedef u64 (*btf_bpf_sysctl_get_new_value)(struct bpf_sysctl_kern *, char *, si typedef u64 (*btf_bpf_sysctl_set_new_value)(struct bpf_sysctl_kern *, const char *, size_t); +typedef u64 (*btf_bpf_get_netns_cookie_sockopt)(struct bpf_sockopt_kern *); + enum sock_type { SOCK_STREAM = 1, SOCK_DGRAM = 2, @@ -38551,37 +39136,6 @@ enum sock_type { SOCK_PACKET = 10, }; -enum { - IPPROTO_IP = 0, - IPPROTO_ICMP = 1, - IPPROTO_IGMP = 2, - IPPROTO_IPIP = 4, - IPPROTO_TCP = 6, - IPPROTO_EGP = 8, - IPPROTO_PUP = 12, - IPPROTO_UDP = 17, - IPPROTO_IDP = 22, - IPPROTO_TP = 29, - IPPROTO_DCCP = 33, - IPPROTO_IPV6 = 41, - IPPROTO_RSVP = 46, - IPPROTO_GRE = 47, - IPPROTO_ESP = 50, - IPPROTO_AH = 51, - IPPROTO_MTP = 92, - IPPROTO_BEETPH = 94, - IPPROTO_ENCAP = 98, - IPPROTO_PIM = 103, - IPPROTO_COMP = 108, - IPPROTO_SCTP = 132, - IPPROTO_UDPLITE = 136, - IPPROTO_MPLS = 137, - IPPROTO_ETHERNET = 143, - IPPROTO_RAW = 255, - IPPROTO_MPTCP = 262, - IPPROTO_MAX = 263, -}; - enum sock_flags { SOCK_DEAD = 0, SOCK_DONE = 1, @@ -38617,6 +39171,15 @@ struct reuseport_array { struct sock *ptrs[0]; }; +struct bpf_dummy_ops_state { + int val; +}; + +struct bpf_dummy_ops { + int (*test_1)(struct bpf_dummy_ops_state *); + int (*test_2)(struct bpf_dummy_ops_state *, int, short unsigned int, char, long unsigned int); +}; + enum bpf_struct_ops_state { BPF_STRUCT_OPS_STATE_INIT = 0, BPF_STRUCT_OPS_STATE_INUSE = 1, @@ -38638,16 +39201,34 @@ struct bpf_struct_ops_value { struct bpf_struct_ops_map { struct bpf_map map; + struct callback_head rcu; const struct bpf_struct_ops *st_ops; struct mutex lock; struct bpf_prog **progs; void *image; struct bpf_struct_ops_value *uvalue; - long: 64; - long: 64; struct bpf_struct_ops_value kvalue; }; +struct bpf_struct_ops_bpf_dummy_ops { + refcount_t refcnt; + enum bpf_struct_ops_state state; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + struct bpf_dummy_ops data; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; +}; + struct bpf_struct_ops_tcp_congestion_ops { refcount_t refcnt; enum bpf_struct_ops_state state; @@ -38661,6 +39242,16 @@ struct bpf_struct_ops_tcp_congestion_ops { struct tcp_congestion_ops data; }; +enum { + BPF_STRUCT_OPS_TYPE_bpf_dummy_ops = 0, + BPF_STRUCT_OPS_TYPE_tcp_congestion_ops = 1, + __NR_BPF_STRUCT_OPS_TYPE = 2, +}; + +enum { + BPF_F_BPRM_SECUREEXEC = 1, +}; + struct sembuf { short unsigned int sem_num; short int sem_op; @@ -38685,14 +39276,52 @@ struct __key_reference_with_attributes; typedef struct __key_reference_with_attributes *key_ref_t; -enum { - BPF_F_BPRM_SECUREEXEC = 1, +enum hash_algo { + HASH_ALGO_MD4 = 0, + HASH_ALGO_MD5 = 1, + HASH_ALGO_SHA1 = 2, + HASH_ALGO_RIPE_MD_160 = 3, + HASH_ALGO_SHA256 = 4, + HASH_ALGO_SHA384 = 5, + HASH_ALGO_SHA512 = 6, + HASH_ALGO_SHA224 = 7, + HASH_ALGO_RIPE_MD_128 = 8, + HASH_ALGO_RIPE_MD_256 = 9, + HASH_ALGO_RIPE_MD_320 = 10, + HASH_ALGO_WP_256 = 11, + HASH_ALGO_WP_384 = 12, + HASH_ALGO_WP_512 = 13, + HASH_ALGO_TGR_128 = 14, + HASH_ALGO_TGR_160 = 15, + HASH_ALGO_TGR_192 = 16, + HASH_ALGO_SM3_256 = 17, + HASH_ALGO_STREEBOG_256 = 18, + HASH_ALGO_STREEBOG_512 = 19, + HASH_ALGO__LAST = 20, }; typedef u64 (*btf_bpf_bprm_opts_set)(struct linux_binprm *, u64); typedef u64 (*btf_bpf_ima_inode_hash)(struct inode *, void *, u32); +enum libbpf_print_level { + LIBBPF_WARN = 0, + LIBBPF_INFO = 1, + LIBBPF_DEBUG = 2, +}; + +struct bpf_core_relo_res { + __u32 orig_val; + __u32 new_val; + bool poison; + bool validate; + bool fail_memsz_adjust; + __u32 orig_sz; + __u32 orig_type_id; + __u32 new_sz; + __u32 new_type_id; +}; + struct static_call_tramp_key { s32 tramp; s32 key; @@ -38747,7 +39376,8 @@ enum perf_event_type { PERF_RECORD_BPF_EVENT = 18, PERF_RECORD_CGROUP = 19, PERF_RECORD_TEXT_POKE = 20, - PERF_RECORD_MAX = 21, + PERF_RECORD_AUX_OUTPUT_HW_ID = 21, + PERF_RECORD_MAX = 22, }; struct swevent_hlist { @@ -39019,12 +39649,17 @@ enum { }; struct perf_aux_event { + struct perf_event_header header; + u64 hw_id; +}; + +struct perf_aux_event___2 { struct perf_event_header header; u32 pid; u32 tid; }; -struct perf_aux_event___2 { +struct perf_aux_event___3 { struct perf_event_header header; u64 offset; u64 size; @@ -39221,6 +39856,8 @@ struct pkcs7_message; struct asymmetric_key_subtype; +typedef struct pglist_data pg_data_t; + typedef void (*xa_update_node_t)(struct xa_node *); struct xa_state { @@ -39258,11 +39895,11 @@ enum mapping_flags { AS_UNEVICTABLE = 3, AS_EXITING = 4, AS_NO_WRITEBACK_TAGS = 5, - AS_THP_SUPPORT = 6, + AS_LARGE_FOLIO_SUPPORT = 6, }; struct wait_page_key { - struct page *page; + struct folio *folio; int bit_nr; int page_match; }; @@ -39363,6 +40000,7 @@ struct kmem_cache { struct reciprocal_value reciprocal_size; unsigned int offset; unsigned int cpu_partial; + unsigned int cpu_partial_slabs; struct kmem_cache_order_objects oo; struct kmem_cache_order_objects max; struct kmem_cache_order_objects min; @@ -39381,11 +40019,37 @@ struct kmem_cache { struct kmem_cache_node *node[64]; }; +struct slab { + long unsigned int __page_flags; + union { + struct list_head slab_list; + struct callback_head callback_head; + struct { + struct slab *next; + int slabs; + }; + }; + struct kmem_cache *slab_cache; + void *freelist; + union { + long unsigned int counters; + struct { + unsigned int inuse: 16; + unsigned int objects: 15; + unsigned int frozen: 1; + }; + }; + unsigned int __unused; + atomic_t __page_refcount; + long unsigned int memcg_data; +}; + struct kmem_cache_cpu { void **freelist; long unsigned int tid; - struct page *page; - struct page *partial; + struct slab *slab; + struct slab *partial; + local_lock_t lock; }; struct kmem_cache_node { @@ -39561,11 +40225,6 @@ enum wb_state { WB_start_all = 3, }; -enum { - BLK_RW_ASYNC = 0, - BLK_RW_SYNC = 1, -}; - struct wb_lock_cookie { bool locked; long unsigned int flags; @@ -39594,11 +40253,22 @@ struct dirty_throttle_control { long unsigned int pos_ratio; }; -typedef struct pglist_data pg_data_t; +enum migrate_reason { + MR_COMPACTION = 0, + MR_MEMORY_FAILURE = 1, + MR_MEMORY_HOTPLUG = 2, + MR_SYSCALL = 3, + MR_MEMPOLICY_MBIND = 4, + MR_NUMA_MISPLACED = 5, + MR_CONTIG_RANGE = 6, + MR_LONGTERM_PIN = 7, + MR_DEMOTION = 8, + MR_TYPES = 9, +}; struct trace_event_raw_mm_lru_insertion { struct trace_entry ent; - struct page *page; + struct folio *folio; long unsigned int pfn; enum lru_list lru; long unsigned int flags; @@ -39607,7 +40277,7 @@ struct trace_event_raw_mm_lru_insertion { struct trace_event_raw_mm_lru_activate { struct trace_entry ent; - struct page *page; + struct folio *folio; long unsigned int pfn; char __data[0]; }; @@ -39616,9 +40286,9 @@ struct trace_event_data_offsets_mm_lru_insertion {}; struct trace_event_data_offsets_mm_lru_activate {}; -typedef void (*btf_trace_mm_lru_insertion)(void *, struct page *); +typedef void (*btf_trace_mm_lru_insertion)(void *, struct folio *); -typedef void (*btf_trace_mm_lru_activate)(void *, struct page *); +typedef void (*btf_trace_mm_lru_activate)(void *, struct folio *); struct lru_rotate { local_lock_t lock; @@ -39675,16 +40345,10 @@ enum ttu_flags { TTU_RMAP_LOCKED = 128, }; -enum migrate_reason { - MR_COMPACTION = 0, - MR_MEMORY_FAILURE = 1, - MR_MEMORY_HOTPLUG = 2, - MR_SYSCALL = 3, - MR_MEMPOLICY_MBIND = 4, - MR_NUMA_MISPLACED = 5, - MR_CONTIG_RANGE = 6, - MR_LONGTERM_PIN = 7, - MR_TYPES = 8, +struct migration_target_control { + int nid; + nodemask_t *nmask; + gfp_t gfp_mask; }; struct trace_event_raw_mm_vmscan_kswapd_sleep { @@ -39807,6 +40471,15 @@ struct trace_event_raw_mm_vmscan_node_reclaim_begin { char __data[0]; }; +struct trace_event_raw_mm_vmscan_throttled { + struct trace_entry ent; + int nid; + int usec_timeout; + int usec_delayed; + int reason; + char __data[0]; +}; + struct trace_event_data_offsets_mm_vmscan_kswapd_sleep {}; struct trace_event_data_offsets_mm_vmscan_kswapd_wake {}; @@ -39831,6 +40504,8 @@ struct trace_event_data_offsets_mm_vmscan_lru_shrink_active {}; struct trace_event_data_offsets_mm_vmscan_node_reclaim_begin {}; +struct trace_event_data_offsets_mm_vmscan_throttled {}; + typedef void (*btf_trace_mm_vmscan_kswapd_sleep)(void *, int); typedef void (*btf_trace_mm_vmscan_kswapd_wake)(void *, int, int, int); @@ -39865,6 +40540,8 @@ typedef void (*btf_trace_mm_vmscan_node_reclaim_begin)(void *, int, int, gfp_t); typedef void (*btf_trace_mm_vmscan_node_reclaim_end)(void *, long unsigned int); +typedef void (*btf_trace_mm_vmscan_throttled)(void *, int, int, int, int); + struct scan_control { long unsigned int nr_to_reclaim; nodemask_t *nodemask; @@ -39883,6 +40560,7 @@ struct scan_control { unsigned int compaction_ready: 1; unsigned int cache_trim_mode: 1; unsigned int file_is_tiny: 1; + unsigned int no_demotion: 1; s8 order; s8 priority; s8 reclaim_idx; @@ -39951,7 +40629,8 @@ enum { MPOL_BIND = 2, MPOL_INTERLEAVE = 3, MPOL_LOCAL = 4, - MPOL_MAX = 5, + MPOL_PREFERRED_MANY = 5, + MPOL_MAX = 6, }; struct shared_policy { @@ -39977,6 +40656,7 @@ struct shmem_inode_info { long unsigned int flags; long unsigned int alloced; long unsigned int swapped; + long unsigned int fallocend; struct list_head shrinklist; struct list_head swaplist; struct shared_policy policy; @@ -39990,7 +40670,7 @@ struct shmem_sb_info { struct percpu_counter used_blocks; long unsigned int max_inodes; long unsigned int free_inodes; - spinlock_t stat_lock; + raw_spinlock_t stat_lock; umode_t mode; unsigned char huge; kuid_t uid; @@ -40006,11 +40686,10 @@ struct shmem_sb_info { enum sgp_type { SGP_READ = 0, - SGP_CACHE = 1, - SGP_NOHUGE = 2, - SGP_HUGE = 3, - SGP_WRITE = 4, - SGP_FALLOC = 5, + SGP_NOALLOC = 1, + SGP_CACHE = 2, + SGP_WRITE = 3, + SGP_FALLOC = 4, }; enum fid_type { @@ -40372,7 +41051,7 @@ struct slabinfo { struct kmem_obj_info { void *kp_ptr; - struct page *kp_page; + struct slab *kp_slab; void *kp_objp; long unsigned int kp_data_offset; struct kmem_cache *kp_slab_cache; @@ -40550,18 +41229,12 @@ typedef struct { long unsigned int pd; } hugepd_t; -struct migration_target_control { - int nid; - nodemask_t *nmask; - gfp_t gfp_mask; -}; - struct follow_page_context { struct dev_pagemap *pgmap; unsigned int page_mask; }; -struct trace_event_raw_mmap_lock_start_locking { +struct trace_event_raw_mmap_lock { struct trace_entry ent; struct mm_struct *mm; u32 __data_loc_memcg_path; @@ -40578,15 +41251,7 @@ struct trace_event_raw_mmap_lock_acquire_returned { char __data[0]; }; -struct trace_event_raw_mmap_lock_released { - struct trace_entry ent; - struct mm_struct *mm; - u32 __data_loc_memcg_path; - bool write; - char __data[0]; -}; - -struct trace_event_data_offsets_mmap_lock_start_locking { +struct trace_event_data_offsets_mmap_lock { u32 memcg_path; }; @@ -40594,16 +41259,12 @@ struct trace_event_data_offsets_mmap_lock_acquire_returned { u32 memcg_path; }; -struct trace_event_data_offsets_mmap_lock_released { - u32 memcg_path; -}; - typedef void (*btf_trace_mmap_lock_start_locking)(void *, struct mm_struct *, const char *, bool); -typedef void (*btf_trace_mmap_lock_acquire_returned)(void *, struct mm_struct *, const char *, bool, bool); - typedef void (*btf_trace_mmap_lock_released)(void *, struct mm_struct *, const char *, bool); +typedef void (*btf_trace_mmap_lock_acquire_returned)(void *, struct mm_struct *, const char *, bool, bool); + struct memcg_path { local_lock_t lock; char *buf; @@ -40613,9 +41274,7 @@ struct memcg_path { typedef unsigned int pgtbl_mod_mask; struct zap_details { - struct address_space *check_mapping; - long unsigned int first_index; - long unsigned int last_index; + struct address_space *zap_mapping; struct page *single_page; }; @@ -40804,7 +41463,8 @@ enum { BIO_TRACKED = 10, BIO_REMAPPED = 11, BIO_ZONE_WRITE_LOCKED = 12, - BIO_FLAG_LAST = 13, + BIO_PERCPU_CACHE = 13, + BIO_FLAG_LAST = 14, }; struct vma_swap_readahead { @@ -40879,6 +41539,12 @@ enum string_size_units { STRING_UNITS_2 = 1, }; +enum mcopy_atomic_mode { + MCOPY_ATOMIC_NORMAL = 0, + MCOPY_ATOMIC_ZEROPAGE = 1, + MCOPY_ATOMIC_CONTINUE = 2, +}; + enum { SUBPAGE_INDEX_SUBPOOL = 1, SUBPAGE_INDEX_CGROUP = 2, @@ -40961,6 +41627,8 @@ struct sp_node { struct mempolicy *policy; }; +typedef u32 compat_ulong_t; + struct mempolicy_operations { int (*create)(struct mempolicy *, const nodemask_t *); void (*rebind)(struct mempolicy *, const nodemask_t *); @@ -41045,8 +41713,6 @@ enum get_ksm_page_flags { GET_KSM_PAGE_TRYLOCK = 2, }; -typedef u32 depot_stack_handle_t; - enum stat_item { ALLOC_FASTPATH = 0, ALLOC_SLOWPATH = 1, @@ -41079,7 +41745,7 @@ enum stat_item { struct track { long unsigned int addr; - depot_stack_handle_t handle; + long unsigned int addrs[16]; int cpu; int pid; long unsigned int when; @@ -41090,8 +41756,14 @@ enum track_item { TRACK_FREE = 1, }; +struct slub_flush_work { + struct work_struct work; + struct kmem_cache *s; + bool skip; +}; + struct detached_freelist { - struct page *page; + struct slab *slab; void *tail; void *freelist; int cnt; @@ -41114,6 +41786,7 @@ struct loc_track { long unsigned int max; long unsigned int count; struct location *loc; + loff_t idx; }; enum slab_stat_type { @@ -41187,6 +41860,15 @@ enum bh_state_bits { BH_PrivateStart = 16, }; +typedef u32 compat_uptr_t; + +struct memory_notify { + long unsigned int start_pfn; + long unsigned int nr_pages; + int status_change_nid_normal; + int status_change_nid; +}; + struct trace_event_raw_mm_migrate_pages { struct trace_entry ent; long unsigned int succeeded; @@ -41425,7 +42107,7 @@ struct uncharge_gather { long unsigned int nr_memory; long unsigned int pgpgout; long unsigned int nr_kmem; - struct page *dummy_page; + int nid; }; struct numa_stat { @@ -41535,7 +42217,7 @@ struct page_state { long unsigned int mask; long unsigned int res; enum mf_action_page_type type; - int (*action)(struct page *, long unsigned int); + int (*action)(struct page_state *, struct page *); }; struct memory_failure_entry { @@ -41654,16 +42336,12 @@ enum { PAGE_REPORTING_ACTIVE = 2, }; -struct open_how { - __u64 flags; - __u64 mode; - __u64 resolve; -}; - enum fsnotify_data_type { FSNOTIFY_EVENT_NONE = 0, FSNOTIFY_EVENT_PATH = 1, FSNOTIFY_EVENT_INODE = 2, + FSNOTIFY_EVENT_DENTRY = 3, + FSNOTIFY_EVENT_ERROR = 4, }; struct fs_context_operations___2; @@ -41969,8 +42647,6 @@ struct fsxattr { unsigned char fsx_pad[8]; }; -typedef int get_block_t(struct inode *, sector_t, struct buffer_head *, int); - struct fiemap_extent; struct fiemap_extent_info { @@ -42222,7 +42898,7 @@ struct wb_writeback_work { struct wb_completion *done; }; -struct trace_event_raw_writeback_page_template { +struct trace_event_raw_writeback_folio_template { struct trace_entry ent; char name[32]; ino_t ino; @@ -42431,7 +43107,7 @@ struct trace_event_raw_writeback_inode_template { char __data[0]; }; -struct trace_event_data_offsets_writeback_page_template {}; +struct trace_event_data_offsets_writeback_folio_template {}; struct trace_event_data_offsets_writeback_dirty_inode_template {}; @@ -42471,9 +43147,9 @@ struct trace_event_data_offsets_writeback_single_inode_template {}; struct trace_event_data_offsets_writeback_inode_template {}; -typedef void (*btf_trace_writeback_dirty_page)(void *, struct page *, struct address_space *); +typedef void (*btf_trace_writeback_dirty_folio)(void *, struct folio *, struct address_space *); -typedef void (*btf_trace_wait_on_page_writeback)(void *, struct page *, struct address_space *); +typedef void (*btf_trace_folio_wait_writeback)(void *, struct folio *, struct address_space *); typedef void (*btf_trace_writeback_mark_inode_dirty)(void *, struct inode *, int); @@ -42485,7 +43161,7 @@ typedef void (*btf_trace_inode_foreign_history)(void *, struct inode *, struct w typedef void (*btf_trace_inode_switch_wbs)(void *, struct inode *, struct bdi_writeback *, struct bdi_writeback *); -typedef void (*btf_trace_track_foreign_dirty)(void *, struct page *, struct bdi_writeback *); +typedef void (*btf_trace_track_foreign_dirty)(void *, struct folio *, struct bdi_writeback *); typedef void (*btf_trace_flush_foreign)(void *, struct bdi_writeback *, unsigned int, unsigned int); @@ -42523,8 +43199,6 @@ typedef void (*btf_trace_writeback_sb_inodes_requeue)(void *, struct inode *); typedef void (*btf_trace_writeback_congestion_wait)(void *, unsigned int, unsigned int); -typedef void (*btf_trace_writeback_wait_iff_congested)(void *, unsigned int, unsigned int); - typedef void (*btf_trace_writeback_single_inode_start)(void *, struct inode *, struct writeback_control *, long unsigned int); typedef void (*btf_trace_writeback_single_inode)(void *, struct inode *, struct writeback_control *, long unsigned int); @@ -42648,6 +43322,8 @@ enum fsconfig_command { FSCONFIG_CMD_RECONFIGURE = 7, }; +typedef int get_block_t(struct inode *, sector_t, struct buffer_head *, int); + struct dax_device; struct iomap_page_ops; @@ -42666,8 +43342,15 @@ struct iomap___2 { }; struct iomap_page_ops { - int (*page_prepare)(struct inode *, loff_t, unsigned int, struct iomap___2 *); - void (*page_done)(struct inode *, loff_t, unsigned int, struct page *, struct iomap___2 *); + int (*page_prepare)(struct inode *, loff_t, unsigned int); + void (*page_done)(struct inode *, loff_t, unsigned int, struct page *); +}; + +enum hctx_type { + HCTX_TYPE_DEFAULT = 0, + HCTX_TYPE_READ = 1, + HCTX_TYPE_POLL = 2, + HCTX_MAX_TYPES = 3, }; struct decrypt_bh_ctx { @@ -42684,58 +43367,6 @@ struct bh_accounting { int ratelimit; }; -enum stat_group { - STAT_READ = 0, - STAT_WRITE = 1, - STAT_DISCARD = 2, - STAT_FLUSH = 3, - NR_STAT_GROUPS = 4, -}; - -enum { - DISK_EVENT_MEDIA_CHANGE = 1, - DISK_EVENT_EJECT_REQUEST = 2, -}; - -struct blk_integrity_profile; - -struct blk_integrity { - const struct blk_integrity_profile *profile; - unsigned char flags; - unsigned char tuple_size; - unsigned char interval_exp; - unsigned char tag_size; -}; - -enum { - BIOSET_NEED_BVECS = 1, - BIOSET_NEED_RESCUER = 2, -}; - -struct bdev_inode { - struct block_device bdev; - struct inode vfs_inode; -}; - -struct blkdev_dio { - union { - struct kiocb *iocb; - struct task_struct *waiter; - }; - size_t size; - atomic_t ref; - bool multi_bio: 1; - bool should_dirty: 1; - bool is_sync: 1; - struct bio bio; -}; - -struct bd_holder_disk { - struct list_head list; - struct gendisk *disk; - int refcnt; -}; - typedef int dio_iodone_t(struct kiocb *, loff_t, ssize_t, void *); typedef void dio_submit_t(struct bio *, struct inode *, loff_t); @@ -42777,7 +43408,6 @@ struct dio { int flags; int op; int op_flags; - blk_qc_t bio_cookie; struct gendisk *bio_disk; struct inode *inode; loff_t i_size; @@ -42887,6 +43517,12 @@ struct proc_fs_opts { const char *str; }; +struct fs_error_report { + int error; + struct inode *inode; + struct super_block *sb; +}; + struct file_handle { __u32 handle_bytes; int handle_type; @@ -43071,6 +43707,120 @@ struct eventfd_ctx___2 { int id; }; +struct userfaultfd_ctx { + wait_queue_head_t fault_pending_wqh; + wait_queue_head_t fault_wqh; + wait_queue_head_t fd_wqh; + wait_queue_head_t event_wqh; + seqcount_spinlock_t refile_seq; + refcount_t refcount; + unsigned int flags; + unsigned int features; + bool released; + atomic_t mmap_changing; + struct mm_struct *mm; +}; + +struct uffd_msg { + __u8 event; + __u8 reserved1; + __u16 reserved2; + __u32 reserved3; + union { + struct { + __u64 flags; + __u64 address; + union { + __u32 ptid; + } feat; + } pagefault; + struct { + __u32 ufd; + } fork; + struct { + __u64 from; + __u64 to; + __u64 len; + } remap; + struct { + __u64 start; + __u64 end; + } remove; + struct { + __u64 reserved1; + __u64 reserved2; + __u64 reserved3; + } reserved; + } arg; +}; + +struct uffdio_api { + __u64 api; + __u64 features; + __u64 ioctls; +}; + +struct uffdio_range { + __u64 start; + __u64 len; +}; + +struct uffdio_register { + struct uffdio_range range; + __u64 mode; + __u64 ioctls; +}; + +struct uffdio_copy { + __u64 dst; + __u64 src; + __u64 len; + __u64 mode; + __s64 copy; +}; + +struct uffdio_zeropage { + struct uffdio_range range; + __u64 mode; + __s64 zeropage; +}; + +struct uffdio_writeprotect { + struct uffdio_range range; + __u64 mode; +}; + +struct uffdio_continue { + struct uffdio_range range; + __u64 mode; + __s64 mapped; +}; + +struct userfaultfd_fork_ctx { + struct userfaultfd_ctx *orig; + struct userfaultfd_ctx *new; + struct list_head list; +}; + +struct userfaultfd_unmap_ctx { + struct userfaultfd_ctx *ctx; + long unsigned int start; + long unsigned int end; + struct list_head list; +}; + +struct userfaultfd_wait_queue { + struct uffd_msg msg; + wait_queue_entry_t wq; + struct userfaultfd_ctx *ctx; + bool waken; +}; + +struct userfaultfd_wake_range { + long unsigned int start; + long unsigned int len; +}; + struct kioctx; struct kioctx_table { @@ -43226,8 +43976,9 @@ struct poll_iocb { struct file *file; struct wait_queue_head *head; __poll_t events; - bool done; bool cancelled; + bool work_scheduled; + bool work_need_resched; struct wait_queue_entry wait; struct work_struct work; }; @@ -43250,6 +44001,7 @@ struct aio_kiocb { struct aio_poll_table { struct poll_table_struct pt; struct aio_kiocb *iocb; + bool queued; int error; }; @@ -43285,8 +44037,14 @@ struct io_uring_task { atomic_t in_idle; spinlock_t task_lock; struct io_wq_work_list task_list; - long unsigned int task_state; struct callback_head task_work; + bool task_running; +}; + +struct iov_iter_state { + size_t iov_offset; + size_t count; + long unsigned int nr_segs; }; struct user_msghdr { @@ -43305,8 +44063,6 @@ typedef s32 compat_int_t; typedef u32 compat_uint_t; -typedef u32 compat_uptr_t; - struct compat_msghdr { compat_uptr_t msg_name; compat_int_t msg_namelen; @@ -43333,6 +44089,233 @@ struct unix_skb_parms { u32 consumed; }; +struct io_uring_sqe { + __u8 opcode; + __u8 flags; + __u16 ioprio; + __s32 fd; + union { + __u64 off; + __u64 addr2; + }; + union { + __u64 addr; + __u64 splice_off_in; + }; + __u32 len; + union { + __kernel_rwf_t rw_flags; + __u32 fsync_flags; + __u16 poll_events; + __u32 poll32_events; + __u32 sync_range_flags; + __u32 msg_flags; + __u32 timeout_flags; + __u32 accept_flags; + __u32 cancel_flags; + __u32 open_flags; + __u32 statx_flags; + __u32 fadvise_advice; + __u32 splice_flags; + __u32 rename_flags; + __u32 unlink_flags; + __u32 hardlink_flags; + }; + __u64 user_data; + union { + __u16 buf_index; + __u16 buf_group; + }; + __u16 personality; + union { + __s32 splice_fd_in; + __u32 file_index; + }; + __u64 __pad2[2]; +}; + +enum { + IOSQE_FIXED_FILE_BIT = 0, + IOSQE_IO_DRAIN_BIT = 1, + IOSQE_IO_LINK_BIT = 2, + IOSQE_IO_HARDLINK_BIT = 3, + IOSQE_ASYNC_BIT = 4, + IOSQE_BUFFER_SELECT_BIT = 5, +}; + +enum { + IORING_OP_NOP = 0, + IORING_OP_READV = 1, + IORING_OP_WRITEV = 2, + IORING_OP_FSYNC = 3, + IORING_OP_READ_FIXED = 4, + IORING_OP_WRITE_FIXED = 5, + IORING_OP_POLL_ADD = 6, + IORING_OP_POLL_REMOVE = 7, + IORING_OP_SYNC_FILE_RANGE = 8, + IORING_OP_SENDMSG = 9, + IORING_OP_RECVMSG = 10, + IORING_OP_TIMEOUT = 11, + IORING_OP_TIMEOUT_REMOVE = 12, + IORING_OP_ACCEPT = 13, + IORING_OP_ASYNC_CANCEL = 14, + IORING_OP_LINK_TIMEOUT = 15, + IORING_OP_CONNECT = 16, + IORING_OP_FALLOCATE = 17, + IORING_OP_OPENAT = 18, + IORING_OP_CLOSE = 19, + IORING_OP_FILES_UPDATE = 20, + IORING_OP_STATX = 21, + IORING_OP_READ = 22, + IORING_OP_WRITE = 23, + IORING_OP_FADVISE = 24, + IORING_OP_MADVISE = 25, + IORING_OP_SEND = 26, + IORING_OP_RECV = 27, + IORING_OP_OPENAT2 = 28, + IORING_OP_EPOLL_CTL = 29, + IORING_OP_SPLICE = 30, + IORING_OP_PROVIDE_BUFFERS = 31, + IORING_OP_REMOVE_BUFFERS = 32, + IORING_OP_TEE = 33, + IORING_OP_SHUTDOWN = 34, + IORING_OP_RENAMEAT = 35, + IORING_OP_UNLINKAT = 36, + IORING_OP_MKDIRAT = 37, + IORING_OP_SYMLINKAT = 38, + IORING_OP_LINKAT = 39, + IORING_OP_LAST = 40, +}; + +struct io_uring_cqe { + __u64 user_data; + __s32 res; + __u32 flags; +}; + +enum { + IORING_CQE_BUFFER_SHIFT = 16, +}; + +struct io_sqring_offsets { + __u32 head; + __u32 tail; + __u32 ring_mask; + __u32 ring_entries; + __u32 flags; + __u32 dropped; + __u32 array; + __u32 resv1; + __u64 resv2; +}; + +struct io_cqring_offsets { + __u32 head; + __u32 tail; + __u32 ring_mask; + __u32 ring_entries; + __u32 overflow; + __u32 cqes; + __u32 flags; + __u32 resv1; + __u64 resv2; +}; + +struct io_uring_params { + __u32 sq_entries; + __u32 cq_entries; + __u32 flags; + __u32 sq_thread_cpu; + __u32 sq_thread_idle; + __u32 features; + __u32 wq_fd; + __u32 resv[3]; + struct io_sqring_offsets sq_off; + struct io_cqring_offsets cq_off; +}; + +enum { + IORING_REGISTER_BUFFERS = 0, + IORING_UNREGISTER_BUFFERS = 1, + IORING_REGISTER_FILES = 2, + IORING_UNREGISTER_FILES = 3, + IORING_REGISTER_EVENTFD = 4, + IORING_UNREGISTER_EVENTFD = 5, + IORING_REGISTER_FILES_UPDATE = 6, + IORING_REGISTER_EVENTFD_ASYNC = 7, + IORING_REGISTER_PROBE = 8, + IORING_REGISTER_PERSONALITY = 9, + IORING_UNREGISTER_PERSONALITY = 10, + IORING_REGISTER_RESTRICTIONS = 11, + IORING_REGISTER_ENABLE_RINGS = 12, + IORING_REGISTER_FILES2 = 13, + IORING_REGISTER_FILES_UPDATE2 = 14, + IORING_REGISTER_BUFFERS2 = 15, + IORING_REGISTER_BUFFERS_UPDATE = 16, + IORING_REGISTER_IOWQ_AFF = 17, + IORING_UNREGISTER_IOWQ_AFF = 18, + IORING_REGISTER_IOWQ_MAX_WORKERS = 19, + IORING_REGISTER_LAST = 20, +}; + +struct io_uring_rsrc_register { + __u32 nr; + __u32 resv; + __u64 resv2; + __u64 data; + __u64 tags; +}; + +struct io_uring_rsrc_update2 { + __u32 offset; + __u32 resv; + __u64 data; + __u64 tags; + __u32 nr; + __u32 resv2; +}; + +struct io_uring_probe_op { + __u8 op; + __u8 resv; + __u16 flags; + __u32 resv2; +}; + +struct io_uring_probe { + __u8 last_op; + __u8 ops_len; + __u16 resv; + __u32 resv2[3]; + struct io_uring_probe_op ops[0]; +}; + +struct io_uring_restriction { + __u16 opcode; + union { + __u8 register_op; + __u8 sqe_op; + __u8 sqe_flags; + }; + __u8 resv; + __u32 resv2[3]; +}; + +enum { + IORING_RESTRICTION_REGISTER_OP = 0, + IORING_RESTRICTION_SQE_OP = 1, + IORING_RESTRICTION_SQE_FLAGS_ALLOWED = 2, + IORING_RESTRICTION_SQE_FLAGS_REQUIRED = 3, + IORING_RESTRICTION_LAST = 4, +}; + +struct io_uring_getevents_arg { + __u64 sigmask; + __u32 sigmask_sz; + __u32 pad; + __u64 ts; +}; + struct trace_event_raw_io_uring_create { struct trace_entry ent; int fd; @@ -43416,7 +44399,7 @@ struct trace_event_raw_io_uring_complete { struct trace_entry ent; void *ctx; u64 user_data; - long int res; + int res; unsigned int cflags; char __data[0]; }; @@ -43471,6 +44454,25 @@ struct trace_event_raw_io_uring_task_run { char __data[0]; }; +struct trace_event_raw_io_uring_req_failed { + struct trace_entry ent; + u8 opcode; + u8 flags; + u8 ioprio; + u64 off; + u64 addr; + u32 len; + u32 op_flags; + u64 user_data; + u16 buf_index; + u16 personality; + u32 file_index; + u64 pad1; + u64 pad2; + int error; + char __data[0]; +}; + struct trace_event_data_offsets_io_uring_create {}; struct trace_event_data_offsets_io_uring_register {}; @@ -43499,6 +44501,8 @@ struct trace_event_data_offsets_io_uring_task_add {}; struct trace_event_data_offsets_io_uring_task_run {}; +struct trace_event_data_offsets_io_uring_req_failed {}; + typedef void (*btf_trace_io_uring_create)(void *, int, void *, u32, u32, u32); typedef void (*btf_trace_io_uring_register)(void *, void *, unsigned int, unsigned int, unsigned int, bool, long int); @@ -43515,7 +44519,7 @@ typedef void (*btf_trace_io_uring_cqring_wait)(void *, void *, int); typedef void (*btf_trace_io_uring_fail_link)(void *, void *, void *); -typedef void (*btf_trace_io_uring_complete)(void *, void *, u64, long int, unsigned int); +typedef void (*btf_trace_io_uring_complete)(void *, void *, u64, int, unsigned int); typedef void (*btf_trace_io_uring_submit_sqe)(void *, void *, void *, u8, u64, u32, bool, bool); @@ -43527,224 +44531,7 @@ typedef void (*btf_trace_io_uring_task_add)(void *, void *, u8, u64, int); typedef void (*btf_trace_io_uring_task_run)(void *, void *, void *, u8, u64); -struct io_uring_sqe { - __u8 opcode; - __u8 flags; - __u16 ioprio; - __s32 fd; - union { - __u64 off; - __u64 addr2; - }; - union { - __u64 addr; - __u64 splice_off_in; - }; - __u32 len; - union { - __kernel_rwf_t rw_flags; - __u32 fsync_flags; - __u16 poll_events; - __u32 poll32_events; - __u32 sync_range_flags; - __u32 msg_flags; - __u32 timeout_flags; - __u32 accept_flags; - __u32 cancel_flags; - __u32 open_flags; - __u32 statx_flags; - __u32 fadvise_advice; - __u32 splice_flags; - __u32 rename_flags; - __u32 unlink_flags; - }; - __u64 user_data; - union { - __u16 buf_index; - __u16 buf_group; - }; - __u16 personality; - __s32 splice_fd_in; - __u64 __pad2[2]; -}; - -enum { - IOSQE_FIXED_FILE_BIT = 0, - IOSQE_IO_DRAIN_BIT = 1, - IOSQE_IO_LINK_BIT = 2, - IOSQE_IO_HARDLINK_BIT = 3, - IOSQE_ASYNC_BIT = 4, - IOSQE_BUFFER_SELECT_BIT = 5, -}; - -enum { - IORING_OP_NOP = 0, - IORING_OP_READV = 1, - IORING_OP_WRITEV = 2, - IORING_OP_FSYNC = 3, - IORING_OP_READ_FIXED = 4, - IORING_OP_WRITE_FIXED = 5, - IORING_OP_POLL_ADD = 6, - IORING_OP_POLL_REMOVE = 7, - IORING_OP_SYNC_FILE_RANGE = 8, - IORING_OP_SENDMSG = 9, - IORING_OP_RECVMSG = 10, - IORING_OP_TIMEOUT = 11, - IORING_OP_TIMEOUT_REMOVE = 12, - IORING_OP_ACCEPT = 13, - IORING_OP_ASYNC_CANCEL = 14, - IORING_OP_LINK_TIMEOUT = 15, - IORING_OP_CONNECT = 16, - IORING_OP_FALLOCATE = 17, - IORING_OP_OPENAT = 18, - IORING_OP_CLOSE = 19, - IORING_OP_FILES_UPDATE = 20, - IORING_OP_STATX = 21, - IORING_OP_READ = 22, - IORING_OP_WRITE = 23, - IORING_OP_FADVISE = 24, - IORING_OP_MADVISE = 25, - IORING_OP_SEND = 26, - IORING_OP_RECV = 27, - IORING_OP_OPENAT2 = 28, - IORING_OP_EPOLL_CTL = 29, - IORING_OP_SPLICE = 30, - IORING_OP_PROVIDE_BUFFERS = 31, - IORING_OP_REMOVE_BUFFERS = 32, - IORING_OP_TEE = 33, - IORING_OP_SHUTDOWN = 34, - IORING_OP_RENAMEAT = 35, - IORING_OP_UNLINKAT = 36, - IORING_OP_LAST = 37, -}; - -struct io_uring_cqe { - __u64 user_data; - __s32 res; - __u32 flags; -}; - -enum { - IORING_CQE_BUFFER_SHIFT = 16, -}; - -struct io_sqring_offsets { - __u32 head; - __u32 tail; - __u32 ring_mask; - __u32 ring_entries; - __u32 flags; - __u32 dropped; - __u32 array; - __u32 resv1; - __u64 resv2; -}; - -struct io_cqring_offsets { - __u32 head; - __u32 tail; - __u32 ring_mask; - __u32 ring_entries; - __u32 overflow; - __u32 cqes; - __u32 flags; - __u32 resv1; - __u64 resv2; -}; - -struct io_uring_params { - __u32 sq_entries; - __u32 cq_entries; - __u32 flags; - __u32 sq_thread_cpu; - __u32 sq_thread_idle; - __u32 features; - __u32 wq_fd; - __u32 resv[3]; - struct io_sqring_offsets sq_off; - struct io_cqring_offsets cq_off; -}; - -enum { - IORING_REGISTER_BUFFERS = 0, - IORING_UNREGISTER_BUFFERS = 1, - IORING_REGISTER_FILES = 2, - IORING_UNREGISTER_FILES = 3, - IORING_REGISTER_EVENTFD = 4, - IORING_UNREGISTER_EVENTFD = 5, - IORING_REGISTER_FILES_UPDATE = 6, - IORING_REGISTER_EVENTFD_ASYNC = 7, - IORING_REGISTER_PROBE = 8, - IORING_REGISTER_PERSONALITY = 9, - IORING_UNREGISTER_PERSONALITY = 10, - IORING_REGISTER_RESTRICTIONS = 11, - IORING_REGISTER_ENABLE_RINGS = 12, - IORING_REGISTER_FILES2 = 13, - IORING_REGISTER_FILES_UPDATE2 = 14, - IORING_REGISTER_BUFFERS2 = 15, - IORING_REGISTER_BUFFERS_UPDATE = 16, - IORING_REGISTER_IOWQ_AFF = 17, - IORING_UNREGISTER_IOWQ_AFF = 18, - IORING_REGISTER_LAST = 19, -}; - -struct io_uring_rsrc_register { - __u32 nr; - __u32 resv; - __u64 resv2; - __u64 data; - __u64 tags; -}; - -struct io_uring_rsrc_update2 { - __u32 offset; - __u32 resv; - __u64 data; - __u64 tags; - __u32 nr; - __u32 resv2; -}; - -struct io_uring_probe_op { - __u8 op; - __u8 resv; - __u16 flags; - __u32 resv2; -}; - -struct io_uring_probe { - __u8 last_op; - __u8 ops_len; - __u16 resv; - __u32 resv2[3]; - struct io_uring_probe_op ops[0]; -}; - -struct io_uring_restriction { - __u16 opcode; - union { - __u8 register_op; - __u8 sqe_op; - __u8 sqe_flags; - }; - __u8 resv; - __u32 resv2[3]; -}; - -enum { - IORING_RESTRICTION_REGISTER_OP = 0, - IORING_RESTRICTION_SQE_OP = 1, - IORING_RESTRICTION_SQE_FLAGS_ALLOWED = 2, - IORING_RESTRICTION_SQE_FLAGS_REQUIRED = 3, - IORING_RESTRICTION_LAST = 4, -}; - -struct io_uring_getevents_arg { - __u64 sigmask; - __u32 sigmask_sz; - __u32 pad; - __u64 ts; -}; +typedef void (*btf_trace_io_uring_req_failed)(void *, const struct io_uring_sqe *, int); enum { IO_WQ_WORK_CANCEL = 1, @@ -43817,8 +44604,9 @@ struct io_rings { }; enum io_uring_cmd_flags { - IO_URING_F_NONBLOCK = 1, - IO_URING_F_COMPLETE_DEFER = 2, + IO_URING_F_COMPLETE_DEFER = 1, + IO_URING_F_UNLOCKED = 2, + IO_URING_F_NONBLOCK = 2147483648, }; struct io_mapped_ubuf { @@ -43849,7 +44637,7 @@ struct io_rsrc_put { }; struct io_file_table { - struct io_fixed_file **files; + struct io_fixed_file *files; }; struct io_rsrc_data; @@ -43882,23 +44670,14 @@ struct io_submit_link { struct io_kiocb *last; }; -struct io_comp_state { - struct io_kiocb *reqs[32]; - unsigned int nr; - struct list_head free_list; -}; - struct io_submit_state { - struct blk_plug plug; + struct io_wq_work_node free_list; + struct io_wq_work_list compl_reqs; struct io_submit_link link; - void *reqs[32]; - unsigned int free_reqs; bool plug_started; - struct io_comp_state comp; - struct file *file; - unsigned int fd; - unsigned int file_refs; - unsigned int ios_left; + bool need_plug; + short unsigned int submit_nr; + struct blk_plug plug; }; struct io_restriction { @@ -43936,12 +44715,14 @@ struct io_ring_ctx { unsigned int sq_entries; struct list_head defer_list; struct io_rsrc_node *rsrc_node; + int rsrc_cached_refs; struct io_file_table file_table; unsigned int nr_user_files; unsigned int nr_user_bufs; struct io_mapped_ubuf **user_bufs; struct io_submit_state submit_state; struct list_head timeout_list; + struct list_head ltimeout_list; struct list_head cq_overflow_list; struct xarray io_buffers; struct xarray personalities; @@ -43954,7 +44735,7 @@ struct io_ring_ctx { long: 64; long: 64; }; - struct list_head locked_free_list; + struct io_wq_work_list locked_free_list; unsigned int locked_free_nr; const struct cred *sq_creds; struct io_sq_data *sq_data; @@ -43971,24 +44752,18 @@ struct io_ring_ctx { unsigned int cached_cq_tail; unsigned int cq_entries; struct eventfd_ctx *cq_ev_fd; - struct wait_queue_head poll_wait; struct wait_queue_head cq_wait; unsigned int cq_extra; atomic_t cq_timeouts; - struct fasync_struct *cq_fasync; unsigned int cq_last_tm_flush; long: 32; long: 64; long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; }; struct { spinlock_t completion_lock; - struct list_head iopoll_list; + spinlock_t timeout_lock; + struct io_wq_work_list iopoll_list; struct hlist_head *cancel_hash; unsigned int cancel_hash_bits; bool poll_multi_queue; @@ -44019,8 +44794,16 @@ struct io_ring_ctx { struct work_struct exit_work; struct list_head tctx_list; struct completion ref_comp; + u32 iowq_limits[2]; + bool iowq_limits_set; }; long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; }; struct io_buffer { @@ -44079,6 +44862,7 @@ struct io_accept { struct sockaddr *addr; int *addr_len; int flags; + u32 file_slot; long unsigned int nofile; }; @@ -44101,6 +44885,7 @@ struct io_timeout { u32 target_seq; struct list_head list; struct io_kiocb *head; + struct io_kiocb *prev; }; struct io_timeout_rem { @@ -44108,6 +44893,7 @@ struct io_timeout_rem { u64 addr; struct timespec64 ts; u32 flags; + bool ltimeout; }; struct io_connect { @@ -44126,12 +44912,12 @@ struct io_sr_msg { int msg_flags; int bgid; size_t len; - struct io_buffer *kbuf; }; struct io_open { struct file *file; int dfd; + u32 file_slot; struct filename *filename; struct open_how how; long unsigned int nofile; @@ -44140,6 +44926,7 @@ struct io_open { struct io_close { struct file *file; int fd; + u32 file_slot; }; struct io_rsrc_update { @@ -44219,13 +45006,30 @@ struct io_unlink { struct filename *filename; }; -struct io_completion { +struct io_mkdir { struct file *file; - struct list_head list; - u32 cflags; + int dfd; + umode_t mode; + struct filename *filename; }; -typedef void (*io_req_tw_func_t)(struct io_kiocb *); +struct io_symlink { + struct file *file; + int new_dfd; + struct filename *oldpath; + struct filename *newpath; +}; + +struct io_hardlink { + struct file *file; + int old_dfd; + int new_dfd; + struct filename *oldpath; + struct filename *newpath; + int flags; +}; + +typedef void (*io_req_tw_func_t)(struct io_kiocb *, bool *); struct io_task_work { union { @@ -44262,27 +45066,31 @@ struct io_kiocb { struct io_shutdown shutdown; struct io_rename rename; struct io_unlink unlink; - struct io_completion compl; + struct io_mkdir mkdir; + struct io_symlink symlink; + struct io_hardlink hardlink; }; - void *async_data; u8 opcode; u8 iopoll_completed; u16 buf_index; - u32 result; - struct io_ring_ctx *ctx; unsigned int flags; - atomic_t refs; - struct task_struct *task; u64 user_data; - struct io_kiocb *link; + u32 result; + u32 cflags; + struct io_ring_ctx *ctx; + struct task_struct *task; struct percpu_ref *fixed_rsrc_refs; - struct list_head inflight_entry; + struct io_mapped_ubuf *imu; + struct io_wq_work_node comp_list; + atomic_t refs; + struct io_kiocb *link; struct io_task_work io_task_work; struct hlist_node hash_node; struct async_poll *apoll; + void *async_data; struct io_wq_work work; const struct cred *creds; - struct io_mapped_ubuf *imu; + struct io_buffer *kbuf; }; struct io_timeout_data { @@ -44290,6 +45098,7 @@ struct io_timeout_data { struct hrtimer timer; struct timespec64 ts; enum hrtimer_mode mode; + u32 flags; }; struct io_async_connect { @@ -44304,10 +45113,15 @@ struct io_async_msghdr { struct __kernel_sockaddr_storage addr; }; -struct io_async_rw { - struct iovec fast_iov[8]; - const struct iovec *free_iovec; +struct io_rw_state { struct iov_iter iter; + struct iov_iter_state iter_state; + struct iovec fast_iov[8]; +}; + +struct io_async_rw { + struct io_rw_state s; + const struct iovec *free_iovec; size_t bytes_done; struct wait_page_queue wpq; }; @@ -44327,13 +45141,13 @@ enum { REQ_F_NEED_CLEANUP_BIT = 13, REQ_F_POLLED_BIT = 14, REQ_F_BUFFER_SELECTED_BIT = 15, - REQ_F_LTIMEOUT_ACTIVE_BIT = 16, - REQ_F_COMPLETE_INLINE_BIT = 17, - REQ_F_REISSUE_BIT = 18, - REQ_F_DONT_REISSUE_BIT = 19, - REQ_F_CREDS_BIT = 20, - REQ_F_ASYNC_READ_BIT = 21, - REQ_F_ASYNC_WRITE_BIT = 22, + REQ_F_COMPLETE_INLINE_BIT = 16, + REQ_F_REISSUE_BIT = 17, + REQ_F_CREDS_BIT = 18, + REQ_F_REFCOUNT_BIT = 19, + REQ_F_ARM_LTIMEOUT_BIT = 20, + REQ_F_ASYNC_DATA_BIT = 21, + REQ_F_SUPPORT_NOWAIT_BIT = 22, REQ_F_ISREG_BIT = 23, __REQ_F_LAST_BIT = 24, }; @@ -44353,14 +45167,14 @@ enum { REQ_F_NEED_CLEANUP = 8192, REQ_F_POLLED = 16384, REQ_F_BUFFER_SELECTED = 32768, - REQ_F_LTIMEOUT_ACTIVE = 65536, - REQ_F_COMPLETE_INLINE = 131072, - REQ_F_REISSUE = 262144, - REQ_F_DONT_REISSUE = 524288, - REQ_F_ASYNC_READ = 2097152, - REQ_F_ASYNC_WRITE = 4194304, + REQ_F_COMPLETE_INLINE = 65536, + REQ_F_REISSUE = 131072, + REQ_F_SUPPORT_NOWAIT = 4194304, REQ_F_ISREG = 8388608, - REQ_F_CREDS = 1048576, + REQ_F_CREDS = 262144, + REQ_F_REFCOUNT = 524288, + REQ_F_ARM_LTIMEOUT = 1048576, + REQ_F_ASYNC_DATA = 2097152, }; struct async_poll { @@ -44387,26 +45201,22 @@ struct io_defer_entry { struct io_op_def { unsigned int needs_file: 1; + unsigned int plug: 1; unsigned int hash_reg_file: 1; unsigned int unbound_nonreg_file: 1; - unsigned int not_supported: 1; unsigned int pollin: 1; unsigned int pollout: 1; unsigned int buffer_select: 1; unsigned int needs_async_setup: 1; - unsigned int plug: 1; + unsigned int not_supported: 1; + unsigned int audit_skip: 1; short unsigned int async_size; }; -struct req_batch { - struct task_struct *task; - int task_refs; - int ctx_refs; -}; - struct io_poll_table { struct poll_table_struct pt; struct io_kiocb *req; + int nr_entries; int error; }; @@ -44424,7 +45234,7 @@ struct io_cancel_data { struct io_wait_queue { struct wait_queue_entry wq; struct io_ring_ctx *ctx; - unsigned int to_wait; + unsigned int cq_tail; unsigned int nr_timeouts; }; @@ -44441,14 +45251,18 @@ struct io_task_cancel { struct creds; +enum { + IO_WQ_BOUND = 0, + IO_WQ_UNBOUND = 1, +}; + typedef bool work_cancel_fn(struct io_wq_work *, void *); enum { IO_WORKER_F_UP = 1, IO_WORKER_F_RUNNING = 2, IO_WORKER_F_FREE = 4, - IO_WORKER_F_FIXED = 8, - IO_WORKER_F_BOUND = 16, + IO_WORKER_F_BOUND = 8, }; enum { @@ -44456,7 +45270,7 @@ enum { }; enum { - IO_WQE_FLAG_STALLED = 1, + IO_ACCT_STALLED_BIT = 0, }; struct io_wqe; @@ -44471,7 +45285,13 @@ struct io_worker { struct io_wq_work *cur_work; spinlock_t lock; struct completion ref_done; - struct callback_head rcu; + long unsigned int create_state; + struct callback_head create_work; + int create_index; + union { + struct callback_head rcu; + struct work_struct work; + }; }; struct io_wqe_acct { @@ -44479,24 +45299,16 @@ struct io_wqe_acct { unsigned int max_workers; int index; atomic_t nr_running; + struct io_wq_work_list work_list; + long unsigned int flags; }; struct io_wq___2; struct io_wqe { - struct { - raw_spinlock_t lock; - struct io_wq_work_list work_list; - unsigned int flags; - long: 32; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - }; - int node; + raw_spinlock_t lock; struct io_wqe_acct acct[2]; + int node; struct hlist_nulls_head free_list; struct list_head all_list; struct wait_queue_entry wait; @@ -44508,6 +45320,7 @@ struct io_wqe { enum { IO_WQ_ACCT_BOUND = 0, IO_WQ_ACCT_UNBOUND = 1, + IO_WQ_ACCT_NR = 2, }; struct io_wq___2 { @@ -44530,12 +45343,6 @@ struct io_cb_cancel_data { bool cancel_all; }; -struct create_worker_data { - struct callback_head work; - struct io_wqe *wqe; - int index; -}; - struct online_data { unsigned int cpu; bool online; @@ -44781,6 +45588,16 @@ struct core_name { int size; }; +struct iomap_iter { + struct inode *inode; + loff_t pos; + u64 len; + s64 processed; + unsigned int flags; + struct iomap___2 iomap; + struct iomap___2 srcmap; +}; + struct trace_event_raw_iomap_readpage_class { struct trace_entry ent; dev_t dev; @@ -44794,8 +45611,8 @@ struct trace_event_raw_iomap_range_class { dev_t dev; u64 ino; loff_t size; - long unsigned int offset; - unsigned int length; + loff_t offset; + u64 length; char __data[0]; }; @@ -44812,15 +45629,14 @@ struct trace_event_raw_iomap_class { char __data[0]; }; -struct trace_event_raw_iomap_apply { +struct trace_event_raw_iomap_iter { struct trace_entry ent; dev_t dev; u64 ino; loff_t pos; - loff_t length; + u64 length; unsigned int flags; const void *ops; - void *actor; long unsigned int caller; char __data[0]; }; @@ -44831,33 +45647,37 @@ struct trace_event_data_offsets_iomap_range_class {}; struct trace_event_data_offsets_iomap_class {}; -struct trace_event_data_offsets_iomap_apply {}; +struct trace_event_data_offsets_iomap_iter {}; typedef void (*btf_trace_iomap_readpage)(void *, struct inode *, int); typedef void (*btf_trace_iomap_readahead)(void *, struct inode *, int); -typedef void (*btf_trace_iomap_writepage)(void *, struct inode *, long unsigned int, unsigned int); +typedef void (*btf_trace_iomap_writepage)(void *, struct inode *, loff_t, u64); -typedef void (*btf_trace_iomap_releasepage)(void *, struct inode *, long unsigned int, unsigned int); +typedef void (*btf_trace_iomap_releasepage)(void *, struct inode *, loff_t, u64); -typedef void (*btf_trace_iomap_invalidatepage)(void *, struct inode *, long unsigned int, unsigned int); +typedef void (*btf_trace_iomap_invalidatepage)(void *, struct inode *, loff_t, u64); -typedef void (*btf_trace_iomap_dio_invalidate_fail)(void *, struct inode *, long unsigned int, unsigned int); +typedef void (*btf_trace_iomap_dio_invalidate_fail)(void *, struct inode *, loff_t, u64); -typedef void (*btf_trace_iomap_apply_dstmap)(void *, struct inode *, struct iomap___2 *); +typedef void (*btf_trace_iomap_iter_dstmap)(void *, struct inode *, struct iomap___2 *); -typedef void (*btf_trace_iomap_apply_srcmap)(void *, struct inode *, struct iomap___2 *); +typedef void (*btf_trace_iomap_iter_srcmap)(void *, struct inode *, struct iomap___2 *); -typedef void (*btf_trace_iomap_apply)(void *, struct inode *, loff_t, loff_t, unsigned int, const void *, void *, long unsigned int); +typedef void (*btf_trace_iomap_iter)(void *, struct iomap_iter *, const void *, long unsigned int); + +enum { + BIOSET_NEED_BVECS = 1, + BIOSET_NEED_RESCUER = 2, + BIOSET_PERCPU_CACHE = 4, +}; struct iomap_ops { int (*iomap_begin)(struct inode *, loff_t, loff_t, unsigned int, struct iomap___2 *, struct iomap___2 *); int (*iomap_end)(struct inode *, loff_t, loff_t, ssize_t, unsigned int, struct iomap___2 *); }; -typedef loff_t (*iomap_actor_t)(struct inode *, loff_t, loff_t, void *, struct iomap___2 *, struct iomap___2 *); - struct iomap_ioend { struct list_head io_list; u16 io_type; @@ -44897,13 +45717,9 @@ struct iomap_readpage_ctx { struct readahead_control *rac; }; -enum { - IOMAP_WRITE_F_UNSHARE = 1, -}; - struct iomap_dio_ops { int (*end_io)(struct kiocb *, ssize_t, int, unsigned int); - blk_qc_t (*submit_io)(struct inode *, struct iomap___2 *, struct bio *, loff_t); + void (*submit_io)(const struct iomap_iter *, struct bio *, loff_t); }; struct iomap_dio { @@ -44914,13 +45730,13 @@ struct iomap_dio { atomic_t ref; unsigned int flags; int error; + size_t done_before; bool wait_for_completion; union { struct { struct iov_iter *iter; struct task_struct *waiter; - struct request_queue *last_queue; - blk_qc_t cookie; + struct bio *poll_bio; } submit; struct { struct work_struct work; @@ -44928,11 +45744,6 @@ struct iomap_dio { }; }; -struct fiemap_ctx { - struct fiemap_extent_info *fi; - struct iomap___2 prev; -}; - struct iomap_swapfile_info { struct iomap___2 iomap; struct swap_info_struct *sis; @@ -44972,7 +45783,6 @@ struct mem_size_stats { u64 pss_shmem; u64 pss_locked; u64 swap_pss; - bool check_shmem_swap; }; enum clear_refs_types { @@ -45051,6 +45861,12 @@ struct syscall_info { struct seccomp_data data; }; +enum resctrl_conf_type { + CDP_NONE = 0, + CDP_CODE = 1, + CDP_DATA = 2, +}; + typedef struct dentry *instantiate_t(struct dentry *, struct task_struct *, const void *); struct pid_entry { @@ -45090,6 +45906,7 @@ struct sysctl_alias { struct seq_net_private { struct net *net; + netns_tracker ns_tracker; }; struct bpf_iter_aux_info___2; @@ -45159,6 +45976,186 @@ struct pts_fs_info { struct dentry *ptmx_dentry; }; +enum netfs_read_source { + NETFS_FILL_WITH_ZEROES = 0, + NETFS_DOWNLOAD_FROM_SERVER = 1, + NETFS_READ_FROM_CACHE = 2, + NETFS_INVALID_READ = 3, +}; + +typedef void (*netfs_io_terminated_t)(void *, ssize_t, bool); + +struct netfs_cache_ops; + +struct netfs_cache_resources { + const struct netfs_cache_ops *ops; + void *cache_priv; + void *cache_priv2; + unsigned int debug_id; +}; + +struct netfs_read_subrequest; + +struct netfs_cache_ops { + void (*end_operation)(struct netfs_cache_resources *); + int (*read)(struct netfs_cache_resources *, loff_t, struct iov_iter *, bool, netfs_io_terminated_t, void *); + int (*write)(struct netfs_cache_resources *, loff_t, struct iov_iter *, netfs_io_terminated_t, void *); + void (*expand_readahead)(struct netfs_cache_resources *, loff_t *, size_t *, loff_t); + enum netfs_read_source (*prepare_read)(struct netfs_read_subrequest *, loff_t); + int (*prepare_write)(struct netfs_cache_resources *, loff_t *, size_t *, loff_t); +}; + +struct netfs_read_request; + +struct netfs_read_subrequest { + struct netfs_read_request *rreq; + struct list_head rreq_link; + loff_t start; + size_t len; + size_t transferred; + refcount_t usage; + short int error; + short unsigned int debug_index; + enum netfs_read_source source; + long unsigned int flags; +}; + +struct netfs_read_request_ops; + +struct netfs_read_request { + struct work_struct work; + struct inode *inode; + struct address_space *mapping; + struct netfs_cache_resources cache_resources; + struct list_head subrequests; + void *netfs_priv; + unsigned int debug_id; + atomic_t nr_rd_ops; + atomic_t nr_wr_ops; + size_t submitted; + size_t len; + short int error; + loff_t i_size; + loff_t start; + long unsigned int no_unlock_folio; + refcount_t usage; + long unsigned int flags; + const struct netfs_read_request_ops *netfs_ops; +}; + +struct netfs_read_request_ops { + bool (*is_cache_enabled)(struct inode *); + void (*init_rreq)(struct netfs_read_request *, struct file *); + int (*begin_cache_operation)(struct netfs_read_request *); + void (*expand_readahead)(struct netfs_read_request *); + bool (*clamp_length)(struct netfs_read_subrequest *); + void (*issue_op)(struct netfs_read_subrequest *); + bool (*is_still_valid)(struct netfs_read_request *); + int (*check_write_begin)(struct file *, loff_t, unsigned int, struct folio *, void **); + void (*done)(struct netfs_read_request *); + void (*cleanup)(struct address_space *, void *); +}; + +enum netfs_read_trace { + netfs_read_trace_expanded = 0, + netfs_read_trace_readahead = 1, + netfs_read_trace_readpage = 2, + netfs_read_trace_write_begin = 3, +}; + +enum netfs_rreq_trace { + netfs_rreq_trace_assess = 0, + netfs_rreq_trace_done = 1, + netfs_rreq_trace_free = 2, + netfs_rreq_trace_resubmit = 3, + netfs_rreq_trace_unlock = 4, + netfs_rreq_trace_unmark = 5, + netfs_rreq_trace_write = 6, +}; + +enum netfs_sreq_trace { + netfs_sreq_trace_download_instead = 0, + netfs_sreq_trace_free = 1, + netfs_sreq_trace_prepare = 2, + netfs_sreq_trace_resubmit_short = 3, + netfs_sreq_trace_submit = 4, + netfs_sreq_trace_terminated = 5, + netfs_sreq_trace_write = 6, + netfs_sreq_trace_write_skip = 7, + netfs_sreq_trace_write_term = 8, +}; + +enum netfs_failure { + netfs_fail_check_write_begin = 0, + netfs_fail_copy_to_cache = 1, + netfs_fail_read = 2, + netfs_fail_short_readpage = 3, + netfs_fail_short_write_begin = 4, + netfs_fail_prepare_write = 5, +}; + +struct trace_event_raw_netfs_read { + struct trace_entry ent; + unsigned int rreq; + unsigned int cookie; + loff_t start; + size_t len; + enum netfs_read_trace what; + char __data[0]; +}; + +struct trace_event_raw_netfs_rreq { + struct trace_entry ent; + unsigned int rreq; + short unsigned int flags; + enum netfs_rreq_trace what; + char __data[0]; +}; + +struct trace_event_raw_netfs_sreq { + struct trace_entry ent; + unsigned int rreq; + short unsigned int index; + short int error; + short unsigned int flags; + enum netfs_read_source source; + enum netfs_sreq_trace what; + size_t len; + size_t transferred; + loff_t start; + char __data[0]; +}; + +struct trace_event_raw_netfs_failure { + struct trace_entry ent; + unsigned int rreq; + short unsigned int index; + short int error; + short unsigned int flags; + enum netfs_read_source source; + enum netfs_failure what; + size_t len; + size_t transferred; + loff_t start; + char __data[0]; +}; + +struct trace_event_data_offsets_netfs_read {}; + +struct trace_event_data_offsets_netfs_rreq {}; + +struct trace_event_data_offsets_netfs_sreq {}; + +struct trace_event_data_offsets_netfs_failure {}; + +typedef void (*btf_trace_netfs_read)(void *, struct netfs_read_request *, loff_t, size_t, enum netfs_read_trace); + +typedef void (*btf_trace_netfs_rreq)(void *, struct netfs_read_request *, enum netfs_rreq_trace); + +typedef void (*btf_trace_netfs_sreq)(void *, struct netfs_read_subrequest *, enum netfs_sreq_trace); + +typedef void (*btf_trace_netfs_failure)(void *, struct netfs_read_request *, struct netfs_read_subrequest *, int, enum netfs_failure); + typedef unsigned int tid_t; struct transaction_chp_stats_s { @@ -45642,7 +46639,10 @@ struct ext4_inode_info { ext4_lblk_t i_dir_start_lookup; long unsigned int i_flags; struct rw_semaphore xattr_sem; - struct list_head i_orphan; + union { + struct list_head i_orphan; + unsigned int i_orphan_idx; + }; struct list_head i_fc_list; ext4_lblk_t i_fc_lblk_start; ext4_lblk_t i_fc_lblk_len; @@ -45651,7 +46651,6 @@ struct ext4_inode_info { struct mutex i_fc_lock; loff_t i_disksize; struct rw_semaphore i_data_sem; - struct rw_semaphore i_mmap_sem; struct inode vfs_inode; struct jbd2_inode *jinode; spinlock_t i_raw_lock; @@ -45783,10 +46782,27 @@ struct ext4_super_block { __u8 s_last_error_errcode; __le16 s_encoding; __le16 s_encoding_flags; - __le32 s_reserved[95]; + __le32 s_orphan_file_inum; + __le32 s_reserved[94]; __le32 s_checksum; }; +struct ext4_journal_trigger { + struct jbd2_buffer_trigger_type tr_triggers; + struct super_block *sb; +}; + +struct ext4_orphan_block { + atomic_t ob_free_entries; + struct buffer_head *ob_bh; +}; + +struct ext4_orphan_info { + int of_blocks; + __u32 of_csum_seed; + struct ext4_orphan_block *of_binfo; +}; + struct mb_cache___2; struct ext4_group_info; @@ -45844,9 +46860,10 @@ struct ext4_sb_info { struct super_block *s_sb; struct buffer_head *s_mmp_bh; struct journal_s *s_journal; - struct list_head s_orphan; - struct mutex s_orphan_lock; long unsigned int s_ext4_flags; + struct mutex s_orphan_lock; + struct list_head s_orphan; + struct ext4_orphan_info s_orphan_info; long unsigned int s_commit_interval; u32 s_max_batch_time; u32 s_min_batch_time; @@ -45861,6 +46878,9 @@ struct ext4_sb_info { unsigned int s_group_info_size; unsigned int s_mb_free_pending; struct list_head s_freed_data_list; + struct list_head s_discard_list; + struct work_struct s_discard_work; + atomic_t s_retry_alloc_pending; struct rb_root s_mb_avg_fragment_size_root; rwlock_t s_mb_rb_lock; struct list_head *s_mb_largest_free_orders; @@ -45920,13 +46940,8 @@ struct ext4_sb_info { struct mb_cache___2 *s_ea_block_cache; struct mb_cache___2 *s_ea_inode_cache; long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; spinlock_t s_es_lock; + struct ext4_journal_trigger s_journal_triggers[1]; struct ratelimit_state s_err_ratelimit_state; struct ratelimit_state s_warning_ratelimit_state; struct ratelimit_state s_msg_ratelimit_state; @@ -45962,9 +46977,6 @@ struct ext4_sb_info { struct ext4_fc_stats s_fc_stats; u64 s_fc_avg_commit_time; struct ext4_fc_replay_state s_fc_replay_state; - long: 64; - long: 64; - long: 64; }; struct ext4_group_info { @@ -46078,6 +47090,11 @@ enum { EXT4_FC_REASON_MAX = 10, }; +enum ext4_journal_trigger_type { + EXT4_JTR_ORPHAN_FILE = 0, + EXT4_JTR_NONE = 1, +}; + struct ext4_dir_entry_hash { __le32 hash; __le32 minor_hash; @@ -46114,6 +47131,11 @@ struct fname { char name[0]; }; +enum { + BLK_RW_ASYNC = 0, + BLK_RW_SYNC = 1, +}; + enum SHIFT_DIRECTION { SHIFT_LEFT = 0, SHIFT_RIGHT = 1, @@ -46131,7 +47153,7 @@ struct ext4_io_end { struct inode *inode; struct bio *bio; unsigned int flag; - atomic_t count; + refcount_t count; struct list_head list_vec; }; @@ -46159,6 +47181,7 @@ enum { EXT4_STATE_LUSTRE_EA_INODE = 9, EXT4_STATE_VERITY_IN_PROGRESS = 10, EXT4_STATE_FC_COMMITTING = 11, + EXT4_STATE_ORPHAN_FILE = 12, }; struct ext4_iloc { @@ -46753,6 +47776,14 @@ struct ext4_new_flex_group_data { ext4_group_t count; }; +enum stat_group { + STAT_READ = 0, + STAT_WRITE = 1, + STAT_DISCARD = 2, + STAT_FLUSH = 3, + NR_STAT_GROUPS = 4, +}; + struct ext4_lazy_init { long unsigned int li_state; struct list_head li_request_list; @@ -48273,6 +49304,11 @@ struct dentry_info_args { char *dname; }; +struct ext4_orphan_block_tail { + __le32 ob_magic; + __le32 ob_checksum; +}; + typedef struct { __le16 e_tag; __le16 e_perm; @@ -48846,6 +49882,53 @@ struct dotl_iattr_map { int p9_iattr_valid; }; +enum fscache_checkaux { + FSCACHE_CHECKAUX_OKAY = 0, + FSCACHE_CHECKAUX_NEEDS_UPDATE = 1, + FSCACHE_CHECKAUX_OBSOLETE = 2, +}; + +struct fscache_cache_tag; + +struct fscache_cookie_def { + char name[16]; + uint8_t type; + struct fscache_cache_tag * (*select_cache)(const void *, const void *); + enum fscache_checkaux (*check_aux)(void *, const void *, uint16_t, loff_t); + void (*get_context)(void *, void *); + void (*put_context)(void *, void *); + void (*mark_page_cached)(void *, struct address_space *, struct page *); +}; + +struct fscache_cookie { + refcount_t ref; + atomic_t n_children; + atomic_t n_active; + unsigned int debug_id; + spinlock_t lock; + spinlock_t stores_lock; + struct hlist_head backing_objects; + const struct fscache_cookie_def *def; + struct fscache_cookie *parent; + struct hlist_bl_node hash_link; + struct list_head proc_link; + void *netfs_data; + struct xarray stores; + long unsigned int flags; + u8 type; + u8 key_len; + u8 aux_len; + u32 key_hash; + union { + void *key; + u8 inline_key[16]; + }; + union { + void *aux; + u8 inline_aux[8]; + }; +}; + struct p9_flock { u8 type; u32 flags; @@ -49313,7 +50396,7 @@ struct shmid_kernel { struct ucounts *mlock_ucounts; struct task_struct *shm_creator; struct list_head shm_clist; - long: 64; + struct ipc_namespace *ns; long: 64; long: 64; long: 64; @@ -49536,13 +50619,13 @@ struct vfs_ns_cap_data { __le32 rootid; }; -struct sctp_endpoint; +struct sctp_association; union security_list_options { - int (*binder_set_context_mgr)(struct task_struct *); - int (*binder_transaction)(struct task_struct *, struct task_struct *); - int (*binder_transfer_binder)(struct task_struct *, struct task_struct *); - int (*binder_transfer_file)(struct task_struct *, struct task_struct *, struct file *); + int (*binder_set_context_mgr)(const struct cred *); + int (*binder_transaction)(const struct cred *, const struct cred *); + int (*binder_transfer_binder)(const struct cred *, const struct cred *); + int (*binder_transfer_file)(const struct cred *, const struct cred *, struct file *); int (*ptrace_access_check)(struct task_struct *, unsigned int); int (*ptrace_traceme)(struct task_struct *); int (*capget)(struct task_struct *, kernel_cap_t *, kernel_cap_t *, kernel_cap_t *); @@ -49577,7 +50660,7 @@ union security_list_options { int (*sb_clone_mnt_opts)(const struct super_block *, struct super_block *, long unsigned int, long unsigned int *); int (*sb_add_mnt_opt)(const char *, const char *, int, void **); int (*move_mount)(const struct path *, const struct path *); - int (*dentry_init_security)(struct dentry *, int, const struct qstr *, void **, u32 *); + int (*dentry_init_security)(struct dentry *, int, const struct qstr *, const char **, void **, u32 *); int (*dentry_create_files_as)(struct dentry *, int, struct qstr *, const struct cred *, struct cred *); int (*path_notify)(const struct path *, u64, unsigned int); int (*inode_alloc_security)(struct inode *); @@ -49725,9 +50808,9 @@ union security_list_options { int (*tun_dev_attach_queue)(void *); int (*tun_dev_attach)(struct sock *, void *); int (*tun_dev_open)(void *); - int (*sctp_assoc_request)(struct sctp_endpoint *, struct sk_buff *); + int (*sctp_assoc_request)(struct sctp_association *, struct sk_buff *); int (*sctp_bind_connect)(struct sock *, int, struct sockaddr *, int); - void (*sctp_sk_clone)(struct sctp_endpoint *, struct sock *, struct sock *); + void (*sctp_sk_clone)(struct sctp_association *, struct sock *, struct sock *); int (*key_alloc)(struct key *, const struct cred *, long unsigned int); void (*key_free)(struct key *); int (*key_permission)(key_ref_t, const struct cred *, enum key_need_perm); @@ -49749,6 +50832,8 @@ union security_list_options { void (*perf_event_free)(struct perf_event *); int (*perf_event_read)(struct perf_event *); int (*perf_event_write)(struct perf_event *); + int (*uring_override_creds)(const struct cred *); + int (*uring_sqpoll)(); }; struct security_hook_list { @@ -49902,6 +50987,7 @@ enum ethtool_link_ext_state { ETHTOOL_LINK_EXT_STATE_CALIBRATION_FAILURE = 7, ETHTOOL_LINK_EXT_STATE_POWER_BUDGET_EXCEEDED = 8, ETHTOOL_LINK_EXT_STATE_OVERHEAT = 9, + ETHTOOL_LINK_EXT_STATE_MODULE = 10, }; enum ethtool_link_ext_substate_autoneg { @@ -49931,6 +51017,8 @@ enum ethtool_link_ext_substate_link_logical_mismatch { enum ethtool_link_ext_substate_bad_signal_integrity { ETHTOOL_LINK_EXT_SUBSTATE_BSI_LARGE_NUMBER_OF_PHYSICAL_ERRORS = 1, ETHTOOL_LINK_EXT_SUBSTATE_BSI_UNSUPPORTED_RATE = 2, + ETHTOOL_LINK_EXT_SUBSTATE_BSI_SERDES_REFERENCE_CLOCK_LOST = 3, + ETHTOOL_LINK_EXT_SUBSTATE_BSI_SERDES_ALOS = 4, }; enum ethtool_link_ext_substate_cable_issue { @@ -49938,6 +51026,20 @@ enum ethtool_link_ext_substate_cable_issue { ETHTOOL_LINK_EXT_SUBSTATE_CI_CABLE_TEST_FAILURE = 2, }; +enum ethtool_link_ext_substate_module { + ETHTOOL_LINK_EXT_SUBSTATE_MODULE_CMIS_NOT_READY = 1, +}; + +enum ethtool_module_power_mode_policy { + ETHTOOL_MODULE_POWER_MODE_POLICY_HIGH = 1, + ETHTOOL_MODULE_POWER_MODE_POLICY_AUTO = 2, +}; + +enum ethtool_module_power_mode { + ETHTOOL_MODULE_POWER_MODE_LOW = 1, + ETHTOOL_MODULE_POWER_MODE_HIGH = 2, +}; + struct ethtool_test { __u32 cmd; __u32 flags; @@ -50192,6 +51294,10 @@ struct ethtool_link_settings { __u32 link_mode_masks[0]; }; +struct kernel_ethtool_ringparam { + u32 rx_buf_len; +}; + struct ethtool_link_ext_state_info { enum ethtool_link_ext_state link_ext_state; union { @@ -50200,6 +51306,7 @@ struct ethtool_link_ext_state_info { enum ethtool_link_ext_substate_link_logical_mismatch link_logical_mismatch; enum ethtool_link_ext_substate_bad_signal_integrity bad_signal_integrity; enum ethtool_link_ext_substate_cable_issue cable_issue; + enum ethtool_link_ext_substate_module module; u8 __link_ext_substate; }; }; @@ -50214,6 +51321,11 @@ struct ethtool_link_ksettings { u32 lanes; }; +struct kernel_ethtool_coalesce { + u8 use_cqe_mode_tx; + u8 use_cqe_mode_rx; +}; + struct ethtool_eth_mac_stats { u64 FramesTransmittedOK; u64 SingleCollisionFrames; @@ -50288,6 +51400,11 @@ struct ethtool_module_eeprom { u8 *data; }; +struct ethtool_module_power_mode_params { + enum ethtool_module_power_mode_policy policy; + enum ethtool_module_power_mode mode; +}; + enum ib_uverbs_write_cmds { IB_USER_VERBS_CMD_GET_CONTEXT = 0, IB_USER_VERBS_CMD_QUERY_DEVICE = 1, @@ -50663,8 +51780,6 @@ struct sctp_bind_addr { }; struct sctp_ep_common { - struct hlist_node node; - int hashent; enum sctp_endpoint_type type; refcount_t refcnt; bool dead; @@ -50674,33 +51789,13 @@ struct sctp_ep_common { struct sctp_bind_addr bind_addr; }; -struct crypto_shash___2; +typedef __s32 sctp_assoc_t; -struct sctp_hmac_algo_param; - -struct sctp_chunks_param; - -struct sctp_endpoint { - struct sctp_ep_common base; - struct list_head asocs; - __u8 secret_key[32]; - __u8 *digest; - __u32 sndbuf_policy; - __u32 rcvbuf_policy; - struct crypto_shash___2 **auth_hmacs; - struct sctp_hmac_algo_param *auth_hmacs_list; - struct sctp_chunks_param *auth_chunk_list; - struct list_head endpoint_shared_keys; - __u16 active_key_id; - __u8 ecn_enable: 1; - __u8 auth_enable: 1; - __u8 intl_enable: 1; - __u8 prsctp_enable: 1; - __u8 asconf_enable: 1; - __u8 reconf_enable: 1; - __u8 strreset_enable; - u32 secid; - u32 peer_secid; +struct sockaddr_in { + __kernel_sa_family_t sin_family; + __be16 sin_port; + struct in_addr sin_addr; + unsigned char __pad[8]; }; struct sockaddr_in6 { @@ -50711,588 +51806,10 @@ struct sockaddr_in6 { __u32 sin6_scope_id; }; -struct sockaddr_in { - __kernel_sa_family_t sin_family; - __be16 sin_port; - struct in_addr sin_addr; - unsigned char __pad[8]; -}; - -struct nf_hook_state; - -typedef unsigned int nf_hookfn(void *, struct sk_buff *, const struct nf_hook_state *); - -struct nf_hook_entry { - nf_hookfn *hook; - void *priv; -}; - -struct nf_hook_entries { - u16 num_hook_entries; - struct nf_hook_entry hooks[0]; -}; - -struct nf_hook_state { - u8 hook; - u8 pf; - struct net_device *in; - struct net_device *out; - struct sock *sk; - struct net *net; - int (*okfn)(struct net *, struct sock *, struct sk_buff *); -}; - -enum nf_hook_ops_type { - NF_HOOK_OP_UNDEFINED = 0, - NF_HOOK_OP_NF_TABLES = 1, -}; - -struct nf_hook_ops { - nf_hookfn *hook; - struct net_device *dev; - void *priv; - u8 pf; - enum nf_hook_ops_type hook_ops_type: 8; - unsigned int hooknum; - int priority; -}; - -struct nf_conn; - -enum nf_nat_manip_type; - -enum ip_conntrack_dir; - -struct nf_nat_hook { - int (*parse_nat_setup)(struct nf_conn *, enum nf_nat_manip_type, const struct nlattr *); - void (*decode_session)(struct sk_buff *, struct flowi *); - unsigned int (*manip_pkt)(struct sk_buff *, struct nf_conn *, enum nf_nat_manip_type, enum ip_conntrack_dir); -}; - -struct nf_conntrack_tuple; - -struct nf_conntrack; - -struct nf_ct_hook { - int (*update)(struct net *, struct sk_buff *); - void (*destroy)(struct nf_conntrack *); - bool (*get_tuple_skb)(struct nf_conntrack_tuple *, const struct sk_buff *); -}; - -enum ip_conntrack_info { - IP_CT_ESTABLISHED = 0, - IP_CT_RELATED = 1, - IP_CT_NEW = 2, - IP_CT_IS_REPLY = 3, - IP_CT_ESTABLISHED_REPLY = 3, - IP_CT_RELATED_REPLY = 4, - IP_CT_NUMBER = 5, - IP_CT_UNTRACKED = 7, -}; - -struct nfnl_ct_hook { - size_t (*build_size)(const struct nf_conn *); - int (*build)(struct sk_buff *, struct nf_conn *, enum ip_conntrack_info, u_int16_t, u_int16_t); - int (*parse)(const struct nlattr *, struct nf_conn *); - int (*attach_expect)(const struct nlattr *, struct nf_conn *, u32, u32); - void (*seq_adjust)(struct sk_buff *, struct nf_conn *, enum ip_conntrack_info, s32); -}; - -enum nf_ip_hook_priorities { - NF_IP_PRI_FIRST = 2147483648, - NF_IP_PRI_RAW_BEFORE_DEFRAG = 4294966846, - NF_IP_PRI_CONNTRACK_DEFRAG = 4294966896, - NF_IP_PRI_RAW = 4294966996, - NF_IP_PRI_SELINUX_FIRST = 4294967071, - NF_IP_PRI_CONNTRACK = 4294967096, - NF_IP_PRI_MANGLE = 4294967146, - NF_IP_PRI_NAT_DST = 4294967196, - NF_IP_PRI_FILTER = 0, - NF_IP_PRI_SECURITY = 50, - NF_IP_PRI_NAT_SRC = 100, - NF_IP_PRI_SELINUX_LAST = 225, - NF_IP_PRI_CONNTRACK_HELPER = 300, - NF_IP_PRI_CONNTRACK_CONFIRM = 2147483647, - NF_IP_PRI_LAST = 2147483647, -}; - -enum nf_ip6_hook_priorities { - NF_IP6_PRI_FIRST = 2147483648, - NF_IP6_PRI_RAW_BEFORE_DEFRAG = 4294966846, - NF_IP6_PRI_CONNTRACK_DEFRAG = 4294966896, - NF_IP6_PRI_RAW = 4294966996, - NF_IP6_PRI_SELINUX_FIRST = 4294967071, - NF_IP6_PRI_CONNTRACK = 4294967096, - NF_IP6_PRI_MANGLE = 4294967146, - NF_IP6_PRI_NAT_DST = 4294967196, - NF_IP6_PRI_FILTER = 0, - NF_IP6_PRI_SECURITY = 50, - NF_IP6_PRI_NAT_SRC = 100, - NF_IP6_PRI_SELINUX_LAST = 225, - NF_IP6_PRI_CONNTRACK_HELPER = 300, - NF_IP6_PRI_LAST = 2147483647, -}; - -struct socket_alloc { - struct socket socket; - struct inode vfs_inode; - long: 64; - long: 64; - long: 64; - long: 64; -}; - -struct ip_options { - __be32 faddr; - __be32 nexthop; - unsigned char optlen; - unsigned char srr; - unsigned char rr; - unsigned char ts; - unsigned char is_strictroute: 1; - unsigned char srr_is_hit: 1; - unsigned char is_changed: 1; - unsigned char rr_needaddr: 1; - unsigned char ts_needtime: 1; - unsigned char ts_needaddr: 1; - unsigned char router_alert; - unsigned char cipso; - unsigned char __pad2; - unsigned char __data[0]; -}; - -struct ip_options_rcu { - struct callback_head rcu; - struct ip_options opt; -}; - -struct ipv6_opt_hdr; - -struct ipv6_rt_hdr; - -struct ipv6_txoptions { - refcount_t refcnt; - int tot_len; - __u16 opt_flen; - __u16 opt_nflen; - struct ipv6_opt_hdr *hopopt; - struct ipv6_opt_hdr *dst0opt; - struct ipv6_rt_hdr *srcrt; - struct ipv6_opt_hdr *dst1opt; - struct callback_head rcu; -}; - -struct inet_cork { - unsigned int flags; - __be32 addr; - struct ip_options *opt; - unsigned int fragsize; - int length; - struct dst_entry *dst; - u8 tx_flags; - __u8 ttl; - __s16 tos; - char priority; - __u16 gso_size; - u64 transmit_time; - u32 mark; -}; - -struct inet_cork_full { - struct inet_cork base; - struct flowi fl; -}; - -struct ipv6_pinfo; - -struct ip_mc_socklist; - -struct inet_sock { - struct sock sk; - struct ipv6_pinfo *pinet6; - __be32 inet_saddr; - __s16 uc_ttl; - __u16 cmsg_flags; - __be16 inet_sport; - __u16 inet_id; - struct ip_options_rcu *inet_opt; - int rx_dst_ifindex; - __u8 tos; - __u8 min_ttl; - __u8 mc_ttl; - __u8 pmtudisc; - __u8 recverr: 1; - __u8 is_icsk: 1; - __u8 freebind: 1; - __u8 hdrincl: 1; - __u8 mc_loop: 1; - __u8 transparent: 1; - __u8 mc_all: 1; - __u8 nodefrag: 1; - __u8 bind_address_no_port: 1; - __u8 recverr_rfc4884: 1; - __u8 defer_connect: 1; - __u8 rcv_tos; - __u8 convert_csum; - int uc_index; - int mc_index; - __be32 mc_addr; - struct ip_mc_socklist *mc_list; - struct inet_cork_full cork; -}; - -struct in6_pktinfo { - struct in6_addr ipi6_addr; - int ipi6_ifindex; -}; - -struct inet6_cork { - struct ipv6_txoptions *opt; - u8 hop_limit; - u8 tclass; -}; - -struct ipv6_mc_socklist; - -struct ipv6_ac_socklist; - -struct ipv6_fl_socklist; - -struct ipv6_pinfo { - struct in6_addr saddr; - struct in6_pktinfo sticky_pktinfo; - const struct in6_addr *daddr_cache; - const struct in6_addr *saddr_cache; - __be32 flow_label; - __u32 frag_size; - __u16 __unused_1: 7; - __s16 hop_limit: 9; - __u16 mc_loop: 1; - __u16 __unused_2: 6; - __s16 mcast_hops: 9; - int ucast_oif; - int mcast_oif; - union { - struct { - __u16 srcrt: 1; - __u16 osrcrt: 1; - __u16 rxinfo: 1; - __u16 rxoinfo: 1; - __u16 rxhlim: 1; - __u16 rxohlim: 1; - __u16 hopopts: 1; - __u16 ohopopts: 1; - __u16 dstopts: 1; - __u16 odstopts: 1; - __u16 rxflow: 1; - __u16 rxtclass: 1; - __u16 rxpmtu: 1; - __u16 rxorigdstaddr: 1; - __u16 recvfragsize: 1; - } bits; - __u16 all; - } rxopt; - __u16 recverr: 1; - __u16 sndflow: 1; - __u16 repflow: 1; - __u16 pmtudisc: 3; - __u16 padding: 1; - __u16 srcprefs: 3; - __u16 dontfrag: 1; - __u16 autoflowlabel: 1; - __u16 autoflowlabel_set: 1; - __u16 mc_all: 1; - __u16 recverr_rfc4884: 1; - __u16 rtalert_isolate: 1; - __u8 min_hopcount; - __u8 tclass; - __be32 rcv_flowinfo; - __u32 dst_cookie; - __u32 rx_dst_cookie; - struct ipv6_mc_socklist *ipv6_mc_list; - struct ipv6_ac_socklist *ipv6_ac_list; - struct ipv6_fl_socklist *ipv6_fl_list; - struct ipv6_txoptions *opt; - struct sk_buff *pktoptions; - struct sk_buff *rxpmtu; - struct inet6_cork cork; -}; - -struct tcphdr { - __be16 source; - __be16 dest; - __be32 seq; - __be32 ack_seq; - __u16 res1: 4; - __u16 doff: 4; - __u16 fin: 1; - __u16 syn: 1; - __u16 rst: 1; - __u16 psh: 1; - __u16 ack: 1; - __u16 urg: 1; - __u16 ece: 1; - __u16 cwr: 1; - __be16 window; - __sum16 check; - __be16 urg_ptr; -}; - -struct iphdr { - __u8 ihl: 4; - __u8 version: 4; - __u8 tos; - __be16 tot_len; - __be16 id; - __be16 frag_off; - __u8 ttl; - __u8 protocol; - __sum16 check; - __be32 saddr; - __be32 daddr; -}; - -struct ipv6_rt_hdr { - __u8 nexthdr; - __u8 hdrlen; - __u8 type; - __u8 segments_left; -}; - -struct ipv6_opt_hdr { - __u8 nexthdr; - __u8 hdrlen; -}; - -struct ipv6hdr { - __u8 priority: 4; - __u8 version: 4; - __u8 flow_lbl[3]; - __be16 payload_len; - __u8 nexthdr; - __u8 hop_limit; - struct in6_addr saddr; - struct in6_addr daddr; -}; - -struct udphdr { - __be16 source; - __be16 dest; - __be16 len; - __sum16 check; -}; - -struct inet6_skb_parm { - int iif; - __be16 ra; - __u16 dst0; - __u16 srcrt; - __u16 dst1; - __u16 lastopt; - __u16 nhoff; - __u16 flags; - __u16 dsthao; - __u16 frag_max_size; -}; - -struct ip6_sf_socklist; - -struct ipv6_mc_socklist { - struct in6_addr addr; - int ifindex; - unsigned int sfmode; - struct ipv6_mc_socklist *next; - struct ip6_sf_socklist *sflist; - struct callback_head rcu; -}; - -struct ipv6_ac_socklist { - struct in6_addr acl_addr; - int acl_ifindex; - struct ipv6_ac_socklist *acl_next; -}; - -struct ip6_flowlabel; - -struct ipv6_fl_socklist { - struct ipv6_fl_socklist *next; - struct ip6_flowlabel *fl; - struct callback_head rcu; -}; - -struct ip6_sf_socklist { - unsigned int sl_max; - unsigned int sl_count; - struct callback_head rcu; - struct in6_addr sl_addr[0]; -}; - -struct ip6_flowlabel { - struct ip6_flowlabel *next; - __be32 label; - atomic_t users; - struct in6_addr dst; - struct ipv6_txoptions *opt; - long unsigned int linger; - struct callback_head rcu; - u8 share; - union { - struct pid *pid; - kuid_t uid; - } owner; - long unsigned int lastuse; - long unsigned int expires; - struct net *fl_net; -}; - -struct inet_skb_parm { - int iif; - struct ip_options opt; - u16 flags; - u16 frag_max_size; -}; - -struct nf_ipv6_ops { - void (*route_input)(struct sk_buff *); - int (*fragment)(struct net *, struct sock *, struct sk_buff *, int (*)(struct net *, struct sock *, struct sk_buff *)); - int (*reroute)(struct sk_buff *, const struct nf_queue_entry *); -}; - -struct nf_queue_entry { - struct list_head list; - struct sk_buff *skb; - unsigned int id; - unsigned int hook_index; - struct nf_hook_state state; - u16 size; -}; - -struct tty_file_private { - struct tty_struct *tty; - struct file *file; - struct list_head list; -}; - -struct icmp_err { - int errno; - unsigned int fatal: 1; -}; - -struct netlbl_lsm_cache { - refcount_t refcount; - void (*free)(const void *); - void *data; -}; - -struct netlbl_lsm_catmap { - u32 startbit; - u64 bitmap[4]; - struct netlbl_lsm_catmap *next; -}; - -struct netlbl_lsm_secattr { - u32 flags; - u32 type; - char *domain; - struct netlbl_lsm_cache *cache; - struct { - struct { - struct netlbl_lsm_catmap *cat; - u32 lvl; - } mls; - u32 secid; - } attr; -}; - -struct dccp_hdr { - __be16 dccph_sport; - __be16 dccph_dport; - __u8 dccph_doff; - __u8 dccph_cscov: 4; - __u8 dccph_ccval: 4; - __sum16 dccph_checksum; - __u8 dccph_x: 1; - __u8 dccph_type: 4; - __u8 dccph_reserved: 3; - __u8 dccph_seq2; - __be16 dccph_seq; -}; - -enum dccp_state { - DCCP_OPEN = 1, - DCCP_REQUESTING = 2, - DCCP_LISTEN = 10, - DCCP_RESPOND = 3, - DCCP_ACTIVE_CLOSEREQ = 4, - DCCP_PASSIVE_CLOSE = 8, - DCCP_CLOSING = 11, - DCCP_TIME_WAIT = 6, - DCCP_CLOSED = 7, - DCCP_NEW_SYN_RECV = 12, - DCCP_PARTOPEN = 13, - DCCP_PASSIVE_CLOSEREQ = 14, - DCCP_MAX_STATES = 15, -}; - -typedef __s32 sctp_assoc_t; - -enum sctp_msg_flags { - MSG_NOTIFICATION = 32768, -}; - -struct sctp_initmsg { - __u16 sinit_num_ostreams; - __u16 sinit_max_instreams; - __u16 sinit_max_attempts; - __u16 sinit_max_init_timeo; -}; - -struct sctp_sndrcvinfo { - __u16 sinfo_stream; - __u16 sinfo_ssn; - __u16 sinfo_flags; - __u32 sinfo_ppid; - __u32 sinfo_context; - __u32 sinfo_timetolive; - __u32 sinfo_tsn; - __u32 sinfo_cumtsn; - sctp_assoc_t sinfo_assoc_id; -}; - -struct sctp_rtoinfo { - sctp_assoc_t srto_assoc_id; - __u32 srto_initial; - __u32 srto_max; - __u32 srto_min; -}; - -struct sctp_assocparams { - sctp_assoc_t sasoc_assoc_id; - __u16 sasoc_asocmaxrxt; - __u16 sasoc_number_peer_destinations; - __u32 sasoc_peer_rwnd; - __u32 sasoc_local_rwnd; - __u32 sasoc_cookie_life; -}; - -struct sctp_paddrparams { - sctp_assoc_t spp_assoc_id; - struct __kernel_sockaddr_storage spp_address; - __u32 spp_hbinterval; - __u16 spp_pathmaxrxt; - __u32 spp_pathmtu; - __u32 spp_sackdelay; - __u32 spp_flags; - __u32 spp_ipv6_flowlabel; - __u8 spp_dscp; - char: 8; -} __attribute__((packed)); - -struct sctphdr { - __be16 source; - __be16 dest; - __be32 vtag; - __le32 checksum; +union sctp_addr { + struct sockaddr_in v4; + struct sockaddr_in6 v6; + struct sockaddr sa; }; struct sctp_chunkhdr { @@ -51301,86 +51818,6 @@ struct sctp_chunkhdr { __be16 length; }; -enum sctp_cid { - SCTP_CID_DATA = 0, - SCTP_CID_INIT = 1, - SCTP_CID_INIT_ACK = 2, - SCTP_CID_SACK = 3, - SCTP_CID_HEARTBEAT = 4, - SCTP_CID_HEARTBEAT_ACK = 5, - SCTP_CID_ABORT = 6, - SCTP_CID_SHUTDOWN = 7, - SCTP_CID_SHUTDOWN_ACK = 8, - SCTP_CID_ERROR = 9, - SCTP_CID_COOKIE_ECHO = 10, - SCTP_CID_COOKIE_ACK = 11, - SCTP_CID_ECN_ECNE = 12, - SCTP_CID_ECN_CWR = 13, - SCTP_CID_SHUTDOWN_COMPLETE = 14, - SCTP_CID_AUTH = 15, - SCTP_CID_I_DATA = 64, - SCTP_CID_FWD_TSN = 192, - SCTP_CID_ASCONF = 193, - SCTP_CID_I_FWD_TSN = 194, - SCTP_CID_ASCONF_ACK = 128, - SCTP_CID_RECONF = 130, - SCTP_CID_PAD = 132, -}; - -struct sctp_paramhdr { - __be16 type; - __be16 length; -}; - -enum sctp_param { - SCTP_PARAM_HEARTBEAT_INFO = 256, - SCTP_PARAM_IPV4_ADDRESS = 1280, - SCTP_PARAM_IPV6_ADDRESS = 1536, - SCTP_PARAM_STATE_COOKIE = 1792, - SCTP_PARAM_UNRECOGNIZED_PARAMETERS = 2048, - SCTP_PARAM_COOKIE_PRESERVATIVE = 2304, - SCTP_PARAM_HOST_NAME_ADDRESS = 2816, - SCTP_PARAM_SUPPORTED_ADDRESS_TYPES = 3072, - SCTP_PARAM_ECN_CAPABLE = 128, - SCTP_PARAM_RANDOM = 640, - SCTP_PARAM_CHUNKS = 896, - SCTP_PARAM_HMAC_ALGO = 1152, - SCTP_PARAM_SUPPORTED_EXT = 2176, - SCTP_PARAM_FWD_TSN_SUPPORT = 192, - SCTP_PARAM_ADD_IP = 448, - SCTP_PARAM_DEL_IP = 704, - SCTP_PARAM_ERR_CAUSE = 960, - SCTP_PARAM_SET_PRIMARY = 1216, - SCTP_PARAM_SUCCESS_REPORT = 1472, - SCTP_PARAM_ADAPTATION_LAYER_IND = 1728, - SCTP_PARAM_RESET_OUT_REQUEST = 3328, - SCTP_PARAM_RESET_IN_REQUEST = 3584, - SCTP_PARAM_RESET_TSN_REQUEST = 3840, - SCTP_PARAM_RESET_RESPONSE = 4096, - SCTP_PARAM_RESET_ADD_OUT_STREAMS = 4352, - SCTP_PARAM_RESET_ADD_IN_STREAMS = 4608, -}; - -struct sctp_datahdr { - __be32 tsn; - __be16 stream; - __be16 ssn; - __u32 ppid; - __u8 payload[0]; -}; - -struct sctp_idatahdr { - __be32 tsn; - __be16 stream; - __be16 reserved; - __be32 mid; - union { - __u32 ppid; - __be32 fsn; - }; - __u8 payload[0]; -}; - struct sctp_inithdr { __be32 init_tag; __be32 a_rwnd; @@ -51395,145 +51832,6 @@ struct sctp_init_chunk { struct sctp_inithdr init_hdr; }; -struct sctp_ipv4addr_param { - struct sctp_paramhdr param_hdr; - struct in_addr addr; -}; - -struct sctp_ipv6addr_param { - struct sctp_paramhdr param_hdr; - struct in6_addr addr; -}; - -struct sctp_cookie_preserve_param { - struct sctp_paramhdr param_hdr; - __be32 lifespan_increment; -}; - -struct sctp_hostname_param { - struct sctp_paramhdr param_hdr; - uint8_t hostname[0]; -}; - -struct sctp_supported_addrs_param { - struct sctp_paramhdr param_hdr; - __be16 types[0]; -}; - -struct sctp_adaptation_ind_param { - struct sctp_paramhdr param_hdr; - __be32 adaptation_ind; -}; - -struct sctp_supported_ext_param { - struct sctp_paramhdr param_hdr; - __u8 chunks[0]; -}; - -struct sctp_random_param { - struct sctp_paramhdr param_hdr; - __u8 random_val[0]; -}; - -struct sctp_chunks_param { - struct sctp_paramhdr param_hdr; - __u8 chunks[0]; -}; - -struct sctp_hmac_algo_param { - struct sctp_paramhdr param_hdr; - __be16 hmac_ids[0]; -}; - -struct sctp_cookie_param { - struct sctp_paramhdr p; - __u8 body[0]; -}; - -struct sctp_gap_ack_block { - __be16 start; - __be16 end; -}; - -union sctp_sack_variable { - struct sctp_gap_ack_block gab; - __be32 dup; -}; - -struct sctp_sackhdr { - __be32 cum_tsn_ack; - __be32 a_rwnd; - __be16 num_gap_ack_blocks; - __be16 num_dup_tsns; - union sctp_sack_variable variable[0]; -}; - -struct sctp_heartbeathdr { - struct sctp_paramhdr info; -}; - -struct sctp_shutdownhdr { - __be32 cum_tsn_ack; -}; - -struct sctp_errhdr { - __be16 cause; - __be16 length; - __u8 variable[0]; -}; - -struct sctp_ecnehdr { - __be32 lowest_tsn; -}; - -struct sctp_cwrhdr { - __be32 lowest_tsn; -}; - -struct sctp_fwdtsn_skip { - __be16 stream; - __be16 ssn; -}; - -struct sctp_fwdtsn_hdr { - __be32 new_cum_tsn; - struct sctp_fwdtsn_skip skip[0]; -}; - -struct sctp_ifwdtsn_skip { - __be16 stream; - __u8 reserved; - __u8 flags; - __be32 mid; -}; - -struct sctp_ifwdtsn_hdr { - __be32 new_cum_tsn; - struct sctp_ifwdtsn_skip skip[0]; -}; - -struct sctp_addip_param { - struct sctp_paramhdr param_hdr; - __be32 crr_id; -}; - -struct sctp_addiphdr { - __be32 serial; - __u8 params[0]; -}; - -struct sctp_authhdr { - __be16 shkey_id; - __be16 hmac_id; - __u8 hmac[0]; -}; - -union sctp_addr { - struct sockaddr_in v4; - struct sockaddr_in6 v6; - struct sockaddr sa; -}; - struct sctp_cookie { __u32 my_vtag; __u32 peer_vtag; @@ -51637,8 +51935,6 @@ struct sctp_stream { struct sctp_sched_ops; -struct sctp_association; - struct sctp_outq { struct sctp_association *asoc; struct list_head out_chunk_list; @@ -51681,8 +51977,16 @@ struct sctp_priv_assoc_stats { __u64 ictrlchunks; }; +struct sctp_endpoint; + struct sctp_transport; +struct sctp_random_param; + +struct sctp_chunks_param; + +struct sctp_hmac_algo_param; + struct sctp_auth_bytes; struct sctp_shared_key; @@ -51815,9 +52119,954 @@ struct sctp_association { __u16 subscribe; __u64 abandoned_unsent[3]; __u64 abandoned_sent[3]; + u32 secid; + u32 peer_secid; struct callback_head rcu; }; +enum ip_conntrack_info { + IP_CT_ESTABLISHED = 0, + IP_CT_RELATED = 1, + IP_CT_NEW = 2, + IP_CT_IS_REPLY = 3, + IP_CT_ESTABLISHED_REPLY = 3, + IP_CT_RELATED_REPLY = 4, + IP_CT_NUMBER = 5, + IP_CT_UNTRACKED = 7, +}; + +struct nf_conntrack { + refcount_t use; +}; + +struct nf_hook_state; + +typedef unsigned int nf_hookfn(void *, struct sk_buff *, const struct nf_hook_state *); + +struct nf_hook_entry { + nf_hookfn *hook; + void *priv; +}; + +struct nf_hook_entries { + u16 num_hook_entries; + struct nf_hook_entry hooks[0]; +}; + +struct nf_hook_state { + u8 hook; + u8 pf; + struct net_device *in; + struct net_device *out; + struct sock *sk; + struct net *net; + int (*okfn)(struct net *, struct sock *, struct sk_buff *); +}; + +enum nf_hook_ops_type { + NF_HOOK_OP_UNDEFINED = 0, + NF_HOOK_OP_NF_TABLES = 1, +}; + +struct nf_hook_ops { + nf_hookfn *hook; + struct net_device *dev; + void *priv; + u8 pf; + enum nf_hook_ops_type hook_ops_type: 8; + unsigned int hooknum; + int priority; +}; + +enum nf_nat_manip_type { + NF_NAT_MANIP_SRC = 0, + NF_NAT_MANIP_DST = 1, +}; + +struct nf_conn; + +struct nf_nat_hook { + int (*parse_nat_setup)(struct nf_conn *, enum nf_nat_manip_type, const struct nlattr *); + void (*decode_session)(struct sk_buff *, struct flowi *); + unsigned int (*manip_pkt)(struct sk_buff *, struct nf_conn *, enum nf_nat_manip_type, enum ip_conntrack_dir); +}; + +union nf_inet_addr { + __u32 all[4]; + __be32 ip; + __be32 ip6[4]; + struct in_addr in; + struct in6_addr in6; +}; + +union nf_conntrack_man_proto { + __be16 all; + struct { + __be16 port; + } tcp; + struct { + __be16 port; + } udp; + struct { + __be16 id; + } icmp; + struct { + __be16 port; + } dccp; + struct { + __be16 port; + } sctp; + struct { + __be16 key; + } gre; +}; + +struct nf_conntrack_man { + union nf_inet_addr u3; + union nf_conntrack_man_proto u; + u_int16_t l3num; +}; + +struct nf_conntrack_tuple { + struct nf_conntrack_man src; + struct { + union nf_inet_addr u3; + union { + __be16 all; + struct { + __be16 port; + } tcp; + struct { + __be16 port; + } udp; + struct { + u_int8_t type; + u_int8_t code; + } icmp; + struct { + __be16 port; + } dccp; + struct { + __be16 port; + } sctp; + struct { + __be16 key; + } gre; + } u; + u_int8_t protonum; + u_int8_t dir; + } dst; +}; + +struct nf_conntrack_tuple_hash { + struct hlist_nulls_node hnnode; + struct nf_conntrack_tuple tuple; +}; + +typedef u64 u_int64_t; + +struct nf_ct_dccp { + u_int8_t role[2]; + u_int8_t state; + u_int8_t last_pkt; + u_int8_t last_dir; + u_int64_t handshake_seq; +}; + +struct ip_ct_sctp { + enum sctp_conntrack state; + __be32 vtag[2]; + u8 last_dir; + u8 flags; +}; + +typedef u32 u_int32_t; + +struct ip_ct_tcp_state { + u_int32_t td_end; + u_int32_t td_maxend; + u_int32_t td_maxwin; + u_int32_t td_maxack; + u_int8_t td_scale; + u_int8_t flags; +}; + +struct ip_ct_tcp { + struct ip_ct_tcp_state seen[2]; + u_int8_t state; + u_int8_t last_dir; + u_int8_t retrans; + u_int8_t last_index; + u_int32_t last_seq; + u_int32_t last_ack; + u_int32_t last_end; + u_int16_t last_win; + u_int8_t last_wscale; + u_int8_t last_flags; +}; + +struct nf_ct_udp { + long unsigned int stream_ts; +}; + +struct nf_ct_gre { + unsigned int stream_timeout; + unsigned int timeout; +}; + +union nf_conntrack_proto { + struct nf_ct_dccp dccp; + struct ip_ct_sctp sctp; + struct ip_ct_tcp tcp; + struct nf_ct_udp udp; + struct nf_ct_gre gre; + unsigned int tmpl_padto; +}; + +struct nf_ct_ext; + +struct nf_conn { + struct nf_conntrack ct_general; + spinlock_t lock; + u32 timeout; + struct nf_conntrack_tuple_hash tuplehash[2]; + long unsigned int status; + u16 cpu; + u16 local_origin: 1; + possible_net_t ct_net; + struct { } __nfct_init_offset; + struct nf_conn *master; + struct nf_ct_ext *ext; + union nf_conntrack_proto proto; +}; + +struct nf_conntrack_zone { + u16 id; + u8 flags; + u8 dir; +}; + +struct nf_ct_hook { + int (*update)(struct net *, struct sk_buff *); + void (*destroy)(struct nf_conntrack *); + bool (*get_tuple_skb)(struct nf_conntrack_tuple *, const struct sk_buff *); + void (*attach)(struct sk_buff *, const struct sk_buff *); +}; + +struct nfnl_ct_hook { + size_t (*build_size)(const struct nf_conn *); + int (*build)(struct sk_buff *, struct nf_conn *, enum ip_conntrack_info, u_int16_t, u_int16_t); + int (*parse)(const struct nlattr *, struct nf_conn *); + int (*attach_expect)(const struct nlattr *, struct nf_conn *, u32, u32); + void (*seq_adjust)(struct sk_buff *, struct nf_conn *, enum ip_conntrack_info, s32); +}; + +enum nf_ip_hook_priorities { + NF_IP_PRI_FIRST = 2147483648, + NF_IP_PRI_RAW_BEFORE_DEFRAG = 4294966846, + NF_IP_PRI_CONNTRACK_DEFRAG = 4294966896, + NF_IP_PRI_RAW = 4294966996, + NF_IP_PRI_SELINUX_FIRST = 4294967071, + NF_IP_PRI_CONNTRACK = 4294967096, + NF_IP_PRI_MANGLE = 4294967146, + NF_IP_PRI_NAT_DST = 4294967196, + NF_IP_PRI_FILTER = 0, + NF_IP_PRI_SECURITY = 50, + NF_IP_PRI_NAT_SRC = 100, + NF_IP_PRI_SELINUX_LAST = 225, + NF_IP_PRI_CONNTRACK_HELPER = 300, + NF_IP_PRI_CONNTRACK_CONFIRM = 2147483647, + NF_IP_PRI_LAST = 2147483647, +}; + +enum nf_ip6_hook_priorities { + NF_IP6_PRI_FIRST = 2147483648, + NF_IP6_PRI_RAW_BEFORE_DEFRAG = 4294966846, + NF_IP6_PRI_CONNTRACK_DEFRAG = 4294966896, + NF_IP6_PRI_RAW = 4294966996, + NF_IP6_PRI_SELINUX_FIRST = 4294967071, + NF_IP6_PRI_CONNTRACK = 4294967096, + NF_IP6_PRI_MANGLE = 4294967146, + NF_IP6_PRI_NAT_DST = 4294967196, + NF_IP6_PRI_FILTER = 0, + NF_IP6_PRI_SECURITY = 50, + NF_IP6_PRI_NAT_SRC = 100, + NF_IP6_PRI_SELINUX_LAST = 225, + NF_IP6_PRI_CONNTRACK_HELPER = 300, + NF_IP6_PRI_LAST = 2147483647, +}; + +struct socket_alloc { + struct socket socket; + struct inode vfs_inode; + long: 64; +}; + +struct ip_options { + __be32 faddr; + __be32 nexthop; + unsigned char optlen; + unsigned char srr; + unsigned char rr; + unsigned char ts; + unsigned char is_strictroute: 1; + unsigned char srr_is_hit: 1; + unsigned char is_changed: 1; + unsigned char rr_needaddr: 1; + unsigned char ts_needtime: 1; + unsigned char ts_needaddr: 1; + unsigned char router_alert; + unsigned char cipso; + unsigned char __pad2; + unsigned char __data[0]; +}; + +struct ip_options_rcu { + struct callback_head rcu; + struct ip_options opt; +}; + +struct ipv6_opt_hdr; + +struct ipv6_rt_hdr; + +struct ipv6_txoptions { + refcount_t refcnt; + int tot_len; + __u16 opt_flen; + __u16 opt_nflen; + struct ipv6_opt_hdr *hopopt; + struct ipv6_opt_hdr *dst0opt; + struct ipv6_rt_hdr *srcrt; + struct ipv6_opt_hdr *dst1opt; + struct callback_head rcu; +}; + +struct inet_cork { + unsigned int flags; + __be32 addr; + struct ip_options *opt; + unsigned int fragsize; + int length; + struct dst_entry *dst; + u8 tx_flags; + __u8 ttl; + __s16 tos; + char priority; + __u16 gso_size; + u64 transmit_time; + u32 mark; +}; + +struct inet_cork_full { + struct inet_cork base; + struct flowi fl; +}; + +struct ipv6_pinfo; + +struct ip_mc_socklist; + +struct inet_sock { + struct sock sk; + struct ipv6_pinfo *pinet6; + __be32 inet_saddr; + __s16 uc_ttl; + __u16 cmsg_flags; + struct ip_options_rcu *inet_opt; + __be16 inet_sport; + __u16 inet_id; + __u8 tos; + __u8 min_ttl; + __u8 mc_ttl; + __u8 pmtudisc; + __u8 recverr: 1; + __u8 is_icsk: 1; + __u8 freebind: 1; + __u8 hdrincl: 1; + __u8 mc_loop: 1; + __u8 transparent: 1; + __u8 mc_all: 1; + __u8 nodefrag: 1; + __u8 bind_address_no_port: 1; + __u8 recverr_rfc4884: 1; + __u8 defer_connect: 1; + __u8 rcv_tos; + __u8 convert_csum; + int uc_index; + int mc_index; + __be32 mc_addr; + struct ip_mc_socklist *mc_list; + struct inet_cork_full cork; +}; + +struct in6_pktinfo { + struct in6_addr ipi6_addr; + int ipi6_ifindex; +}; + +struct inet6_cork { + struct ipv6_txoptions *opt; + u8 hop_limit; + u8 tclass; +}; + +struct ipv6_mc_socklist; + +struct ipv6_ac_socklist; + +struct ipv6_fl_socklist; + +struct ipv6_pinfo { + struct in6_addr saddr; + struct in6_pktinfo sticky_pktinfo; + const struct in6_addr *daddr_cache; + const struct in6_addr *saddr_cache; + __be32 flow_label; + __u32 frag_size; + __u16 __unused_1: 7; + __s16 hop_limit: 9; + __u16 mc_loop: 1; + __u16 __unused_2: 6; + __s16 mcast_hops: 9; + int ucast_oif; + int mcast_oif; + union { + struct { + __u16 srcrt: 1; + __u16 osrcrt: 1; + __u16 rxinfo: 1; + __u16 rxoinfo: 1; + __u16 rxhlim: 1; + __u16 rxohlim: 1; + __u16 hopopts: 1; + __u16 ohopopts: 1; + __u16 dstopts: 1; + __u16 odstopts: 1; + __u16 rxflow: 1; + __u16 rxtclass: 1; + __u16 rxpmtu: 1; + __u16 rxorigdstaddr: 1; + __u16 recvfragsize: 1; + } bits; + __u16 all; + } rxopt; + __u16 recverr: 1; + __u16 sndflow: 1; + __u16 repflow: 1; + __u16 pmtudisc: 3; + __u16 padding: 1; + __u16 srcprefs: 3; + __u16 dontfrag: 1; + __u16 autoflowlabel: 1; + __u16 autoflowlabel_set: 1; + __u16 mc_all: 1; + __u16 recverr_rfc4884: 1; + __u16 rtalert_isolate: 1; + __u8 min_hopcount; + __u8 tclass; + __be32 rcv_flowinfo; + __u32 dst_cookie; + struct ipv6_mc_socklist *ipv6_mc_list; + struct ipv6_ac_socklist *ipv6_ac_list; + struct ipv6_fl_socklist *ipv6_fl_list; + struct ipv6_txoptions *opt; + struct sk_buff *pktoptions; + struct sk_buff *rxpmtu; + struct inet6_cork cork; +}; + +struct tcphdr { + __be16 source; + __be16 dest; + __be32 seq; + __be32 ack_seq; + __u16 res1: 4; + __u16 doff: 4; + __u16 fin: 1; + __u16 syn: 1; + __u16 rst: 1; + __u16 psh: 1; + __u16 ack: 1; + __u16 urg: 1; + __u16 ece: 1; + __u16 cwr: 1; + __be16 window; + __sum16 check; + __be16 urg_ptr; +}; + +struct iphdr { + __u8 ihl: 4; + __u8 version: 4; + __u8 tos; + __be16 tot_len; + __be16 id; + __be16 frag_off; + __u8 ttl; + __u8 protocol; + __sum16 check; + __be32 saddr; + __be32 daddr; +}; + +struct ipv6_rt_hdr { + __u8 nexthdr; + __u8 hdrlen; + __u8 type; + __u8 segments_left; +}; + +struct ipv6_opt_hdr { + __u8 nexthdr; + __u8 hdrlen; +}; + +struct ipv6hdr { + __u8 priority: 4; + __u8 version: 4; + __u8 flow_lbl[3]; + __be16 payload_len; + __u8 nexthdr; + __u8 hop_limit; + struct in6_addr saddr; + struct in6_addr daddr; +}; + +struct udphdr { + __be16 source; + __be16 dest; + __be16 len; + __sum16 check; +}; + +struct inet6_skb_parm { + int iif; + __be16 ra; + __u16 dst0; + __u16 srcrt; + __u16 dst1; + __u16 lastopt; + __u16 nhoff; + __u16 flags; + __u16 dsthao; + __u16 frag_max_size; + __u16 srhoff; +}; + +struct ip6_sf_socklist; + +struct ipv6_mc_socklist { + struct in6_addr addr; + int ifindex; + unsigned int sfmode; + struct ipv6_mc_socklist *next; + struct ip6_sf_socklist *sflist; + struct callback_head rcu; +}; + +struct ipv6_ac_socklist { + struct in6_addr acl_addr; + int acl_ifindex; + struct ipv6_ac_socklist *acl_next; +}; + +struct ip6_flowlabel; + +struct ipv6_fl_socklist { + struct ipv6_fl_socklist *next; + struct ip6_flowlabel *fl; + struct callback_head rcu; +}; + +struct ip6_sf_socklist { + unsigned int sl_max; + unsigned int sl_count; + struct callback_head rcu; + struct in6_addr sl_addr[0]; +}; + +struct ip6_flowlabel { + struct ip6_flowlabel *next; + __be32 label; + atomic_t users; + struct in6_addr dst; + struct ipv6_txoptions *opt; + long unsigned int linger; + struct callback_head rcu; + u8 share; + union { + struct pid *pid; + kuid_t uid; + } owner; + long unsigned int lastuse; + long unsigned int expires; + struct net *fl_net; +}; + +struct inet_skb_parm { + int iif; + struct ip_options opt; + u16 flags; + u16 frag_max_size; +}; + +struct nf_queue_entry; + +struct nf_ipv6_ops { + void (*route_input)(struct sk_buff *); + int (*fragment)(struct net *, struct sock *, struct sk_buff *, int (*)(struct net *, struct sock *, struct sk_buff *)); + int (*reroute)(struct sk_buff *, const struct nf_queue_entry *); +}; + +struct nf_queue_entry { + struct list_head list; + struct sk_buff *skb; + unsigned int id; + unsigned int hook_index; + struct nf_hook_state state; + u16 size; +}; + +struct tty_file_private { + struct tty_struct *tty; + struct file *file; + struct list_head list; +}; + +struct icmp_err { + int errno; + unsigned int fatal: 1; +}; + +struct netlbl_lsm_cache { + refcount_t refcount; + void (*free)(const void *); + void *data; +}; + +struct netlbl_lsm_catmap { + u32 startbit; + u64 bitmap[4]; + struct netlbl_lsm_catmap *next; +}; + +struct netlbl_lsm_secattr { + u32 flags; + u32 type; + char *domain; + struct netlbl_lsm_cache *cache; + struct { + struct { + struct netlbl_lsm_catmap *cat; + u32 lvl; + } mls; + u32 secid; + } attr; +}; + +struct dccp_hdr { + __be16 dccph_sport; + __be16 dccph_dport; + __u8 dccph_doff; + __u8 dccph_cscov: 4; + __u8 dccph_ccval: 4; + __sum16 dccph_checksum; + __u8 dccph_x: 1; + __u8 dccph_type: 4; + __u8 dccph_reserved: 3; + __u8 dccph_seq2; + __be16 dccph_seq; +}; + +enum dccp_state { + DCCP_OPEN = 1, + DCCP_REQUESTING = 2, + DCCP_LISTEN = 10, + DCCP_RESPOND = 3, + DCCP_ACTIVE_CLOSEREQ = 4, + DCCP_PASSIVE_CLOSE = 8, + DCCP_CLOSING = 11, + DCCP_TIME_WAIT = 6, + DCCP_CLOSED = 7, + DCCP_NEW_SYN_RECV = 12, + DCCP_PARTOPEN = 13, + DCCP_PASSIVE_CLOSEREQ = 14, + DCCP_MAX_STATES = 15, +}; + +enum sctp_msg_flags { + MSG_NOTIFICATION = 32768, +}; + +struct sctp_initmsg { + __u16 sinit_num_ostreams; + __u16 sinit_max_instreams; + __u16 sinit_max_attempts; + __u16 sinit_max_init_timeo; +}; + +struct sctp_sndrcvinfo { + __u16 sinfo_stream; + __u16 sinfo_ssn; + __u16 sinfo_flags; + __u32 sinfo_ppid; + __u32 sinfo_context; + __u32 sinfo_timetolive; + __u32 sinfo_tsn; + __u32 sinfo_cumtsn; + sctp_assoc_t sinfo_assoc_id; +}; + +struct sctp_rtoinfo { + sctp_assoc_t srto_assoc_id; + __u32 srto_initial; + __u32 srto_max; + __u32 srto_min; +}; + +struct sctp_assocparams { + sctp_assoc_t sasoc_assoc_id; + __u16 sasoc_asocmaxrxt; + __u16 sasoc_number_peer_destinations; + __u32 sasoc_peer_rwnd; + __u32 sasoc_local_rwnd; + __u32 sasoc_cookie_life; +}; + +struct sctp_paddrparams { + sctp_assoc_t spp_assoc_id; + struct __kernel_sockaddr_storage spp_address; + __u32 spp_hbinterval; + __u16 spp_pathmaxrxt; + __u32 spp_pathmtu; + __u32 spp_sackdelay; + __u32 spp_flags; + __u32 spp_ipv6_flowlabel; + __u8 spp_dscp; + char: 8; +} __attribute__((packed)); + +struct sctphdr { + __be16 source; + __be16 dest; + __be32 vtag; + __le32 checksum; +}; + +enum sctp_cid { + SCTP_CID_DATA = 0, + SCTP_CID_INIT = 1, + SCTP_CID_INIT_ACK = 2, + SCTP_CID_SACK = 3, + SCTP_CID_HEARTBEAT = 4, + SCTP_CID_HEARTBEAT_ACK = 5, + SCTP_CID_ABORT = 6, + SCTP_CID_SHUTDOWN = 7, + SCTP_CID_SHUTDOWN_ACK = 8, + SCTP_CID_ERROR = 9, + SCTP_CID_COOKIE_ECHO = 10, + SCTP_CID_COOKIE_ACK = 11, + SCTP_CID_ECN_ECNE = 12, + SCTP_CID_ECN_CWR = 13, + SCTP_CID_SHUTDOWN_COMPLETE = 14, + SCTP_CID_AUTH = 15, + SCTP_CID_I_DATA = 64, + SCTP_CID_FWD_TSN = 192, + SCTP_CID_ASCONF = 193, + SCTP_CID_I_FWD_TSN = 194, + SCTP_CID_ASCONF_ACK = 128, + SCTP_CID_RECONF = 130, + SCTP_CID_PAD = 132, +}; + +struct sctp_paramhdr { + __be16 type; + __be16 length; +}; + +enum sctp_param { + SCTP_PARAM_HEARTBEAT_INFO = 256, + SCTP_PARAM_IPV4_ADDRESS = 1280, + SCTP_PARAM_IPV6_ADDRESS = 1536, + SCTP_PARAM_STATE_COOKIE = 1792, + SCTP_PARAM_UNRECOGNIZED_PARAMETERS = 2048, + SCTP_PARAM_COOKIE_PRESERVATIVE = 2304, + SCTP_PARAM_HOST_NAME_ADDRESS = 2816, + SCTP_PARAM_SUPPORTED_ADDRESS_TYPES = 3072, + SCTP_PARAM_ECN_CAPABLE = 128, + SCTP_PARAM_RANDOM = 640, + SCTP_PARAM_CHUNKS = 896, + SCTP_PARAM_HMAC_ALGO = 1152, + SCTP_PARAM_SUPPORTED_EXT = 2176, + SCTP_PARAM_FWD_TSN_SUPPORT = 192, + SCTP_PARAM_ADD_IP = 448, + SCTP_PARAM_DEL_IP = 704, + SCTP_PARAM_ERR_CAUSE = 960, + SCTP_PARAM_SET_PRIMARY = 1216, + SCTP_PARAM_SUCCESS_REPORT = 1472, + SCTP_PARAM_ADAPTATION_LAYER_IND = 1728, + SCTP_PARAM_RESET_OUT_REQUEST = 3328, + SCTP_PARAM_RESET_IN_REQUEST = 3584, + SCTP_PARAM_RESET_TSN_REQUEST = 3840, + SCTP_PARAM_RESET_RESPONSE = 4096, + SCTP_PARAM_RESET_ADD_OUT_STREAMS = 4352, + SCTP_PARAM_RESET_ADD_IN_STREAMS = 4608, +}; + +struct sctp_datahdr { + __be32 tsn; + __be16 stream; + __be16 ssn; + __u32 ppid; + __u8 payload[0]; +}; + +struct sctp_idatahdr { + __be32 tsn; + __be16 stream; + __be16 reserved; + __be32 mid; + union { + __u32 ppid; + __be32 fsn; + }; + __u8 payload[0]; +}; + +struct sctp_ipv4addr_param { + struct sctp_paramhdr param_hdr; + struct in_addr addr; +}; + +struct sctp_ipv6addr_param { + struct sctp_paramhdr param_hdr; + struct in6_addr addr; +}; + +struct sctp_cookie_preserve_param { + struct sctp_paramhdr param_hdr; + __be32 lifespan_increment; +}; + +struct sctp_hostname_param { + struct sctp_paramhdr param_hdr; + uint8_t hostname[0]; +}; + +struct sctp_supported_addrs_param { + struct sctp_paramhdr param_hdr; + __be16 types[0]; +}; + +struct sctp_adaptation_ind_param { + struct sctp_paramhdr param_hdr; + __be32 adaptation_ind; +}; + +struct sctp_supported_ext_param { + struct sctp_paramhdr param_hdr; + __u8 chunks[0]; +}; + +struct sctp_random_param { + struct sctp_paramhdr param_hdr; + __u8 random_val[0]; +}; + +struct sctp_chunks_param { + struct sctp_paramhdr param_hdr; + __u8 chunks[0]; +}; + +struct sctp_hmac_algo_param { + struct sctp_paramhdr param_hdr; + __be16 hmac_ids[0]; +}; + +struct sctp_cookie_param { + struct sctp_paramhdr p; + __u8 body[0]; +}; + +struct sctp_gap_ack_block { + __be16 start; + __be16 end; +}; + +union sctp_sack_variable { + struct sctp_gap_ack_block gab; + __be32 dup; +}; + +struct sctp_sackhdr { + __be32 cum_tsn_ack; + __be32 a_rwnd; + __be16 num_gap_ack_blocks; + __be16 num_dup_tsns; + union sctp_sack_variable variable[0]; +}; + +struct sctp_heartbeathdr { + struct sctp_paramhdr info; +}; + +struct sctp_shutdownhdr { + __be32 cum_tsn_ack; +}; + +struct sctp_errhdr { + __be16 cause; + __be16 length; + __u8 variable[0]; +}; + +struct sctp_ecnehdr { + __be32 lowest_tsn; +}; + +struct sctp_cwrhdr { + __be32 lowest_tsn; +}; + +struct sctp_fwdtsn_skip { + __be16 stream; + __be16 ssn; +}; + +struct sctp_fwdtsn_hdr { + __be32 new_cum_tsn; + struct sctp_fwdtsn_skip skip[0]; +}; + +struct sctp_ifwdtsn_skip { + __be16 stream; + __u8 reserved; + __u8 flags; + __be32 mid; +}; + +struct sctp_ifwdtsn_hdr { + __be32 new_cum_tsn; + struct sctp_ifwdtsn_skip skip[0]; +}; + +struct sctp_addip_param { + struct sctp_paramhdr param_hdr; + __be32 crr_id; +}; + +struct sctp_addiphdr { + __be32 serial; + __u8 params[0]; +}; + +struct sctp_authhdr { + __be16 shkey_id; + __be16 hmac_id; + __u8 hmac[0]; +}; + struct sctp_auth_bytes { refcount_t refcnt; __u32 len; @@ -52032,6 +53281,8 @@ enum sctp_socket_type { SCTP_SOCKET_TCP = 2, }; +struct crypto_shash___2; + struct sctp_pf; struct sctp_sock { @@ -52114,6 +53365,30 @@ struct sctp_pf { struct sctp_af *af; }; +struct sctp_endpoint { + struct sctp_ep_common base; + struct hlist_node node; + int hashent; + struct list_head asocs; + __u8 secret_key[32]; + __u8 *digest; + __u32 sndbuf_policy; + __u32 rcvbuf_policy; + struct crypto_shash___2 **auth_hmacs; + struct sctp_hmac_algo_param *auth_hmacs_list; + struct sctp_chunks_param *auth_chunk_list; + struct list_head endpoint_shared_keys; + __u16 active_key_id; + __u8 ecn_enable: 1; + __u8 auth_enable: 1; + __u8 intl_enable: 1; + __u8 prsctp_enable: 1; + __u8 asconf_enable: 1; + __u8 reconf_enable: 1; + __u8 strreset_enable; + struct callback_head rcu; +}; + struct sctp_signed_cookie { __u8 signature[32]; __u32 __pad; @@ -52242,8 +53517,7 @@ struct sctp_transport { __u16 pmtu; __u16 probe_size; __u16 probe_high; - __u8 probe_count: 3; - __u8 raise_count: 5; + __u8 probe_count; __u8 state; } pl; __u64 hb_nonce; @@ -52479,7 +53753,9 @@ enum { XFRM_MSG_NEWSPDINFO = 36, XFRM_MSG_GETSPDINFO = 37, XFRM_MSG_MAPPING = 38, - __XFRM_MSG_MAX = 39, + XFRM_MSG_SETDEFAULT = 39, + XFRM_MSG_GETDEFAULT = 40, + __XFRM_MSG_MAX = 41, }; enum { @@ -52554,6 +53830,14 @@ enum { __RTM_MAX = 119, }; +enum { + INET_DIAG_REQ_NONE = 0, + INET_DIAG_REQ_BYTECODE = 1, + INET_DIAG_REQ_SK_BPF_STORAGES = 2, + INET_DIAG_REQ_PROTOCOL = 3, + __INET_DIAG_REQ_MAX = 4, +}; + struct nlmsg_perm { u16 nlmsg_type; u32 perm; @@ -52649,6 +53933,8 @@ struct hashtab_info { u32 max_chain_len; }; +struct path___2; + struct hashtab_key_params { u32 (*hash)(const void *); int (*cmp)(const void *, const void *); @@ -53047,7 +54333,6 @@ struct sockaddr_un { struct unix_address { refcount_t refcnt; int len; - unsigned int hash; struct sockaddr_un name[0]; }; @@ -53074,8 +54359,7 @@ struct unix_sock { struct socket_wq peer_wq; wait_queue_entry_t peer_wake; struct scm_stat scm_stat; - long: 32; - long: 64; + struct sk_buff *oob_skb; long: 64; }; @@ -53313,30 +54597,6 @@ enum ima_fs_flags { IMA_FS_BUSY = 0, }; -enum hash_algo { - HASH_ALGO_MD4 = 0, - HASH_ALGO_MD5 = 1, - HASH_ALGO_SHA1 = 2, - HASH_ALGO_RIPE_MD_160 = 3, - HASH_ALGO_SHA256 = 4, - HASH_ALGO_SHA384 = 5, - HASH_ALGO_SHA512 = 6, - HASH_ALGO_SHA224 = 7, - HASH_ALGO_RIPE_MD_128 = 8, - HASH_ALGO_RIPE_MD_256 = 9, - HASH_ALGO_RIPE_MD_320 = 10, - HASH_ALGO_WP_256 = 11, - HASH_ALGO_WP_384 = 12, - HASH_ALGO_WP_512 = 13, - HASH_ALGO_TGR_128 = 14, - HASH_ALGO_TGR_160 = 15, - HASH_ALGO_TGR_192 = 16, - HASH_ALGO_SM3_256 = 17, - HASH_ALGO_STREEBOG_256 = 18, - HASH_ALGO_STREEBOG_512 = 19, - HASH_ALGO__LAST = 20, -}; - enum tpm_duration { TPM_SHORT = 0, TPM_MEDIUM = 1, @@ -53370,7 +54630,8 @@ enum ima_hooks { KEXEC_CMDLINE = 11, KEY_CHECK = 12, CRITICAL_DATA = 13, - MAX_CHECK = 14, + SETXATTR_CHECK = 14, + MAX_CHECK = 15, }; struct crypto_async_request; @@ -53491,10 +54752,15 @@ struct ima_rule_entry { long unsigned int fsmagic; uuid_t fsuuid; kuid_t uid; + kgid_t gid; kuid_t fowner; + kgid_t fgroup; bool (*uid_op)(kuid_t, kuid_t); + bool (*gid_op)(kgid_t, kgid_t); bool (*fowner_op)(kuid_t, kuid_t); + bool (*fgroup_op)(kgid_t, kgid_t); int pcr; + unsigned int allowed_algos; struct { void *rule; char *args_p; @@ -53506,7 +54772,7 @@ struct ima_rule_entry { struct ima_template_desc *template; }; -enum { +enum policy_opt { Opt_measure = 0, Opt_dont_measure = 1, Opt_appraise = 2, @@ -53527,21 +54793,31 @@ enum { Opt_fsuuid = 17, Opt_uid_eq = 18, Opt_euid_eq = 19, - Opt_fowner_eq = 20, - Opt_uid_gt = 21, - Opt_euid_gt = 22, - Opt_fowner_gt = 23, - Opt_uid_lt = 24, - Opt_euid_lt = 25, - Opt_fowner_lt = 26, - Opt_appraise_type = 27, - Opt_appraise_flag = 28, - Opt_permit_directio = 29, - Opt_pcr = 30, - Opt_template = 31, - Opt_keyrings = 32, - Opt_label = 33, - Opt_err___6 = 34, + Opt_gid_eq = 20, + Opt_egid_eq = 21, + Opt_fowner_eq = 22, + Opt_fgroup_eq = 23, + Opt_uid_gt = 24, + Opt_euid_gt = 25, + Opt_gid_gt = 26, + Opt_egid_gt = 27, + Opt_fowner_gt = 28, + Opt_fgroup_gt = 29, + Opt_uid_lt = 30, + Opt_euid_lt = 31, + Opt_gid_lt = 32, + Opt_egid_lt = 33, + Opt_fowner_lt = 34, + Opt_fgroup_lt = 35, + Opt_appraise_type = 36, + Opt_appraise_flag = 37, + Opt_appraise_algos = 38, + Opt_permit_directio = 39, + Opt_pcr = 40, + Opt_template = 41, + Opt_keyrings = 42, + Opt_label = 43, + Opt_err___6 = 44, }; enum { @@ -53630,6 +54906,7 @@ struct crypto_larval { struct crypto_alg *adult; struct completion completion; u32 mask; + bool test_started; }; struct crypto_cipher { @@ -54307,6 +55584,14 @@ struct sha256_state { u8 buf[64]; }; +struct sha512_state { + u64 state[8]; + u64 count[2]; + u8 buf[128]; +}; + +typedef void sha512_block_fn(struct sha512_state *, const u8 *, int); + enum blake2b_lengths { BLAKE2B_BLOCK_SIZE = 128, BLAKE2B_HASH_SIZE = 64, @@ -54740,9 +56025,9 @@ struct asymmetric_key_subtype___2 { struct public_key_signature { struct asymmetric_key_id *auth_ids[2]; u8 *s; - u32 s_size; u8 *digest; - u8 digest_size; + u32 s_size; + u32 digest_size; const char *pkey_algo; const char *hash_algo; const char *encoding; @@ -54799,59 +56084,64 @@ enum OID { OID_msOutlookExpress = 38, OID_ntlmssp = 39, OID_spnego = 40, - OID_certAuthInfoAccess = 41, - OID_sha1 = 42, - OID_id_ansip384r1 = 43, - OID_sha256 = 44, - OID_sha384 = 45, - OID_sha512 = 46, - OID_sha224 = 47, - OID_commonName = 48, - OID_surname = 49, - OID_countryName = 50, - OID_locality = 51, - OID_stateOrProvinceName = 52, - OID_organizationName = 53, - OID_organizationUnitName = 54, - OID_title = 55, - OID_description = 56, - OID_name = 57, - OID_givenName = 58, - OID_initials = 59, - OID_generationalQualifier = 60, - OID_subjectKeyIdentifier = 61, - OID_keyUsage = 62, - OID_subjectAltName = 63, - OID_issuerAltName = 64, - OID_basicConstraints = 65, - OID_crlDistributionPoints = 66, - OID_certPolicies = 67, - OID_authorityKeyIdentifier = 68, - OID_extKeyUsage = 69, - OID_gostCPSignA = 70, - OID_gostCPSignB = 71, - OID_gostCPSignC = 72, - OID_gost2012PKey256 = 73, - OID_gost2012PKey512 = 74, - OID_gost2012Digest256 = 75, - OID_gost2012Digest512 = 76, - OID_gost2012Signature256 = 77, - OID_gost2012Signature512 = 78, - OID_gostTC26Sign256A = 79, - OID_gostTC26Sign256B = 80, - OID_gostTC26Sign256C = 81, - OID_gostTC26Sign256D = 82, - OID_gostTC26Sign512A = 83, - OID_gostTC26Sign512B = 84, - OID_gostTC26Sign512C = 85, - OID_sm2 = 86, - OID_sm3 = 87, - OID_SM2_with_SM3 = 88, - OID_sm3WithRSAEncryption = 89, - OID_TPMLoadableKey = 90, - OID_TPMImportableKey = 91, - OID_TPMSealedData = 92, - OID__NR = 93, + OID_IAKerb = 41, + OID_PKU2U = 42, + OID_Scram = 43, + OID_certAuthInfoAccess = 44, + OID_sha1 = 45, + OID_id_ansip384r1 = 46, + OID_sha256 = 47, + OID_sha384 = 48, + OID_sha512 = 49, + OID_sha224 = 50, + OID_commonName = 51, + OID_surname = 52, + OID_countryName = 53, + OID_locality = 54, + OID_stateOrProvinceName = 55, + OID_organizationName = 56, + OID_organizationUnitName = 57, + OID_title = 58, + OID_description = 59, + OID_name = 60, + OID_givenName = 61, + OID_initials = 62, + OID_generationalQualifier = 63, + OID_subjectKeyIdentifier = 64, + OID_keyUsage = 65, + OID_subjectAltName = 66, + OID_issuerAltName = 67, + OID_basicConstraints = 68, + OID_crlDistributionPoints = 69, + OID_certPolicies = 70, + OID_authorityKeyIdentifier = 71, + OID_extKeyUsage = 72, + OID_NetlogonMechanism = 73, + OID_appleLocalKdcSupported = 74, + OID_gostCPSignA = 75, + OID_gostCPSignB = 76, + OID_gostCPSignC = 77, + OID_gost2012PKey256 = 78, + OID_gost2012PKey512 = 79, + OID_gost2012Digest256 = 80, + OID_gost2012Digest512 = 81, + OID_gost2012Signature256 = 82, + OID_gost2012Signature512 = 83, + OID_gostTC26Sign256A = 84, + OID_gostTC26Sign256B = 85, + OID_gostTC26Sign256C = 86, + OID_gostTC26Sign256D = 87, + OID_gostTC26Sign512A = 88, + OID_gostTC26Sign512B = 89, + OID_gostTC26Sign512C = 90, + OID_sm2 = 91, + OID_sm3 = 92, + OID_SM2_with_SM3 = 93, + OID_sm3WithRSAEncryption = 94, + OID_TPMLoadableKey = 95, + OID_TPMImportableKey = 96, + OID_TPMSealedData = 97, + OID__NR = 98, }; struct public_key { @@ -55014,6 +56304,79 @@ struct pkcs7_parse_context { bool expect_skid; }; +enum { + DISK_EVENT_MEDIA_CHANGE = 1, + DISK_EVENT_EJECT_REQUEST = 2, +}; + +struct blk_integrity_profile; + +struct blk_integrity { + const struct blk_integrity_profile *profile; + unsigned char flags; + unsigned char tuple_size; + unsigned char interval_exp; + unsigned char tag_size; +}; + +struct blk_integrity_iter; + +typedef blk_status_t integrity_processing_fn(struct blk_integrity_iter *); + +typedef void integrity_prepare_fn(struct request *); + +typedef void integrity_complete_fn(struct request *, unsigned int); + +struct blk_integrity_profile { + integrity_processing_fn *generate_fn; + integrity_processing_fn *verify_fn; + integrity_prepare_fn *prepare_fn; + integrity_complete_fn *complete_fn; + const char *name; +}; + +struct blk_integrity_iter { + void *prot_buf; + void *data_buf; + sector_t seed; + unsigned int data_size; + short unsigned int interval; + const char *disk_name; +}; + +struct bdev_inode { + struct block_device bdev; + struct inode vfs_inode; +}; + +enum { + DIO_SHOULD_DIRTY = 1, + DIO_IS_SYNC = 2, +}; + +struct blkdev_dio { + union { + struct kiocb *iocb; + struct task_struct *waiter; + }; + size_t size; + atomic_t ref; + unsigned int flags; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + struct bio bio; + long: 64; + long: 64; +}; + +struct bio_alloc_cache { + struct bio_list free_list; + unsigned int nr; +}; + enum rq_qos_id { RQ_QOS_WBT = 0, RQ_QOS_LATENCY = 1, @@ -55031,13 +56394,6 @@ struct rq_qos { struct dentry *debugfs_dir; }; -enum hctx_type { - HCTX_TYPE_DEFAULT = 0, - HCTX_TYPE_READ = 1, - HCTX_TYPE_POLL = 2, - HCTX_MAX_TYPES = 3, -}; - struct rq_qos_ops { void (*throttle)(struct rq_qos *, struct bio *); void (*track)(struct rq_qos *, struct request *, struct bio *); @@ -55072,6 +56428,7 @@ enum { BLK_MQ_F_TAG_HCTX_SHARED = 8, BLK_MQ_F_BLOCKING = 32, BLK_MQ_F_NO_SCHED = 64, + BLK_MQ_F_NO_SCHED_BY_DEFAULT = 128, BLK_MQ_F_ALLOC_POLICY_START_BIT = 8, BLK_MQ_F_ALLOC_POLICY_BITS = 1, BLK_MQ_S_STOPPED = 0, @@ -55090,6 +56447,7 @@ enum { }; enum rpm_status { + RPM_INVALID = 4294967295, RPM_ACTIVE = 0, RPM_RESUMING = 1, RPM_SUSPENDED = 2, @@ -55112,6 +56470,45 @@ enum { BLK_MQ_REQ_PM = 4, }; +typedef struct blkcg_policy_data *blkcg_pol_alloc_cpd_fn(gfp_t); + +typedef void blkcg_pol_init_cpd_fn(struct blkcg_policy_data *); + +typedef void blkcg_pol_free_cpd_fn(struct blkcg_policy_data *); + +typedef void blkcg_pol_bind_cpd_fn(struct blkcg_policy_data *); + +typedef struct blkg_policy_data *blkcg_pol_alloc_pd_fn(gfp_t, struct request_queue *, struct blkcg *); + +typedef void blkcg_pol_init_pd_fn(struct blkg_policy_data *); + +typedef void blkcg_pol_online_pd_fn(struct blkg_policy_data *); + +typedef void blkcg_pol_offline_pd_fn(struct blkg_policy_data *); + +typedef void blkcg_pol_free_pd_fn(struct blkg_policy_data *); + +typedef void blkcg_pol_reset_pd_stats_fn(struct blkg_policy_data *); + +typedef bool blkcg_pol_stat_pd_fn(struct blkg_policy_data *, struct seq_file *); + +struct blkcg_policy { + int plid; + struct cftype *dfl_cftypes; + struct cftype *legacy_cftypes; + blkcg_pol_alloc_cpd_fn *cpd_alloc_fn; + blkcg_pol_init_cpd_fn *cpd_init_fn; + blkcg_pol_free_cpd_fn *cpd_free_fn; + blkcg_pol_bind_cpd_fn *cpd_bind_fn; + blkcg_pol_alloc_pd_fn *pd_alloc_fn; + blkcg_pol_init_pd_fn *pd_init_fn; + blkcg_pol_online_pd_fn *pd_online_fn; + blkcg_pol_offline_pd_fn *pd_offline_fn; + blkcg_pol_free_pd_fn *pd_free_fn; + blkcg_pol_reset_pd_stats_fn *pd_reset_stats_fn; + blkcg_pol_stat_pd_fn *pd_stat_fn; +}; + struct trace_event_raw_block_buffer { struct trace_entry ent; dev_t dev; @@ -55253,7 +56650,7 @@ typedef void (*btf_trace_block_dirty_buffer)(void *, struct buffer_head *); typedef void (*btf_trace_block_rq_requeue)(void *, struct request *); -typedef void (*btf_trace_block_rq_complete)(void *, struct request *, int, unsigned int); +typedef void (*btf_trace_block_rq_complete)(void *, struct request *, blk_status_t, unsigned int); typedef void (*btf_trace_block_rq_insert)(void *, struct request *); @@ -55289,6 +56686,83 @@ enum { BLK_MQ_TAG_MAX = 4294967294, }; +enum blkg_rwstat_type { + BLKG_RWSTAT_READ = 0, + BLKG_RWSTAT_WRITE = 1, + BLKG_RWSTAT_SYNC = 2, + BLKG_RWSTAT_ASYNC = 3, + BLKG_RWSTAT_DISCARD = 4, + BLKG_RWSTAT_NR = 5, + BLKG_RWSTAT_TOTAL = 5, +}; + +struct blkg_rwstat { + struct percpu_counter cpu_cnt[5]; + atomic64_t aux_cnt[5]; +}; + +struct throtl_grp; + +struct throtl_qnode { + struct list_head node; + struct bio_list bios; + struct throtl_grp *tg; +}; + +struct throtl_service_queue { + struct throtl_service_queue *parent_sq; + struct list_head queued[2]; + unsigned int nr_queued[2]; + struct rb_root_cached pending_tree; + unsigned int nr_pending; + long unsigned int first_pending_disptime; + struct timer_list pending_timer; +}; + +struct throtl_grp { + struct blkg_policy_data pd; + struct rb_node rb_node; + struct throtl_data *td; + struct throtl_service_queue service_queue; + struct throtl_qnode qnode_on_self[2]; + struct throtl_qnode qnode_on_parent[2]; + long unsigned int disptime; + unsigned int flags; + bool has_rules[2]; + uint64_t bps[4]; + uint64_t bps_conf[4]; + unsigned int iops[4]; + unsigned int iops_conf[4]; + uint64_t bytes_disp[2]; + unsigned int io_disp[2]; + long unsigned int last_low_overflow_time[2]; + uint64_t last_bytes_disp[2]; + unsigned int last_io_disp[2]; + long unsigned int last_check_time; + long unsigned int latency_target; + long unsigned int latency_target_conf; + long unsigned int slice_start[2]; + long unsigned int slice_end[2]; + long unsigned int last_finish_time; + long unsigned int checked_last_finish_time; + long unsigned int avg_idletime; + long unsigned int idletime_threshold; + long unsigned int idletime_threshold_conf; + unsigned int bio_cnt; + unsigned int bad_bio_cnt; + long unsigned int bio_cnt_reset_time; + atomic_t io_split_cnt[2]; + atomic_t last_io_split_cnt[2]; + struct blkg_rwstat stat_bytes; + struct blkg_rwstat stat_ios; +}; + +enum { + LIMIT_LOW = 0, + LIMIT_MAX = 1, + LIMIT_CNT = 2, +}; + struct queue_sysfs_entry { struct attribute attr; ssize_t (*show)(struct request_queue *, char *); @@ -55404,12 +56878,6 @@ struct blk_queue_stats { bool enable_accounting; }; -struct blk_mq_ctx_sysfs_entry { - struct attribute attr; - ssize_t (*show)(struct blk_mq_ctx *, char *); - ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t); -}; - struct blk_mq_hw_ctx_sysfs_entry { struct attribute attr; ssize_t (*show)(struct blk_mq_hw_ctx *, char *); @@ -55515,7 +56983,7 @@ enum { }; struct parsed_partitions { - struct block_device *bdev; + struct gendisk *disk; char name[32]; struct { sector_t from; @@ -56032,160 +57500,61 @@ struct disk_events { struct delayed_work dwork; }; -enum sam_status { - SAM_STAT_GOOD = 0, - SAM_STAT_CHECK_CONDITION = 2, - SAM_STAT_CONDITION_MET = 4, - SAM_STAT_BUSY = 8, - SAM_STAT_INTERMEDIATE = 16, - SAM_STAT_INTERMEDIATE_CONDITION_MET = 20, - SAM_STAT_RESERVATION_CONFLICT = 24, - SAM_STAT_COMMAND_TERMINATED = 34, - SAM_STAT_TASK_SET_FULL = 40, - SAM_STAT_ACA_ACTIVE = 48, - SAM_STAT_TASK_ABORTED = 64, +struct blk_ia_range_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct blk_independent_access_range *, char *); }; -struct request_sense; - -struct cdrom_generic_command { - unsigned char cmd[12]; - unsigned char *buffer; - unsigned int buflen; - int stat; - struct request_sense *sense; - unsigned char data_direction; - int quiet; - int timeout; - union { - void *reserved[1]; - void *unused; - }; +struct sg_io_v4 { + __s32 guard; + __u32 protocol; + __u32 subprotocol; + __u32 request_len; + __u64 request; + __u64 request_tag; + __u32 request_attr; + __u32 request_priority; + __u32 request_extra; + __u32 max_response_len; + __u64 response; + __u32 dout_iovec_count; + __u32 dout_xfer_len; + __u32 din_iovec_count; + __u32 din_xfer_len; + __u64 dout_xferp; + __u64 din_xferp; + __u32 timeout; + __u32 flags; + __u64 usr_ptr; + __u32 spare_in; + __u32 driver_status; + __u32 transport_status; + __u32 device_status; + __u32 retry_delay; + __u32 info; + __u32 duration; + __u32 response_len; + __s32 din_resid; + __s32 dout_resid; + __u64 generated_tag; + __u32 spare_out; + __u32 padding; }; -struct request_sense { - __u8 error_code: 7; - __u8 valid: 1; - __u8 segment_number; - __u8 sense_key: 4; - __u8 reserved2: 1; - __u8 ili: 1; - __u8 reserved1: 2; - __u8 information[4]; - __u8 add_sense_len; - __u8 command_info[4]; - __u8 asc; - __u8 ascq; - __u8 fruc; - __u8 sks[3]; - __u8 asb[46]; -}; - -enum scsi_msg_byte { - COMMAND_COMPLETE = 0, - EXTENDED_MESSAGE = 1, - SAVE_POINTERS = 2, - RESTORE_POINTERS = 3, - DISCONNECT = 4, - INITIATOR_ERROR = 5, - ABORT_TASK_SET = 6, - MESSAGE_REJECT = 7, - NOP___2 = 8, - MSG_PARITY_ERROR = 9, - LINKED_CMD_COMPLETE = 10, - LINKED_FLG_CMD_COMPLETE = 11, - TARGET_RESET = 12, - ABORT_TASK = 13, - CLEAR_TASK_SET = 14, - INITIATE_RECOVERY = 15, - RELEASE_RECOVERY = 16, - TERMINATE_IO_PROC = 17, - CLEAR_ACA = 22, - LOGICAL_UNIT_RESET = 23, - SIMPLE_QUEUE_TAG = 32, - HEAD_OF_QUEUE_TAG = 33, - ORDERED_QUEUE_TAG = 34, - IGNORE_WIDE_RESIDUE = 35, - ACA = 36, - QAS_REQUEST = 85, - BUS_DEVICE_RESET = 12, - ABORT = 6, -}; - -struct scsi_ioctl_command { - unsigned int inlen; - unsigned int outlen; - unsigned char data[0]; -}; - -enum scsi_device_event { - SDEV_EVT_MEDIA_CHANGE = 1, - SDEV_EVT_INQUIRY_CHANGE_REPORTED = 2, - SDEV_EVT_CAPACITY_CHANGE_REPORTED = 3, - SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED = 4, - SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED = 5, - SDEV_EVT_LUN_CHANGE_REPORTED = 6, - SDEV_EVT_ALUA_STATE_CHANGE_REPORTED = 7, - SDEV_EVT_POWER_ON_RESET_OCCURRED = 8, - SDEV_EVT_FIRST = 1, - SDEV_EVT_LAST = 8, - SDEV_EVT_MAXBITS = 9, -}; - -struct scsi_request { - unsigned char __cmd[16]; - unsigned char *cmd; - short unsigned int cmd_len; - int result; - unsigned int sense_len; - unsigned int resid_len; - int retries; - void *sense; -}; - -struct sg_io_hdr { - int interface_id; - int dxfer_direction; - unsigned char cmd_len; - unsigned char mx_sb_len; - short unsigned int iovec_count; - unsigned int dxfer_len; - void *dxferp; - unsigned char *cmdp; - void *sbp; - unsigned int timeout; - unsigned int flags; - int pack_id; - void *usr_ptr; - unsigned char status; - unsigned char masked_status; - unsigned char msg_status; - unsigned char sb_len_wr; - short unsigned int host_status; - short unsigned int driver_status; - int resid; - unsigned int duration; - unsigned int info; -}; - -struct blk_cmd_filter { - long unsigned int read_ok[4]; - long unsigned int write_ok[4]; -}; - -enum { - OMAX_SB_LEN = 16, -}; +typedef int bsg_sg_io_fn(struct request_queue *, struct sg_io_v4 *, fmode_t, unsigned int); struct bsg_device { struct request_queue *queue; - spinlock_t lock; - struct hlist_node dev_list; - refcount_t ref_count; - char name[20]; + struct device device; + struct cdev cdev; int max_queue; + unsigned int timeout; + unsigned int reserved_size; + bsg_sg_io_fn *sg_io_fn; }; +struct blk_integrity_profile___2; + struct bsg_job; typedef int bsg_job_fn(struct bsg_job *); @@ -56215,49 +57584,27 @@ struct bsg_job { typedef enum blk_eh_timer_return bsg_timeout_fn(struct request *); -struct bsg_set { - struct blk_mq_tag_set tag_set; - bsg_job_fn *job_fn; - bsg_timeout_fn *timeout_fn; +enum scsi_device_event { + SDEV_EVT_MEDIA_CHANGE = 1, + SDEV_EVT_INQUIRY_CHANGE_REPORTED = 2, + SDEV_EVT_CAPACITY_CHANGE_REPORTED = 3, + SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED = 4, + SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED = 5, + SDEV_EVT_LUN_CHANGE_REPORTED = 6, + SDEV_EVT_ALUA_STATE_CHANGE_REPORTED = 7, + SDEV_EVT_POWER_ON_RESET_OCCURRED = 8, + SDEV_EVT_FIRST = 1, + SDEV_EVT_LAST = 8, + SDEV_EVT_MAXBITS = 9, }; -typedef struct blkcg_policy_data *blkcg_pol_alloc_cpd_fn(gfp_t); +struct bsg_device___2; -typedef void blkcg_pol_init_cpd_fn(struct blkcg_policy_data *); - -typedef void blkcg_pol_free_cpd_fn(struct blkcg_policy_data *); - -typedef void blkcg_pol_bind_cpd_fn(struct blkcg_policy_data *); - -typedef struct blkg_policy_data *blkcg_pol_alloc_pd_fn(gfp_t, struct request_queue *, struct blkcg *); - -typedef void blkcg_pol_init_pd_fn(struct blkg_policy_data *); - -typedef void blkcg_pol_online_pd_fn(struct blkg_policy_data *); - -typedef void blkcg_pol_offline_pd_fn(struct blkg_policy_data *); - -typedef void blkcg_pol_free_pd_fn(struct blkg_policy_data *); - -typedef void blkcg_pol_reset_pd_stats_fn(struct blkg_policy_data *); - -typedef size_t blkcg_pol_stat_pd_fn(struct blkg_policy_data *, char *, size_t); - -struct blkcg_policy { - int plid; - struct cftype *dfl_cftypes; - struct cftype *legacy_cftypes; - blkcg_pol_alloc_cpd_fn *cpd_alloc_fn; - blkcg_pol_init_cpd_fn *cpd_init_fn; - blkcg_pol_free_cpd_fn *cpd_free_fn; - blkcg_pol_bind_cpd_fn *cpd_bind_fn; - blkcg_pol_alloc_pd_fn *pd_alloc_fn; - blkcg_pol_init_pd_fn *pd_init_fn; - blkcg_pol_online_pd_fn *pd_online_fn; - blkcg_pol_offline_pd_fn *pd_offline_fn; - blkcg_pol_free_pd_fn *pd_free_fn; - blkcg_pol_reset_pd_stats_fn *pd_reset_stats_fn; - blkcg_pol_stat_pd_fn *pd_stat_fn; +struct bsg_set { + struct blk_mq_tag_set tag_set; + struct bsg_device___2 *bd; + bsg_job_fn *job_fn; + bsg_timeout_fn *timeout_fn; }; struct blkg_conf_ctx { @@ -56266,35 +57613,10 @@ struct blkg_conf_ctx { char *body; }; -enum blkg_rwstat_type { - BLKG_RWSTAT_READ = 0, - BLKG_RWSTAT_WRITE = 1, - BLKG_RWSTAT_SYNC = 2, - BLKG_RWSTAT_ASYNC = 3, - BLKG_RWSTAT_DISCARD = 4, - BLKG_RWSTAT_NR = 5, - BLKG_RWSTAT_TOTAL = 5, -}; - -struct blkg_rwstat { - struct percpu_counter cpu_cnt[5]; - atomic64_t aux_cnt[5]; -}; - struct blkg_rwstat_sample { u64 cnt[5]; }; -struct throtl_service_queue { - struct throtl_service_queue *parent_sq; - struct list_head queued[2]; - unsigned int nr_queued[2]; - struct rb_root_cached pending_tree; - unsigned int nr_pending; - long unsigned int first_pending_disptime; - struct timer_list pending_timer; -}; - struct latency_bucket { long unsigned int total_latency; int samples; @@ -56324,61 +57646,11 @@ struct throtl_data { bool track_bio_latency; }; -struct throtl_grp; - -struct throtl_qnode { - struct list_head node; - struct bio_list bios; - struct throtl_grp *tg; -}; - -struct throtl_grp { - struct blkg_policy_data pd; - struct rb_node rb_node; - struct throtl_data *td; - struct throtl_service_queue service_queue; - struct throtl_qnode qnode_on_self[2]; - struct throtl_qnode qnode_on_parent[2]; - long unsigned int disptime; - unsigned int flags; - bool has_rules[2]; - uint64_t bps[4]; - uint64_t bps_conf[4]; - unsigned int iops[4]; - unsigned int iops_conf[4]; - uint64_t bytes_disp[2]; - unsigned int io_disp[2]; - long unsigned int last_low_overflow_time[2]; - uint64_t last_bytes_disp[2]; - unsigned int last_io_disp[2]; - long unsigned int last_check_time; - long unsigned int latency_target; - long unsigned int latency_target_conf; - long unsigned int slice_start[2]; - long unsigned int slice_end[2]; - long unsigned int last_finish_time; - long unsigned int checked_last_finish_time; - long unsigned int avg_idletime; - long unsigned int idletime_threshold; - long unsigned int idletime_threshold_conf; - unsigned int bio_cnt; - unsigned int bad_bio_cnt; - long unsigned int bio_cnt_reset_time; - struct blkg_rwstat stat_bytes; - struct blkg_rwstat stat_ios; -}; - enum tg_state_flags { THROTL_TG_PENDING = 1, THROTL_TG_WAS_EMPTY = 2, }; -enum { - LIMIT_LOW = 0, - LIMIT_MAX = 1, - LIMIT_CNT = 2, -}; - struct blk_iolatency { struct rq_qos rqos; struct timer_list timer; @@ -56425,22 +57697,6 @@ struct iolatency_grp { struct child_latency_info child_lat; }; -struct io_stats_per_prio { - local_t inserted; - local_t merged; - local_t dispatched; - local_t completed; -}; - -struct blkcg_io_stats { - struct io_stats_per_prio stats[4]; -}; - -struct dd_blkcg { - struct blkcg_policy_data cpd; - struct blkcg_io_stats *stats; -}; - enum dd_data_dir { DD_READ = 0, DD_WRITE = 1, @@ -56461,8 +57717,11 @@ enum { DD_PRIO_COUNT = 3, }; -struct io_stats { - struct io_stats_per_prio stats[3]; +struct io_stats_per_prio { + uint32_t inserted; + uint32_t merged; + uint32_t dispatched; + atomic_t completed; }; struct dd_per_prio { @@ -56470,29 +57729,24 @@ struct dd_per_prio { struct rb_root sort_list[2]; struct list_head fifo_list[2]; struct request *next_rq[2]; + struct io_stats_per_prio stats; }; struct deadline_data { - struct request_queue *queue; struct dd_per_prio per_prio[3]; enum dd_data_dir last_dir; unsigned int batching; unsigned int starved; - struct io_stats *stats; int fifo_expire[2]; int fifo_batch; int writes_starved; int front_merges; u32 async_depth; - int aging_expire; + int prio_aging_expire; spinlock_t lock; spinlock_t zone_lock; }; -struct dd_blkg { - struct blkg_policy_data pd; -}; - struct trace_event_raw_kyber_latency { struct trace_entry ent; dev_t dev; @@ -56526,11 +57780,11 @@ struct trace_event_data_offsets_kyber_adjust {}; struct trace_event_data_offsets_kyber_throttled {}; -typedef void (*btf_trace_kyber_latency)(void *, struct request_queue *, const char *, const char *, unsigned int, unsigned int, unsigned int, unsigned int); +typedef void (*btf_trace_kyber_latency)(void *, dev_t, const char *, const char *, unsigned int, unsigned int, unsigned int, unsigned int); -typedef void (*btf_trace_kyber_adjust)(void *, struct request_queue *, const char *, unsigned int); +typedef void (*btf_trace_kyber_adjust)(void *, dev_t, const char *, unsigned int); -typedef void (*btf_trace_kyber_throttled)(void *, struct request_queue *, const char *); +typedef void (*btf_trace_kyber_throttled)(void *, dev_t, const char *); enum { KYBER_READ = 0, @@ -56566,6 +57820,7 @@ struct kyber_ctx_queue { struct kyber_queue_data { struct request_queue *q; + dev_t dev; struct sbitmap_queue domain_tokens[4]; unsigned int async_depth; struct kyber_cpu_latency *cpu_latency; @@ -56621,6 +57876,7 @@ struct virtio_device { bool config_enabled; bool config_change_pending; spinlock_t config_lock; + spinlock_t vqs_list_lock; struct device dev; struct virtio_device_id id; const struct virtio_config_ops *config; @@ -56635,6 +57891,7 @@ typedef void vq_callback_t(struct virtqueue *); struct virtio_shm_region; struct virtio_config_ops { + void (*enable_cbs)(struct virtio_device *); void (*get)(struct virtio_device *, unsigned int, void *, unsigned int); void (*set)(struct virtio_device *, unsigned int, const void *, unsigned int); u32 (*generation)(struct virtio_device *); @@ -56690,6 +57947,12 @@ enum { REG_OP_RELEASE = 2, }; +struct sg_append_table { + struct sg_table sgt; + struct scatterlist *prv; + unsigned int total_nents; +}; + typedef struct scatterlist *sg_alloc_fn(unsigned int, gfp_t); typedef void sg_free_fn(struct scatterlist *, unsigned int); @@ -56753,6 +58016,7 @@ union nested_table { struct once_work { struct work_struct work; struct static_key_true *key; + struct module *module; }; struct genradix_iter { @@ -56774,6 +58038,38 @@ struct reciprocal_value_adv { bool is_wide_m; }; +enum blake2s_lengths { + BLAKE2S_BLOCK_SIZE = 64, + BLAKE2S_HASH_SIZE = 32, + BLAKE2S_KEY_SIZE = 32, + BLAKE2S_128_HASH_SIZE = 16, + BLAKE2S_160_HASH_SIZE = 20, + BLAKE2S_224_HASH_SIZE = 28, + BLAKE2S_256_HASH_SIZE = 32, +}; + +struct blake2s_state { + u32 h[8]; + u32 t[2]; + u32 f[2]; + u8 buf[64]; + unsigned int buflen; + unsigned int outlen; +}; + +enum blake2s_iv { + BLAKE2S_IV0 = 1779033703, + BLAKE2S_IV1 = 3144134277, + BLAKE2S_IV2 = 1013904242, + BLAKE2S_IV3 = 2773480762, + BLAKE2S_IV4 = 1359893119, + BLAKE2S_IV5 = 2600822924, + BLAKE2S_IV6 = 528734635, + BLAKE2S_IV7 = 1541459225, +}; + +typedef void (*blake2s_compress_t)(struct blake2s_state *, const u8 *, size_t, u32); + enum devm_ioremap_type { DEVM_IOREMAP = 0, DEVM_IOREMAP_UC = 1, @@ -56785,6 +58081,11 @@ struct pcim_iomap_devres { void *table[6]; }; +struct arch_io_reserve_memtype_wc_devres { + resource_size_t start; + resource_size_t size; +}; + enum assoc_array_walk_status { assoc_array_walk_tree_empty = 0, assoc_array_walk_found_terminal_node = 1, @@ -56994,11 +58295,108 @@ typedef enum { partial_decode = 1, } earlyEnd_directive; +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall = 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_maxCode = 120, +} ZSTD_ErrorCode; + +struct ZSTD_DCtx_s; + +typedef struct ZSTD_DCtx_s ZSTD_DCtx; + +struct ZSTD_inBuffer_s { + const void *src; + size_t size; + size_t pos; +}; + +typedef struct ZSTD_inBuffer_s ZSTD_inBuffer; + +struct ZSTD_outBuffer_s { + void *dst; + size_t size; + size_t pos; +}; + +typedef struct ZSTD_outBuffer_s ZSTD_outBuffer; + +typedef ZSTD_DCtx ZSTD_DStream; + +typedef void * (*ZSTD_allocFunction)(void *, size_t); + +typedef void (*ZSTD_freeFunction)(void *, void *); + +typedef struct { + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; + void *opaque; +} ZSTD_customMem; + +typedef enum { + ZSTD_frame = 0, + ZSTD_skippableFrame = 1, +} ZSTD_frameType_e; + +typedef struct { + long long unsigned int frameContentSize; + long long unsigned int windowSize; + unsigned int blockSizeMax; + ZSTD_frameType_e frameType; + unsigned int headerSize; + unsigned int dictID; + unsigned int checksumFlag; +} ZSTD_frameHeader; + +typedef ZSTD_ErrorCode zstd_error_code; + +typedef ZSTD_DCtx zstd_dctx; + +typedef ZSTD_inBuffer zstd_in_buffer; + +typedef ZSTD_outBuffer zstd_out_buffer; + +typedef ZSTD_DStream zstd_dstream; + +typedef ZSTD_frameHeader zstd_frame_header; + +typedef ZSTD_ErrorCode ERR_enum; + +typedef s16 int16_t; + +typedef int16_t S16; + typedef struct { size_t bitContainer; unsigned int bitsConsumed; const char *ptr; const char *start; + const char *limitPtr; } BIT_DStream_t; typedef enum { @@ -57008,6 +58406,29 @@ typedef enum { BIT_DStream_overflow = 3, } BIT_DStream_status; +typedef unsigned int FSE_DTable; + +typedef struct { + size_t state; + const void *table; +} FSE_DState_t; + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; + +typedef struct { + short unsigned int newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; + +typedef struct { + short int ncount[256]; + FSE_DTable dtable[1]; +} FSE_DecompressWksp; + typedef U32 HUF_DTable; typedef struct { @@ -57020,13 +58441,21 @@ typedef struct { typedef struct { BYTE byte; BYTE nbBits; -} HUF_DEltX2; +} HUF_DEltX1; + +typedef struct { + U32 rankVal[16]; + U32 rankStart[16]; + U32 statsWksp[218]; + BYTE symbols[256]; + BYTE huffWeight[256]; +} HUF_ReadDTableX1_Workspace; typedef struct { U16 sequence; BYTE nbBits; BYTE length; -} HUF_DEltX4; +} HUF_DEltX2; typedef struct { BYTE symbol; @@ -57035,30 +58464,35 @@ typedef struct { typedef U32 rankValCol_t[13]; +typedef struct { + U32 rankVal[156]; + U32 rankStats[13]; + U32 rankStart0[14]; + sortedSymbol_t sortedSymbol[256]; + BYTE weightList[256]; + U32 calleeWksp[218]; +} HUF_ReadDTableX2_Workspace; + typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; -typedef s16 int16_t; - -typedef unsigned int FSE_DTable; +typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; +} ZSTD_seqSymbol; typedef struct { - FSE_DTable LLTable[513]; - FSE_DTable OFTable[257]; - FSE_DTable MLTable[513]; + ZSTD_seqSymbol LLTable[513]; + ZSTD_seqSymbol OFTable[257]; + ZSTD_seqSymbol MLTable[513]; HUF_DTable hufTable[4097]; - U64 workspace[384]; U32 rep[3]; -} ZSTD_entropyTables_t; - -typedef struct { - long long unsigned int frameContentSize; - unsigned int windowSize; - unsigned int dictID; - unsigned int checksumFlag; -} ZSTD_frameParams; + U32 workspace[157]; +} ZSTD_entropyDTables_t; typedef enum { bt_raw = 0, @@ -57078,72 +58512,36 @@ typedef enum { ZSTDds_skipFrame = 7, } ZSTD_dStage; -typedef void * (*ZSTD_allocFunction)(void *, size_t); +typedef enum { + ZSTD_f_zstd1 = 0, + ZSTD_f_zstd1_magicless = 1, +} ZSTD_format_e; -typedef void (*ZSTD_freeFunction)(void *, void *); +typedef enum { + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1, +} ZSTD_forceIgnoreChecksum_e; -typedef struct { - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; - void *opaque; -} ZSTD_customMem; +typedef enum { + ZSTD_use_indefinitely = 4294967295, + ZSTD_dont_use = 0, + ZSTD_use_once = 1, +} ZSTD_dictUses_e; -struct ZSTD_DCtx_s { - const FSE_DTable *LLTptr; - const FSE_DTable *MLTptr; - const FSE_DTable *OFTptr; - const HUF_DTable *HUFptr; - ZSTD_entropyTables_t entropy; - const void *previousDstEnd; - const void *base; - const void *vBase; - const void *dictEnd; - size_t expected; - ZSTD_frameParams fParams; - blockType_e bType; - ZSTD_dStage stage; - U32 litEntropy; - U32 fseEntropy; - struct xxh64_state xxhState; - size_t headerSize; - U32 dictID; - const BYTE *litPtr; - ZSTD_customMem customMem; - size_t litSize; - size_t rleSize; - BYTE litBuffer[131080]; - BYTE headerBuffer[18]; -}; - -typedef struct ZSTD_DCtx_s ZSTD_DCtx; - -struct ZSTD_DDict_s { - void *dictBuffer; - const void *dictContent; - size_t dictSize; - ZSTD_entropyTables_t entropy; - U32 dictID; - U32 entropyPresent; - ZSTD_customMem cMem; -}; +struct ZSTD_DDict_s; typedef struct ZSTD_DDict_s ZSTD_DDict; -struct ZSTD_inBuffer_s { - const void *src; - size_t size; - size_t pos; -}; +typedef struct { + const ZSTD_DDict **ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; -typedef struct ZSTD_inBuffer_s ZSTD_inBuffer; - -struct ZSTD_outBuffer_s { - void *dst; - size_t size; - size_t pos; -}; - -typedef struct ZSTD_outBuffer_s ZSTD_outBuffer; +typedef enum { + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1, +} ZSTD_refMultipleDDicts_e; typedef enum { zdss_init = 0, @@ -57153,12 +58551,49 @@ typedef enum { zdss_flush = 4, } ZSTD_dStreamStage; -struct ZSTD_DStream_s { - ZSTD_DCtx *dctx; +typedef enum { + ZSTD_bm_buffered = 0, + ZSTD_bm_stable = 1, +} ZSTD_bufferMode_e; + +struct ZSTD_DCtx_s___2 { + const ZSTD_seqSymbol *LLTptr; + const ZSTD_seqSymbol *MLTptr; + const ZSTD_seqSymbol *OFTptr; + const HUF_DTable *HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[640]; + const void *previousDstEnd; + const void *prefixStart; + const void *virtualStart; + const void *dictEnd; + size_t expected; + ZSTD_frameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + struct xxh64_state xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; + U32 validateChecksum; + const BYTE *litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; + int bmi2; ZSTD_DDict *ddictLocal; const ZSTD_DDict *ddict; - ZSTD_frameParams fParams; - ZSTD_dStreamStage stage; + U32 dictID; + int ddictIsCold; + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet *ddictSet; + ZSTD_refMultipleDDicts_e refMultipleDDicts; + ZSTD_dStreamStage streamStage; char *inBuff; size_t inBuffSize; size_t inPos; @@ -57167,17 +58602,70 @@ struct ZSTD_DStream_s { size_t outBuffSize; size_t outStart; size_t outEnd; - size_t blockSize; - BYTE headerBuffer[18]; size_t lhSize; - ZSTD_customMem customMem; void *legacyContext; U32 previousLegacyVersion; U32 legacyVersion; U32 hostageByte; + int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; + BYTE litBuffer[131104]; + BYTE headerBuffer[18]; + size_t oversizedDuration; }; -typedef struct ZSTD_DStream_s ZSTD_DStream; +typedef struct ZSTD_DCtx_s___2 ZSTD_DCtx___2; + +struct ZSTD_DDict_s { + void *dictBuffer; + const void *dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; + +typedef enum { + ZSTD_dct_auto = 0, + ZSTD_dct_rawContent = 1, + ZSTD_dct_fullDict = 2, +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, + ZSTD_dlm_byRef = 1, +} ZSTD_dictLoadMethod_e; + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3, +} ZSTD_ResetDirective; + +typedef enum { + ZSTD_d_windowLogMax = 100, + ZSTD_d_experimentalParam1 = 1000, + ZSTD_d_experimentalParam2 = 1001, + ZSTD_d_experimentalParam3 = 1002, + ZSTD_d_experimentalParam4 = 1003, +} ZSTD_dParameter; + +typedef ZSTD_DCtx___2 ZSTD_DStream___2; typedef enum { ZSTDnit_frameHeader = 0, @@ -57188,25 +58676,16 @@ typedef enum { ZSTDnit_skippableFrame = 5, } ZSTD_nextInputType_e; -typedef int16_t S16; - -typedef uintptr_t uPtrDiff; +typedef struct { + size_t compressedSize; + long long unsigned int decompressedBound; +} ZSTD_frameSizeInfo; typedef struct { - size_t state; - const void *table; -} FSE_DState_t; - -typedef struct { - U16 tableLog; - U16 fastMode; -} FSE_DTableHeader; - -typedef struct { - short unsigned int newState; - unsigned char symbol; - unsigned char nbBits; -} FSE_decode_t; + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; typedef enum { set_basic = 0, @@ -57215,16 +58694,15 @@ typedef enum { set_repeat = 3, } symbolEncodingType_e; -typedef struct { - blockType_e blockType; - U32 lastBlock; - U32 origSize; -} blockProperties_t; +typedef enum { + ZSTD_no_overlap = 0, + ZSTD_overlap_src_before_dst = 1, +} ZSTD_overlap_e; -typedef union { - FSE_decode_t realData; - U32 alignedBy4; -} FSE_decode_t4; +typedef struct { + U32 fastMode; + U32 tableLog; +} ZSTD_seqSymbol_header; typedef struct { size_t litLength; @@ -57234,20 +58712,30 @@ typedef struct { } seq_t; typedef struct { - BIT_DStream_t DStream; - FSE_DState_t stateLL; - FSE_DState_t stateOffb; - FSE_DState_t stateML; - size_t prevOffset[3]; - const BYTE *base; - size_t pos; - uPtrDiff gotoDict; -} seqState_t; + size_t state; + const ZSTD_seqSymbol *table; +} ZSTD_fseState; typedef struct { - void *ptr; - const void *end; -} ZSTD_stack; + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[3]; + const BYTE *prefixStart; + const BYTE *dictEnd; + size_t pos; +} seqState_t; + +typedef enum { + ZSTD_lo_isRegularOffset = 0, + ZSTD_lo_isLongOffset = 1, +} ZSTD_longOffset_e; + +typedef enum { + ZSTD_p_noPrefetch = 0, + ZSTD_p_prefetch = 1, +} ZSTD_prefetch_e; enum xz_mode { XZ_SINGLE = 0, @@ -57636,6 +59124,13 @@ struct karatsuba_ctx { typedef long int mpi_limb_signed_t; +struct sg_pool { + size_t size; + char *name; + struct kmem_cache *slab; + mempool_t *pool; +}; + struct irq_poll; typedef int irq_poll_fn(struct irq_poll *, int); @@ -57652,6 +59147,8 @@ enum { IRQ_POLL_F_DISABLE = 1, }; +typedef u32 depot_stack_handle_t; + union handle_parts { depot_stack_handle_t handle; struct { @@ -57849,40 +59346,6 @@ struct cstate { struct xz_dec___2; -typedef enum { - ZSTD_error_no_error = 0, - ZSTD_error_GENERIC = 1, - ZSTD_error_prefix_unknown = 2, - ZSTD_error_version_unsupported = 3, - ZSTD_error_parameter_unknown = 4, - ZSTD_error_frameParameter_unsupported = 5, - ZSTD_error_frameParameter_unsupportedBy32bits = 6, - ZSTD_error_frameParameter_windowTooLarge = 7, - ZSTD_error_compressionParameter_unsupported = 8, - ZSTD_error_init_missing = 9, - ZSTD_error_memory_allocation = 10, - ZSTD_error_stage_wrong = 11, - ZSTD_error_dstSize_tooSmall = 12, - ZSTD_error_srcSize_wrong = 13, - ZSTD_error_corruption_detected = 14, - ZSTD_error_checksum_wrong = 15, - ZSTD_error_tableLog_tooLarge = 16, - ZSTD_error_maxSymbolValue_tooLarge = 17, - ZSTD_error_maxSymbolValue_tooSmall = 18, - ZSTD_error_dictionary_corrupted = 19, - ZSTD_error_dictionary_wrong = 20, - ZSTD_error_dictionaryCreation_failed = 21, - ZSTD_error_maxCode = 22, -} ZSTD_ErrorCode; - -struct ZSTD_DCtx_s___2; - -typedef struct ZSTD_DCtx_s___2 ZSTD_DCtx___2; - -struct ZSTD_DStream_s___2; - -typedef struct ZSTD_DStream_s___2 ZSTD_DStream___2; - struct cpio_data { void *data; size_t size; @@ -58090,6 +59553,16 @@ enum { struct in6_addr___2; +enum mmio_type { + MMIO_DECODE_FAILED = 0, + MMIO_WRITE = 1, + MMIO_WRITE_IMM = 2, + MMIO_READ = 3, + MMIO_READ_ZERO_EXTEND = 4, + MMIO_READ_SIGN_EXTEND = 5, + MMIO_MOVS = 6, +}; + enum reg_type { REG_TYPE_RM = 0, REG_TYPE_REG = 1, @@ -58328,6 +59801,7 @@ enum pci_dev_flags { PCI_DEV_FLAGS_BRIDGE_XLATE_ROOT = 512, PCI_DEV_FLAGS_NO_FLR_RESET = 1024, PCI_DEV_FLAGS_NO_RELAXED_ORDERING = 2048, + PCI_DEV_FLAGS_HAS_MSI_MASKING = 4096, }; enum pci_bus_flags { @@ -58373,6 +59847,7 @@ struct pci_host_bridge { struct pci_ops *child_ops; void *sysdata; int busnr; + int domain_nr; struct list_head windows; struct list_head dma_ranges; u8 (*swizzle_irq)(struct pci_dev *, u8 *); @@ -58397,7 +59872,6 @@ struct pci_host_bridge { long: 64; long: 64; long: 64; - long: 64; long unsigned int private[0]; }; @@ -58434,7 +59908,8 @@ struct acpi_device_flags { u32 coherent_dma: 1; u32 cca_seen: 1; u32 enumeration_by_parent: 1; - u32 reserved: 19; + u32 honor_deps: 1; + u32 reserved: 18; }; typedef char acpi_bus_id[8]; @@ -58491,6 +59966,7 @@ struct acpi_device_power { int state; struct acpi_device_power_flags flags; struct acpi_device_power_state states[5]; + u8 state_for_enumeration; }; struct acpi_device_wakeup_flags { @@ -58586,7 +60062,7 @@ struct hotplug_slot_ops { int (*get_attention_status)(struct hotplug_slot *, u8 *); int (*get_latch_status)(struct hotplug_slot *, u8 *); int (*get_adapter_status)(struct hotplug_slot *, u8 *); - int (*reset_slot)(struct hotplug_slot *, int); + int (*reset_slot)(struct hotplug_slot *, bool); }; typedef u64 acpi_io_address; @@ -58736,6 +60212,8 @@ enum pcie_link_width { PCIE_LNK_WIDTH_UNKNOWN = 255, }; +typedef int (*arch_set_vga_state_t)(struct pci_dev *, bool, unsigned int, u32); + struct pci_cap_saved_data { u16 cap_nr; bool cap_extended; @@ -58748,17 +60226,9 @@ struct pci_cap_saved_state { struct pci_cap_saved_data cap; }; -typedef int (*arch_set_vga_state_t)(struct pci_dev *, bool, unsigned int, u32); - -struct pci_platform_pm_ops { - bool (*bridge_d3)(struct pci_dev *); - bool (*is_manageable)(struct pci_dev *); - int (*set_state)(struct pci_dev *, pci_power_t); - pci_power_t (*get_state)(struct pci_dev *); - void (*refresh_state)(struct pci_dev *); - pci_power_t (*choose_state)(struct pci_dev *); - int (*set_wakeup)(struct pci_dev *, bool); - bool (*need_resume)(struct pci_dev *); +struct pci_reset_fn_method { + int (*reset_fn)(struct pci_dev *, bool); + char *name; }; struct pci_pme_device { @@ -58786,15 +60256,6 @@ struct driver_attribute { ssize_t (*store)(struct device_driver *, const char *, size_t); }; -enum pci_ers_result { - PCI_ERS_RESULT_NONE = 1, - PCI_ERS_RESULT_CAN_RECOVER = 2, - PCI_ERS_RESULT_NEED_RESET = 3, - PCI_ERS_RESULT_DISCONNECT = 4, - PCI_ERS_RESULT_RECOVERED = 5, - PCI_ERS_RESULT_NO_AER_DRIVER = 6, -}; - enum dev_dma_attr { DEV_DMA_NOT_SUPPORTED = 0, DEV_DMA_NON_COHERENT = 1, @@ -58818,7 +60279,7 @@ struct pcie_port_service_driver { int (*resume)(struct pcie_device *); int (*runtime_suspend)(struct pcie_device *); int (*runtime_resume)(struct pcie_device *); - void (*error_resume)(struct pci_dev *); + int (*slot_reset)(struct pcie_device *); int port_type; u32 service; struct device_driver driver; @@ -58845,23 +60306,6 @@ enum pci_mmap_api { PCI_MMAP_PROCFS = 1, }; -struct pci_vpd_ops; - -struct pci_vpd { - const struct pci_vpd_ops *ops; - struct mutex lock; - unsigned int len; - u16 flag; - u8 cap; - unsigned int busy: 1; - unsigned int valid: 1; -}; - -struct pci_vpd_ops { - ssize_t (*read)(struct pci_dev *, loff_t, size_t, void *); - ssize_t (*write)(struct pci_dev *, loff_t, size_t, const void *); -}; - struct pci_dev_resource { struct list_head list; struct resource *res; @@ -58897,6 +60341,17 @@ struct portdrv_service_data { u32 service; }; +typedef int (*pcie_callback_t)(struct pcie_device *); + +enum pci_ers_result { + PCI_ERS_RESULT_NONE = 1, + PCI_ERS_RESULT_CAN_RECOVER = 2, + PCI_ERS_RESULT_NEED_RESET = 3, + PCI_ERS_RESULT_DISCONNECT = 4, + PCI_ERS_RESULT_RECOVERED = 5, + PCI_ERS_RESULT_NO_AER_DRIVER = 6, +}; + struct walk_rcec_data { struct pci_dev *rcec; int (*user_callback)(struct pci_dev *, void *); @@ -58947,15 +60402,6 @@ struct acpi_buffer { void *pointer; }; -struct acpi_bus_type { - struct list_head list; - const char *name; - bool (*match)(struct device *); - struct acpi_device * (*find_companion)(struct device *); - void (*setup)(struct device *); - void (*cleanup)(struct device *); -}; - struct acpi_pci_root { struct acpi_device *device; struct pci_bus *bus; @@ -59198,7 +60644,7 @@ struct ntb_ctrl_regs { struct pci_dev_reset_methods { u16 vendor; u16 device; - int (*reset)(struct pci_dev *, int); + int (*reset)(struct pci_dev *, bool); }; struct pci_dev_acs_enabled { @@ -59280,6 +60726,71 @@ struct linux_logo { const unsigned char *data; }; +struct fb_fix_screeninfo { + char id[16]; + long unsigned int smem_start; + __u32 smem_len; + __u32 type; + __u32 type_aux; + __u32 visual; + __u16 xpanstep; + __u16 ypanstep; + __u16 ywrapstep; + __u32 line_length; + long unsigned int mmio_start; + __u32 mmio_len; + __u32 accel; + __u16 capabilities; + __u16 reserved[2]; +}; + +struct fb_bitfield { + __u32 offset; + __u32 length; + __u32 msb_right; +}; + +struct fb_var_screeninfo { + __u32 xres; + __u32 yres; + __u32 xres_virtual; + __u32 yres_virtual; + __u32 xoffset; + __u32 yoffset; + __u32 bits_per_pixel; + __u32 grayscale; + struct fb_bitfield red; + struct fb_bitfield green; + struct fb_bitfield blue; + struct fb_bitfield transp; + __u32 nonstd; + __u32 activate; + __u32 height; + __u32 width; + __u32 accel_flags; + __u32 pixclock; + __u32 left_margin; + __u32 right_margin; + __u32 upper_margin; + __u32 lower_margin; + __u32 hsync_len; + __u32 vsync_len; + __u32 sync; + __u32 vmode; + __u32 rotate; + __u32 colorspace; + __u32 reserved[4]; +}; + +struct fb_cmap { + __u32 start; + __u32 len; + __u16 *red; + __u16 *green; + __u16 *blue; + __u16 *transp; +}; + enum { FB_BLANK_UNBLANK = 0, FB_BLANK_NORMAL = 1, @@ -59288,11 +60799,278 @@ enum { FB_BLANK_POWERDOWN = 4, }; +struct fb_copyarea { + __u32 dx; + __u32 dy; + __u32 width; + __u32 height; + __u32 sx; + __u32 sy; +}; + +struct fb_fillrect { + __u32 dx; + __u32 dy; + __u32 width; + __u32 height; + __u32 color; + __u32 rop; +}; + +struct fb_image { + __u32 dx; + __u32 dy; + __u32 width; + __u32 height; + __u32 fg_color; + __u32 bg_color; + __u8 depth; + const char *data; + struct fb_cmap cmap; +}; + +struct fbcurpos { + __u16 x; + __u16 y; +}; + +struct fb_cursor { + __u16 set; + __u16 enable; + __u16 rop; + const char *mask; + struct fbcurpos hot; + struct fb_image image; +}; + +struct fb_chroma { + __u32 redx; + __u32 greenx; + __u32 bluex; + __u32 whitex; + __u32 redy; + __u32 greeny; + __u32 bluey; + __u32 whitey; +}; + +struct fb_videomode; + +struct fb_monspecs { + struct fb_chroma chroma; + struct fb_videomode *modedb; + __u8 manufacturer[4]; + __u8 monitor[14]; + __u8 serial_no[14]; + __u8 ascii[14]; + __u32 modedb_len; + __u32 model; + __u32 serial; + __u32 year; + __u32 week; + __u32 hfmin; + __u32 hfmax; + __u32 dclkmin; + __u32 dclkmax; + __u16 input; + __u16 dpms; + __u16 signal; + __u16 vfmin; + __u16 vfmax; + __u16 gamma; + __u16 gtf: 1; + __u16 misc; + __u8 version; + __u8 revision; + __u8 max_x; + __u8 max_y; +}; + +struct fb_videomode { + const char *name; + u32 refresh; + u32 xres; + u32 yres; + u32 pixclock; + u32 left_margin; + u32 right_margin; + u32 upper_margin; + u32 lower_margin; + u32 hsync_len; + u32 vsync_len; + u32 sync; + u32 vmode; + u32 flag; +}; + +struct fb_info; + struct fb_event { struct fb_info *info; void *data; }; +struct fb_pixmap { + u8 *addr; + u32 size; + u32 offset; + u32 buf_align; + u32 scan_align; + u32 access_align; + u32 flags; + u32 blit_x; + u32 blit_y; + void (*writeio)(struct fb_info *, void *, void *, unsigned int); + void (*readio)(struct fb_info *, void *, void *, unsigned int); +}; + +struct fb_ops; + +struct fb_tile_ops; + +struct apertures_struct; + +struct fb_info { + refcount_t count; + int node; + int flags; + int fbcon_rotate_hint; + struct mutex lock; + struct mutex mm_lock; + struct fb_var_screeninfo var; + struct fb_fix_screeninfo fix; + struct fb_monspecs monspecs; + struct work_struct queue; + struct fb_pixmap pixmap; + struct fb_pixmap sprite; + struct fb_cmap cmap; + struct list_head modelist; + struct fb_videomode *mode; + const struct fb_ops *fbops; + struct device *device; + struct device *dev; + int class_flag; + struct fb_tile_ops *tileops; + union { + char *screen_base; + char *screen_buffer; + }; + long unsigned int screen_size; + void *pseudo_palette; + u32 state; + void *fbcon_par; + void *par; + struct apertures_struct *apertures; + bool skip_vt_switch; +}; + +struct fb_blit_caps { + u32 x; + u32 y; + u32 len; + u32 flags; +}; + +struct fb_ops { + struct module *owner; + int (*fb_open)(struct fb_info *, int); + int (*fb_release)(struct fb_info *, int); + ssize_t (*fb_read)(struct fb_info *, char *, size_t, loff_t *); + ssize_t (*fb_write)(struct fb_info *, const char *, size_t, loff_t *); + int (*fb_check_var)(struct fb_var_screeninfo *, struct fb_info *); + int (*fb_set_par)(struct fb_info *); + int (*fb_setcolreg)(unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, struct fb_info *); + int (*fb_setcmap)(struct fb_cmap *, struct fb_info *); + int (*fb_blank)(int, struct fb_info *); + int (*fb_pan_display)(struct fb_var_screeninfo *, struct fb_info *); + void (*fb_fillrect)(struct fb_info *, const struct fb_fillrect *); + void (*fb_copyarea)(struct fb_info *, const struct fb_copyarea *); + void (*fb_imageblit)(struct fb_info *, const struct fb_image *); + int (*fb_cursor)(struct fb_info *, struct fb_cursor *); + int (*fb_sync)(struct fb_info *); + int (*fb_ioctl)(struct fb_info *, unsigned int, long unsigned int); + int (*fb_compat_ioctl)(struct fb_info *, unsigned int, long unsigned int); + int (*fb_mmap)(struct fb_info *, struct vm_area_struct *); + void (*fb_get_caps)(struct fb_info *, struct fb_blit_caps *, struct fb_var_screeninfo *); + void (*fb_destroy)(struct fb_info *); + int (*fb_debug_enter)(struct fb_info *); + int (*fb_debug_leave)(struct fb_info *); +}; + +struct fb_tilemap { + __u32 width; + __u32 height; + __u32 depth; + __u32 length; + const __u8 *data; +}; + +struct fb_tilerect { + __u32 sx; + __u32 sy; + __u32 width; + __u32 height; + __u32 index; + __u32 fg; + __u32 bg; + __u32 rop; +}; + +struct fb_tilearea { + __u32 sx; + __u32 sy; + __u32 dx; + __u32 dy; + __u32 width; + __u32 height; +}; + +struct fb_tileblit { + __u32 sx; + __u32 sy; + __u32 width; + __u32 height; + __u32 fg; + __u32 bg; + __u32 length; + __u32 *indices; +}; + +struct fb_tilecursor { + __u32 sx; + __u32 sy; + __u32 mode; + __u32 shape; + __u32 fg; + __u32 bg; +}; + +struct fb_tile_ops { + void (*fb_settile)(struct fb_info *, struct fb_tilemap *); + void (*fb_tilecopy)(struct fb_info *, struct fb_tilearea *); + void (*fb_tilefill)(struct fb_info *, struct fb_tilerect *); + void (*fb_tileblit)(struct fb_info *, struct fb_tileblit *); + void (*fb_tilecursor)(struct fb_info *, struct fb_tilecursor *); + int (*fb_get_tilemax)(struct fb_info *); +}; + +struct aperture { + resource_size_t base; + resource_size_t size; +}; + +struct apertures_struct { + unsigned int count; + struct aperture ranges[0]; +}; + +struct dmt_videomode { + u32 dmt_id; + u32 std_2byte_code; + u32 cvt_3byte_code; + const struct fb_videomode *mode; +}; + enum backlight_update_reason { BACKLIGHT_UPDATE_HOTKEY = 0, BACKLIGHT_UPDATE_SYSFS = 1, @@ -59425,7 +61203,6 @@ struct fb_con2fbmap { struct fbcon_display { const u_char *fontdata; int userfont; - u_short scrollmode; u_short inverse; short int yscroll; int vrows; @@ -59448,7 +61225,6 @@ struct fbcon_display { }; struct fbcon_ops { - void (*bmove)(struct vc_data *, struct fb_info *, int, int, int, int, int, int); void (*clear)(struct vc_data *, struct fb_info *, int, int, int, int); void (*putcs)(struct vc_data *, struct fb_info *, const short unsigned int *, int, int, int, int, int); void (*clear_margins)(struct vc_data *, struct fb_info *, int, int); @@ -59558,6 +61334,12 @@ struct acpi_madt_generic_distributor { typedef int (*acpi_tbl_table_handler)(struct acpi_table_header *); +enum acpi_ec_event_state { + EC_EVENT_READY = 0, + EC_EVENT_IN_PROGRESS = 1, + EC_EVENT_COMPLETE = 2, +}; + struct transaction; struct acpi_ec { @@ -59576,7 +61358,10 @@ struct acpi_ec { spinlock_t lock; struct work_struct work; long unsigned int timestamp; - long unsigned int nr_pending_queries; + enum acpi_ec_event_state event_state; + unsigned int events_to_process; + unsigned int events_in_progress; + unsigned int queries_in_progress; bool busy_polling; unsigned int polling_guard; }; @@ -59860,6 +61645,7 @@ struct acpi_object_region { union acpi_operand_object *next; acpi_physical_address address; u32 length; + void *pointer; }; struct acpi_object_notify_common { @@ -60663,6 +62449,14 @@ struct acpi_osc_context { struct acpi_buffer ret; }; +struct acpi_bus_type { + struct list_head list; + const char *name; + bool (*match)(struct device *); + struct acpi_device * (*find_companion)(struct device *); + void (*setup)(struct device *); +}; + struct acpi_pnp_device_id { u32 length; char *string; @@ -61134,6 +62928,7 @@ struct acpi_dep_data { struct list_head node; acpi_handle supplier; acpi_handle consumer; + bool honor_dep; }; enum acpi_reconfig_event { @@ -61166,6 +62961,14 @@ struct resource_win { resource_size_t offset; }; +struct irq_override_cmp { + const struct dmi_system_id *system; + unsigned char irq; + unsigned char triggering; + unsigned char polarity; + unsigned char shareable; +}; + struct res_proc_context { struct list_head *list; int (*preproc)(struct acpi_resource *, void *); @@ -61209,14 +63012,12 @@ enum ec_command { enum { EC_FLAGS_QUERY_ENABLED = 0, - EC_FLAGS_QUERY_PENDING = 1, - EC_FLAGS_QUERY_GUARDING = 2, - EC_FLAGS_EVENT_HANDLER_INSTALLED = 3, - EC_FLAGS_EC_HANDLER_INSTALLED = 4, - EC_FLAGS_QUERY_METHODS_INSTALLED = 5, - EC_FLAGS_STARTED = 6, - EC_FLAGS_STOPPED = 7, - EC_FLAGS_EVENTS_MASKED = 8, + EC_FLAGS_EVENT_HANDLER_INSTALLED = 1, + EC_FLAGS_EC_HANDLER_INSTALLED = 2, + EC_FLAGS_QUERY_METHODS_INSTALLED = 3, + EC_FLAGS_STARTED = 4, + EC_FLAGS_STOPPED = 5, + EC_FLAGS_EVENTS_MASKED = 6, }; struct acpi_ec_query_handler { @@ -61232,6 +63033,7 @@ struct acpi_ec_query { struct transaction transaction; struct work_struct work; struct acpi_ec_query_handler *handler; + struct acpi_ec *ec; }; struct acpi_pci_root_ops; @@ -61326,12 +63128,10 @@ struct acpi_power_dependent_device { struct acpi_power_resource { struct acpi_device device; struct list_head list_node; - char *name; u32 system_level; u32 order; unsigned int ref_count; u8 state; - bool wakeup_enabled; struct mutex resource_lock; struct list_head dependents; }; @@ -61416,11 +63216,13 @@ struct acpi_device_properties { struct list_head list; }; -struct always_present_id { +struct override_status_id { struct acpi_device_id hid[2]; struct x86_cpu_id cpu_ids[2]; struct dmi_system_id dmi_ids[2]; const char *uid; + const char *path; + long long unsigned int status; }; struct acpi_lpat { @@ -61528,6 +63330,80 @@ struct prm_module_info { struct prm_handler_info handlers[0]; }; +struct acpi_pcc_info { + u8 subspace_id; + u16 length; + u8 *internal_buffer; +}; + +struct mbox_chan; + +struct mbox_chan_ops { + int (*send_data)(struct mbox_chan *, void *); + int (*flush)(struct mbox_chan *, long unsigned int); + int (*startup)(struct mbox_chan *); + void (*shutdown)(struct mbox_chan *); + bool (*last_tx_done)(struct mbox_chan *); + bool (*peek_data)(struct mbox_chan *); +}; + +struct mbox_controller; + +struct mbox_client; + +struct mbox_chan { + struct mbox_controller *mbox; + unsigned int txdone_method; + struct mbox_client *cl; + struct completion tx_complete; + void *active_req; + unsigned int msg_count; + unsigned int msg_free; + void *msg_data[20]; + spinlock_t lock; + void *con_priv; +}; + +struct mbox_controller { + struct device *dev; + const struct mbox_chan_ops *ops; + struct mbox_chan *chans; + int num_chans; + bool txdone_irq; + bool txdone_poll; + unsigned int txpoll_period; + struct mbox_chan * (*of_xlate)(struct mbox_controller *, const struct of_phandle_args *); + struct hrtimer poll_hrt; + struct list_head node; +}; + +struct mbox_client { + struct device *dev; + bool tx_block; + long unsigned int tx_tout; + bool knows_txdone; + void (*rx_callback)(struct mbox_client *, void *); + void (*tx_prepare)(struct mbox_client *, void *); + void (*tx_done)(struct mbox_client *, void *, int); +}; + +struct pcc_mbox_chan { + struct mbox_chan *mchan; + u64 shmem_base_addr; + u64 shmem_size; + u32 latency; + u32 max_access_rate; + u16 min_turnaround_time; +}; + +struct pcc_data { + struct pcc_mbox_chan *pcc_chan; + void *pcc_comm_addr; + struct completion done; + struct mbox_client cl; + struct acpi_pcc_info ctx; +}; + enum { ACPI_REFCLASS_LOCAL = 0, ACPI_REFCLASS_ARG = 1, @@ -61700,6 +63576,10 @@ struct acpi_reg_walk_info { acpi_adr_space_type space_id; }; +struct acpi_data_table_space_context { + void *pointer; +}; + enum { AML_FIELD_UPDATE_PRESERVE = 0, AML_FIELD_UPDATE_WRITE_AS_ONES = 32, @@ -62332,7 +64212,8 @@ enum acpi_srat_type { ACPI_SRAT_TYPE_GICC_AFFINITY = 3, ACPI_SRAT_TYPE_GIC_ITS_AFFINITY = 4, ACPI_SRAT_TYPE_GENERIC_AFFINITY = 5, - ACPI_SRAT_TYPE_RESERVED = 6, + ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY = 6, + ACPI_SRAT_TYPE_RESERVED = 7, }; struct acpi_srat_mem_affinity { @@ -62373,78 +64254,12 @@ struct acpi_pci_ioapic { struct list_head list; }; -struct acpi_pcct_hw_reduced { - struct acpi_subtable_header header; - u32 platform_interrupt; - u8 flags; - u8 reserved; - u64 base_address; - u64 length; - struct acpi_generic_address doorbell_register; - u64 preserve_mask; - u64 write_mask; - u32 latency; - u32 max_access_rate; - u16 min_turnaround_time; -} __attribute__((packed)); - struct acpi_pcct_shared_memory { u32 signature; u16 command; u16 status; }; -struct mbox_chan; - -struct mbox_chan_ops { - int (*send_data)(struct mbox_chan *, void *); - int (*flush)(struct mbox_chan *, long unsigned int); - int (*startup)(struct mbox_chan *); - void (*shutdown)(struct mbox_chan *); - bool (*last_tx_done)(struct mbox_chan *); - bool (*peek_data)(struct mbox_chan *); -}; - -struct mbox_controller; - -struct mbox_client; - -struct mbox_chan { - struct mbox_controller *mbox; - unsigned int txdone_method; - struct mbox_client *cl; - struct completion tx_complete; - void *active_req; - unsigned int msg_count; - unsigned int msg_free; - void *msg_data[20]; - spinlock_t lock; - void *con_priv; -}; - -struct mbox_controller { - struct device *dev; - const struct mbox_chan_ops *ops; - struct mbox_chan *chans; - int num_chans; - bool txdone_irq; - bool txdone_poll; - unsigned int txpoll_period; - struct mbox_chan * (*of_xlate)(struct mbox_controller *, const struct of_phandle_args *); - struct hrtimer poll_hrt; - struct list_head node; -}; - -struct mbox_client { - struct device *dev; - bool tx_block; - long unsigned int tx_tout; - bool knows_txdone; - void (*rx_callback)(struct mbox_client *, void *); - void (*tx_prepare)(struct mbox_client *, void *); - void (*tx_done)(struct mbox_client *, void *, int); -}; - struct cpc_register_resource { acpi_object_type type; u64 *sys_mem_vaddr; @@ -62512,7 +64327,7 @@ struct cppc_cpudata { }; struct cppc_pcc_data { - struct mbox_chan *pcc_channel; + struct pcc_mbox_chan *pcc_channel; void *pcc_comm_addr; bool pcc_channel_acquired; unsigned int deadline_us; @@ -63102,6 +64917,13 @@ struct virtio_pci_modern_device { struct virtio_device_id id; }; +struct virtio_pci_legacy_device { + struct pci_dev *pci_dev; + u8 *isr; + void *ioaddr; + struct virtio_device_id id; +}; + struct virtio_pci_vq_info { struct virtqueue *vq; struct list_head node; @@ -63111,14 +64933,16 @@ struct virtio_pci_vq_info { struct virtio_pci_device { struct virtio_device vdev; struct pci_dev *pci_dev; + struct virtio_pci_legacy_device ldev; struct virtio_pci_modern_device mdev; + bool is_legacy; u8 *isr; - void *ioaddr; spinlock_t lock; struct list_head virtqueues; struct virtio_pci_vq_info **vqs; int msix_enabled; int intx_enabled; + bool intx_soft_enabled; cpumask_var_t *msix_affinity_masks; char (*msix_names)[256]; unsigned int msix_vectors; @@ -63752,6 +65576,9 @@ enum { ESnonstd = 10, ESpalette = 11, ESosc = 12, + ESapc = 13, + ESpm = 14, + ESdcs = 15, }; struct interval { @@ -63938,6 +65765,7 @@ enum dma_slave_buswidth { DMA_SLAVE_BUSWIDTH_16_BYTES = 16, DMA_SLAVE_BUSWIDTH_32_BYTES = 32, DMA_SLAVE_BUSWIDTH_64_BYTES = 64, + DMA_SLAVE_BUSWIDTH_128_BYTES = 128, }; struct dma_slave_config { @@ -64332,11 +66160,6 @@ struct timedia_struct { const short unsigned int *ids; }; -struct quatech_feature { - u16 devid; - bool amcc; -}; - enum pci_board_num_t { pbn_default = 0, pbn_b0_1_115200 = 1, @@ -64470,6 +66293,7 @@ struct software_node { struct exar8250_platform { int (*rs485_config)(struct uart_port *, struct serial_rs485 *); int (*register_gpio)(struct pci_dev *, struct uart_8250_port *); + void (*unregister_gpio)(struct uart_8250_port *); }; struct exar8250; @@ -64501,6 +66325,13 @@ struct timer_rand_state { long int last_delta2; }; +enum chacha_constants { + CHACHA_CONSTANT_EXPA = 1634760805, + CHACHA_CONSTANT_ND_3 = 857760878, + CHACHA_CONSTANT_2_BY = 2036477234, + CHACHA_CONSTANT_TE_K = 1797285236, +}; + struct trace_event_raw_add_device_randomness { struct trace_entry ent; int bytes; @@ -64646,7 +66477,7 @@ struct entropy_store { short unsigned int input_rotate; int entropy_count; unsigned int last_data_init: 1; - __u8 last_data[10]; + __u8 last_data[16]; }; struct fast_pool { @@ -65511,6 +67342,8 @@ enum tis_defaults { TIS_MEM_LEN = 20480, TIS_SHORT_TIMEOUT = 750, TIS_LONG_TIMEOUT = 2000, + TIS_TIMEOUT_MIN_ATML = 14700, + TIS_TIMEOUT_MAX_ATML = 15000, }; enum tpm_tis_flags { @@ -65532,6 +67365,8 @@ struct tpm_tis_data { wait_queue_head_t read_queue; const struct tpm_tis_phy_ops *phy_ops; short unsigned int rng_quality; + unsigned int timeout_min; + unsigned int timeout_max; }; struct tpm_tis_phy_ops { @@ -65654,9 +67489,7 @@ struct vga_device { unsigned int io_norm_cnt; unsigned int mem_norm_cnt; bool bridge_has_one_vga; - void *cookie; - void (*irq_set_state)(void *, bool); - unsigned int (*set_vga_decode)(void *, bool); + unsigned int (*set_decode)(struct pci_dev *, bool); }; struct vga_arb_user_card { @@ -66223,12 +68056,6 @@ struct firmware { void *priv; }; -struct builtin_fw { - char *name; - void *data; - long unsigned int size; -}; - enum fw_opt { FW_OPT_UEVENT = 1, FW_OPT_NOWAIT = 2, @@ -66296,6 +68123,12 @@ struct fw_sysfs { struct firmware *fw; }; +struct builtin_fw { + char *name; + void *data; + long unsigned int size; +}; + typedef void (*node_registration_func_t)(struct node *); struct node_access_nodes { @@ -66314,6 +68147,7 @@ typedef void (*irq_write_msi_msg_t)(struct msi_desc *, struct msi_msg *); struct platform_msi_priv_data { struct device *dev; void *host_data; + const struct attribute_group **msi_irq_groups; msi_alloc_info_t arg; irq_write_msi_msg_t write_msg; int devid; @@ -66399,26 +68233,15 @@ enum { Lo_deleting = 3, }; -struct loop_func_table; - struct loop_device { int lo_number; atomic_t lo_refcnt; loff_t lo_offset; loff_t lo_sizelimit; int lo_flags; - int (*transfer)(struct loop_device *, int, struct page *, unsigned int, struct page *, unsigned int, int, sector_t); char lo_file_name[64]; - char lo_crypt_name[64]; - char lo_encrypt_key[32]; - int lo_encrypt_key_size; - struct loop_func_table *lo_encryption; - __u32 lo_init[2]; - kuid_t lo_key_owner; - int (*ioctl)(struct loop_device *, int, long unsigned int); struct file *lo_backing_file; struct block_device *lo_device; - void *key_data; gfp_t old_gfp_mask; spinlock_t lo_lock; int lo_state; @@ -66435,15 +68258,7 @@ struct loop_device { struct blk_mq_tag_set tag_set; struct gendisk *lo_disk; struct mutex lo_mutex; -}; - -struct loop_func_table { - int number; - int (*transfer)(struct loop_device *, int, struct page *, unsigned int, struct page *, unsigned int, int, sector_t); - int (*init)(struct loop_device *, const struct loop_info64 *); - int (*release)(struct loop_device *); - int (*ioctl)(struct loop_device *, int, long unsigned int); - struct module *owner; + bool idr_visible; }; struct loop_cmd { @@ -66534,6 +68349,7 @@ struct virtio_blk { struct virtblk_req { struct virtio_blk_outhdr out_hdr; u8 status; + struct sg_table sg_table; struct scatterlist sg[0]; }; @@ -66581,13 +68397,6 @@ struct dma_fence_ops { void (*timeline_value_str)(struct dma_fence *, char *, int); }; -enum dma_fence_flag_bits { - DMA_FENCE_FLAG_SIGNALED_BIT = 0, - DMA_FENCE_FLAG_TIMESTAMP_BIT = 1, - DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT = 2, - DMA_FENCE_FLAG_USER_BITS = 3, -}; - struct dma_fence_cb; typedef void (*dma_fence_func_t)(struct dma_fence *, struct dma_fence_cb *); @@ -66641,8 +68450,8 @@ struct dma_buf { void *priv; struct dma_resv *resv; wait_queue_head_t poll; - struct dma_buf_poll_cb_t cb_excl; - struct dma_buf_poll_cb_t cb_shared; + struct dma_buf_poll_cb_t cb_in; + struct dma_buf_poll_cb_t cb_out; }; struct dma_buf_attach_ops; @@ -66690,6 +68499,17 @@ struct dma_resv_list { struct dma_fence *shared[0]; }; +struct dma_resv_iter { + struct dma_resv *obj; + bool all_fences; + struct dma_fence *fence; + unsigned int seq; + unsigned int index; + struct dma_resv_list *fences; + unsigned int shared_count; + bool is_restarted; +}; + struct dma_buf_sync { __u64 flags; }; @@ -66699,6 +68519,13 @@ struct dma_buf_list { struct mutex lock; }; +enum dma_fence_flag_bits { + DMA_FENCE_FLAG_SIGNALED_BIT = 0, + DMA_FENCE_FLAG_TIMESTAMP_BIT = 1, + DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT = 2, + DMA_FENCE_FLAG_USER_BITS = 3, +}; + struct trace_event_raw_dma_fence { struct trace_entry ent; u32 __data_loc_driver; @@ -66750,25 +68577,14 @@ struct dma_fence_array { struct dma_fence_chain { struct dma_fence base; - spinlock_t lock; struct dma_fence *prev; u64 prev_seqno; struct dma_fence *fence; - struct dma_fence_cb cb; - struct irq_work work; -}; - -enum seqno_fence_condition { - SEQNO_FENCE_WAIT_GEQUAL = 0, - SEQNO_FENCE_WAIT_NONZERO = 1, -}; - -struct seqno_fence { - struct dma_fence base; - const struct dma_fence_ops *ops; - struct dma_buf *sync_buf; - uint32_t seqno_ofs; - enum seqno_fence_condition condition; + union { + struct dma_fence_cb cb; + struct irq_work work; + }; + spinlock_t lock; }; struct sync_file { @@ -66806,27 +68622,6 @@ struct sync_file_info { __u64 sync_fence_info; }; -struct scsi_lun { - __u8 scsi_lun[8]; -}; - -struct scsi_sense_hdr { - u8 response_code; - u8 sense_key; - u8 asc; - u8 ascq; - u8 byte4; - u8 byte5; - u8 byte6; - u8 additional_length; -}; - -struct ip_mreqn { - struct in_addr imr_multiaddr; - struct in_addr imr_address; - int imr_ifindex; -}; - enum { NETIF_F_SG_BIT = 0, NETIF_F_IP_CSUM_BIT = 1, @@ -67009,45 +68804,9 @@ struct skb_shared_info { skb_frag_t frags[17]; }; -enum { - IPV4_DEVCONF_FORWARDING = 1, - IPV4_DEVCONF_MC_FORWARDING = 2, - IPV4_DEVCONF_PROXY_ARP = 3, - IPV4_DEVCONF_ACCEPT_REDIRECTS = 4, - IPV4_DEVCONF_SECURE_REDIRECTS = 5, - IPV4_DEVCONF_SEND_REDIRECTS = 6, - IPV4_DEVCONF_SHARED_MEDIA = 7, - IPV4_DEVCONF_RP_FILTER = 8, - IPV4_DEVCONF_ACCEPT_SOURCE_ROUTE = 9, - IPV4_DEVCONF_BOOTP_RELAY = 10, - IPV4_DEVCONF_LOG_MARTIANS = 11, - IPV4_DEVCONF_TAG = 12, - IPV4_DEVCONF_ARPFILTER = 13, - IPV4_DEVCONF_MEDIUM_ID = 14, - IPV4_DEVCONF_NOXFRM = 15, - IPV4_DEVCONF_NOPOLICY = 16, - IPV4_DEVCONF_FORCE_IGMP_VERSION = 17, - IPV4_DEVCONF_ARP_ANNOUNCE = 18, - IPV4_DEVCONF_ARP_IGNORE = 19, - IPV4_DEVCONF_PROMOTE_SECONDARIES = 20, - IPV4_DEVCONF_ARP_ACCEPT = 21, - IPV4_DEVCONF_ARP_NOTIFY = 22, - IPV4_DEVCONF_ACCEPT_LOCAL = 23, - IPV4_DEVCONF_SRC_VMARK = 24, - IPV4_DEVCONF_PROXY_ARP_PVLAN = 25, - IPV4_DEVCONF_ROUTE_LOCALNET = 26, - IPV4_DEVCONF_IGMPV2_UNSOLICITED_REPORT_INTERVAL = 27, - IPV4_DEVCONF_IGMPV3_UNSOLICITED_REPORT_INTERVAL = 28, - IPV4_DEVCONF_IGNORE_ROUTES_WITH_LINKDOWN = 29, - IPV4_DEVCONF_DROP_UNICAST_IN_L2_MULTICAST = 30, - IPV4_DEVCONF_DROP_GRATUITOUS_ARP = 31, - IPV4_DEVCONF_BC_FORWARDING = 32, - __IPV4_DEVCONF_MAX = 33, -}; - struct ipv4_devconf { void *sysctl; - int data[32]; + int data[33]; long unsigned int state[1]; }; @@ -67062,6 +68821,7 @@ struct fnhe_hash_bucket; struct fib_nh_common { struct net_device *nhc_dev; + netdevice_tracker nhc_dev_tracker; int nhc_oif; unsigned char nhc_scope; u8 nhc_family; @@ -67173,6 +68933,12 @@ struct fib6_table { unsigned int fib_seq; }; +struct ip_mreqn { + struct in_addr imr_multiaddr; + struct in_addr imr_address; + int imr_ifindex; +}; + typedef union { __be32 a4; __be32 a6[4]; @@ -67315,7 +69081,8 @@ enum xfrm_attr_type_t { XFRMA_SET_MARK = 29, XFRMA_SET_MARK_MASK = 30, XFRMA_IF_ID = 31, - __XFRMA_MAX = 32, + XFRMA_MTIMER_THRESH = 32, + __XFRMA_MAX = 33, }; struct xfrm_mark { @@ -67366,47 +69133,13 @@ enum netdev_state_t { __LINK_STATE_TESTING = 5, }; -enum netdev_priv_flags { - IFF_802_1Q_VLAN = 1, - IFF_EBRIDGE = 2, - IFF_BONDING = 4, - IFF_ISATAP = 8, - IFF_WAN_HDLC = 16, - IFF_XMIT_DST_RELEASE = 32, - IFF_DONT_BRIDGE = 64, - IFF_DISABLE_NETPOLL = 128, - IFF_MACVLAN_PORT = 256, - IFF_BRIDGE_PORT = 512, - IFF_OVS_DATAPATH = 1024, - IFF_TX_SKB_SHARING = 2048, - IFF_UNICAST_FLT = 4096, - IFF_TEAM_PORT = 8192, - IFF_SUPP_NOFCS = 16384, - IFF_LIVE_ADDR_CHANGE = 32768, - IFF_MACVLAN = 65536, - IFF_XMIT_DST_RELEASE_PERM = 131072, - IFF_L3MDEV_MASTER = 262144, - IFF_NO_QUEUE = 524288, - IFF_OPENVSWITCH = 1048576, - IFF_L3MDEV_SLAVE = 2097152, - IFF_TEAM = 4194304, - IFF_RXFH_CONFIGURED = 8388608, - IFF_PHONY_HEADROOM = 16777216, - IFF_MACSEC = 33554432, - IFF_NO_RX_HANDLER = 67108864, - IFF_FAILOVER = 134217728, - IFF_FAILOVER_SLAVE = 268435456, - IFF_L3MDEV_RX_HANDLER = 536870912, - IFF_LIVE_RENAME_OK = 1073741824, - IFF_TX_SKB_NO_LINEAR = 2147483648, -}; - struct in_ifaddr; struct ip_mc_list; struct in_device { struct net_device *dev; + netdevice_tracker dev_tracker; refcount_t refcnt; int dead; struct in_ifaddr *ifa_list; @@ -67422,7 +69155,7 @@ struct in_device { long unsigned int mr_qri; unsigned char mr_qrv; unsigned char mr_gq_running; - unsigned char mr_ifc_count; + u32 mr_ifc_count; struct timer_list mr_gq_timer; struct timer_list mr_ifc_timer; struct neigh_parms *arp_parms; @@ -67498,6 +69231,43 @@ struct qdisc_walker { int (*fn)(struct Qdisc *, long unsigned int, struct qdisc_walker *); }; +enum { + IPV4_DEVCONF_FORWARDING = 1, + IPV4_DEVCONF_MC_FORWARDING = 2, + IPV4_DEVCONF_PROXY_ARP = 3, + IPV4_DEVCONF_ACCEPT_REDIRECTS = 4, + IPV4_DEVCONF_SECURE_REDIRECTS = 5, + IPV4_DEVCONF_SEND_REDIRECTS = 6, + IPV4_DEVCONF_SHARED_MEDIA = 7, + IPV4_DEVCONF_RP_FILTER = 8, + IPV4_DEVCONF_ACCEPT_SOURCE_ROUTE = 9, + IPV4_DEVCONF_BOOTP_RELAY = 10, + IPV4_DEVCONF_LOG_MARTIANS = 11, + IPV4_DEVCONF_TAG = 12, + IPV4_DEVCONF_ARPFILTER = 13, + IPV4_DEVCONF_MEDIUM_ID = 14, + IPV4_DEVCONF_NOXFRM = 15, + IPV4_DEVCONF_NOPOLICY = 16, + IPV4_DEVCONF_FORCE_IGMP_VERSION = 17, + IPV4_DEVCONF_ARP_ANNOUNCE = 18, + IPV4_DEVCONF_ARP_IGNORE = 19, + IPV4_DEVCONF_PROMOTE_SECONDARIES = 20, + IPV4_DEVCONF_ARP_ACCEPT = 21, + IPV4_DEVCONF_ARP_NOTIFY = 22, + IPV4_DEVCONF_ACCEPT_LOCAL = 23, + IPV4_DEVCONF_SRC_VMARK = 24, + IPV4_DEVCONF_PROXY_ARP_PVLAN = 25, + IPV4_DEVCONF_ROUTE_LOCALNET = 26, + IPV4_DEVCONF_IGMPV2_UNSOLICITED_REPORT_INTERVAL = 27, + IPV4_DEVCONF_IGMPV3_UNSOLICITED_REPORT_INTERVAL = 28, + IPV4_DEVCONF_IGNORE_ROUTES_WITH_LINKDOWN = 29, + IPV4_DEVCONF_DROP_UNICAST_IN_L2_MULTICAST = 30, + IPV4_DEVCONF_DROP_GRATUITOUS_ARP = 31, + IPV4_DEVCONF_BC_FORWARDING = 32, + IPV4_DEVCONF_ARP_EVICT_NOCARRIER = 33, + __IPV4_DEVCONF_MAX = 34, +}; + struct xfrm_state_walk { struct list_head all; u8 state; @@ -67515,6 +69285,7 @@ enum xfrm_replay_mode { struct xfrm_state_offload { struct net_device *dev; + netdevice_tracker dev_tracker; struct net_device *real_dev; long unsigned int offload_handle; unsigned int num_exthdrs; @@ -67570,6 +69341,9 @@ struct xfrm_state { struct xfrm_algo *calg; struct xfrm_algo_aead *aead; const char *geniv; + __be16 new_mapping_sport; + u32 new_mapping; + u32 mapping_maxage; struct xfrm_encap_tmpl *encap; struct sock *encap_sk; xfrm_address_t *coaddr; @@ -67600,6 +69374,39 @@ struct xfrm_state { void *data; }; +enum { + SOF_TIMESTAMPING_TX_HARDWARE = 1, + SOF_TIMESTAMPING_TX_SOFTWARE = 2, + SOF_TIMESTAMPING_RX_HARDWARE = 4, + SOF_TIMESTAMPING_RX_SOFTWARE = 8, + SOF_TIMESTAMPING_SOFTWARE = 16, + SOF_TIMESTAMPING_SYS_HARDWARE = 32, + SOF_TIMESTAMPING_RAW_HARDWARE = 64, + SOF_TIMESTAMPING_OPT_ID = 128, + SOF_TIMESTAMPING_TX_SCHED = 256, + SOF_TIMESTAMPING_TX_ACK = 512, + SOF_TIMESTAMPING_OPT_CMSG = 1024, + SOF_TIMESTAMPING_OPT_TSONLY = 2048, + SOF_TIMESTAMPING_OPT_STATS = 4096, + SOF_TIMESTAMPING_OPT_PKTINFO = 8192, + SOF_TIMESTAMPING_OPT_TX_SWHW = 16384, + SOF_TIMESTAMPING_BIND_PHC = 32768, + SOF_TIMESTAMPING_LAST = 32768, + SOF_TIMESTAMPING_MASK = 65535, +}; + +struct hwtstamp_config { + int flags; + int tx_type; + int rx_filter; +}; + +enum hwtstamp_flags { + HWTSTAMP_FLAG_BONDED_PHC_INDEX = 1, + HWTSTAMP_FLAG_LAST = 1, + HWTSTAMP_FLAG_MASK = 1, +}; + struct xfrm_policy_walk_entry { struct list_head all; u8 dead; @@ -68068,7 +69875,7 @@ struct rlb_client_info { struct alb_bond_info { struct tlb_client_info *tx_hashtbl; u32 unbalanced_load; - int tx_rebalance_counter; + atomic_t tx_rebalance_counter; int lp_counter; int rlb_enabled; struct rlb_client_info *rx_hashtbl; @@ -68113,7 +69920,9 @@ enum { BOND_OPT_AD_USER_PORT_KEY = 27, BOND_OPT_NUM_PEER_NOTIF_ALIAS = 28, BOND_OPT_PEER_NOTIF_DELAY = 29, - BOND_OPT_LAST = 30, + BOND_OPT_LACP_ACTIVE = 30, + BOND_OPT_MISSED_MAX = 31, + BOND_OPT_LAST = 32, }; struct bond_opt_value { @@ -68127,6 +69936,7 @@ struct bond_params { int xmit_policy; int miimon; u8 num_peer_notif; + u8 missed_max; int arp_interval; int arp_validate; int arp_all_targets; @@ -68135,6 +69945,7 @@ struct bond_params { int updelay; int downdelay; int peer_notif_delay; + int lacp_active; int lacp_fast; unsigned int min_links; int ad_select; @@ -68185,11 +69996,7 @@ struct bonding { struct delayed_work slave_arr_work; struct dentry *debug_dir; struct rtnl_link_stats64 bond_stats; -}; - -struct bond_parm_tbl { - char *modename; - int mode; + struct bpf_prog *xdp_prog; }; struct bond_up_slave { @@ -68287,6 +70094,7 @@ enum ad_link_speed_type { struct netdev_hw_addr { struct list_head list; + struct rb_node node; unsigned char addr[32]; unsigned char type; bool global_use; @@ -68296,59 +70104,6 @@ struct netdev_hw_addr { struct callback_head callback_head; }; -struct packet_type { - __be16 type; - bool ignore_outgoing; - struct net_device *dev; - int (*func)(struct sk_buff *, struct net_device *, struct packet_type *, struct net_device *); - void (*list_func)(struct list_head *, struct packet_type *, struct net_device *); - bool (*id_match)(struct packet_type *, struct sock *); - void *af_packet_priv; - struct list_head list; -}; - -struct llc_sap; - -struct datalink_proto { - unsigned char type[8]; - struct llc_sap *sap; - short unsigned int header_length; - int (*rcvfunc)(struct sk_buff *, struct net_device *, struct packet_type *, struct net_device *); - int (*request)(struct datalink_proto *, struct sk_buff *, unsigned char *); - struct list_head node; -}; - -struct ipx_address { - __be32 net; - __u8 node[6]; - __be16 sock; -}; - -struct ipxhdr { - __be16 ipx_checksum; - __be16 ipx_pktsize; - __u8 ipx_tctrl; - __u8 ipx_type; - struct ipx_address ipx_dest; - struct ipx_address ipx_source; -} __attribute__((packed)); - -struct ipx_interface { - __be32 if_netnum; - unsigned char if_node[6]; - refcount_t refcnt; - struct net_device *if_dev; - struct datalink_proto *if_dlink; - __be16 if_dlink_type; - short unsigned int if_sknum; - struct hlist_head if_sklist; - spinlock_t if_sklist_lock; - int if_ipx_offset; - unsigned char if_internal; - unsigned char if_primary; - struct list_head node; -}; - struct learning_pkt { u8 mac_dst[6]; u8 mac_src[6]; @@ -68371,7 +70126,7 @@ struct arp_pkt { struct alb_walk_data { struct bonding *bond; struct slave *slave; - u8 *mac_addr; + const u8 *mac_addr; bool strict_match; }; @@ -68450,7 +70205,8 @@ enum { IFLA_PROTO_DOWN_REASON = 55, IFLA_PARENT_DEV_NAME = 56, IFLA_PARENT_DEV_BUS_NAME = 57, - __IFLA_MAX = 58, + IFLA_GRO_MAX_SIZE = 58, + __IFLA_MAX = 59, }; enum { @@ -68483,7 +70239,9 @@ enum { IFLA_BOND_AD_ACTOR_SYSTEM = 26, IFLA_BOND_TLB_DYNAMIC_LB = 27, IFLA_BOND_PEER_NOTIF_DELAY = 28, - __IFLA_BOND_MAX = 29, + IFLA_BOND_AD_LACP_ACTIVE = 29, + IFLA_BOND_MISSED_MAX = 30, + __IFLA_BOND_MAX = 31, }; enum { @@ -68544,11 +70302,6 @@ enum { BOND_VALFLAG_MAX = 4, }; -struct devprobe2 { - struct net_device * (*probe)(int); - int status; -}; - enum { SKBTX_HW_TSTAMP = 1, SKBTX_SW_TSTAMP = 2, @@ -68565,6 +70318,7 @@ struct flow_keys_basic { enum { SKBFL_ZEROCOPY_ENABLE = 1, SKBFL_SHARED_FRAG = 2, + SKBFL_PURE_ZEROCOPY = 4, }; struct mmpin { @@ -68747,7 +70501,7 @@ struct fib_info { struct hlist_node fib_lhash; struct list_head nh_list; struct net *fib_net; - int fib_treeref; + refcount_t fib_treeref; refcount_t fib_clntref; unsigned int fib_flags; unsigned char fib_dead; @@ -68898,6 +70652,7 @@ typedef struct { struct ax25_dev { struct ax25_dev *next; struct net_device *dev; + netdevice_tracker dev_tracker; struct net_device *forward; struct ctl_table_header *sysheader; int values[14]; @@ -69034,7 +70789,6 @@ struct wpan_phy { possible_net_t _net; long: 64; long: 64; - long: 64; char priv[0]; }; @@ -69117,6 +70871,8 @@ struct tun_struct { struct tun_prog *steering_prog; struct tun_prog *filter_prog; struct ethtool_link_ksettings link_ksettings; + struct file *file; + struct ifreq *ifr; }; struct tun_page { @@ -69300,6 +71056,7 @@ struct failover_ops { struct failover { struct list_head list; struct net_device *failover_dev; + netdevice_tracker dev_tracker; struct failover_ops *ops; }; @@ -69319,6 +71076,7 @@ struct virtnet_sq_stats { u64 xdp_tx; u64 xdp_tx_drops; u64 kicks; + u64 tx_timeouts; }; struct virtnet_rq_stats { @@ -69455,7 +71213,8 @@ enum { NDA_PROTOCOL = 12, NDA_NH_ID = 13, NDA_FDB_EXT_ATTRS = 14, - __NDA_MAX = 15, + NDA_FLAGS_EXT = 15, + __NDA_MAX = 16, }; struct nda_cacheinfo { @@ -69512,37 +71271,6 @@ enum ifla_vxlan_df { VXLAN_DF_MAX = 2, }; -struct napi_gro_cb { - void *frag0; - unsigned int frag0_len; - int data_offset; - u16 flush; - u16 flush_id; - u16 count; - u16 gro_remcsum_start; - long unsigned int age; - u16 proto; - u8 same_flow: 1; - u8 encap_mark: 1; - u8 csum_valid: 1; - u8 csum_cnt: 3; - u8 free: 2; - u8 is_ipv6: 1; - u8 is_fou: 1; - u8 is_atomic: 1; - u8 recursion_counter: 4; - u8 is_flist: 1; - __wsum csum; - struct sk_buff *last; -}; - -typedef struct sk_buff * (*gro_receive_t)(struct list_head *, struct sk_buff *); - -struct gro_remcsum { - int offset; - __wsum delta; -}; - enum { RTN_UNSPEC = 0, RTN_UNICAST = 1, @@ -69594,46 +71322,8 @@ enum rtnetlink_groups { RTNLGRP_IPV6_MROUTE_R = 31, RTNLGRP_NEXTHOP = 32, RTNLGRP_BRVLAN = 33, - __RTNLGRP_MAX = 34, -}; - -struct vlan_hdr { - __be16 h_vlan_TCI; - __be16 h_vlan_encapsulated_proto; -}; - -struct ip_tunnel_key { - __be64 tun_id; - union { - struct { - __be32 src; - __be32 dst; - } ipv4; - struct { - struct in6_addr src; - struct in6_addr dst; - } ipv6; - } u; - __be16 tun_flags; - u8 tos; - u8 ttl; - __be32 label; - __be16 tp_src; - __be16 tp_dst; -}; - -struct dst_cache_pcpu; - -struct dst_cache { - struct dst_cache_pcpu *cache; - long unsigned int reset_ts; -}; - -struct ip_tunnel_info { - struct ip_tunnel_key key; - struct dst_cache dst_cache; - u8 options_len; - u8 mode; + RTNLGRP_MCTP_IFADDR = 34, + __RTNLGRP_MAX = 35, }; struct udp_hslot; @@ -69667,13 +71357,6 @@ struct udp_sock { void (*encap_destroy)(struct sock *); struct sk_buff * (*gro_receive)(struct sock *, struct list_head *, struct sk_buff *); int (*gro_complete)(struct sock *, struct sk_buff *, int); - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; struct sk_buff_head reader_queue; int forward_deficit; long: 32; @@ -69699,7 +71382,50 @@ enum lwtunnel_encap_types { LWTUNNEL_ENCAP_BPF = 6, LWTUNNEL_ENCAP_SEG6_LOCAL = 7, LWTUNNEL_ENCAP_RPL = 8, - __LWTUNNEL_ENCAP_MAX = 9, + LWTUNNEL_ENCAP_IOAM6 = 9, + __LWTUNNEL_ENCAP_MAX = 10, +}; + +struct udp_hslot { + struct hlist_head head; + int count; + spinlock_t lock; +}; + +struct napi_gro_cb { + void *frag0; + unsigned int frag0_len; + int data_offset; + u16 flush; + u16 flush_id; + u16 count; + u16 gro_remcsum_start; + long unsigned int age; + u16 proto; + u8 same_flow: 1; + u8 encap_mark: 1; + u8 csum_valid: 1; + u8 csum_cnt: 3; + u8 free: 2; + u8 is_ipv6: 1; + u8 is_fou: 1; + u8 is_atomic: 1; + u8 recursion_counter: 4; + u8 is_flist: 1; + __wsum csum; + struct sk_buff *last; +}; + +typedef struct sk_buff * (*gro_receive_t)(struct list_head *, struct sk_buff *); + +struct gro_remcsum { + int offset; + __wsum delta; +}; + +struct vlan_hdr { + __be16 h_vlan_TCI; + __be16 h_vlan_encapsulated_proto; }; enum { @@ -69716,6 +71442,13 @@ struct gro_cells { struct gro_cell *cells; }; +struct dst_cache_pcpu; + +struct dst_cache { + struct dst_cache_pcpu *cache; + long unsigned int reset_ts; +}; + enum nexthop_event_type { NEXTHOP_EVENT_DEL = 0, NEXTHOP_EVENT_REPLACE = 1, @@ -69780,10 +71513,31 @@ struct nh_notifier_info { }; }; -struct udp_hslot { - struct hlist_head head; - int count; - spinlock_t lock; +struct ip_tunnel_key { + __be64 tun_id; + union { + struct { + __be32 src; + __be32 dst; + } ipv4; + struct { + struct in6_addr src; + struct in6_addr dst; + } ipv6; + } u; + __be16 tun_flags; + u8 tos; + u8 ttl; + __be32 label; + __be16 tp_src; + __be16 tp_dst; +}; + +struct ip_tunnel_info { + struct ip_tunnel_key key; + struct dst_cache dst_cache; + u8 options_len; + u8 mode; }; struct udp_port_cfg { @@ -69887,6 +71641,8 @@ enum switchdev_notifier_type { SWITCHDEV_VXLAN_FDB_ADD_TO_DEVICE = 12, SWITCHDEV_VXLAN_FDB_DEL_TO_DEVICE = 13, SWITCHDEV_VXLAN_FDB_OFFLOADED = 14, + SWITCHDEV_BRPORT_OFFLOADED = 15, + SWITCHDEV_BRPORT_UNOFFLOADED = 16, }; struct switchdev_notifier_info { @@ -70116,7 +71872,6 @@ struct devlink_port { struct list_head region_list; struct devlink *devlink; unsigned int index; - bool registered; spinlock_t type_lock; enum devlink_port_type type; enum devlink_port_type desired_type; @@ -70144,188 +71899,21 @@ struct tcf_walker { int (*fn)(struct tcf_proto *, void *, struct tcf_walker *); }; -enum devlink_sb_pool_type { - DEVLINK_SB_POOL_TYPE_INGRESS = 0, - DEVLINK_SB_POOL_TYPE_EGRESS = 1, -}; - -enum devlink_sb_threshold_type { - DEVLINK_SB_THRESHOLD_TYPE_STATIC = 0, - DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC = 1, -}; - enum devlink_eswitch_mode { DEVLINK_ESWITCH_MODE_LEGACY = 0, DEVLINK_ESWITCH_MODE_SWITCHDEV = 1, }; -enum devlink_eswitch_encap_mode { - DEVLINK_ESWITCH_ENCAP_MODE_NONE = 0, - DEVLINK_ESWITCH_ENCAP_MODE_BASIC = 1, -}; - enum devlink_rate_type { DEVLINK_RATE_TYPE_LEAF = 0, DEVLINK_RATE_TYPE_NODE = 1, }; -enum devlink_trap_action { - DEVLINK_TRAP_ACTION_DROP = 0, - DEVLINK_TRAP_ACTION_TRAP = 1, - DEVLINK_TRAP_ACTION_MIRROR = 2, -}; - -enum devlink_trap_type { - DEVLINK_TRAP_TYPE_DROP = 0, - DEVLINK_TRAP_TYPE_EXCEPTION = 1, - DEVLINK_TRAP_TYPE_CONTROL = 2, -}; - -enum devlink_reload_action { - DEVLINK_RELOAD_ACTION_UNSPEC = 0, - DEVLINK_RELOAD_ACTION_DRIVER_REINIT = 1, - DEVLINK_RELOAD_ACTION_FW_ACTIVATE = 2, - __DEVLINK_RELOAD_ACTION_MAX = 3, - DEVLINK_RELOAD_ACTION_MAX = 2, -}; - -enum devlink_reload_limit { - DEVLINK_RELOAD_LIMIT_UNSPEC = 0, - DEVLINK_RELOAD_LIMIT_NO_RESET = 1, - __DEVLINK_RELOAD_LIMIT_MAX = 2, - DEVLINK_RELOAD_LIMIT_MAX = 1, -}; - enum devlink_dpipe_field_mapping_type { DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0, DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 1, }; -enum devlink_port_fn_state { - DEVLINK_PORT_FN_STATE_INACTIVE = 0, - DEVLINK_PORT_FN_STATE_ACTIVE = 1, -}; - -enum devlink_port_fn_opstate { - DEVLINK_PORT_FN_OPSTATE_DETACHED = 0, - DEVLINK_PORT_FN_OPSTATE_ATTACHED = 1, -}; - -struct devlink_dev_stats { - u32 reload_stats[6]; - u32 remote_reload_stats[6]; -}; - -struct devlink_dpipe_headers; - -struct devlink_ops; - -struct devlink { - struct list_head list; - struct list_head port_list; - struct list_head rate_list; - struct list_head sb_list; - struct list_head dpipe_table_list; - struct list_head resource_list; - struct list_head param_list; - struct list_head region_list; - struct list_head reporter_list; - struct mutex reporters_lock; - struct devlink_dpipe_headers *dpipe_headers; - struct list_head trap_list; - struct list_head trap_group_list; - struct list_head trap_policer_list; - const struct devlink_ops *ops; - struct xarray snapshot_ids; - struct devlink_dev_stats stats; - struct device *dev; - possible_net_t _net; - struct mutex lock; - u8 reload_failed: 1; - u8 reload_enabled: 1; - u8 registered: 1; - long: 61; - long: 64; - long: 64; - long: 64; - char priv[0]; -}; - -struct devlink_dpipe_header; - -struct devlink_dpipe_headers { - struct devlink_dpipe_header **headers; - unsigned int headers_count; -}; - -struct devlink_sb_pool_info; - -struct devlink_info_req; - -struct devlink_flash_update_params; - -struct devlink_trap; - -struct devlink_trap_group; - -struct devlink_trap_policer; - -struct devlink_port_new_attrs; - -struct devlink_ops { - u32 supported_flash_update_params; - long unsigned int reload_actions; - long unsigned int reload_limits; - int (*reload_down)(struct devlink *, bool, enum devlink_reload_action, enum devlink_reload_limit, struct netlink_ext_ack *); - int (*reload_up)(struct devlink *, enum devlink_reload_action, enum devlink_reload_limit, u32 *, struct netlink_ext_ack *); - int (*port_type_set)(struct devlink_port *, enum devlink_port_type); - int (*port_split)(struct devlink *, unsigned int, unsigned int, struct netlink_ext_ack *); - int (*port_unsplit)(struct devlink *, unsigned int, struct netlink_ext_ack *); - int (*sb_pool_get)(struct devlink *, unsigned int, u16, struct devlink_sb_pool_info *); - int (*sb_pool_set)(struct devlink *, unsigned int, u16, u32, enum devlink_sb_threshold_type, struct netlink_ext_ack *); - int (*sb_port_pool_get)(struct devlink_port *, unsigned int, u16, u32 *); - int (*sb_port_pool_set)(struct devlink_port *, unsigned int, u16, u32, struct netlink_ext_ack *); - int (*sb_tc_pool_bind_get)(struct devlink_port *, unsigned int, u16, enum devlink_sb_pool_type, u16 *, u32 *); - int (*sb_tc_pool_bind_set)(struct devlink_port *, unsigned int, u16, enum devlink_sb_pool_type, u16, u32, struct netlink_ext_ack *); - int (*sb_occ_snapshot)(struct devlink *, unsigned int); - int (*sb_occ_max_clear)(struct devlink *, unsigned int); - int (*sb_occ_port_pool_get)(struct devlink_port *, unsigned int, u16, u32 *, u32 *); - int (*sb_occ_tc_port_bind_get)(struct devlink_port *, unsigned int, u16, enum devlink_sb_pool_type, u32 *, u32 *); - int (*eswitch_mode_get)(struct devlink *, u16 *); - int (*eswitch_mode_set)(struct devlink *, u16, struct netlink_ext_ack *); - int (*eswitch_inline_mode_get)(struct devlink *, u8 *); - int (*eswitch_inline_mode_set)(struct devlink *, u8, struct netlink_ext_ack *); - int (*eswitch_encap_mode_get)(struct devlink *, enum devlink_eswitch_encap_mode *); - int (*eswitch_encap_mode_set)(struct devlink *, enum devlink_eswitch_encap_mode, struct netlink_ext_ack *); - int (*info_get)(struct devlink *, struct devlink_info_req *, struct netlink_ext_ack *); - int (*flash_update)(struct devlink *, struct devlink_flash_update_params *, struct netlink_ext_ack *); - int (*trap_init)(struct devlink *, const struct devlink_trap *, void *); - void (*trap_fini)(struct devlink *, const struct devlink_trap *, void *); - int (*trap_action_set)(struct devlink *, const struct devlink_trap *, enum devlink_trap_action, struct netlink_ext_ack *); - int (*trap_group_init)(struct devlink *, const struct devlink_trap_group *); - int (*trap_group_set)(struct devlink *, const struct devlink_trap_group *, const struct devlink_trap_policer *, struct netlink_ext_ack *); - int (*trap_group_action_set)(struct devlink *, const struct devlink_trap_group *, enum devlink_trap_action, struct netlink_ext_ack *); - int (*trap_drop_counter_get)(struct devlink *, const struct devlink_trap *, u64 *); - int (*trap_policer_init)(struct devlink *, const struct devlink_trap_policer *); - void (*trap_policer_fini)(struct devlink *, const struct devlink_trap_policer *); - int (*trap_policer_set)(struct devlink *, const struct devlink_trap_policer *, u64, u64, struct netlink_ext_ack *); - int (*trap_policer_counter_get)(struct devlink *, const struct devlink_trap_policer *, u64 *); - int (*port_function_hw_addr_get)(struct devlink *, struct devlink_port *, u8 *, int *, struct netlink_ext_ack *); - int (*port_function_hw_addr_set)(struct devlink *, struct devlink_port *, const u8 *, int, struct netlink_ext_ack *); - int (*port_new)(struct devlink *, const struct devlink_port_new_attrs *, struct netlink_ext_ack *, unsigned int *); - int (*port_del)(struct devlink *, unsigned int, struct netlink_ext_ack *); - int (*port_fn_state_get)(struct devlink *, struct devlink_port *, enum devlink_port_fn_state *, enum devlink_port_fn_opstate *, struct netlink_ext_ack *); - int (*port_fn_state_set)(struct devlink *, struct devlink_port *, enum devlink_port_fn_state, struct netlink_ext_ack *); - int (*rate_leaf_tx_share_set)(struct devlink_rate *, void *, u64, struct netlink_ext_ack *); - int (*rate_leaf_tx_max_set)(struct devlink_rate *, void *, u64, struct netlink_ext_ack *); - int (*rate_node_tx_share_set)(struct devlink_rate *, void *, u64, struct netlink_ext_ack *); - int (*rate_node_tx_max_set)(struct devlink_rate *, void *, u64, struct netlink_ext_ack *); - int (*rate_node_new)(struct devlink_rate *, void **, struct netlink_ext_ack *); - int (*rate_node_del)(struct devlink_rate *, void *, struct netlink_ext_ack *); - int (*rate_leaf_parent_set)(struct devlink_rate *, struct devlink_rate *, void *, void *, struct netlink_ext_ack *); - int (*rate_node_parent_set)(struct devlink_rate *, struct devlink_rate *, void *, void *, struct netlink_ext_ack *); -}; - struct devlink_rate { struct list_head list; enum devlink_rate_type type; @@ -70343,24 +71931,6 @@ struct devlink_rate { }; }; -struct devlink_port_new_attrs { - enum devlink_port_flavour flavour; - unsigned int port_index; - u32 controller; - u32 sfnum; - u16 pfnum; - u8 port_index_valid: 1; - u8 controller_valid: 1; - u8 sfnum_valid: 1; -}; - -struct devlink_sb_pool_info { - enum devlink_sb_pool_type pool_type; - u32 size; - enum devlink_sb_threshold_type threshold_type; - u32 cell_size; -}; - struct devlink_dpipe_field { const char *name; unsigned int id; @@ -70376,39 +71946,6 @@ struct devlink_dpipe_header { bool global; }; -struct devlink_flash_update_params { - const struct firmware *fw; - const char *component; - u32 overwrite_mask; -}; - -struct devlink_trap_policer { - u32 id; - u64 init_rate; - u64 init_burst; - u64 max_rate; - u64 min_rate; - u64 max_burst; - u64 min_burst; -}; - -struct devlink_trap_group { - const char *name; - u16 id; - bool generic; - u32 init_policer_id; -}; - -struct devlink_trap { - enum devlink_trap_type type; - enum devlink_trap_action init_action; - bool generic; - u16 id; - const char *name; - u16 init_group_id; - u32 metadata_cap; -}; - struct nsim_sa { struct xfrm_state *xs; __be32 ipaddr[4]; @@ -70437,6 +71974,7 @@ struct nsim_ethtool_pauseparam { struct nsim_ethtool { u32 get_err; u32 set_err; + u32 channels; struct nsim_ethtool_pauseparam pauseparam; struct ethtool_coalesce coalesce; struct ethtool_ringparam ring; @@ -70496,6 +72034,8 @@ struct nsim_fib_data; struct nsim_trap_data; +struct nsim_vf_config; + struct nsim_dev { struct nsim_bus_dev *nsim_bus_dev; struct nsim_fib_data *fib_data; @@ -70503,8 +72043,9 @@ struct nsim_dev { struct dentry *ddir; struct dentry *ports_ddir; struct dentry *take_snapshot; - struct dentry *max_vfs; struct dentry *nodes_ddir; + struct mutex vfs_lock; + struct nsim_vf_config *vfconfigs; struct bpf_offload_dev *bpf_dev; bool bpf_bind_accept; bool bpf_bind_verifier_accept; @@ -70560,8 +72101,6 @@ struct nsim_dev_port { struct netdevsim *ns; }; -struct nsim_vf_config; - struct nsim_bus_dev { struct device dev; struct list_head list; @@ -70570,9 +72109,8 @@ struct nsim_bus_dev { struct net *initial_net; unsigned int max_vfs; unsigned int num_vfs; - struct mutex vfs_lock; - struct nsim_vf_config *vfconfigs; struct mutex nsim_bus_reload_lock; + bool in_reload; bool init; }; @@ -70589,6 +72127,21 @@ struct nsim_vf_config { bool rss_query_enabled; }; +enum devlink_sb_pool_type { + DEVLINK_SB_POOL_TYPE_INGRESS = 0, + DEVLINK_SB_POOL_TYPE_EGRESS = 1, +}; + +enum devlink_sb_threshold_type { + DEVLINK_SB_THRESHOLD_TYPE_STATIC = 0, + DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC = 1, +}; + +enum devlink_eswitch_encap_mode { + DEVLINK_ESWITCH_ENCAP_MODE_NONE = 0, + DEVLINK_ESWITCH_ENCAP_MODE_BASIC = 1, +}; + enum devlink_param_cmode { DEVLINK_PARAM_CMODE_RUNTIME = 0, DEVLINK_PARAM_CMODE_DRIVERINIT = 1, @@ -70597,10 +72150,65 @@ enum devlink_param_cmode { DEVLINK_PARAM_CMODE_MAX = 2, }; +enum devlink_trap_action { + DEVLINK_TRAP_ACTION_DROP = 0, + DEVLINK_TRAP_ACTION_TRAP = 1, + DEVLINK_TRAP_ACTION_MIRROR = 2, +}; + +enum devlink_trap_type { + DEVLINK_TRAP_TYPE_DROP = 0, + DEVLINK_TRAP_TYPE_EXCEPTION = 1, + DEVLINK_TRAP_TYPE_CONTROL = 2, +}; + +enum devlink_reload_action { + DEVLINK_RELOAD_ACTION_UNSPEC = 0, + DEVLINK_RELOAD_ACTION_DRIVER_REINIT = 1, + DEVLINK_RELOAD_ACTION_FW_ACTIVATE = 2, + __DEVLINK_RELOAD_ACTION_MAX = 3, + DEVLINK_RELOAD_ACTION_MAX = 2, +}; + +enum devlink_reload_limit { + DEVLINK_RELOAD_LIMIT_UNSPEC = 0, + DEVLINK_RELOAD_LIMIT_NO_RESET = 1, + __DEVLINK_RELOAD_LIMIT_MAX = 2, + DEVLINK_RELOAD_LIMIT_MAX = 1, +}; + enum devlink_resource_unit { DEVLINK_RESOURCE_UNIT_ENTRY = 0, }; +enum devlink_port_fn_state { + DEVLINK_PORT_FN_STATE_INACTIVE = 0, + DEVLINK_PORT_FN_STATE_ACTIVE = 1, +}; + +enum devlink_port_fn_opstate { + DEVLINK_PORT_FN_OPSTATE_DETACHED = 0, + DEVLINK_PORT_FN_OPSTATE_ATTACHED = 1, +}; + +struct devlink_port_new_attrs { + enum devlink_port_flavour flavour; + unsigned int port_index; + u32 controller; + u32 sfnum; + u16 pfnum; + u8 port_index_valid: 1; + u8 controller_valid: 1; + u8 sfnum_valid: 1; +}; + +struct devlink_sb_pool_info { + enum devlink_sb_pool_type pool_type; + u32 size; + enum devlink_sb_threshold_type threshold_type; + u32 cell_size; +}; + struct devlink_resource_size_params { u64 size_min; u64 size_max; @@ -70652,8 +72260,20 @@ enum devlink_param_generic_id { DEVLINK_PARAM_GENERIC_ID_RESET_DEV_ON_DRV_PROBE = 8, DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE = 9, DEVLINK_PARAM_GENERIC_ID_ENABLE_REMOTE_DEV_RESET = 10, - __DEVLINK_PARAM_GENERIC_ID_MAX = 11, - DEVLINK_PARAM_GENERIC_ID_MAX = 10, + DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH = 11, + DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA = 12, + DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET = 13, + DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP = 14, + DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE = 15, + DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE = 16, + __DEVLINK_PARAM_GENERIC_ID_MAX = 17, + DEVLINK_PARAM_GENERIC_ID_MAX = 16, +}; + +struct devlink_flash_update_params { + const struct firmware *fw; + const char *component; + u32 overwrite_mask; }; struct devlink_region_ops { @@ -70663,6 +72283,33 @@ struct devlink_region_ops { void *priv; }; +struct devlink_trap_policer { + u32 id; + u64 init_rate; + u64 init_burst; + u64 max_rate; + u64 min_rate; + u64 max_burst; + u64 min_burst; +}; + +struct devlink_trap_group { + const char *name; + u16 id; + bool generic; + u32 init_policer_id; +}; + +struct devlink_trap { + enum devlink_trap_type type; + enum devlink_trap_action init_action; + bool generic; + u16 id; + const char *name; + u16 init_group_id; + u32 metadata_cap; +}; + enum devlink_trap_generic_id { DEVLINK_TRAP_GENERIC_ID_SMAC_MC = 0, DEVLINK_TRAP_GENERIC_ID_VLAN_TAG_MISMATCH = 1, @@ -70791,6 +72438,66 @@ enum devlink_trap_group_generic_id { DEVLINK_TRAP_GROUP_GENERIC_ID_MAX = 25, }; +enum { + DEVLINK_F_RELOAD = 1, +}; + +struct devlink_info_req; + +struct devlink_ops { + u32 supported_flash_update_params; + long unsigned int reload_actions; + long unsigned int reload_limits; + int (*reload_down)(struct devlink *, bool, enum devlink_reload_action, enum devlink_reload_limit, struct netlink_ext_ack *); + int (*reload_up)(struct devlink *, enum devlink_reload_action, enum devlink_reload_limit, u32 *, struct netlink_ext_ack *); + int (*port_type_set)(struct devlink_port *, enum devlink_port_type); + int (*port_split)(struct devlink *, unsigned int, unsigned int, struct netlink_ext_ack *); + int (*port_unsplit)(struct devlink *, unsigned int, struct netlink_ext_ack *); + int (*sb_pool_get)(struct devlink *, unsigned int, u16, struct devlink_sb_pool_info *); + int (*sb_pool_set)(struct devlink *, unsigned int, u16, u32, enum devlink_sb_threshold_type, struct netlink_ext_ack *); + int (*sb_port_pool_get)(struct devlink_port *, unsigned int, u16, u32 *); + int (*sb_port_pool_set)(struct devlink_port *, unsigned int, u16, u32, struct netlink_ext_ack *); + int (*sb_tc_pool_bind_get)(struct devlink_port *, unsigned int, u16, enum devlink_sb_pool_type, u16 *, u32 *); + int (*sb_tc_pool_bind_set)(struct devlink_port *, unsigned int, u16, enum devlink_sb_pool_type, u16, u32, struct netlink_ext_ack *); + int (*sb_occ_snapshot)(struct devlink *, unsigned int); + int (*sb_occ_max_clear)(struct devlink *, unsigned int); + int (*sb_occ_port_pool_get)(struct devlink_port *, unsigned int, u16, u32 *, u32 *); + int (*sb_occ_tc_port_bind_get)(struct devlink_port *, unsigned int, u16, enum devlink_sb_pool_type, u32 *, u32 *); + int (*eswitch_mode_get)(struct devlink *, u16 *); + int (*eswitch_mode_set)(struct devlink *, u16, struct netlink_ext_ack *); + int (*eswitch_inline_mode_get)(struct devlink *, u8 *); + int (*eswitch_inline_mode_set)(struct devlink *, u8, struct netlink_ext_ack *); + int (*eswitch_encap_mode_get)(struct devlink *, enum devlink_eswitch_encap_mode *); + int (*eswitch_encap_mode_set)(struct devlink *, enum devlink_eswitch_encap_mode, struct netlink_ext_ack *); + int (*info_get)(struct devlink *, struct devlink_info_req *, struct netlink_ext_ack *); + int (*flash_update)(struct devlink *, struct devlink_flash_update_params *, struct netlink_ext_ack *); + int (*trap_init)(struct devlink *, const struct devlink_trap *, void *); + void (*trap_fini)(struct devlink *, const struct devlink_trap *, void *); + int (*trap_action_set)(struct devlink *, const struct devlink_trap *, enum devlink_trap_action, struct netlink_ext_ack *); + int (*trap_group_init)(struct devlink *, const struct devlink_trap_group *); + int (*trap_group_set)(struct devlink *, const struct devlink_trap_group *, const struct devlink_trap_policer *, struct netlink_ext_ack *); + int (*trap_group_action_set)(struct devlink *, const struct devlink_trap_group *, enum devlink_trap_action, struct netlink_ext_ack *); + int (*trap_drop_counter_get)(struct devlink *, const struct devlink_trap *, u64 *); + int (*trap_policer_init)(struct devlink *, const struct devlink_trap_policer *); + void (*trap_policer_fini)(struct devlink *, const struct devlink_trap_policer *); + int (*trap_policer_set)(struct devlink *, const struct devlink_trap_policer *, u64, u64, struct netlink_ext_ack *); + int (*trap_policer_counter_get)(struct devlink *, const struct devlink_trap_policer *, u64 *); + int (*port_function_hw_addr_get)(struct devlink_port *, u8 *, int *, struct netlink_ext_ack *); + int (*port_function_hw_addr_set)(struct devlink_port *, const u8 *, int, struct netlink_ext_ack *); + int (*port_new)(struct devlink *, const struct devlink_port_new_attrs *, struct netlink_ext_ack *, unsigned int *); + int (*port_del)(struct devlink *, unsigned int, struct netlink_ext_ack *); + int (*port_fn_state_get)(struct devlink_port *, enum devlink_port_fn_state *, enum devlink_port_fn_opstate *, struct netlink_ext_ack *); + int (*port_fn_state_set)(struct devlink_port *, enum devlink_port_fn_state, struct netlink_ext_ack *); + int (*rate_leaf_tx_share_set)(struct devlink_rate *, void *, u64, struct netlink_ext_ack *); + int (*rate_leaf_tx_max_set)(struct devlink_rate *, void *, u64, struct netlink_ext_ack *); + int (*rate_node_tx_share_set)(struct devlink_rate *, void *, u64, struct netlink_ext_ack *); + int (*rate_node_tx_max_set)(struct devlink_rate *, void *, u64, struct netlink_ext_ack *); + int (*rate_node_new)(struct devlink_rate *, void **, struct netlink_ext_ack *); + int (*rate_node_del)(struct devlink_rate *, void *, struct netlink_ext_ack *); + int (*rate_leaf_parent_set)(struct devlink_rate *, struct devlink_rate *, void *, void *, struct netlink_ext_ack *); + int (*rate_node_parent_set)(struct devlink_rate *, struct devlink_rate *, void *, void *, struct netlink_ext_ack *); +}; + enum nsim_resource_id { NSIM_RESOURCE_NONE = 0, NSIM_RESOURCE_IPV4 = 1, @@ -70814,8 +72521,8 @@ struct nsim_trap_data { }; enum nsim_devlink_param_id { - NSIM_DEVLINK_PARAM_ID_BASE = 10, - NSIM_DEVLINK_PARAM_ID_TEST1 = 11, + NSIM_DEVLINK_PARAM_ID_BASE = 16, + NSIM_DEVLINK_PARAM_ID_TEST1 = 17, }; struct nsim_trap_item { @@ -71057,13 +72764,14 @@ struct tc_action { atomic_t tcfa_bindcnt; int tcfa_action; struct tcf_t tcfa_tm; - struct gnet_stats_basic_packed tcfa_bstats; - struct gnet_stats_basic_packed tcfa_bstats_hw; + long: 64; + struct gnet_stats_basic_sync tcfa_bstats; + struct gnet_stats_basic_sync tcfa_bstats_hw; struct gnet_stats_queue tcfa_qstats; struct net_rate_estimator *tcfa_rate_est; spinlock_t tcfa_lock; - struct gnet_stats_basic_cpu *cpu_bstats; - struct gnet_stats_basic_cpu *cpu_bstats_hw; + struct gnet_stats_basic_sync *cpu_bstats; + struct gnet_stats_basic_sync *cpu_bstats_hw; struct gnet_stats_queue *cpu_qstats; struct tc_cookie *act_cookie; struct tcf_chain *goto_chain; @@ -71071,6 +72779,9 @@ struct tc_action { u8 hw_stats; u8 used_hw_stats; bool used_hw_stats_valid; + u32 in_hw_count; + long: 32; + long: 64; }; typedef void (*tc_action_priv_destructor)(void *); @@ -71087,12 +72798,13 @@ struct tc_action_ops { int (*dump)(struct sk_buff *, struct tc_action *, int, int); void (*cleanup)(struct tc_action *); int (*lookup)(struct net *, struct tc_action **, u32); - int (*init)(struct net *, struct nlattr *, struct nlattr *, struct tc_action **, int, int, bool, struct tcf_proto *, u32, struct netlink_ext_ack *); + int (*init)(struct net *, struct nlattr *, struct nlattr *, struct tc_action **, struct tcf_proto *, u32, struct netlink_ext_ack *); int (*walk)(struct net *, struct sk_buff *, struct netlink_callback *, int, const struct tc_action_ops *, struct netlink_ext_ack *); void (*stats_update)(struct tc_action *, u64, u64, u64, u64, bool); size_t (*get_fill_size)(const struct tc_action *); struct net_device * (*get_dev)(const struct tc_action *, tc_action_priv_destructor *); struct psample_group * (*get_psample_group)(const struct tc_action *, tc_action_priv_destructor *); + int (*offload_act_setup)(struct tc_action *, void *, u32 *, bool); }; struct tc_cookie { @@ -71106,6 +72818,7 @@ struct tcf_exts { int nr_actions; struct tc_action **actions; struct net *net; + netns_tracker ns_tracker; int action; int police; }; @@ -71363,6 +73076,286 @@ struct evdev_client { struct input_event buffer[0]; }; +struct lirc_scancode { + __u64 timestamp; + __u16 flags; + __u16 rc_proto; + __u32 keycode; + __u64 scancode; +}; + +enum rc_proto { + RC_PROTO_UNKNOWN = 0, + RC_PROTO_OTHER = 1, + RC_PROTO_RC5 = 2, + RC_PROTO_RC5X_20 = 3, + RC_PROTO_RC5_SZ = 4, + RC_PROTO_JVC = 5, + RC_PROTO_SONY12 = 6, + RC_PROTO_SONY15 = 7, + RC_PROTO_SONY20 = 8, + RC_PROTO_NEC = 9, + RC_PROTO_NECX = 10, + RC_PROTO_NEC32 = 11, + RC_PROTO_SANYO = 12, + RC_PROTO_MCIR2_KBD = 13, + RC_PROTO_MCIR2_MSE = 14, + RC_PROTO_RC6_0 = 15, + RC_PROTO_RC6_6A_20 = 16, + RC_PROTO_RC6_6A_24 = 17, + RC_PROTO_RC6_6A_32 = 18, + RC_PROTO_RC6_MCE = 19, + RC_PROTO_SHARP = 20, + RC_PROTO_XMP = 21, + RC_PROTO_CEC = 22, + RC_PROTO_IMON = 23, + RC_PROTO_RCMM12 = 24, + RC_PROTO_RCMM24 = 25, + RC_PROTO_RCMM32 = 26, + RC_PROTO_XBOX_DVD = 27, + RC_PROTO_MAX = 27, +}; + +struct rc_map_table { + u64 scancode; + u32 keycode; +}; + +struct rc_map { + struct rc_map_table *scan; + unsigned int size; + unsigned int len; + unsigned int alloc; + enum rc_proto rc_proto; + const char *name; + spinlock_t lock; +}; + +struct rc_map_list { + struct list_head list; + struct rc_map map; +}; + +enum rc_driver_type { + RC_DRIVER_SCANCODE = 0, + RC_DRIVER_IR_RAW = 1, + RC_DRIVER_IR_RAW_TX = 2, +}; + +struct rc_scancode_filter { + u32 data; + u32 mask; +}; + +enum rc_filter_type { + RC_FILTER_NORMAL = 0, + RC_FILTER_WAKEUP = 1, + RC_FILTER_MAX = 2, +}; + +struct ir_raw_event_ctrl; + +struct rc_dev { + struct device dev; + bool managed_alloc; + const struct attribute_group *sysfs_groups[5]; + const char *device_name; + const char *input_phys; + struct input_id input_id; + const char *driver_name; + const char *map_name; + struct rc_map rc_map; + struct mutex lock; + unsigned int minor; + struct ir_raw_event_ctrl *raw; + struct input_dev *input_dev; + enum rc_driver_type driver_type; + bool idle; + bool encode_wakeup; + u64 allowed_protocols; + u64 enabled_protocols; + u64 allowed_wakeup_protocols; + enum rc_proto wakeup_protocol; + struct rc_scancode_filter scancode_filter; + struct rc_scancode_filter scancode_wakeup_filter; + u32 scancode_mask; + u32 users; + void *priv; + spinlock_t keylock; + bool keypressed; + long unsigned int keyup_jiffies; + struct timer_list timer_keyup; + struct timer_list timer_repeat; + u32 last_keycode; + enum rc_proto last_protocol; + u64 last_scancode; + u8 last_toggle; + u32 timeout; + u32 min_timeout; + u32 max_timeout; + u32 rx_resolution; + u32 tx_resolution; + struct device lirc_dev; + struct cdev lirc_cdev; + ktime_t gap_start; + u64 gap_duration; + bool gap; + spinlock_t lirc_fh_lock; + struct list_head lirc_fh; + bool registered; + int (*change_protocol)(struct rc_dev *, u64 *); + int (*open)(struct rc_dev *); + void (*close)(struct rc_dev *); + int (*s_tx_mask)(struct rc_dev *, u32); + int (*s_tx_carrier)(struct rc_dev *, u32); + int (*s_tx_duty_cycle)(struct rc_dev *, u32); + int (*s_rx_carrier_range)(struct rc_dev *, u32, u32); + int (*tx_ir)(struct rc_dev *, unsigned int *, unsigned int); + void (*s_idle)(struct rc_dev *, bool); + int (*s_wideband_receiver)(struct rc_dev *, int); + int (*s_carrier_report)(struct rc_dev *, int); + int (*s_filter)(struct rc_dev *, struct rc_scancode_filter *); + int (*s_wakeup_filter)(struct rc_dev *, struct rc_scancode_filter *); + int (*s_timeout)(struct rc_dev *, unsigned int); +}; + +struct ir_raw_event { + union { + u32 duration; + u32 carrier; + }; + u8 duty_cycle; + unsigned int pulse: 1; + unsigned int reset: 1; + unsigned int timeout: 1; + unsigned int carrier_report: 1; +}; + +struct ir_raw_event_ctrl { + struct list_head list; + struct task_struct *thread; + struct { + union { + struct __kfifo kfifo; + struct ir_raw_event *type; + const struct ir_raw_event *const_type; + char (*rectype)[0]; + struct ir_raw_event *ptr; + const struct ir_raw_event *ptr_const; + }; + struct ir_raw_event buf[512]; + } kfifo; + ktime_t last_event; + struct rc_dev *dev; + spinlock_t edge_spinlock; + struct timer_list edge_handle; + struct ir_raw_event prev_ev; + struct ir_raw_event this_ev; + u32 bpf_sample; + struct bpf_prog_array *progs; +}; + +enum led_brightness { + LED_OFF = 0, + LED_ON = 1, + LED_HALF = 127, + LED_FULL = 255, +}; + +struct led_trigger {}; + +struct rc_filter_attribute { + struct device_attribute attr; + enum rc_filter_type type; + bool mask; +}; + +struct ir_raw_handler { + struct list_head list; + u64 protocols; + int (*decode)(struct rc_dev *, struct ir_raw_event); + int (*encode)(enum rc_proto, u32, struct ir_raw_event *, unsigned int); + u32 carrier; + u32 min_timeout; + int (*raw_register)(struct rc_dev *); + int (*raw_unregister)(struct rc_dev *); +}; + +struct ir_raw_timings_manchester { + unsigned int leader_pulse; + unsigned int leader_space; + unsigned int clock; + unsigned int invert: 1; + unsigned int trailer_space; +}; + +struct ir_raw_timings_pd { + unsigned int header_pulse; + unsigned int header_space; + unsigned int bit_pulse; + unsigned int bit_space[2]; + unsigned int trailer_pulse; + unsigned int trailer_space; + unsigned int msb_first: 1; +}; + +struct ir_raw_timings_pl { + unsigned int header_pulse; + unsigned int bit_space; + unsigned int bit_pulse[2]; + unsigned int trailer_space; + unsigned int msb_first: 1; +}; + +struct lirc_fh { + struct list_head list; + struct rc_dev *rc; + int carrier_low; + struct { + union { + struct __kfifo kfifo; + unsigned int *type; + const unsigned int *const_type; + char (*rectype)[0]; + unsigned int *ptr; + const unsigned int *ptr_const; + }; + unsigned int buf[0]; + } rawir; + struct { + union { + struct __kfifo kfifo; + struct lirc_scancode *type; + const struct lirc_scancode *const_type; + char (*rectype)[0]; + struct lirc_scancode *ptr; + const struct lirc_scancode *ptr_const; + }; + struct lirc_scancode buf[0]; + } scancodes; + wait_queue_head_t wait_poll; + u8 send_mode; + u8 rec_mode; +}; + +typedef u64 (*btf_bpf_rc_repeat)(u32 *); + +typedef u64 (*btf_bpf_rc_keydown)(u32 *, u32, u64, u32); + +typedef u64 (*btf_bpf_rc_pointer_rel)(u32 *, s32, s32); + +struct loopback_dev { + struct rc_dev *dev; + u32 txmask; + u32 txcarrier; + u32 txduty; + bool idle; + bool wideband; + bool carrierreport; + u32 rxcarriermin; + u32 rxcarriermax; +}; + struct pps_ktime { __s64 sec; __s32 nsec; @@ -71605,6 +73598,16 @@ struct ptp_vclock { spinlock_t lock; }; +enum { + POWER_SUPPLY_TECHNOLOGY_UNKNOWN = 0, + POWER_SUPPLY_TECHNOLOGY_NiMH = 1, + POWER_SUPPLY_TECHNOLOGY_LION = 2, + POWER_SUPPLY_TECHNOLOGY_LIPO = 3, + POWER_SUPPLY_TECHNOLOGY_LiFe = 4, + POWER_SUPPLY_TECHNOLOGY_NiCd = 5, + POWER_SUPPLY_TECHNOLOGY_LiMn = 6, +}; + enum power_supply_property { POWER_SUPPLY_PROP_STATUS = 0, POWER_SUPPLY_PROP_CHARGE_TYPE = 1, @@ -71839,6 +73842,7 @@ struct power_supply_resistance_temp_table { }; struct power_supply_battery_info { + unsigned int technology; int energy_full_design_uwh; int charge_full_design_uah; int voltage_min_design_uv; @@ -71972,16 +73976,6 @@ enum { POWER_SUPPLY_HEALTH_HOT = 13, }; -enum { - POWER_SUPPLY_TECHNOLOGY_UNKNOWN = 0, - POWER_SUPPLY_TECHNOLOGY_NiMH = 1, - POWER_SUPPLY_TECHNOLOGY_LION = 2, - POWER_SUPPLY_TECHNOLOGY_LIPO = 3, - POWER_SUPPLY_TECHNOLOGY_LiFe = 4, - POWER_SUPPLY_TECHNOLOGY_NiCd = 5, - POWER_SUPPLY_TECHNOLOGY_LiMn = 6, -}; - enum { POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN = 0, POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL = 1, @@ -72155,10 +74149,10 @@ struct cpufreq_driver { int (*exit)(struct cpufreq_policy *); int (*suspend)(struct cpufreq_policy *); int (*resume)(struct cpufreq_policy *); - void (*ready)(struct cpufreq_policy *); struct freq_attr **attr; bool boost_enabled; int (*set_boost)(struct cpufreq_policy *, int); + void (*register_em)(struct cpufreq_policy *); }; struct cpufreq_stats { @@ -72403,6 +74397,7 @@ struct cpudata { unsigned int sched_flags; u32 hwp_boost_min; bool suspended; + struct delayed_work hwp_notify_work; }; struct pstate_funcs { @@ -72411,11 +74406,20 @@ struct pstate_funcs { int (*get_min)(); int (*get_turbo)(); int (*get_scaling)(); + int (*get_cpu_scaling)(int); int (*get_aperf_mperf_shift)(); u64 (*get_val)(struct cpudata *, int); void (*get_vid)(struct cpudata *); }; +enum energy_perf_value_index { + EPP_INDEX_DEFAULT = 0, + EPP_INDEX_PERFORMANCE = 1, + EPP_INDEX_BALANCE_PERFORMANCE = 2, + EPP_INDEX_BALANCE_POWERSAVE = 3, + EPP_INDEX_POWERSAVE = 4, +}; + enum { PSS = 0, PPC = 1, @@ -72568,6 +74572,22 @@ struct memmap_attribute { ssize_t (*show)(struct firmware_map_entry *, char *); }; +struct simplefb_platform_data { + u32 width; + u32 height; + u32 stride; + const char *format; +}; + +struct efifb_dmi_info { + char *optname; + long unsigned int base; + int stride; + int width; + int height; + int flags; +}; + typedef efi_status_t efi_query_variable_store_t(u32, long unsigned int, bool); typedef struct { @@ -72698,6 +74718,72 @@ struct map_attribute { ssize_t (*show)(struct efi_runtime_map_entry *, char *); }; +struct of_bus; + +struct of_pci_range_parser { + struct device_node *node; + struct of_bus *bus; + const __be32 *range; + const __be32 *end; + int na; + int ns; + int pna; + bool dma; +}; + +struct of_pci_range { + union { + u64 pci_addr; + u64 bus_addr; + }; + u64 cpu_addr; + u64 size; + u32 flags; +}; + +enum { + M_I17 = 0, + M_I20 = 1, + M_I20_SR = 2, + M_I24 = 3, + M_I24_8_1 = 4, + M_I24_10_1 = 5, + M_I27_11_1 = 6, + M_MINI = 7, + M_MINI_3_1 = 8, + M_MINI_4_1 = 9, + M_MB = 10, + M_MB_2 = 11, + M_MB_3 = 12, + M_MB_5_1 = 13, + M_MB_6_1 = 14, + M_MB_7_1 = 15, + M_MB_SR = 16, + M_MBA = 17, + M_MBA_3 = 18, + M_MBP = 19, + M_MBP_2 = 20, + M_MBP_2_2 = 21, + M_MBP_SR = 22, + M_MBP_4 = 23, + M_MBP_5_1 = 24, + M_MBP_5_2 = 25, + M_MBP_5_3 = 26, + M_MBP_6_1 = 27, + M_MBP_6_2 = 28, + M_MBP_7_1 = 29, + M_MBP_8_2 = 30, + M_UNKNOWN = 31, +}; + +enum { + OVERRIDE_NONE = 0, + OVERRIDE_BASE = 1, + OVERRIDE_STRIDE = 2, + OVERRIDE_HEIGHT = 4, + OVERRIDE_WIDTH = 8, +}; + struct hid_device_id { __u16 bus; __u16 group; @@ -73023,107 +75109,6 @@ struct apple_key_translation { u8 flags; }; -struct usb_interface_descriptor { - __u8 bLength; - __u8 bDescriptorType; - __u8 bInterfaceNumber; - __u8 bAlternateSetting; - __u8 bNumEndpoints; - __u8 bInterfaceClass; - __u8 bInterfaceSubClass; - __u8 bInterfaceProtocol; - __u8 iInterface; -}; - -struct usb_endpoint_descriptor { - __u8 bLength; - __u8 bDescriptorType; - __u8 bEndpointAddress; - __u8 bmAttributes; - __le16 wMaxPacketSize; - __u8 bInterval; - __u8 bRefresh; - __u8 bSynchAddress; -} __attribute__((packed)); - -struct usb_ssp_isoc_ep_comp_descriptor { - __u8 bLength; - __u8 bDescriptorType; - __le16 wReseved; - __le32 dwBytesPerInterval; -}; - -struct usb_ss_ep_comp_descriptor { - __u8 bLength; - __u8 bDescriptorType; - __u8 bMaxBurst; - __u8 bmAttributes; - __le16 wBytesPerInterval; -}; - -struct usb_interface_assoc_descriptor { - __u8 bLength; - __u8 bDescriptorType; - __u8 bFirstInterface; - __u8 bInterfaceCount; - __u8 bFunctionClass; - __u8 bFunctionSubClass; - __u8 bFunctionProtocol; - __u8 iFunction; -}; - -struct ep_device; - -struct usb_host_endpoint { - struct usb_endpoint_descriptor desc; - struct usb_ss_ep_comp_descriptor ss_ep_comp; - struct usb_ssp_isoc_ep_comp_descriptor ssp_isoc_ep_comp; - char: 8; - struct list_head urb_list; - void *hcpriv; - struct ep_device *ep_dev; - unsigned char *extra; - int extralen; - int enabled; - int streams; - int: 32; -} __attribute__((packed)); - -struct usb_host_interface { - struct usb_interface_descriptor desc; - int extralen; - unsigned char *extra; - struct usb_host_endpoint *endpoint; - char *string; -}; - -enum usb_interface_condition { - USB_INTERFACE_UNBOUND = 0, - USB_INTERFACE_BINDING = 1, - USB_INTERFACE_BOUND = 2, - USB_INTERFACE_UNBINDING = 3, -}; - -struct usb_interface { - struct usb_host_interface *altsetting; - struct usb_host_interface *cur_altsetting; - unsigned int num_altsetting; - struct usb_interface_assoc_descriptor *intf_assoc; - int minor; - enum usb_interface_condition condition; - unsigned int sysfs_files_created: 1; - unsigned int ep_devs_created: 1; - unsigned int unregistering: 1; - unsigned int needs_remote_wakeup: 1; - unsigned int needs_altsetting0: 1; - unsigned int needs_binding: 1; - unsigned int resetting_device: 1; - unsigned int authorized: 1; - struct device dev; - struct device *usb_dev; - struct work_struct reset_ws; -}; - struct ms_data { long unsigned int quirks; struct hid_device *hdev; @@ -73207,6 +75192,21 @@ struct acpi_pcct_subspace { u16 min_turnaround_time; } __attribute__((packed)); +struct acpi_pcct_hw_reduced { + struct acpi_subtable_header header; + u32 platform_interrupt; + u8 flags; + u8 reserved; + u64 base_address; + u64 length; + struct acpi_generic_address doorbell_register; + u64 preserve_mask; + u64 write_mask; + u32 latency; + u32 max_access_rate; + u16 min_turnaround_time; +} __attribute__((packed)); + struct acpi_pcct_hw_reduced_type2 { struct acpi_subtable_header header; u32 platform_interrupt; @@ -73225,6 +75225,50 @@ struct acpi_pcct_hw_reduced_type2 { u64 ack_write_mask; } __attribute__((packed)); +struct acpi_pcct_ext_pcc_master { + struct acpi_subtable_header header; + u32 platform_interrupt; + u8 flags; + u8 reserved1; + u64 base_address; + u32 length; + struct acpi_generic_address doorbell_register; + u64 preserve_mask; + u64 write_mask; + u32 latency; + u32 max_access_rate; + u32 min_turnaround_time; + struct acpi_generic_address platform_ack_register; + u64 ack_preserve_mask; + u64 ack_set_mask; + u64 reserved2; + struct acpi_generic_address cmd_complete_register; + u64 cmd_complete_mask; + struct acpi_generic_address cmd_update_register; + u64 cmd_update_preserve_mask; + u64 cmd_update_set_mask; + struct acpi_generic_address error_status_register; + u64 error_status_mask; +} __attribute__((packed)); + +struct pcc_chan_reg { + void *vaddr; + struct acpi_generic_address *gas; + u64 preserve_mask; + u64 set_mask; + u64 status_mask; +}; + +struct pcc_chan_info { + struct pcc_mbox_chan chan; + struct pcc_chan_reg db; + struct pcc_chan_reg plat_irq_ack; + struct pcc_chan_reg cmd_complete; + struct pcc_chan_reg cmd_update; + struct pcc_chan_reg error; + int plat_irq; +}; + struct cper_sec_proc_arm { u32 validation_bits; u16 err_info_num; @@ -73365,6 +75409,8 @@ typedef int (*nvmem_reg_read_t)(void *, unsigned int, void *, size_t); typedef int (*nvmem_reg_write_t)(void *, unsigned int, void *, size_t); +typedef int (*nvmem_cell_post_process_t)(void *, const char *, unsigned int, void *, size_t); + enum nvmem_type { NVMEM_TYPE_UNKNOWN = 0, NVMEM_TYPE_EEPROM = 1, @@ -73396,6 +75442,7 @@ struct nvmem_config { bool no_of_node; nvmem_reg_read_t reg_read; nvmem_reg_write_t reg_write; + nvmem_cell_post_process_t cell_post_process; int size; int word_size; int stride; @@ -73430,11 +75477,12 @@ struct nvmem_device { unsigned int nkeepout; nvmem_reg_read_t reg_read; nvmem_reg_write_t reg_write; + nvmem_cell_post_process_t cell_post_process; struct gpio_desc *wp_gpio; void *priv; }; -struct nvmem_cell { +struct nvmem_cell_entry { const char *name; int offset; int bytes; @@ -73445,6 +75493,11 @@ struct nvmem_cell { struct list_head node; }; +struct nvmem_cell { + struct nvmem_cell_entry *entry; + const char *id; +}; + struct net_device_devres { struct net_device *ndev; }; @@ -73468,12 +75521,42 @@ struct scm_timestamping_internal { struct timespec64 ts[3]; }; -struct ifconf { - int ifc_len; +typedef u32 compat_caddr_t; + +struct compat_ifmap { + compat_ulong_t mem_start; + compat_ulong_t mem_end; + short unsigned int base_addr; + unsigned char irq; + unsigned char dma; + unsigned char port; +}; + +struct compat_if_settings { + unsigned int type; + unsigned int size; + compat_uptr_t ifs_ifsu; +}; + +struct compat_ifreq { union { - char *ifcu_buf; - struct ifreq *ifcu_req; - } ifc_ifcu; + char ifrn_name[16]; + } ifr_ifrn; + union { + struct sockaddr ifru_addr; + struct sockaddr ifru_dstaddr; + struct sockaddr ifru_broadaddr; + struct sockaddr ifru_netmask; + struct sockaddr ifru_hwaddr; + short int ifru_flags; + compat_int_t ifru_ivalue; + compat_int_t ifru_mtu; + struct compat_ifmap ifru_map; + char ifru_slave[16]; + char ifru_newname[16]; + compat_caddr_t ifru_data; + struct compat_if_settings ifru_settings; + } ifr_ifru; }; enum sock_shutdown_cmd { @@ -73487,27 +75570,6 @@ struct compat_mmsghdr { compat_uint_t msg_len; }; -enum { - SOF_TIMESTAMPING_TX_HARDWARE = 1, - SOF_TIMESTAMPING_TX_SOFTWARE = 2, - SOF_TIMESTAMPING_RX_HARDWARE = 4, - SOF_TIMESTAMPING_RX_SOFTWARE = 8, - SOF_TIMESTAMPING_SOFTWARE = 16, - SOF_TIMESTAMPING_SYS_HARDWARE = 32, - SOF_TIMESTAMPING_RAW_HARDWARE = 64, - SOF_TIMESTAMPING_OPT_ID = 128, - SOF_TIMESTAMPING_TX_SCHED = 256, - SOF_TIMESTAMPING_TX_ACK = 512, - SOF_TIMESTAMPING_OPT_CMSG = 1024, - SOF_TIMESTAMPING_OPT_TSONLY = 2048, - SOF_TIMESTAMPING_OPT_STATS = 4096, - SOF_TIMESTAMPING_OPT_PKTINFO = 8192, - SOF_TIMESTAMPING_OPT_TX_SWHW = 16384, - SOF_TIMESTAMPING_BIND_PHC = 32768, - SOF_TIMESTAMPING_LAST = 32768, - SOF_TIMESTAMPING_MASK = 65535, -}; - struct scm_ts_pktinfo { __u32 if_index; __u32 pkt_length; @@ -73549,6 +75611,8 @@ struct sock_exterr_skb { u8 unused: 7; }; +struct net_bridge; + struct used_address { struct __kernel_sockaddr_storage name; unsigned int name_len; @@ -73565,10 +75629,6 @@ struct ucred { __u32 gid; }; -struct prot_inuse { - int val[64]; -}; - struct offload_callbacks { struct sk_buff * (*gso_segment)(struct sk_buff *, netdev_features_t); struct sk_buff * (*gro_receive)(struct list_head *, struct sk_buff *); @@ -74166,11 +76226,11 @@ struct gnet_estimator { unsigned char ewma_log; }; -struct net_rate_estimator { - struct gnet_stats_basic_packed *bstats; +struct net_rate_estimator___2 { + struct gnet_stats_basic_sync *bstats; spinlock_t *stats_lock; - seqcount_t *running; - struct gnet_stats_basic_cpu *cpu_bstats; + bool running; + struct gnet_stats_basic_sync *cpu_bstats; u8 ewma_log; u8 intvl_log; seqcount_t seq; @@ -74297,6 +76357,13 @@ struct flow_dissector_key_meta { u16 ingress_iftype; }; +struct flow_dissector_key_ct { + u16 ct_state; + u16 ct_zone; + u32 ct_mark; + u32 ct_labels[4]; +}; + struct flow_dissector_key_hash { u32 hash; }; @@ -74305,6 +76372,21 @@ struct flow_keys_digest { u8 data[16]; }; +enum { + TCA_FLOWER_KEY_CT_FLAGS_NEW = 1, + TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED = 2, + TCA_FLOWER_KEY_CT_FLAGS_RELATED = 4, + TCA_FLOWER_KEY_CT_FLAGS_TRACKED = 8, + TCA_FLOWER_KEY_CT_FLAGS_INVALID = 16, + TCA_FLOWER_KEY_CT_FLAGS_REPLY = 32, + __TCA_FLOWER_KEY_CT_FLAGS_MAX = 33, +}; + +union tcp_word_hdr { + struct tcphdr hdr; + __be32 words[5]; +}; + enum bpf_ret_code { BPF_OK = 0, BPF_DROP = 2, @@ -74318,11 +76400,6 @@ enum { BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP = 4, }; -union tcp_word_hdr { - struct tcphdr hdr; - __be32 words[5]; -}; - struct gre_base_hdr { __be16 flags; __be16 protocol; @@ -74437,6 +76514,44 @@ struct batadv_unicast_packet { __u8 dest[6]; }; +struct nf_conntrack_l4proto { + u_int8_t l4proto; + bool allow_clash; + u16 nlattr_size; + bool (*can_early_drop)(const struct nf_conn *); + int (*to_nlattr)(struct sk_buff *, struct nlattr *, struct nf_conn *, bool); + int (*from_nlattr)(struct nlattr **, struct nf_conn *); + int (*tuple_to_nlattr)(struct sk_buff *, const struct nf_conntrack_tuple *); + unsigned int (*nlattr_tuple_size)(); + int (*nlattr_to_tuple)(struct nlattr **, struct nf_conntrack_tuple *, u_int32_t); + const struct nla_policy *nla_policy; + struct { + int (*nlattr_to_obj)(struct nlattr **, struct net *, void *); + int (*obj_to_nlattr)(struct sk_buff *, const void *); + u16 obj_size; + u16 nlattr_max; + const struct nla_policy *nla_policy; + } ctnl_timeout; + void (*print_conntrack)(struct seq_file *, struct nf_conn *); +}; + +struct nf_ct_ext { + u8 offset[3]; + u8 len; + char data[0]; +}; + +enum nf_ct_ext_id { + NF_CT_EXT_HELPER = 0, + NF_CT_EXT_SEQADJ = 1, + NF_CT_EXT_ACCT = 2, + NF_CT_EXT_NUM = 3, +}; + +struct nf_conn_labels { + long unsigned int bits[2]; +}; + struct _flow_keys_digest_data { __be16 n_proto; u8 ip_proto; @@ -74446,15 +76561,10 @@ struct _flow_keys_digest_data { __be32 dst; }; -struct tc_skb_ext { - __u32 chain; - __u16 mru; - bool post_ct; -}; - enum nf_dev_hooks { NF_NETDEV_INGRESS = 0, - NF_NETDEV_NUMHOOKS = 1, + NF_NETDEV_EGRESS = 1, + NF_NETDEV_NUMHOOKS = 2, }; enum { @@ -74467,11 +76577,6 @@ enum { IF_OPER_UP = 6, }; -struct netdev_boot_setup { - char name[16]; - struct ifmap map; -}; - enum { NAPIF_STATE_SCHED = 1, NAPIF_STATE_MISSED = 2, @@ -74485,16 +76590,6 @@ enum { NAPIF_STATE_SCHED_THREADED = 512, }; -enum gro_result { - GRO_MERGED = 0, - GRO_MERGED_FREE = 1, - GRO_HELD = 2, - GRO_NORMAL = 3, - GRO_CONSUMED = 4, -}; - -typedef enum gro_result gro_result_t; - struct net_device_path_stack { int num_paths; struct net_device_path path[5]; @@ -74518,10 +76613,15 @@ struct netdev_net_notifier { struct notifier_block *nb; }; -struct packet_offload { +struct packet_type { __be16 type; - u16 priority; - struct offload_callbacks callbacks; + bool ignore_outgoing; + struct net_device *dev; + netdevice_tracker dev_tracker; + int (*func)(struct sk_buff *, struct net_device *, struct packet_type *, struct net_device *); + void (*list_func)(struct list_head *, struct packet_type *, struct net_device *); + bool (*id_match)(struct packet_type *, struct sock *); + void *af_packet_priv; struct list_head list; }; @@ -74574,12 +76674,26 @@ enum qdisc_state_t { __QDISC_STATE_DRAINING = 3, }; +enum qdisc_state2_t { + __QDISC_STATE2_RUNNING = 0, +}; + +struct tc_skb_cb { + struct qdisc_skb_cb qdisc_cb; + u16 mru; + u8 post_ct: 1; + u8 post_ct_snat: 1; + u8 post_ct_dnat: 1; + u16 zone; +}; + struct dev_kfree_skb_cb { enum skb_free_reason reason; }; struct netdev_adjacent { struct net_device *dev; + netdevice_tracker dev_tracker; bool master; bool ignore; u16 ref_nr; @@ -75054,7 +77168,7 @@ struct rtnl_af_ops { int family; int (*fill_link_af)(struct sk_buff *, const struct net_device *, u32); size_t (*get_link_af_size)(const struct net_device *, u32); - int (*validate_link_af)(const struct net_device *, const struct nlattr *); + int (*validate_link_af)(const struct net_device *, const struct nlattr *, struct netlink_ext_ack *); int (*set_link_af)(struct net_device *, const struct nlattr *, struct netlink_ext_ack *); int (*fill_stats_af)(struct sk_buff *, const struct net_device *); size_t (*get_stats_af_size)(const struct net_device *); @@ -75083,6 +77197,70 @@ struct seg6_pernet_data { struct in6_addr *tun_src; }; +enum rt_class_t { + RT_TABLE_UNSPEC = 0, + RT_TABLE_COMPAT = 252, + RT_TABLE_DEFAULT = 253, + RT_TABLE_MAIN = 254, + RT_TABLE_LOCAL = 255, + RT_TABLE_MAX = 4294967295, +}; + +struct inet_timewait_sock { + struct sock_common __tw_common; + __u32 tw_mark; + volatile unsigned char tw_substate; + unsigned char tw_rcv_wscale; + __be16 tw_sport; + unsigned int tw_kill: 1; + unsigned int tw_transparent: 1; + unsigned int tw_flowlabel: 20; + unsigned int tw_pad: 2; + unsigned int tw_tos: 8; + u32 tw_txhash; + u32 tw_priority; + struct timer_list tw_timer; + struct inet_bind_bucket *tw_tb; +}; + +struct tcp_timewait_sock { + struct inet_timewait_sock tw_sk; + u32 tw_rcv_wnd; + u32 tw_ts_offset; + u32 tw_ts_recent; + u32 tw_last_oow_ack_time; + int tw_ts_recent_stamp; + u32 tw_tx_delay; + struct tcp_md5sig_key *tw_md5_key; +}; + +struct udp6_sock { + struct udp_sock udp; + struct ipv6_pinfo inet6; + long: 64; + long: 64; + long: 64; + long: 64; +}; + +struct tcp6_sock { + struct tcp_sock tcp; + struct ipv6_pinfo inet6; +}; + +struct fib_result { + __be32 prefix; + unsigned char prefixlen; + unsigned char nh_sel; + unsigned char type; + unsigned char scope; + u32 tclassid; + struct fib_nh_common *nhc; + struct fib_info *fi; + struct fib_table *table; + struct hlist_head *fa_head; +}; + enum { BPF_F_RECOMPUTE_CSUM = 1, BPF_F_INVALIDATE_HASH = 2, @@ -75336,15 +77514,6 @@ enum bpf_check_mtu_ret { BPF_MTU_CHK_RET_SEGS_TOOBIG = 2, }; -enum rt_class_t { - RT_TABLE_UNSPEC = 0, - RT_TABLE_COMPAT = 252, - RT_TABLE_DEFAULT = 253, - RT_TABLE_MAIN = 254, - RT_TABLE_LOCAL = 255, - RT_TABLE_MAX = 4294967295, -}; - struct compat_sock_fprog { u16 len; compat_uptr_t filter; @@ -75352,61 +77521,6 @@ struct compat_sock_fprog { typedef int (*bpf_aux_classic_check_t)(struct sock_filter *, unsigned int); -struct inet_timewait_sock { - struct sock_common __tw_common; - __u32 tw_mark; - volatile unsigned char tw_substate; - unsigned char tw_rcv_wscale; - __be16 tw_sport; - unsigned int tw_kill: 1; - unsigned int tw_transparent: 1; - unsigned int tw_flowlabel: 20; - unsigned int tw_pad: 2; - unsigned int tw_tos: 8; - u32 tw_txhash; - u32 tw_priority; - struct timer_list tw_timer; - struct inet_bind_bucket *tw_tb; -}; - -struct tcp_timewait_sock { - struct inet_timewait_sock tw_sk; - u32 tw_rcv_wnd; - u32 tw_ts_offset; - u32 tw_ts_recent; - u32 tw_last_oow_ack_time; - int tw_ts_recent_stamp; - u32 tw_tx_delay; - struct tcp_md5sig_key *tw_md5_key; -}; - -struct udp6_sock { - struct udp_sock udp; - struct ipv6_pinfo inet6; - long: 64; - long: 64; - long: 64; - long: 64; -}; - -struct tcp6_sock { - struct tcp_sock tcp; - struct ipv6_pinfo inet6; -}; - -struct fib_result { - __be32 prefix; - unsigned char prefixlen; - unsigned char nh_sel; - unsigned char type; - unsigned char scope; - u32 tclassid; - struct fib_nh_common *nhc; - struct fib_info *fi; - struct fib_table *table; - struct hlist_head *fa_head; -}; - struct tcp_skb_cb { __u32 seq; __u32 end_seq; @@ -75427,9 +77541,9 @@ struct tcp_skb_cb { __u32 ack_seq; union { struct { - __u32 in_flight: 30; __u32 is_app_limited: 1; - __u32 unused: 1; + __u32 delivered_ce: 20; + __u32 unused: 11; __u32 delivered; u64 first_tx_mstamp; u64 delivered_mstamp; @@ -75446,6 +77560,17 @@ struct strp_msg { int offset; }; +struct _strp_msg { + struct strp_msg strp; + int accum_len; +}; + +struct sk_skb_cb { + unsigned char data[20]; + struct _strp_msg strp; + u64 temp_reg; +}; + struct xdp_umem { void *addrs; u64 size; @@ -75565,6 +77690,22 @@ struct tls12_crypto_info_chacha20_poly1305 { unsigned char rec_seq[8]; }; +struct tls12_crypto_info_sm4_gcm { + struct tls_crypto_info info; + unsigned char iv[8]; + unsigned char key[16]; + unsigned char salt[4]; + unsigned char rec_seq[8]; +}; + +struct tls12_crypto_info_sm4_ccm { + struct tls_crypto_info info; + unsigned char iv[8]; + unsigned char key[16]; + unsigned char salt[4]; + unsigned char rec_seq[8]; +}; + struct tls_sw_context_rx { struct crypto_aead *aead_recv; struct crypto_wait async_wait; @@ -75591,6 +77732,8 @@ union tls_crypto_context { struct tls12_crypto_info_aes_gcm_128 aes_gcm_128; struct tls12_crypto_info_aes_gcm_256 aes_gcm_256; struct tls12_crypto_info_chacha20_poly1305 chacha20_poly1305; + struct tls12_crypto_info_sm4_gcm sm4_gcm; + struct tls12_crypto_info_sm4_ccm sm4_ccm; }; }; @@ -75787,8 +77930,16 @@ typedef u64 (*btf_bpf_get_netns_cookie_sock)(struct sock *); typedef u64 (*btf_bpf_get_netns_cookie_sock_addr)(struct bpf_sock_addr_kern *); +typedef u64 (*btf_bpf_get_netns_cookie_sock_ops)(struct bpf_sock_ops_kern *); + +typedef u64 (*btf_bpf_get_netns_cookie_sk_msg)(struct sk_msg *); + typedef u64 (*btf_bpf_get_socket_uid)(struct sk_buff *); +typedef u64 (*btf_bpf_sk_setsockopt)(struct sock *, int, int, char *, int); + +typedef u64 (*btf_bpf_sk_getsockopt)(struct sock *, int, int, char *, int); + typedef u64 (*btf_bpf_sock_addr_setsockopt)(struct bpf_sock_addr_kern *, int, int, char *, int); typedef u64 (*btf_bpf_sock_addr_getsockopt)(struct bpf_sock_addr_kern *, int, int, char *, int); @@ -75877,15 +78028,11 @@ typedef u64 (*btf_bpf_skc_to_tcp_request_sock)(struct sock *); typedef u64 (*btf_bpf_skc_to_udp6_sock)(struct sock *); +typedef u64 (*btf_bpf_skc_to_unix_sock)(struct sock *); + typedef u64 (*btf_bpf_sock_from_file)(struct file *); -enum { - INET_DIAG_REQ_NONE = 0, - INET_DIAG_REQ_BYTECODE = 1, - INET_DIAG_REQ_SK_BPF_STORAGES = 2, - INET_DIAG_REQ_PROTOCOL = 3, - __INET_DIAG_REQ_MAX = 4, -}; +struct unix_sock___2; struct sock_diag_req { __u8 sdiag_family; @@ -75904,12 +78051,12 @@ struct broadcast_sk { struct work_struct work; }; -typedef int gifconf_func_t(struct net_device *, char *, int, int); - -struct hwtstamp_config { - int flags; - int tx_type; - int rx_filter; +struct ifconf { + int ifc_len; + union { + char *ifcu_buf; + struct ifreq *ifcu_req; + } ifc_ifcu; }; enum hwtstamp_tx_types { @@ -75940,6 +78087,11 @@ enum hwtstamp_rx_filters { __HWTSTAMP_FILTER_CNT = 16, }; +struct compat_ifconf { + compat_int_t ifc_len; + compat_caddr_t ifcbuf; +}; + struct tso_t { int next_frag_idx; int size; @@ -75992,6 +78144,7 @@ struct xsk_buff_pool { u32 free_heads_cnt; u32 headroom; u32 chunk_size; + u32 chunk_shift; u32 frame_len; u8 cached_need_wakeup; bool uses_need_wakeup; @@ -76012,18 +78165,10 @@ struct xdp_buff_xsk { dma_addr_t dma; dma_addr_t frame_dma; struct xsk_buff_pool *pool; - bool unaligned; u64 orig_addr; struct list_head free_list_node; }; -struct flow_dissector_key_ct { - u16 ct_state; - u16 ct_zone; - u32 ct_mark; - u32 ct_labels[4]; -}; - struct flow_match { struct flow_dissector *dissector; void *mask; @@ -76164,6 +78309,7 @@ struct action_gate_entry; struct flow_action_entry { enum flow_action_id id; + u32 hw_index; enum flow_action_hw_stats hw_stats; action_destr destructor; void *destructor_priv; @@ -76198,7 +78344,6 @@ struct flow_action_entry { bool truncate; } sample; struct { - u32 index; u32 burst; u64 rate_bytes_ps; u64 burst_pkt; @@ -76233,7 +78378,6 @@ struct flow_action_entry { u8 ttl; } mpls_mangle; struct { - u32 index; s32 prio; u64 basetime; u64 cycletime; @@ -76258,6 +78402,15 @@ struct flow_rule { struct flow_action action; }; +struct flow_stats { + u64 pkts; + u64 bytes; + u64 drops; + u64 lastused; + enum flow_action_hw_stats used_hw_stats; + bool used_hw_stats_valid; +}; + enum flow_block_command { FLOW_BLOCK_BIND = 0, FLOW_BLOCK_UNBIND = 1, @@ -76282,6 +78435,7 @@ struct flow_block_offload { struct list_head *driver_block_list; struct netlink_ext_ack *extack; struct Qdisc *sch; + struct list_head *cb_list_head; }; struct flow_block_cb; @@ -76307,6 +78461,21 @@ struct flow_block_cb { unsigned int refcnt; }; +enum offload_act_command { + FLOW_ACT_REPLACE = 0, + FLOW_ACT_DESTROY = 1, + FLOW_ACT_STATS = 2, +}; + +struct flow_offload_action { + struct netlink_ext_ack *extack; + enum offload_act_command command; + enum flow_action_id id; + u32 index; + struct flow_stats stats; + struct flow_action action; +}; + typedef int flow_indr_block_bind_cb_t(struct net_device *, struct Qdisc *, void *, enum tc_setup_type, void *, void *, void (*)(struct flow_block_cb *)); struct flow_indr_dev { @@ -76314,7 +78483,44 @@ struct flow_indr_dev { flow_indr_block_bind_cb_t *cb; void *cb_priv; refcount_t refcnt; - struct callback_head rcu; +}; + +struct flow_indir_dev_info { + void *data; + struct net_device *dev; + struct Qdisc *sch; + enum tc_setup_type type; + void (*cleanup)(struct flow_block_cb *); + struct list_head list; + enum flow_block_command command; + enum flow_block_binder_type binder_type; + struct list_head *cb_list; +}; + +struct tc_skb_ext { + __u32 chain; + __u16 mru; + __u16 zone; + u8 post_ct: 1; + u8 post_ct_snat: 1; + u8 post_ct_dnat: 1; +}; + +enum gro_result { + GRO_MERGED = 0, + GRO_MERGED_FREE = 1, + GRO_HELD = 2, + GRO_NORMAL = 3, + GRO_CONSUMED = 4, +}; + +typedef enum gro_result gro_result_t; + +struct packet_offload { + __be16 type; + u16 priority; + struct offload_callbacks callbacks; + struct list_head list; }; struct netdev_queue_attribute { @@ -76380,6 +78586,7 @@ struct trace_event_raw_kfree_skb { void *skbaddr; void *location; short unsigned int protocol; + enum skb_drop_reason reason; char __data[0]; }; @@ -76402,7 +78609,7 @@ struct trace_event_data_offsets_consume_skb {}; struct trace_event_data_offsets_skb_copy_datagram_iovec {}; -typedef void (*btf_trace_kfree_skb)(void *, struct sk_buff *, void *); +typedef void (*btf_trace_kfree_skb)(void *, struct sk_buff *, void *, enum skb_drop_reason); typedef void (*btf_trace_consume_skb)(void *, struct sk_buff *); @@ -76782,6 +78989,17 @@ struct trace_event_raw_qdisc_dequeue { char __data[0]; }; +struct trace_event_raw_qdisc_enqueue { + struct trace_entry ent; + struct Qdisc *qdisc; + const struct netdev_queue *txq; + void *skbaddr; + int ifindex; + u32 handle; + u32 parent; + char __data[0]; +}; + struct trace_event_raw_qdisc_reset { struct trace_entry ent; u32 __data_loc_dev; @@ -76810,6 +79028,8 @@ struct trace_event_raw_qdisc_create { struct trace_event_data_offsets_qdisc_dequeue {}; +struct trace_event_data_offsets_qdisc_enqueue {}; + struct trace_event_data_offsets_qdisc_reset { u32 dev; u32 kind; @@ -76827,6 +79047,8 @@ struct trace_event_data_offsets_qdisc_create { typedef void (*btf_trace_qdisc_dequeue)(void *, struct Qdisc *, const struct netdev_queue *, int, struct sk_buff *); +typedef void (*btf_trace_qdisc_enqueue)(void *, struct Qdisc *, const struct netdev_queue *, struct sk_buff *); + typedef void (*btf_trace_qdisc_reset)(void *, struct Qdisc *); typedef void (*btf_trace_qdisc_destroy)(void *, struct Qdisc *); @@ -77073,11 +79295,14 @@ struct rdma_port_counter { struct mutex lock; }; +struct rdma_stat_desc; + struct rdma_hw_stats { struct mutex lock; long unsigned int timestamp; long unsigned int lifespan; - const char * const *names; + const struct rdma_stat_desc *descs; + long unsigned int *is_disabled; int num_counters; u64 value[0]; }; @@ -77303,7 +79528,7 @@ struct ib_device_ops { int (*modify_srq)(struct ib_srq *, struct ib_srq_attr *, enum ib_srq_attr_mask, struct ib_udata *); int (*query_srq)(struct ib_srq *, struct ib_srq_attr *); int (*destroy_srq)(struct ib_srq *, struct ib_udata *); - struct ib_qp * (*create_qp)(struct ib_pd *, struct ib_qp_init_attr *, struct ib_udata *); + int (*create_qp)(struct ib_qp *, struct ib_qp_init_attr *, struct ib_udata *); int (*modify_qp)(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *); int (*query_qp)(struct ib_qp *, struct ib_qp_attr *, int, struct ib_qp_init_attr *); int (*destroy_qp)(struct ib_qp *, struct ib_udata *); @@ -77352,6 +79577,7 @@ struct ib_device_ops { struct rdma_hw_stats * (*alloc_hw_device_stats)(struct ib_device *); struct rdma_hw_stats * (*alloc_hw_port_stats)(struct ib_device *, u32); int (*get_hw_stats)(struct ib_device *, struct rdma_hw_stats *, u32, int); + int (*modify_hw_stat)(struct ib_device *, u32, unsigned int, bool); int (*fill_res_mr_entry)(struct sk_buff *, struct ib_mr *); int (*fill_res_mr_entry_raw)(struct sk_buff *, struct ib_mr *); int (*fill_res_cq_entry)(struct sk_buff *, struct ib_cq *); @@ -77376,11 +79602,13 @@ struct ib_device_ops { int (*counter_update_stats)(struct rdma_counter *); int (*fill_stat_mr_entry)(struct sk_buff *, struct ib_mr *); int (*query_ucontext)(struct ib_ucontext *, struct uverbs_attr_bundle *); + int (*get_numa_node)(struct ib_device *); size_t size_ib_ah; size_t size_ib_counters; size_t size_ib_cq; size_t size_ib_mw; size_t size_ib_pd; + size_t size_ib_qp; size_t size_ib_rwq_ind_table; size_t size_ib_srq; size_t size_ib_ucontext; @@ -77662,6 +79890,12 @@ enum ib_port_state { IB_PORT_ACTIVE_DEFER = 5, }; +struct rdma_stat_desc { + const char *name; + unsigned int flags; + const void *priv; +}; + struct ib_port_attr { u64 subnet_prefix; enum ib_port_state state; @@ -78941,8 +81175,9 @@ enum devlink_attr { DEVLINK_ATTR_RATE_TX_MAX = 167, DEVLINK_ATTR_RATE_NODE_NAME = 168, DEVLINK_ATTR_RATE_PARENT_NODE_NAME = 169, - __DEVLINK_ATTR_MAX = 170, - DEVLINK_ATTR_MAX = 169, + DEVLINK_ATTR_REGION_MAX_SNAPSHOTS = 170, + __DEVLINK_ATTR_MAX = 171, + DEVLINK_ATTR_MAX = 170, }; enum devlink_dpipe_match_type { @@ -78980,6 +81215,41 @@ enum devlink_port_function_attr { DEVLINK_PORT_FUNCTION_ATTR_MAX = 3, }; +struct devlink_dev_stats { + u32 reload_stats[6]; + u32 remote_reload_stats[6]; +}; + +struct devlink_dpipe_headers; + +struct devlink { + u32 index; + struct list_head port_list; + struct list_head rate_list; + struct list_head sb_list; + struct list_head dpipe_table_list; + struct list_head resource_list; + struct list_head param_list; + struct list_head region_list; + struct list_head reporter_list; + struct mutex reporters_lock; + struct devlink_dpipe_headers *dpipe_headers; + struct list_head trap_list; + struct list_head trap_group_list; + struct list_head trap_policer_list; + const struct devlink_ops *ops; + u64 features; + struct xarray snapshot_ids; + struct devlink_dev_stats stats; + struct device *dev; + possible_net_t _net; + struct mutex lock; + u8 reload_failed: 1; + refcount_t refcount; + struct completion comp; + char priv[0]; +}; + struct devlink_dpipe_match { enum devlink_dpipe_match_type type; unsigned int header_index; @@ -79047,22 +81317,13 @@ struct devlink_dpipe_table_ops { u64 (*size_get)(void *); }; -typedef u64 devlink_resource_occ_get_t(void *); - -struct devlink_resource { - const char *name; - u64 id; - u64 size; - u64 size_new; - bool size_valid; - struct devlink_resource *parent; - struct devlink_resource_size_params size_params; - struct list_head list; - struct list_head resource_list; - devlink_resource_occ_get_t *occ_get; - void *occ_get_priv; +struct devlink_dpipe_headers { + struct devlink_dpipe_header **headers; + unsigned int headers_count; }; +typedef u64 devlink_resource_occ_get_t(void *); + struct devlink_flash_notify { const char *status_msg; const char *component; @@ -79076,7 +81337,6 @@ struct devlink_param_item { const struct devlink_param *param; union devlink_param_value driverinit_value; bool driverinit_value_valid; - bool published; }; struct devlink_port_region_ops { @@ -79120,6 +81380,7 @@ struct devlink_trap_metadata { const char *trap_name; const char *trap_group_name; struct net_device *input_dev; + netdevice_tracker dev_tracker; const struct flow_action_cookie *fa_cookie; enum devlink_trap_type trap_type; }; @@ -79249,6 +81510,20 @@ typedef void (*btf_trace_devlink_health_reporter_state_update)(void *, const str typedef void (*btf_trace_devlink_trap_report)(void *, const struct devlink *, struct sk_buff *, const struct devlink_trap_metadata *); +struct devlink_resource { + const char *name; + u64 id; + u64 size; + u64 size_new; + bool size_valid; + struct devlink_resource *parent; + struct devlink_resource_size_params size_params; + struct list_head list; + struct list_head resource_list; + devlink_resource_occ_get_t *occ_get; + void *occ_get_priv; +}; + struct devlink_sb { struct list_head list; unsigned int index; @@ -79540,7 +81815,7 @@ struct pfifo_fast_priv { }; struct tc_qopt_offload_stats { - struct gnet_stats_basic_packed *bstats; + struct gnet_stats_basic_sync *bstats; struct gnet_stats_queue *qstats; }; @@ -79670,7 +81945,8 @@ enum { TCA_ACT_FLAGS = 7, TCA_ACT_HW_STATS = 8, TCA_ACT_USED_HW_STATS = 9, - __TCA_ACT_MAX = 10, + TCA_ACT_IN_HW_COUNT = 10, + __TCA_ACT_MAX = 11, }; struct psample_group { @@ -79748,157 +82024,179 @@ struct tcf_pedit { unsigned char tcfp_flags; struct tc_pedit_key *tcfp_keys; struct tcf_pedit_key_ex *tcfp_keys_ex; -}; - -struct tcf_mirred { - struct tc_action common; - int tcfm_eaction; - bool tcfm_mac_header_xmit; - struct net_device *tcfm_dev; - struct list_head tcfm_list; -}; - -struct tcf_vlan_params { - int tcfv_action; - unsigned char tcfv_push_dst[6]; - unsigned char tcfv_push_src[6]; - u16 tcfv_push_vid; - __be16 tcfv_push_proto; - u8 tcfv_push_prio; - bool tcfv_push_prio_exists; - struct callback_head rcu; -}; - -struct tcf_vlan { - struct tc_action common; - struct tcf_vlan_params *vlan_p; -}; - -struct tcf_tunnel_key_params { - struct callback_head rcu; - int tcft_action; - struct metadata_dst *tcft_enc_metadata; -}; - -struct tcf_tunnel_key { - struct tc_action common; - struct tcf_tunnel_key_params *params; -}; - -struct tcf_csum_params { - u32 update_flags; - struct callback_head rcu; -}; - -struct tcf_csum { - struct tc_action common; - struct tcf_csum_params *params; -}; - -struct tcf_gact { - struct tc_action common; -}; - -struct tcf_police_params { - int tcfp_result; - u32 tcfp_ewma_rate; - s64 tcfp_burst; - u32 tcfp_mtu; - s64 tcfp_mtu_ptoks; - s64 tcfp_pkt_burst; - struct psched_ratecfg rate; - bool rate_present; - struct psched_ratecfg peak; - bool peak_present; - struct psched_pktrate ppsrate; - bool pps_present; - struct callback_head rcu; -}; - -struct tcf_police { - struct tc_action common; - struct tcf_police_params *params; - spinlock_t tcfp_lock; - s64 tcfp_toks; - s64 tcfp_ptoks; - s64 tcfp_pkttoks; - s64 tcfp_t_c; - long: 64; - long: 64; - long: 64; long: 64; }; -struct tcf_sample { - struct tc_action common; - u32 rate; - bool truncate; - u32 trunc_size; - struct psample_group *psample_group; - u32 psample_group_num; - struct list_head tcfm_list; +struct nf_conntrack_tuple_mask { + struct { + union nf_inet_addr u3; + union nf_conntrack_man_proto u; + } src; }; -struct tcf_skbedit_params { - u32 flags; - u32 priority; - u32 mark; - u32 mask; - u16 queue_mapping; - u16 ptype; +struct nf_conntrack_l4proto___2; + +struct nf_conntrack_helper; + +struct nf_conntrack_expect { + struct hlist_node lnode; + struct hlist_node hnode; + struct nf_conntrack_tuple tuple; + struct nf_conntrack_tuple_mask mask; + void (*expectfn)(struct nf_conn *, struct nf_conntrack_expect *); + struct nf_conntrack_helper *helper; + struct nf_conn *master; + struct timer_list timeout; + refcount_t use; + unsigned int flags; + unsigned int class; struct callback_head rcu; }; -struct tcf_skbedit { - struct tc_action common; - struct tcf_skbedit_params *params; +struct PptpControlHeader { + __be16 messageType; + __u16 reserved; }; -struct tcf_mpls_params { - int tcfm_action; - u32 tcfm_label; - u8 tcfm_tc; - u8 tcfm_ttl; - u8 tcfm_bos; - __be16 tcfm_proto; - struct callback_head rcu; +struct PptpStartSessionRequest { + __be16 protocolVersion; + __u16 reserved1; + __be32 framingCapability; + __be32 bearerCapability; + __be16 maxChannels; + __be16 firmwareRevision; + __u8 hostName[64]; + __u8 vendorString[64]; }; -struct tcf_mpls { - struct tc_action common; - struct tcf_mpls_params *mpls_p; +struct PptpStartSessionReply { + __be16 protocolVersion; + __u8 resultCode; + __u8 generalErrorCode; + __be32 framingCapability; + __be32 bearerCapability; + __be16 maxChannels; + __be16 firmwareRevision; + __u8 hostName[64]; + __u8 vendorString[64]; }; -struct tcfg_gate_entry { - int index; - u8 gate_state; - u32 interval; - s32 ipv; - s32 maxoctets; - struct list_head list; +struct PptpStopSessionRequest { + __u8 reason; + __u8 reserved1; + __u16 reserved2; }; -struct tcf_gate_params { - s32 tcfg_priority; - u64 tcfg_basetime; - u64 tcfg_cycletime; - u64 tcfg_cycletime_ext; - u32 tcfg_flags; - s32 tcfg_clockid; - size_t num_entries; - struct list_head entries; +struct PptpStopSessionReply { + __u8 resultCode; + __u8 generalErrorCode; + __u16 reserved1; }; -struct tcf_gate { - struct tc_action common; - struct tcf_gate_params param; - u8 current_gate_status; - ktime_t current_close_time; - u32 current_entry_octets; - s32 current_max_octets; - struct tcfg_gate_entry *next_entry; - struct hrtimer hitimer; - enum tk_offsets tk_offset; +struct PptpOutCallRequest { + __be16 callID; + __be16 callSerialNumber; + __be32 minBPS; + __be32 maxBPS; + __be32 bearerType; + __be32 framingType; + __be16 packetWindow; + __be16 packetProcDelay; + __be16 phoneNumberLength; + __u16 reserved1; + __u8 phoneNumber[64]; + __u8 subAddress[64]; +}; + +struct PptpOutCallReply { + __be16 callID; + __be16 peersCallID; + __u8 resultCode; + __u8 generalErrorCode; + __be16 causeCode; + __be32 connectSpeed; + __be16 packetWindow; + __be16 packetProcDelay; + __be32 physChannelID; +}; + +struct PptpInCallRequest { + __be16 callID; + __be16 callSerialNumber; + __be32 callBearerType; + __be32 physChannelID; + __be16 dialedNumberLength; + __be16 dialingNumberLength; + __u8 dialedNumber[64]; + __u8 dialingNumber[64]; + __u8 subAddress[64]; +}; + +struct PptpInCallReply { + __be16 callID; + __be16 peersCallID; + __u8 resultCode; + __u8 generalErrorCode; + __be16 packetWindow; + __be16 packetProcDelay; + __u16 reserved; +}; + +struct PptpInCallConnected { + __be16 peersCallID; + __u16 reserved; + __be32 connectSpeed; + __be16 packetWindow; + __be16 packetProcDelay; + __be32 callFramingType; +}; + +struct PptpClearCallRequest { + __be16 callID; + __u16 reserved; +}; + +struct PptpCallDisconnectNotify { + __be16 callID; + __u8 resultCode; + __u8 generalErrorCode; + __be16 causeCode; + __u16 reserved; + __u8 callStatistics[128]; +}; + +struct PptpWanErrorNotify { + __be16 peersCallID; + __u16 reserved; + __be32 crcErrors; + __be32 framingErrors; + __be32 hardwareOverRuns; + __be32 bufferOverRuns; + __be32 timeoutErrors; + __be32 alignmentErrors; +}; + +struct PptpSetLinkInfo { + __be16 peersCallID; + __u16 reserved; + __be32 sendAccm; + __be32 recvAccm; +}; + +union pptp_ctrl_union { + struct PptpStartSessionRequest sreq; + struct PptpStartSessionReply srep; + struct PptpStopSessionRequest streq; + struct PptpStopSessionReply strep; + struct PptpOutCallRequest ocreq; + struct PptpOutCallReply ocack; + struct PptpInCallRequest icreq; + struct PptpInCallReply icack; + struct PptpInCallConnected iccon; + struct PptpClearCallRequest clrreq; + struct PptpCallDisconnectNotify disc; + struct PptpWanErrorNotify wanerr; + struct PptpSetLinkInfo setlink; }; struct tcf_filter_chain_list_item { @@ -79953,6 +82251,11 @@ struct tc_action_net { const struct tc_action_ops *ops; }; +struct tc_act_pernet_id { + struct list_head list; + unsigned int id; +}; + struct tc_act_bpf { __u32 index; __u32 capab; @@ -80039,7 +82342,9 @@ enum { TCA_FQ_CODEL_CE_THRESHOLD = 7, TCA_FQ_CODEL_DROP_BATCH_SIZE = 8, TCA_FQ_CODEL_MEMORY_LIMIT = 9, - __TCA_FQ_CODEL_MAX = 10, + TCA_FQ_CODEL_CE_THRESHOLD_SELECTOR = 10, + TCA_FQ_CODEL_CE_THRESHOLD_MASK = 11, + __TCA_FQ_CODEL_MAX = 12, }; enum { @@ -80086,6 +82391,8 @@ struct codel_params { codel_time_t interval; u32 mtu; bool ecn; + u8 ce_threshold_selector; + u8 ce_threshold_mask; }; struct codel_vars { @@ -80380,8 +82687,6 @@ struct netlink_broadcast_data { gfp_t allocation; struct sk_buff *skb; struct sk_buff *skb2; - int (*tx_filter)(struct sock *, struct sk_buff *, void *); - void *tx_data; }; struct netlink_set_err_data { @@ -80549,12 +82854,63 @@ struct bpf_fentry_test_t { struct bpf_fentry_test_t *a; }; +struct prog_test_ref_kfunc { + int a; + int b; + struct prog_test_ref_kfunc *next; +}; + +struct prog_test_pass1 { + int x0; + struct { + int x1; + struct { + int x2; + struct { + int x3; + }; + }; + }; +}; + +struct prog_test_pass2 { + int len; + short int arr1[4]; + struct { + char arr2[4]; + long unsigned int arr3[8]; + } x; +}; + +struct prog_test_fail1 { + void *p; + int x; +}; + +struct prog_test_fail2 { + int x8; + struct prog_test_pass1 x; +}; + +struct prog_test_fail3 { + int len; + char arr1[2]; + char arr2[0]; +}; + struct bpf_raw_tp_test_run_info { struct bpf_prog *prog; void *ctx; u32 retval; }; +typedef int (*dummy_ops_test_ret_fn)(struct bpf_dummy_ops_state *, ...); + +struct bpf_dummy_ops_test_args { + u64 args[12]; + struct bpf_dummy_ops_state state; +}; + struct ethtool_cmd { __u32 cmd; __u32 supported; @@ -80585,7 +82941,8 @@ enum tunable_id { ETHTOOL_RX_COPYBREAK = 1, ETHTOOL_TX_COPYBREAK = 2, ETHTOOL_PFC_PREVENTION_TOUT = 3, - __ETHTOOL_TUNABLE_COUNT = 4, + ETHTOOL_TX_COPYBREAK_BUF_SIZE = 4, + __ETHTOOL_TUNABLE_COUNT = 5, }; enum tunable_type_id { @@ -80746,8 +83103,10 @@ enum { ETHTOOL_MSG_MODULE_EEPROM_GET_REPLY = 32, ETHTOOL_MSG_STATS_GET_REPLY = 33, ETHTOOL_MSG_PHC_VCLOCKS_GET_REPLY = 34, - __ETHTOOL_MSG_KERNEL_CNT = 35, - ETHTOOL_MSG_KERNEL_MAX = 34, + ETHTOOL_MSG_MODULE_GET_REPLY = 35, + ETHTOOL_MSG_MODULE_NTF = 36, + __ETHTOOL_MSG_KERNEL_CNT = 37, + ETHTOOL_MSG_KERNEL_MAX = 36, }; enum { @@ -80766,6 +83125,14 @@ struct link_mode_info { u8 duplex; }; +struct ethtool_devlink_compat { + struct devlink *devlink; + union { + struct ethtool_flash efl; + struct ethtool_drvinfo info; + }; +}; + struct ethtool_link_usettings { struct ethtool_link_settings base; struct { @@ -80837,8 +83204,10 @@ enum { ETHTOOL_MSG_MODULE_EEPROM_GET = 31, ETHTOOL_MSG_STATS_GET = 32, ETHTOOL_MSG_PHC_VCLOCKS_GET = 33, - __ETHTOOL_MSG_USER_CNT = 34, - ETHTOOL_MSG_USER_MAX = 33, + ETHTOOL_MSG_MODULE_GET = 34, + ETHTOOL_MSG_MODULE_SET = 35, + __ETHTOOL_MSG_USER_CNT = 36, + ETHTOOL_MSG_USER_MAX = 35, }; enum { @@ -80945,8 +83314,9 @@ enum { ETHTOOL_A_RINGS_RX_MINI = 7, ETHTOOL_A_RINGS_RX_JUMBO = 8, ETHTOOL_A_RINGS_TX = 9, - __ETHTOOL_A_RINGS_CNT = 10, - ETHTOOL_A_RINGS_MAX = 9, + ETHTOOL_A_RINGS_RX_BUF_LEN = 10, + __ETHTOOL_A_RINGS_CNT = 11, + ETHTOOL_A_RINGS_MAX = 10, }; enum { @@ -80989,8 +83359,10 @@ enum { ETHTOOL_A_COALESCE_TX_USECS_HIGH = 21, ETHTOOL_A_COALESCE_TX_MAX_FRAMES_HIGH = 22, ETHTOOL_A_COALESCE_RATE_SAMPLE_INTERVAL = 23, - __ETHTOOL_A_COALESCE_CNT = 24, - ETHTOOL_A_COALESCE_MAX = 23, + ETHTOOL_A_COALESCE_USE_CQE_MODE_TX = 24, + ETHTOOL_A_COALESCE_USE_CQE_MODE_RX = 25, + __ETHTOOL_A_COALESCE_CNT = 26, + ETHTOOL_A_COALESCE_MAX = 25, }; enum { @@ -81142,12 +83514,22 @@ enum { ETHTOOL_A_STATS_RMON_MAX = 3, }; +enum { + ETHTOOL_A_MODULE_UNSPEC = 0, + ETHTOOL_A_MODULE_HEADER = 1, + ETHTOOL_A_MODULE_POWER_MODE_POLICY = 2, + ETHTOOL_A_MODULE_POWER_MODE = 3, + __ETHTOOL_A_MODULE_CNT = 4, + ETHTOOL_A_MODULE_MAX = 3, +}; + enum ethtool_multicast_groups { ETHNL_MCGRP_MONITOR = 0, }; struct ethnl_req_info { struct net_device *dev; + netdevice_tracker dev_tracker; u32 flags; }; @@ -81306,9 +83688,14 @@ struct privflags_reply_data { u32 priv_flags; }; +enum ethtool_supported_ring_param { + ETHTOOL_RING_USE_RX_BUF_LEN = 1, +}; + struct rings_reply_data { struct ethnl_reply_data base; struct ethtool_ringparam ringparam; + struct kernel_ethtool_ringparam kernel_ringparam; }; struct channels_reply_data { @@ -81319,6 +83706,7 @@ struct channels_reply_data { struct coalesce_reply_data { struct ethnl_reply_data base; struct ethtool_coalesce coalesce; + struct kernel_ethtool_coalesce kernel_coalesce; u32 supported_params; }; @@ -81528,10 +83916,20 @@ struct stats_req_info { struct stats_reply_data { struct ethnl_reply_data base; - struct ethtool_eth_phy_stats phy_stats; - struct ethtool_eth_mac_stats mac_stats; - struct ethtool_eth_ctrl_stats ctrl_stats; - struct ethtool_rmon_stats rmon_stats; + union { + struct { + struct ethtool_eth_phy_stats phy_stats; + struct ethtool_eth_mac_stats mac_stats; + struct ethtool_eth_ctrl_stats ctrl_stats; + struct ethtool_rmon_stats rmon_stats; + }; + struct { + struct ethtool_eth_phy_stats phy_stats; + struct ethtool_eth_mac_stats mac_stats; + struct ethtool_eth_ctrl_stats ctrl_stats; + struct ethtool_rmon_stats rmon_stats; + } stats; + }; const struct ethtool_rmon_hist_range *rmon_ranges; }; @@ -81541,13 +83939,16 @@ struct phc_vclocks_reply_data { int *index; }; +struct module_reply_data { + struct ethnl_reply_data base; + struct ethtool_module_power_mode_params power; +}; + struct nf_hook_entries_rcu_head { struct callback_head head; void *allocation; }; -typedef u32 u_int32_t; - struct nf_loginfo { u_int8_t type; union { @@ -81582,6 +83983,11 @@ struct ip6_rt_info { u_int32_t mark; }; +struct nf_queue_handler { + int (*outfn)(struct nf_queue_entry *, unsigned int); + void (*nf_hook_drop)(struct net *); +}; + struct nf_sockopt_ops { struct list_head list; u_int8_t pf; @@ -81677,8 +84083,45 @@ enum { NFNL_BATCH_REPLAY = 4, }; -struct nf_conntrack { - atomic_t use; +enum ip_conntrack_status { + IPS_EXPECTED_BIT = 0, + IPS_EXPECTED = 1, + IPS_SEEN_REPLY_BIT = 1, + IPS_SEEN_REPLY = 2, + IPS_ASSURED_BIT = 2, + IPS_ASSURED = 4, + IPS_CONFIRMED_BIT = 3, + IPS_CONFIRMED = 8, + IPS_SRC_NAT_BIT = 4, + IPS_SRC_NAT = 16, + IPS_DST_NAT_BIT = 5, + IPS_DST_NAT = 32, + IPS_NAT_MASK = 48, + IPS_SEQ_ADJUST_BIT = 6, + IPS_SEQ_ADJUST = 64, + IPS_SRC_NAT_DONE_BIT = 7, + IPS_SRC_NAT_DONE = 128, + IPS_DST_NAT_DONE_BIT = 8, + IPS_DST_NAT_DONE = 256, + IPS_NAT_DONE_MASK = 384, + IPS_DYING_BIT = 9, + IPS_DYING = 512, + IPS_FIXED_TIMEOUT_BIT = 10, + IPS_FIXED_TIMEOUT = 1024, + IPS_TEMPLATE_BIT = 11, + IPS_TEMPLATE = 2048, + IPS_UNTRACKED_BIT = 12, + IPS_UNTRACKED = 4096, + IPS_NAT_CLASH_BIT = 12, + IPS_NAT_CLASH = 4096, + IPS_HELPER_BIT = 13, + IPS_HELPER = 8192, + IPS_OFFLOAD_BIT = 14, + IPS_OFFLOAD = 16384, + IPS_HW_OFFLOAD_BIT = 15, + IPS_HW_OFFLOAD = 32768, + IPS_UNCHANGEABLE_MASK = 56313, + __IPS_MAX_BIT = 16, }; enum nfqnl_msg_types { @@ -81905,6 +84348,7 @@ struct nfulnl_instance { struct sk_buff *skb; struct timer_list timer; struct net *net; + netns_tracker ns_tracker; struct user_namespace *peer_user_ns; u32 peer_portid; unsigned int flushtimeout; @@ -81924,6 +84368,347 @@ struct nfnl_log_net { atomic_t global_seq; }; +enum ip_conntrack_events { + IPCT_NEW = 0, + IPCT_RELATED = 1, + IPCT_DESTROY = 2, + IPCT_REPLY = 3, + IPCT_ASSURED = 4, + IPCT_PROTOINFO = 5, + IPCT_HELPER = 6, + IPCT_MARK = 7, + IPCT_SEQADJ = 8, + IPCT_NATSEQADJ = 8, + IPCT_SECMARK = 9, + IPCT_LABEL = 10, + IPCT_SYNPROXY = 11, + __IPCT_MAX = 12, +}; + +struct nf_conntrack_net { + atomic_t count; + unsigned int expect_count; + u8 sysctl_auto_assign_helper; + bool auto_assign_helper_warned; + unsigned int users4; + unsigned int users6; + unsigned int users_bridge; + struct ctl_table_header *sysctl_header; +}; + +struct nf_conntrack_expect_policy; + +struct nf_conntrack_helper { + struct hlist_node hnode; + char name[16]; + refcount_t refcnt; + struct module *me; + const struct nf_conntrack_expect_policy *expect_policy; + struct nf_conntrack_tuple tuple; + int (*help)(struct sk_buff *, unsigned int, struct nf_conn *, enum ip_conntrack_info); + void (*destroy)(struct nf_conn *); + int (*from_nlattr)(struct nlattr *, struct nf_conn *); + int (*to_nlattr)(struct sk_buff *, const struct nf_conn *); + unsigned int expect_class_max; + unsigned int flags; + unsigned int queue_num; + u16 data_len; + char nat_mod_name[16]; +}; + +struct nf_conntrack_expect_policy { + unsigned int max_expected; + unsigned int timeout; + char name[16]; +}; + +enum nf_ct_helper_flags { + NF_CT_HELPER_F_USERSPACE = 1, + NF_CT_HELPER_F_CONFIGURED = 2, +}; + +struct nf_conn_help { + struct nf_conntrack_helper *helper; + struct hlist_head expectations; + u8 expecting[4]; + int: 32; + char data[32]; +}; + +enum nf_ct_ecache_state { + NFCT_ECACHE_UNKNOWN = 0, + NFCT_ECACHE_DESTROY_FAIL = 1, + NFCT_ECACHE_DESTROY_SENT = 2, +}; + +struct nf_conntrack_ecache { + long unsigned int cache; + u16 missed; + u16 ctmask; + u16 expmask; + enum nf_ct_ecache_state state: 8; + u32 portid; +}; + +struct nf_conn_counter { + atomic64_t packets; + atomic64_t bytes; +}; + +struct nf_conn_acct { + struct nf_conn_counter counter[2]; +}; + +struct nf_conn_tstamp { + u_int64_t start; + u_int64_t stop; +}; + +struct nf_ct_timeout { + __u16 l3num; + const struct nf_conntrack_l4proto *l4proto; + char data[0]; +}; + +struct nf_conn_timeout { + struct nf_ct_timeout *timeout; +}; + +struct conntrack_gc_work { + struct delayed_work dwork; + u32 next_bucket; + bool exiting; + bool early_drop; +}; + +struct iter_data { + int (*iter)(struct nf_conn *, void *); + void *data; + struct net *net; +}; + +struct ct_iter_state { + struct seq_net_private p; + struct hlist_nulls_head *hash; + unsigned int htable_size; + unsigned int bucket; + u_int64_t time_now; +}; + +enum nf_ct_sysctl_index { + NF_SYSCTL_CT_MAX = 0, + NF_SYSCTL_CT_COUNT = 1, + NF_SYSCTL_CT_BUCKETS = 2, + NF_SYSCTL_CT_CHECKSUM = 3, + NF_SYSCTL_CT_LOG_INVALID = 4, + NF_SYSCTL_CT_EXPECT_MAX = 5, + NF_SYSCTL_CT_ACCT = 6, + NF_SYSCTL_CT_HELPER = 7, + NF_SYSCTL_CT_PROTO_TIMEOUT_GENERIC = 8, + NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_SYN_SENT = 9, + NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_SYN_RECV = 10, + NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_ESTABLISHED = 11, + NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_FIN_WAIT = 12, + NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_CLOSE_WAIT = 13, + NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_LAST_ACK = 14, + NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_TIME_WAIT = 15, + NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_CLOSE = 16, + NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_RETRANS = 17, + NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_UNACK = 18, + NF_SYSCTL_CT_PROTO_TCP_LOOSE = 19, + NF_SYSCTL_CT_PROTO_TCP_LIBERAL = 20, + NF_SYSCTL_CT_PROTO_TCP_IGNORE_INVALID_RST = 21, + NF_SYSCTL_CT_PROTO_TCP_MAX_RETRANS = 22, + NF_SYSCTL_CT_PROTO_TIMEOUT_UDP = 23, + NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_STREAM = 24, + NF_SYSCTL_CT_PROTO_TIMEOUT_ICMP = 25, + NF_SYSCTL_CT_PROTO_TIMEOUT_ICMPV6 = 26, + NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_CLOSED = 27, + NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_COOKIE_WAIT = 28, + NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_COOKIE_ECHOED = 29, + NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_ESTABLISHED = 30, + NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_SHUTDOWN_SENT = 31, + NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_SHUTDOWN_RECD = 32, + NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_SHUTDOWN_ACK_SENT = 33, + NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_HEARTBEAT_SENT = 34, + NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_HEARTBEAT_ACKED = 35, + NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_REQUEST = 36, + NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_RESPOND = 37, + NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_PARTOPEN = 38, + NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_OPEN = 39, + NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_CLOSEREQ = 40, + NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_CLOSING = 41, + NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_TIMEWAIT = 42, + NF_SYSCTL_CT_PROTO_DCCP_LOOSE = 43, + NF_SYSCTL_CT_LWTUNNEL = 44, + __NF_SYSCTL_CT_LAST_SYSCTL = 45, +}; + +enum ip_conntrack_expect_events { + IPEXP_NEW = 0, + IPEXP_DESTROY = 1, +}; + +struct ct_expect_iter_state { + struct seq_net_private p; + unsigned int bucket; +}; + +struct nf_ct_ext_type { + void (*destroy)(struct nf_conn *); + enum nf_ct_ext_id id; + u8 len; + u8 align; +}; + +struct nf_ct_helper_expectfn { + struct list_head head; + const char *name; + void (*expectfn)(struct nf_conn *, struct nf_conntrack_expect *); +}; + +struct nf_conntrack_nat_helper { + struct list_head list; + char mod_name[16]; + struct module *module; +}; + +struct nf_ct_bridge_info { + struct nf_hook_ops *ops; + unsigned int ops_size; + struct module *me; +}; + +enum { + TCP_FLAG_CWR = 32768, + TCP_FLAG_ECE = 16384, + TCP_FLAG_URG = 8192, + TCP_FLAG_ACK = 4096, + TCP_FLAG_PSH = 2048, + TCP_FLAG_RST = 1024, + TCP_FLAG_SYN = 512, + TCP_FLAG_FIN = 256, + TCP_RESERVED_BITS = 15, + TCP_DATA_OFFSET = 240, +}; + +struct nf_conn_synproxy { + u32 isn; + u32 its; + u32 tsoff; +}; + +enum tcp_bit_set { + TCP_SYN_SET = 0, + TCP_SYNACK_SET = 1, + TCP_FIN_SET = 2, + TCP_ACK_SET = 3, + TCP_RST_SET = 4, + TCP_NONE_SET = 5, +}; + +struct tcp_sack_block_wire { + __be32 start_seq; + __be32 end_seq; +}; + +struct nf_ct_seqadj { + u32 correction_pos; + s32 offset_before; + s32 offset_after; +}; + +struct nf_conn_seqadj { + struct nf_ct_seqadj seq[2]; +}; + +enum ct_dccp_roles { + CT_DCCP_ROLE_CLIENT = 0, + CT_DCCP_ROLE_SERVER = 1, + __CT_DCCP_ROLE_MAX = 2, +}; + +struct dccp_hdr_ext { + __be32 dccph_seq_low; +}; + +struct dccp_hdr_ack_bits { + __be16 dccph_reserved1; + __be16 dccph_ack_nr_high; + __be32 dccph_ack_nr_low; +}; + +enum dccp_pkt_type { + DCCP_PKT_REQUEST = 0, + DCCP_PKT_RESPONSE = 1, + DCCP_PKT_DATA = 2, + DCCP_PKT_ACK = 3, + DCCP_PKT_DATAACK = 4, + DCCP_PKT_CLOSEREQ = 5, + DCCP_PKT_CLOSE = 6, + DCCP_PKT_RESET = 7, + DCCP_PKT_SYNC = 8, + DCCP_PKT_SYNCACK = 9, + DCCP_PKT_INVALID = 10, +}; + +enum { + SCTP_CHUNK_FLAG_T = 1, +}; + +enum { + SCTP_MIB_NUM = 0, + SCTP_MIB_CURRESTAB = 1, + SCTP_MIB_ACTIVEESTABS = 2, + SCTP_MIB_PASSIVEESTABS = 3, + SCTP_MIB_ABORTEDS = 4, + SCTP_MIB_SHUTDOWNS = 5, + SCTP_MIB_OUTOFBLUES = 6, + SCTP_MIB_CHECKSUMERRORS = 7, + SCTP_MIB_OUTCTRLCHUNKS = 8, + SCTP_MIB_OUTORDERCHUNKS = 9, + SCTP_MIB_OUTUNORDERCHUNKS = 10, + SCTP_MIB_INCTRLCHUNKS = 11, + SCTP_MIB_INORDERCHUNKS = 12, + SCTP_MIB_INUNORDERCHUNKS = 13, + SCTP_MIB_FRAGUSRMSGS = 14, + SCTP_MIB_REASMUSRMSGS = 15, + SCTP_MIB_OUTSCTPPACKS = 16, + SCTP_MIB_INSCTPPACKS = 17, + SCTP_MIB_T1_INIT_EXPIREDS = 18, + SCTP_MIB_T1_COOKIE_EXPIREDS = 19, + SCTP_MIB_T2_SHUTDOWN_EXPIREDS = 20, + SCTP_MIB_T3_RTX_EXPIREDS = 21, + SCTP_MIB_T4_RTO_EXPIREDS = 22, + SCTP_MIB_T5_SHUTDOWN_GUARD_EXPIREDS = 23, + SCTP_MIB_DELAY_SACK_EXPIREDS = 24, + SCTP_MIB_AUTOCLOSE_EXPIREDS = 25, + SCTP_MIB_T1_RETRANSMITS = 26, + SCTP_MIB_T3_RETRANSMITS = 27, + SCTP_MIB_PMTUD_RETRANSMITS = 28, + SCTP_MIB_FAST_RETRANSMITS = 29, + SCTP_MIB_IN_PKT_SOFTIRQ = 30, + SCTP_MIB_IN_PKT_BACKLOG = 31, + SCTP_MIB_IN_PKT_DISCARDS = 32, + SCTP_MIB_IN_DATA_CHUNK_DISCARDS = 33, + __SCTP_MIB_MAX = 34, +}; + +enum { + BPF_F_CURRENT_NETNS = 4294967295, +}; + +struct bpf_ct_opts { + s32 netns_id; + s32 error; + u8 l4proto; + u8 reserved[3]; +}; + +enum { + NF_BPF_CT_OPTS_SZ = 12, +}; + struct xt_action_param; struct xt_mtchk_param; @@ -82080,7 +84865,6 @@ struct xt_table { struct module *me; u_int8_t af; int priority; - int (*table_init)(struct net *); const char name[32]; }; @@ -82100,6 +84884,13 @@ struct xt_percpu_counter_alloc_state { const char *mem; }; +struct xt_template { + struct list_head list; + int (*table_init)(struct net *); + struct module *me; + char name[32]; +}; + struct xt_pernet { struct list_head tables[13]; }; @@ -82208,6 +84999,7 @@ struct mr_table_ops { struct vif_device { struct net_device *dev; + netdevice_tracker dev_tracker; long unsigned int bytes_in; long unsigned int bytes_out; long unsigned int pkt_in; @@ -82413,11 +85205,22 @@ struct ip_mreq_source { }; struct ip_msfilter { - __be32 imsf_multiaddr; - __be32 imsf_interface; - __u32 imsf_fmode; - __u32 imsf_numsrc; - __be32 imsf_slist[1]; + union { + struct { + __be32 imsf_multiaddr_aux; + __be32 imsf_interface_aux; + __u32 imsf_fmode_aux; + __u32 imsf_numsrc_aux; + __be32 imsf_slist[1]; + }; + struct { + __be32 imsf_multiaddr; + __be32 imsf_interface; + __u32 imsf_fmode; + __u32 imsf_numsrc; + __be32 imsf_slist_flex[0]; + }; + }; }; struct group_req { @@ -82432,11 +85235,22 @@ struct group_source_req { }; struct group_filter { - __u32 gf_interface; - struct __kernel_sockaddr_storage gf_group; - __u32 gf_fmode; - __u32 gf_numsrc; - struct __kernel_sockaddr_storage gf_slist[1]; + union { + struct { + __u32 gf_interface_aux; + struct __kernel_sockaddr_storage gf_group_aux; + __u32 gf_fmode_aux; + __u32 gf_numsrc_aux; + struct __kernel_sockaddr_storage gf_slist[1]; + }; + struct { + __u32 gf_interface; + struct __kernel_sockaddr_storage gf_group; + __u32 gf_fmode; + __u32 gf_numsrc; + struct __kernel_sockaddr_storage gf_slist_flex[0]; + }; + }; }; struct in_pktinfo { @@ -82457,11 +85271,22 @@ struct compat_group_source_req { } __attribute__((packed)); struct compat_group_filter { - __u32 gf_interface; - struct __kernel_sockaddr_storage gf_group; - __u32 gf_fmode; - __u32 gf_numsrc; - struct __kernel_sockaddr_storage gf_slist[1]; + union { + struct { + __u32 gf_interface_aux; + struct __kernel_sockaddr_storage gf_group_aux; + __u32 gf_fmode_aux; + __u32 gf_numsrc_aux; + struct __kernel_sockaddr_storage gf_slist[1]; + } __attribute__((packed)); + struct { + __u32 gf_interface; + struct __kernel_sockaddr_storage gf_group; + __u32 gf_fmode; + __u32 gf_numsrc; + struct __kernel_sockaddr_storage gf_slist_flex[0]; + } __attribute__((packed)); + }; } __attribute__((packed)); enum { @@ -82514,22 +85339,6 @@ union tcp_cc_info { struct tcp_bbr_info bbr; }; -enum { - BPF_TCP_ESTABLISHED = 1, - BPF_TCP_SYN_SENT = 2, - BPF_TCP_SYN_RECV = 3, - BPF_TCP_FIN_WAIT1 = 4, - BPF_TCP_FIN_WAIT2 = 5, - BPF_TCP_TIME_WAIT = 6, - BPF_TCP_CLOSE = 7, - BPF_TCP_CLOSE_WAIT = 8, - BPF_TCP_LAST_ACK = 9, - BPF_TCP_LISTEN = 10, - BPF_TCP_CLOSING = 11, - BPF_TCP_NEW_SYN_RECV = 12, - BPF_TCP_MAX_STATES = 13, -}; - enum inet_csk_ack_state_t { ICSK_ACK_SCHED = 1, ICSK_ACK_TIMER = 2, @@ -82538,19 +85347,6 @@ enum inet_csk_ack_state_t { ICSK_ACK_NOW = 16, }; -enum { - TCP_FLAG_CWR = 32768, - TCP_FLAG_ECE = 16384, - TCP_FLAG_URG = 8192, - TCP_FLAG_ACK = 4096, - TCP_FLAG_PSH = 2048, - TCP_FLAG_RST = 1024, - TCP_FLAG_SYN = 512, - TCP_FLAG_FIN = 256, - TCP_RESERVED_BITS = 15, - TCP_DATA_OFFSET = 240, -}; - struct tcp_repair_opt { __u32 opt_code; __u32 opt_val; @@ -82675,6 +85471,22 @@ struct tcp_zerocopy_receive { __u32 reserved; }; +enum { + BPF_TCP_ESTABLISHED = 1, + BPF_TCP_SYN_SENT = 2, + BPF_TCP_SYN_RECV = 3, + BPF_TCP_FIN_WAIT1 = 4, + BPF_TCP_FIN_WAIT2 = 5, + BPF_TCP_TIME_WAIT = 6, + BPF_TCP_CLOSE = 7, + BPF_TCP_CLOSE_WAIT = 8, + BPF_TCP_LAST_ACK = 9, + BPF_TCP_LISTEN = 10, + BPF_TCP_CLOSING = 11, + BPF_TCP_NEW_SYN_RECV = 12, + BPF_TCP_MAX_STATES = 13, +}; + struct tcp_md5sig_pool { struct ahash_request *md5_req; void *scratch; @@ -82706,11 +85518,6 @@ enum tcp_fastopen_client_fail { TFO_SYN_RETRANSMITTED = 3, }; -struct tcp_sack_block_wire { - __be32 start_seq; - __be32 end_seq; -}; - enum tcp_queue { TCP_FRAG_IN_WRITE_QUEUE = 0, TCP_FRAG_IN_RTX_QUEUE = 1, @@ -82739,11 +85546,6 @@ enum pkt_hash_types { PKT_HASH_TYPE_L4 = 3, }; -enum { - BPF_WRITE_HDR_TCP_CURRENT_MSS = 1, - BPF_WRITE_HDR_TCP_SYNACK_COOKIE = 2, -}; - enum tsq_flags { TSQF_THROTTLED = 1, TSQF_QUEUED = 2, @@ -82753,6 +85555,11 @@ enum tsq_flags { TCPF_MTU_REDUCED_DEFERRED = 32, }; +enum { + BPF_WRITE_HDR_TCP_CURRENT_MSS = 1, + BPF_WRITE_HDR_TCP_SYNACK_COOKIE = 2, +}; + struct mptcp_out_options {}; struct tcp_out_options { @@ -82811,7 +85618,6 @@ struct tcp_iter_state { struct seq_net_private p; enum tcp_seq_states state; struct sock *syn_wait_sk; - struct tcp_seq_afinfo *bpf_seq_afinfo; int bucket; int offset; int sbucket; @@ -82819,6 +85625,15 @@ struct tcp_iter_state { loff_t last_pos; }; +struct bpf_tcp_iter_state { + struct tcp_iter_state state; + unsigned int cur_sk; + unsigned int end_sk; + unsigned int max_sk; + struct sock **batch; + bool st_bucket_done; +}; + struct bpf_iter__tcp { union { struct bpf_iter_meta *meta; @@ -82961,10 +85776,10 @@ struct inet_protosw { unsigned char flags; }; -typedef struct sk_buff * (*gro_receive_sk_t)(struct sock *, struct list_head *, struct sk_buff *); - typedef struct sock * (*udp_lookup_t)(const struct sk_buff *, __be16, __be16); +typedef struct sk_buff * (*gro_receive_sk_t)(struct sock *, struct list_head *, struct sk_buff *); + struct arpreq { struct sockaddr arp_pa; struct sockaddr arp_ha; @@ -83109,7 +85924,7 @@ struct inet_fill_args { struct devinet_sysctl_table { struct ctl_table_header *sysctl_header; - struct ctl_table devinet_vars[33]; + struct ctl_table devinet_vars[34]; }; struct rtentry { @@ -83529,6 +86344,7 @@ struct ip_tunnel { struct ip_tunnel *next; struct hlist_node hash_node; struct net_device *dev; + netdevice_tracker dev_tracker; struct net *net; long unsigned int err_time; int err_count; @@ -84217,7 +87033,9 @@ enum { enum { TCP_BPF_BASE = 0, TCP_BPF_TX = 1, - TCP_BPF_NUM_CFGS = 2, + TCP_BPF_RX = 2, + TCP_BPF_TXRX = 3, + TCP_BPF_NUM_CFGS = 4, }; enum { @@ -84347,6 +87165,14 @@ struct xfrm4_protocol { typedef u64 (*btf_bpf_tcp_send_ack)(struct tcp_sock *, u32); +struct tls12_crypto_info_aes_ccm_128 { + struct tls_crypto_info info; + unsigned char iv[8]; + unsigned char key[16]; + unsigned char salt[4]; + unsigned char rec_seq[8]; +}; + enum { TLS_INFO_UNSPEC = 0, TLS_INFO_VERSION = 1, @@ -84370,14 +87196,6 @@ enum { TLS_NUM_PROTS = 2, }; -struct tls12_crypto_info_aes_ccm_128 { - struct tls_crypto_info info; - unsigned char iv[8]; - unsigned char key[16]; - unsigned char salt[4]; - unsigned char rec_seq[8]; -}; - struct tls_msg { struct strp_msg rxm; u8 control; @@ -84471,13 +87289,6 @@ typedef void (*btf_trace_tls_device_tx_resync_req)(void *, struct sock *, u32, u typedef void (*btf_trace_tls_device_tx_resync_send)(void *, struct sock *, u32, u8 *); -struct seqcount_mutex { - seqcount_t seqcount; - struct mutex *lock; -}; - -typedef struct seqcount_mutex seqcount_mutex_t; - enum { XFRM_STATE_VOID = 0, XFRM_STATE_ACQ = 1, @@ -84699,6 +87510,7 @@ struct __ip6_tnl_parm { struct ip6_tnl { struct ip6_tnl *next; struct net_device *dev; + netdevice_tracker dev_tracker; struct net *net; struct __ip6_tnl_parm parms; struct flowi fl; @@ -84968,6 +87780,12 @@ struct xfrm_user_offload { __u8 flags; }; +struct xfrm_userpolicy_default { + __u8 in; + __u8 fwd; + __u8 out; +}; + struct xfrm_dump_info { struct sk_buff *in_skb; struct sk_buff *out_skb; @@ -84994,6 +87812,31 @@ struct unix_stream_read_state { unsigned int splice_flags; }; +struct bpf_unix_iter_state { + struct seq_net_private p; + unsigned int cur_sk; + unsigned int end_sk; + unsigned int max_sk; + struct sock **batch; + bool st_bucket_done; +}; + +struct bpf_iter__unix { + union { + struct bpf_iter_meta *meta; + }; + union { + struct unix_sock *unix_sk; + }; + uid_t uid; +}; + +struct ioam6_pernet_data { + struct mutex lock; + struct rhashtable namespaces; + struct rhashtable schemas; +}; + enum flowlabel_reflect { FLOWLABEL_REFLECT_ESTABLISHED = 1, FLOWLABEL_REFLECT_TCP_RESET = 2, @@ -85045,9 +87888,9 @@ struct ipcm6_cookie { struct sockcm_cookie sockc; __s16 hlimit; __s16 tclass; + __u16 gso_size; __s8 dontfrag; struct ipv6_txoptions *opt; - __u16 gso_size; }; enum { @@ -85060,7 +87903,8 @@ enum { IFLA_INET6_ICMP6STATS = 6, IFLA_INET6_TOKEN = 7, IFLA_INET6_ADDR_GEN_MODE = 8, - __IFLA_INET6_MAX = 9, + IFLA_INET6_RA_MTU = 9, + __IFLA_INET6_MAX = 10, }; enum in6_addr_gen_mode { @@ -85160,7 +88004,11 @@ enum { DEVCONF_NDISC_TCLASS = 50, DEVCONF_RPL_SEG_ENABLED = 51, DEVCONF_RA_DEFRTR_METRIC = 52, - DEVCONF_MAX = 53, + DEVCONF_IOAM6_ENABLED = 53, + DEVCONF_IOAM6_ID = 54, + DEVCONF_IOAM6_ID_WIDE = 55, + DEVCONF_NDISC_EVICT_NOCARRIER = 56, + DEVCONF_MAX = 57, }; enum { @@ -85413,6 +88261,7 @@ struct __rt6_probe_work { struct work_struct work; struct in6_addr target; struct net_device *dev; + netdevice_tracker dev_tracker; }; struct fib6_nh_frl_arg { @@ -85735,9 +88584,70 @@ struct ipv6_rpl_sr_hdr { } segments; }; -struct tlvtype_proc { - int type; - bool (*func)(struct sk_buff *, int); +struct ioam6_hdr { + __u8 opt_type; + __u8 opt_len; + char: 8; + __u8 type; +}; + +struct ioam6_trace_hdr { + __be16 namespace_id; + char: 2; + __u8 overflow: 1; + __u8 nodelen: 5; + __u8 remlen: 7; + union { + __be32 type_be32; + struct { + __u32 bit7: 1; + __u32 bit6: 1; + __u32 bit5: 1; + __u32 bit4: 1; + __u32 bit3: 1; + __u32 bit2: 1; + __u32 bit1: 1; + __u32 bit0: 1; + __u32 bit15: 1; + __u32 bit14: 1; + __u32 bit13: 1; + __u32 bit12: 1; + __u32 bit11: 1; + __u32 bit10: 1; + __u32 bit9: 1; + __u32 bit8: 1; + __u32 bit23: 1; + __u32 bit22: 1; + __u32 bit21: 1; + __u32 bit20: 1; + __u32 bit19: 1; + __u32 bit18: 1; + __u32 bit17: 1; + __u32 bit16: 1; + } type; + }; + __u8 data[0]; +}; + +struct ioam6_schema; + +struct ioam6_namespace { + struct rhash_head head; + struct callback_head rcu; + struct ioam6_schema *schema; + __be16 id; + __be32 data; + __be64 data_wide; +}; + +struct ioam6_schema { + struct rhash_head head; + struct callback_head rcu; + struct ioam6_namespace *ns; + u32 id; + int len; + __be32 hdr; + u8 data[0]; }; struct ip6fl_iter_state { @@ -85773,6 +88683,30 @@ enum { __SEG6_CMD_MAX = 5, }; +enum { + IOAM6_ATTR_UNSPEC = 0, + IOAM6_ATTR_NS_ID = 1, + IOAM6_ATTR_NS_DATA = 2, + IOAM6_ATTR_NS_DATA_WIDE = 3, + IOAM6_ATTR_SC_ID = 4, + IOAM6_ATTR_SC_DATA = 5, + IOAM6_ATTR_SC_NONE = 6, + IOAM6_ATTR_PAD = 7, + __IOAM6_ATTR_MAX = 8, +}; + +enum { + IOAM6_CMD_UNSPEC = 0, + IOAM6_CMD_ADD_NAMESPACE = 1, + IOAM6_CMD_DEL_NAMESPACE = 2, + IOAM6_CMD_DUMP_NAMESPACES = 3, + IOAM6_CMD_ADD_SCHEMA = 4, + IOAM6_CMD_DEL_SCHEMA = 5, + IOAM6_CMD_DUMP_SCHEMAS = 6, + IOAM6_CMD_NS_SET_SCHEMA = 7, + __IOAM6_CMD_MAX = 8, +}; + struct xfrm6_protocol { int (*handler)(struct sk_buff *); int (*input_handler)(struct sk_buff *, int, __be32, int); @@ -86067,6 +89001,11 @@ struct ip6t_get_entries { struct ip6t_entry entrytable[0]; }; +struct nft_ct_frag6_pernet { + struct ctl_table_header *nf_frag_frags_hdr; + struct fqdir *fqdir; +}; + struct ip_tunnel_prl { __be32 addr; __u16 flags; @@ -86449,11 +89388,6 @@ struct packet_skb_cb { } sa; }; -struct _strp_msg { - struct strp_msg strp; - int accum_len; -}; - struct vlan_group { unsigned int nr_vlan_devs; struct hlist_node hlist; @@ -86502,6 +89436,7 @@ struct vlan_dev_priv { u16 vlan_id; u16 flags; struct net_device *real_dev; + netdevice_tracker dev_tracker; unsigned char real_dev_addr[6]; struct proc_dir_entry *dent; struct vlan_pcpu_stats *vlan_pcpu_stats; @@ -87579,6 +90514,7 @@ struct irq_router { u16 device; int (*get)(struct pci_dev *, struct pci_dev *, int); int (*set)(struct pci_dev *, struct pci_dev *, int, int); + int (*lvl)(struct pci_dev *, struct pci_dev *, int, int); }; struct irq_router_handler { From 3febb8a16597f4605450f6c947f4deefb446cb8a Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 21 Jan 2022 16:10:20 -0800 Subject: [PATCH 085/131] ci: update s390x blacklist Add bpf_mod_race and bpf_nf selftests to blacklist for s390x. Signed-off-by: Andrii Nakryiko --- travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x | 2 ++ 1 file changed, 2 insertions(+) diff --git a/travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x b/travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x index 11e3f5f..67811c5 100644 --- a/travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x +++ b/travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x @@ -4,6 +4,8 @@ bpf_iter_setsockopt # JIT does not support calling kernel f bloom_filter_map # failed to find kernel BTF type ID of '__x64_sys_getpgid': -3 (?) bpf_tcp_ca # JIT does not support calling kernel function (kfunc) bpf_loop # attaches to __x64_sys_nanosleep +bpf_mod_race # BPF trampoline +bpf_nf # JIT does not support calling kernel function core_read_macros # unknown func bpf_probe_read#4 (overlapping) d_path # failed to auto-attach program 'prog_stat': -524 (trampoline) dummy_st_ops # test_run unexpected error: -524 (errno 524) (trampoline) From 393a058d061d49d5c3055fa9eefafb4c0c31ccc3 Mon Sep 17 00:00:00 2001 From: Evgeny Vereshchagin Date: Sat, 22 Jan 2022 01:05:11 +0000 Subject: [PATCH 086/131] tests: move the fuzzer upstream It should make it easier to start using CFLite or something like that to fuzz libbpf without getting pointless CVEs :-) More importantly, now it's possible to build the fuzzer by just cloning the repository, installing clang and running `./scripts/build-fuzzers.h`: ``` git clone https://github.com/libbpf/libbpf ./scripts/build-fuzzers.h unzip -d CORPUS fuzz/bpf-object-fuzzer_seed_corpus.zip ./out/bpf-object-fuzzer CORPUS ``` It should make it easier (for me at least) to report some elfutils bugs because they are much easier to reproduce manually now. --- fuzz/bpf-object-fuzzer.c | 23 ++++++++++ fuzz/bpf-object-fuzzer_seed_corpus.zip | Bin 0 -> 1091 bytes scripts/build-fuzzers.sh | 60 +++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 fuzz/bpf-object-fuzzer.c create mode 100644 fuzz/bpf-object-fuzzer_seed_corpus.zip create mode 100755 scripts/build-fuzzers.sh diff --git a/fuzz/bpf-object-fuzzer.c b/fuzz/bpf-object-fuzzer.c new file mode 100644 index 0000000..89286e2 --- /dev/null +++ b/fuzz/bpf-object-fuzzer.c @@ -0,0 +1,23 @@ +#include "libbpf.h" + +static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args) +{ + return 0; +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + struct bpf_object *obj = NULL; + DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts); + int err; + + libbpf_set_print(libbpf_print_fn); + + opts.object_name = "fuzz-object"; + obj = bpf_object__open_mem(data, size, &opts); + err = libbpf_get_error(obj); + if (err) + return 0; + + bpf_object__close(obj); + return 0; +} diff --git a/fuzz/bpf-object-fuzzer_seed_corpus.zip b/fuzz/bpf-object-fuzzer_seed_corpus.zip new file mode 100644 index 0000000000000000000000000000000000000000..602b3812c46ce03b56f89b3f8f94f06954ef0015 GIT binary patch literal 1091 zcmWIWW@Zs#U|`^2C}TGZ@pR&yIGLG&!GVi`ftNvsAvZHGGdD3uFR36+FF!PdlYyBb z{B(HUypnZvBhbdi;_vz1YWO(mhZm)A_G(GPvU-id*X-IV;A(f=jw8$_3p`yY2Q)z zwvW-?$!5+%nf;vIdsLWTu8=<>vtY&!^OYA0D<;U4X)~6y{I=YbxX;$`ciAT8{|;y7 zsal^C)%s@@;VUoYwmEC|nf&Ja*Y@(RPr9+KT{g|E_HwFu>}hky{4FxU`xCFkE`IrX zTd3al?Do@Zr@jBV?&7txrH3qXg zR&G|I_UG4OrCCw7rCzp1^>eJRU(qbNR~}Sqb|F!!$Rm5V&6!(Q{{3UGU%Ini?6Z%2 zyLCXOgq}fJTIJ6(s@pb|>^u;rKb5P;vG8Ti!$V$-haOzKWO{7rvmXnl+VY!h4>he} z&RLygW^247NGtcQUeN5vYxM>6XLD8o^Gw~<5*CkJQ^FT2xeaE|wqk_eEIsOF8?tW2vjXQDfnWF~3YWklZ`KUAZ z3`6hgjgJmTo+~>Wy{Ax6&-2HumfJ%2rhIVb?EV|f@qX?N-dXwUKL36`YumK=*ee2G zcFqetn6<6;aH8ARS5;xQ5B{xgIav5>CwNpb+f=<=Et9=nMJA2S!~m!f3IJ?_Q(97yPw_X`|$tUX7@+Ow)Ro%iHYma z#J@fq?_d7=X@~ab{z6}|{|^jvj{W@Jsej-5=O44`9}8?0{=L@yQO_9Q&B!Fjj4N+T m0P{QqENKMMXgQq~lGD+0d4M-78%QxD5C#M3TfpqkzyJUk4eOl% literal 0 HcmV?d00001 diff --git a/scripts/build-fuzzers.sh b/scripts/build-fuzzers.sh new file mode 100755 index 0000000..0591451 --- /dev/null +++ b/scripts/build-fuzzers.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -eux + +SANITIZER=${SANITIZER:-address} +flags="-O1 -fno-omit-frame-pointer -g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=$SANITIZER -fsanitize=fuzzer-no-link" + +export CC=${CC:-clang} +export CFLAGS=${CFLAGS:-$flags} + +export CXX=${CXX:-clang++} +export CXXFLAGS=${CXXFLAGS:-$flags} + +cd "$(dirname -- "$0")/.." + +export OUT=${OUT:-"$(pwd)/out"} +mkdir -p "$OUT" + +export LIB_FUZZING_ENGINE=${LIB_FUZZING_ENGINE:--fsanitize=fuzzer} + +# Ideally libbelf should be built using release tarballs available +# at https://sourceware.org/elfutils/ftp/. Unfortunately sometimes they +# fail to compile (for example, elfutils-0.185 fails to compile with LDFLAGS enabled +# due to https://bugs.gentoo.org/794601) so let's just point the script to +# commits referring to versions of libelf that actually can be built +elfutils=$(mktemp -d) +git clone git://sourceware.org/git/elfutils.git "$elfutils" +( +cd "$elfutils" +git checkout 983e86fd89e8bf02f2d27ba5dce5bf078af4ceda +git log --oneline -1 + +# ASan isn't compatible with -Wl,--no-undefined: https://github.com/google/sanitizers/issues/380 +find -name Makefile.am | xargs sed -i 's/,--no-undefined//' + +# ASan isn't compatible with -Wl,-z,defs either: +# https://clang.llvm.org/docs/AddressSanitizer.html#usage +sed -i 's/^\(ZDEFS_LDFLAGS=\).*/\1/' configure.ac + + +autoreconf -i -f +if ! ./configure --enable-maintainer-mode --disable-debuginfod --disable-libdebuginfod \ + CC="$CC" CFLAGS="-Wno-error $CFLAGS" CXX="$CXX" CXXFLAGS="-Wno-error $CXXFLAGS" LDFLAGS="$CFLAGS"; then + cat config.log + exit 1 +fi + +make -C config -j$(nproc) V=1 +make -C lib -j$(nproc) V=1 +make -C libelf -j$(nproc) V=1 +) + +make -C src BUILD_STATIC_ONLY=y V=1 clean +make -C src -j$(nproc) CFLAGS="-I$elfutils/libelf $CFLAGS" BUILD_STATIC_ONLY=y V=1 + +$CC $CFLAGS -Isrc -Iinclude -Iinclude/uapi -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -c fuzz/bpf-object-fuzzer.c -o bpf-object-fuzzer.o +$CXX $CXXFLAGS $LIB_FUZZING_ENGINE bpf-object-fuzzer.o src/libbpf.a "$elfutils/libelf/libelf.a" -l:libz.a -o "$OUT/bpf-object-fuzzer" + +cp fuzz/bpf-object-fuzzer_seed_corpus.zip "$OUT" + +rm -rf "$elfutils" From f1085fe3c32b082160a07d9442cbc8655875b698 Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 26 Jan 2022 01:14:47 +0100 Subject: [PATCH 087/131] ci: add s390x-self-hosted-builder's user to kvm group RHEL's podman sets /dev/kvm permissions to 0666, while Ubuntu's docker sets them to 0660. Therefore, in order to use KVM from a container, the user within must belong to the kvm group. Signed-off-by: Ilya Leoshkevich --- .../s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/travis-ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile b/travis-ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile index a07f177..d830657 100644 --- a/travis-ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile +++ b/travis-ci/rootfs/s390x-self-hosted-builder/actions-runner-libbpf.Dockerfile @@ -37,6 +37,7 @@ ARG version=2.285.0 RUN useradd -m actions-runner RUN echo "actions-runner ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers RUN echo "Defaults env_keep += \"DEBIAN_FRONTEND\"" >>/etc/sudoers +RUN usermod -a -G kvm actions-runner USER actions-runner ENV USER=actions-runner WORKDIR /home/actions-runner From 0ee425cdd72bb7b3972d8498fc7a2effdf16b17f Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 26 Jan 2022 00:47:50 +0100 Subject: [PATCH 088/131] ci: add Ubuntu instructions for s390x-self-hosted-builder There is a new Ubuntu-based builder, and setup steps for it are slightly different from what we had on the old RHEL-based one. Signed-off-by: Ilya Leoshkevich --- travis-ci/rootfs/s390x-self-hosted-builder/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/travis-ci/rootfs/s390x-self-hosted-builder/README.md b/travis-ci/rootfs/s390x-self-hosted-builder/README.md index 3e9f966..f8a4b86 100644 --- a/travis-ci/rootfs/s390x-self-hosted-builder/README.md +++ b/travis-ci/rootfs/s390x-self-hosted-builder/README.md @@ -10,7 +10,8 @@ the master branch. ### Install prerequisites. ``` -$ sudo dnf install docker +$ sudo dnf install docker # RHEL +$ sudo apt install -y docker.io # Ubuntu ``` ### Add services. @@ -53,14 +54,15 @@ get the latest OS security fixes, use the following commands: $ sudo docker build \ --pull \ -f actions-runner-libbpf.Dockerfile \ - -t iiilinuxibmcom/actions-runner-libbpf + -t iiilinuxibmcom/actions-runner-libbpf \ + . $ sudo systemctl restart actions-runner-libbpf ``` ## Removing persistent data The `actions-runner-libbpf` service stores various temporary data, such as -runner registration information, work directories and logs, in the +runner registration information, work directories and logs, in the `actions-runner-libbpf` volume. In order to remove it and start from scratch, e.g. when upgrading the runner or switching it to a different repository, use the following commands: From b4b6e4dc20f02440df4a937f6ab48809d8136e7f Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 25 Jan 2022 22:19:09 -0800 Subject: [PATCH 089/131] sync: start syncing perf_event.h UAPI header as well This header is necessary for libbpf-sys to generate perf-related Rust bindings. It's more convenient to have it available locally with libbpf. Signed-off-by: Andrii Nakryiko --- include/uapi/linux/perf_event.h | 1388 +++++++++++++++++++++++++++++++ scripts/sync-kernel.sh | 1 + 2 files changed, 1389 insertions(+) create mode 100644 include/uapi/linux/perf_event.h diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h new file mode 100644 index 0000000..bd8860e --- /dev/null +++ b/include/uapi/linux/perf_event.h @@ -0,0 +1,1388 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Performance events: + * + * Copyright (C) 2008-2009, Thomas Gleixner + * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar + * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra + * + * Data type definitions, declarations, prototypes. + * + * Started by: Thomas Gleixner and Ingo Molnar + * + * For licencing details see kernel-base/COPYING + */ +#ifndef _UAPI_LINUX_PERF_EVENT_H +#define _UAPI_LINUX_PERF_EVENT_H + +#include +#include +#include + +/* + * User-space ABI bits: + */ + +/* + * attr.type + */ +enum perf_type_id { + PERF_TYPE_HARDWARE = 0, + PERF_TYPE_SOFTWARE = 1, + PERF_TYPE_TRACEPOINT = 2, + PERF_TYPE_HW_CACHE = 3, + PERF_TYPE_RAW = 4, + PERF_TYPE_BREAKPOINT = 5, + + PERF_TYPE_MAX, /* non-ABI */ +}; + +/* + * attr.config layout for type PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE + * PERF_TYPE_HARDWARE: 0xEEEEEEEE000000AA + * AA: hardware event ID + * EEEEEEEE: PMU type ID + * PERF_TYPE_HW_CACHE: 0xEEEEEEEE00DDCCBB + * BB: hardware cache ID + * CC: hardware cache op ID + * DD: hardware cache op result ID + * EEEEEEEE: PMU type ID + * If the PMU type ID is 0, the PERF_TYPE_RAW will be applied. + */ +#define PERF_PMU_TYPE_SHIFT 32 +#define PERF_HW_EVENT_MASK 0xffffffff + +/* + * Generalized performance event event_id types, used by the + * attr.event_id parameter of the sys_perf_event_open() + * syscall: + */ +enum perf_hw_id { + /* + * Common hardware events, generalized by the kernel: + */ + PERF_COUNT_HW_CPU_CYCLES = 0, + PERF_COUNT_HW_INSTRUCTIONS = 1, + PERF_COUNT_HW_CACHE_REFERENCES = 2, + PERF_COUNT_HW_CACHE_MISSES = 3, + PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4, + PERF_COUNT_HW_BRANCH_MISSES = 5, + PERF_COUNT_HW_BUS_CYCLES = 6, + PERF_COUNT_HW_STALLED_CYCLES_FRONTEND = 7, + PERF_COUNT_HW_STALLED_CYCLES_BACKEND = 8, + PERF_COUNT_HW_REF_CPU_CYCLES = 9, + + PERF_COUNT_HW_MAX, /* non-ABI */ +}; + +/* + * Generalized hardware cache events: + * + * { L1-D, L1-I, LLC, ITLB, DTLB, BPU, NODE } x + * { read, write, prefetch } x + * { accesses, misses } + */ +enum perf_hw_cache_id { + PERF_COUNT_HW_CACHE_L1D = 0, + PERF_COUNT_HW_CACHE_L1I = 1, + PERF_COUNT_HW_CACHE_LL = 2, + PERF_COUNT_HW_CACHE_DTLB = 3, + PERF_COUNT_HW_CACHE_ITLB = 4, + PERF_COUNT_HW_CACHE_BPU = 5, + PERF_COUNT_HW_CACHE_NODE = 6, + + PERF_COUNT_HW_CACHE_MAX, /* non-ABI */ +}; + +enum perf_hw_cache_op_id { + PERF_COUNT_HW_CACHE_OP_READ = 0, + PERF_COUNT_HW_CACHE_OP_WRITE = 1, + PERF_COUNT_HW_CACHE_OP_PREFETCH = 2, + + PERF_COUNT_HW_CACHE_OP_MAX, /* non-ABI */ +}; + +enum perf_hw_cache_op_result_id { + PERF_COUNT_HW_CACHE_RESULT_ACCESS = 0, + PERF_COUNT_HW_CACHE_RESULT_MISS = 1, + + PERF_COUNT_HW_CACHE_RESULT_MAX, /* non-ABI */ +}; + +/* + * Special "software" events provided by the kernel, even if the hardware + * does not support performance events. These events measure various + * physical and sw events of the kernel (and allow the profiling of them as + * well): + */ +enum perf_sw_ids { + PERF_COUNT_SW_CPU_CLOCK = 0, + PERF_COUNT_SW_TASK_CLOCK = 1, + PERF_COUNT_SW_PAGE_FAULTS = 2, + PERF_COUNT_SW_CONTEXT_SWITCHES = 3, + PERF_COUNT_SW_CPU_MIGRATIONS = 4, + PERF_COUNT_SW_PAGE_FAULTS_MIN = 5, + PERF_COUNT_SW_PAGE_FAULTS_MAJ = 6, + PERF_COUNT_SW_ALIGNMENT_FAULTS = 7, + PERF_COUNT_SW_EMULATION_FAULTS = 8, + PERF_COUNT_SW_DUMMY = 9, + PERF_COUNT_SW_BPF_OUTPUT = 10, + PERF_COUNT_SW_CGROUP_SWITCHES = 11, + + PERF_COUNT_SW_MAX, /* non-ABI */ +}; + +/* + * Bits that can be set in attr.sample_type to request information + * in the overflow packets. + */ +enum perf_event_sample_format { + PERF_SAMPLE_IP = 1U << 0, + PERF_SAMPLE_TID = 1U << 1, + PERF_SAMPLE_TIME = 1U << 2, + PERF_SAMPLE_ADDR = 1U << 3, + PERF_SAMPLE_READ = 1U << 4, + PERF_SAMPLE_CALLCHAIN = 1U << 5, + PERF_SAMPLE_ID = 1U << 6, + PERF_SAMPLE_CPU = 1U << 7, + PERF_SAMPLE_PERIOD = 1U << 8, + PERF_SAMPLE_STREAM_ID = 1U << 9, + PERF_SAMPLE_RAW = 1U << 10, + PERF_SAMPLE_BRANCH_STACK = 1U << 11, + PERF_SAMPLE_REGS_USER = 1U << 12, + PERF_SAMPLE_STACK_USER = 1U << 13, + PERF_SAMPLE_WEIGHT = 1U << 14, + PERF_SAMPLE_DATA_SRC = 1U << 15, + PERF_SAMPLE_IDENTIFIER = 1U << 16, + PERF_SAMPLE_TRANSACTION = 1U << 17, + PERF_SAMPLE_REGS_INTR = 1U << 18, + PERF_SAMPLE_PHYS_ADDR = 1U << 19, + PERF_SAMPLE_AUX = 1U << 20, + PERF_SAMPLE_CGROUP = 1U << 21, + PERF_SAMPLE_DATA_PAGE_SIZE = 1U << 22, + PERF_SAMPLE_CODE_PAGE_SIZE = 1U << 23, + PERF_SAMPLE_WEIGHT_STRUCT = 1U << 24, + + PERF_SAMPLE_MAX = 1U << 25, /* non-ABI */ + + __PERF_SAMPLE_CALLCHAIN_EARLY = 1ULL << 63, /* non-ABI; internal use */ +}; + +#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT | PERF_SAMPLE_WEIGHT_STRUCT) +/* + * values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set + * + * If the user does not pass priv level information via branch_sample_type, + * the kernel uses the event's priv level. Branch and event priv levels do + * not have to match. Branch priv level is checked for permissions. + * + * The branch types can be combined, however BRANCH_ANY covers all types + * of branches and therefore it supersedes all the other types. + */ +enum perf_branch_sample_type_shift { + PERF_SAMPLE_BRANCH_USER_SHIFT = 0, /* user branches */ + PERF_SAMPLE_BRANCH_KERNEL_SHIFT = 1, /* kernel branches */ + PERF_SAMPLE_BRANCH_HV_SHIFT = 2, /* hypervisor branches */ + + PERF_SAMPLE_BRANCH_ANY_SHIFT = 3, /* any branch types */ + PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT = 4, /* any call branch */ + PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT = 5, /* any return branch */ + PERF_SAMPLE_BRANCH_IND_CALL_SHIFT = 6, /* indirect calls */ + PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT = 7, /* transaction aborts */ + PERF_SAMPLE_BRANCH_IN_TX_SHIFT = 8, /* in transaction */ + PERF_SAMPLE_BRANCH_NO_TX_SHIFT = 9, /* not in transaction */ + PERF_SAMPLE_BRANCH_COND_SHIFT = 10, /* conditional branches */ + + PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT = 11, /* call/ret stack */ + PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT = 12, /* indirect jumps */ + PERF_SAMPLE_BRANCH_CALL_SHIFT = 13, /* direct call */ + + PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT = 14, /* no flags */ + PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT = 15, /* no cycles */ + + PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT = 16, /* save branch type */ + + PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */ + + PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */ +}; + +enum perf_branch_sample_type { + PERF_SAMPLE_BRANCH_USER = 1U << PERF_SAMPLE_BRANCH_USER_SHIFT, + PERF_SAMPLE_BRANCH_KERNEL = 1U << PERF_SAMPLE_BRANCH_KERNEL_SHIFT, + PERF_SAMPLE_BRANCH_HV = 1U << PERF_SAMPLE_BRANCH_HV_SHIFT, + + PERF_SAMPLE_BRANCH_ANY = 1U << PERF_SAMPLE_BRANCH_ANY_SHIFT, + PERF_SAMPLE_BRANCH_ANY_CALL = 1U << PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT, + PERF_SAMPLE_BRANCH_ANY_RETURN = 1U << PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT, + PERF_SAMPLE_BRANCH_IND_CALL = 1U << PERF_SAMPLE_BRANCH_IND_CALL_SHIFT, + PERF_SAMPLE_BRANCH_ABORT_TX = 1U << PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT, + PERF_SAMPLE_BRANCH_IN_TX = 1U << PERF_SAMPLE_BRANCH_IN_TX_SHIFT, + PERF_SAMPLE_BRANCH_NO_TX = 1U << PERF_SAMPLE_BRANCH_NO_TX_SHIFT, + PERF_SAMPLE_BRANCH_COND = 1U << PERF_SAMPLE_BRANCH_COND_SHIFT, + + PERF_SAMPLE_BRANCH_CALL_STACK = 1U << PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT, + PERF_SAMPLE_BRANCH_IND_JUMP = 1U << PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT, + PERF_SAMPLE_BRANCH_CALL = 1U << PERF_SAMPLE_BRANCH_CALL_SHIFT, + + PERF_SAMPLE_BRANCH_NO_FLAGS = 1U << PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT, + PERF_SAMPLE_BRANCH_NO_CYCLES = 1U << PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT, + + PERF_SAMPLE_BRANCH_TYPE_SAVE = + 1U << PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT, + + PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT, + + PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT, +}; + +/* + * Common flow change classification + */ +enum { + PERF_BR_UNKNOWN = 0, /* unknown */ + PERF_BR_COND = 1, /* conditional */ + PERF_BR_UNCOND = 2, /* unconditional */ + PERF_BR_IND = 3, /* indirect */ + PERF_BR_CALL = 4, /* function call */ + PERF_BR_IND_CALL = 5, /* indirect function call */ + PERF_BR_RET = 6, /* function return */ + PERF_BR_SYSCALL = 7, /* syscall */ + PERF_BR_SYSRET = 8, /* syscall return */ + PERF_BR_COND_CALL = 9, /* conditional function call */ + PERF_BR_COND_RET = 10, /* conditional function return */ + PERF_BR_MAX, +}; + +#define PERF_SAMPLE_BRANCH_PLM_ALL \ + (PERF_SAMPLE_BRANCH_USER|\ + PERF_SAMPLE_BRANCH_KERNEL|\ + PERF_SAMPLE_BRANCH_HV) + +/* + * Values to determine ABI of the registers dump. + */ +enum perf_sample_regs_abi { + PERF_SAMPLE_REGS_ABI_NONE = 0, + PERF_SAMPLE_REGS_ABI_32 = 1, + PERF_SAMPLE_REGS_ABI_64 = 2, +}; + +/* + * Values for the memory transaction event qualifier, mostly for + * abort events. Multiple bits can be set. + */ +enum { + PERF_TXN_ELISION = (1 << 0), /* From elision */ + PERF_TXN_TRANSACTION = (1 << 1), /* From transaction */ + PERF_TXN_SYNC = (1 << 2), /* Instruction is related */ + PERF_TXN_ASYNC = (1 << 3), /* Instruction not related */ + PERF_TXN_RETRY = (1 << 4), /* Retry possible */ + PERF_TXN_CONFLICT = (1 << 5), /* Conflict abort */ + PERF_TXN_CAPACITY_WRITE = (1 << 6), /* Capacity write abort */ + PERF_TXN_CAPACITY_READ = (1 << 7), /* Capacity read abort */ + + PERF_TXN_MAX = (1 << 8), /* non-ABI */ + + /* bits 32..63 are reserved for the abort code */ + + PERF_TXN_ABORT_MASK = (0xffffffffULL << 32), + PERF_TXN_ABORT_SHIFT = 32, +}; + +/* + * The format of the data returned by read() on a perf event fd, + * as specified by attr.read_format: + * + * struct read_format { + * { u64 value; + * { u64 time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED + * { u64 time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING + * { u64 id; } && PERF_FORMAT_ID + * } && !PERF_FORMAT_GROUP + * + * { u64 nr; + * { u64 time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED + * { u64 time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING + * { u64 value; + * { u64 id; } && PERF_FORMAT_ID + * } cntr[nr]; + * } && PERF_FORMAT_GROUP + * }; + */ +enum perf_event_read_format { + PERF_FORMAT_TOTAL_TIME_ENABLED = 1U << 0, + PERF_FORMAT_TOTAL_TIME_RUNNING = 1U << 1, + PERF_FORMAT_ID = 1U << 2, + PERF_FORMAT_GROUP = 1U << 3, + + PERF_FORMAT_MAX = 1U << 4, /* non-ABI */ +}; + +#define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */ +#define PERF_ATTR_SIZE_VER1 72 /* add: config2 */ +#define PERF_ATTR_SIZE_VER2 80 /* add: branch_sample_type */ +#define PERF_ATTR_SIZE_VER3 96 /* add: sample_regs_user */ + /* add: sample_stack_user */ +#define PERF_ATTR_SIZE_VER4 104 /* add: sample_regs_intr */ +#define PERF_ATTR_SIZE_VER5 112 /* add: aux_watermark */ +#define PERF_ATTR_SIZE_VER6 120 /* add: aux_sample_size */ +#define PERF_ATTR_SIZE_VER7 128 /* add: sig_data */ + +/* + * Hardware event_id to monitor via a performance monitoring event: + * + * @sample_max_stack: Max number of frame pointers in a callchain, + * should be < /proc/sys/kernel/perf_event_max_stack + */ +struct perf_event_attr { + + /* + * Major type: hardware/software/tracepoint/etc. + */ + __u32 type; + + /* + * Size of the attr structure, for fwd/bwd compat. + */ + __u32 size; + + /* + * Type specific configuration information. + */ + __u64 config; + + union { + __u64 sample_period; + __u64 sample_freq; + }; + + __u64 sample_type; + __u64 read_format; + + __u64 disabled : 1, /* off by default */ + inherit : 1, /* children inherit it */ + pinned : 1, /* must always be on PMU */ + exclusive : 1, /* only group on PMU */ + exclude_user : 1, /* don't count user */ + exclude_kernel : 1, /* ditto kernel */ + exclude_hv : 1, /* ditto hypervisor */ + exclude_idle : 1, /* don't count when idle */ + mmap : 1, /* include mmap data */ + comm : 1, /* include comm data */ + freq : 1, /* use freq, not period */ + inherit_stat : 1, /* per task counts */ + enable_on_exec : 1, /* next exec enables */ + task : 1, /* trace fork/exit */ + watermark : 1, /* wakeup_watermark */ + /* + * precise_ip: + * + * 0 - SAMPLE_IP can have arbitrary skid + * 1 - SAMPLE_IP must have constant skid + * 2 - SAMPLE_IP requested to have 0 skid + * 3 - SAMPLE_IP must have 0 skid + * + * See also PERF_RECORD_MISC_EXACT_IP + */ + precise_ip : 2, /* skid constraint */ + mmap_data : 1, /* non-exec mmap data */ + sample_id_all : 1, /* sample_type all events */ + + exclude_host : 1, /* don't count in host */ + exclude_guest : 1, /* don't count in guest */ + + exclude_callchain_kernel : 1, /* exclude kernel callchains */ + exclude_callchain_user : 1, /* exclude user callchains */ + mmap2 : 1, /* include mmap with inode data */ + comm_exec : 1, /* flag comm events that are due to an exec */ + use_clockid : 1, /* use @clockid for time fields */ + context_switch : 1, /* context switch data */ + write_backward : 1, /* Write ring buffer from end to beginning */ + namespaces : 1, /* include namespaces data */ + ksymbol : 1, /* include ksymbol events */ + bpf_event : 1, /* include bpf events */ + aux_output : 1, /* generate AUX records instead of events */ + cgroup : 1, /* include cgroup events */ + text_poke : 1, /* include text poke events */ + build_id : 1, /* use build id in mmap2 events */ + inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */ + remove_on_exec : 1, /* event is removed from task on exec */ + sigtrap : 1, /* send synchronous SIGTRAP on event */ + __reserved_1 : 26; + + union { + __u32 wakeup_events; /* wakeup every n events */ + __u32 wakeup_watermark; /* bytes before wakeup */ + }; + + __u32 bp_type; + union { + __u64 bp_addr; + __u64 kprobe_func; /* for perf_kprobe */ + __u64 uprobe_path; /* for perf_uprobe */ + __u64 config1; /* extension of config */ + }; + union { + __u64 bp_len; + __u64 kprobe_addr; /* when kprobe_func == NULL */ + __u64 probe_offset; /* for perf_[k,u]probe */ + __u64 config2; /* extension of config1 */ + }; + __u64 branch_sample_type; /* enum perf_branch_sample_type */ + + /* + * Defines set of user regs to dump on samples. + * See asm/perf_regs.h for details. + */ + __u64 sample_regs_user; + + /* + * Defines size of the user stack to dump on samples. + */ + __u32 sample_stack_user; + + __s32 clockid; + /* + * Defines set of regs to dump for each sample + * state captured on: + * - precise = 0: PMU interrupt + * - precise > 0: sampled instruction + * + * See asm/perf_regs.h for details. + */ + __u64 sample_regs_intr; + + /* + * Wakeup watermark for AUX area + */ + __u32 aux_watermark; + __u16 sample_max_stack; + __u16 __reserved_2; + __u32 aux_sample_size; + __u32 __reserved_3; + + /* + * User provided data if sigtrap=1, passed back to user via + * siginfo_t::si_perf_data, e.g. to permit user to identify the event. + */ + __u64 sig_data; +}; + +/* + * Structure used by below PERF_EVENT_IOC_QUERY_BPF command + * to query bpf programs attached to the same perf tracepoint + * as the given perf event. + */ +struct perf_event_query_bpf { + /* + * The below ids array length + */ + __u32 ids_len; + /* + * Set by the kernel to indicate the number of + * available programs + */ + __u32 prog_cnt; + /* + * User provided buffer to store program ids + */ + __u32 ids[0]; +}; + +/* + * Ioctls that can be done on a perf event fd: + */ +#define PERF_EVENT_IOC_ENABLE _IO ('$', 0) +#define PERF_EVENT_IOC_DISABLE _IO ('$', 1) +#define PERF_EVENT_IOC_REFRESH _IO ('$', 2) +#define PERF_EVENT_IOC_RESET _IO ('$', 3) +#define PERF_EVENT_IOC_PERIOD _IOW('$', 4, __u64) +#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5) +#define PERF_EVENT_IOC_SET_FILTER _IOW('$', 6, char *) +#define PERF_EVENT_IOC_ID _IOR('$', 7, __u64 *) +#define PERF_EVENT_IOC_SET_BPF _IOW('$', 8, __u32) +#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW('$', 9, __u32) +#define PERF_EVENT_IOC_QUERY_BPF _IOWR('$', 10, struct perf_event_query_bpf *) +#define PERF_EVENT_IOC_MODIFY_ATTRIBUTES _IOW('$', 11, struct perf_event_attr *) + +enum perf_event_ioc_flags { + PERF_IOC_FLAG_GROUP = 1U << 0, +}; + +/* + * Structure of the page that can be mapped via mmap + */ +struct perf_event_mmap_page { + __u32 version; /* version number of this structure */ + __u32 compat_version; /* lowest version this is compat with */ + + /* + * Bits needed to read the hw events in user-space. + * + * u32 seq, time_mult, time_shift, index, width; + * u64 count, enabled, running; + * u64 cyc, time_offset; + * s64 pmc = 0; + * + * do { + * seq = pc->lock; + * barrier() + * + * enabled = pc->time_enabled; + * running = pc->time_running; + * + * if (pc->cap_usr_time && enabled != running) { + * cyc = rdtsc(); + * time_offset = pc->time_offset; + * time_mult = pc->time_mult; + * time_shift = pc->time_shift; + * } + * + * index = pc->index; + * count = pc->offset; + * if (pc->cap_user_rdpmc && index) { + * width = pc->pmc_width; + * pmc = rdpmc(index - 1); + * } + * + * barrier(); + * } while (pc->lock != seq); + * + * NOTE: for obvious reason this only works on self-monitoring + * processes. + */ + __u32 lock; /* seqlock for synchronization */ + __u32 index; /* hardware event identifier */ + __s64 offset; /* add to hardware event value */ + __u64 time_enabled; /* time event active */ + __u64 time_running; /* time event on cpu */ + union { + __u64 capabilities; + struct { + __u64 cap_bit0 : 1, /* Always 0, deprecated, see commit 860f085b74e9 */ + cap_bit0_is_deprecated : 1, /* Always 1, signals that bit 0 is zero */ + + cap_user_rdpmc : 1, /* The RDPMC instruction can be used to read counts */ + cap_user_time : 1, /* The time_{shift,mult,offset} fields are used */ + cap_user_time_zero : 1, /* The time_zero field is used */ + cap_user_time_short : 1, /* the time_{cycle,mask} fields are used */ + cap_____res : 58; + }; + }; + + /* + * If cap_user_rdpmc this field provides the bit-width of the value + * read using the rdpmc() or equivalent instruction. This can be used + * to sign extend the result like: + * + * pmc <<= 64 - width; + * pmc >>= 64 - width; // signed shift right + * count += pmc; + */ + __u16 pmc_width; + + /* + * If cap_usr_time the below fields can be used to compute the time + * delta since time_enabled (in ns) using rdtsc or similar. + * + * u64 quot, rem; + * u64 delta; + * + * quot = (cyc >> time_shift); + * rem = cyc & (((u64)1 << time_shift) - 1); + * delta = time_offset + quot * time_mult + + * ((rem * time_mult) >> time_shift); + * + * Where time_offset,time_mult,time_shift and cyc are read in the + * seqcount loop described above. This delta can then be added to + * enabled and possible running (if index), improving the scaling: + * + * enabled += delta; + * if (index) + * running += delta; + * + * quot = count / running; + * rem = count % running; + * count = quot * enabled + (rem * enabled) / running; + */ + __u16 time_shift; + __u32 time_mult; + __u64 time_offset; + /* + * If cap_usr_time_zero, the hardware clock (e.g. TSC) can be calculated + * from sample timestamps. + * + * time = timestamp - time_zero; + * quot = time / time_mult; + * rem = time % time_mult; + * cyc = (quot << time_shift) + (rem << time_shift) / time_mult; + * + * And vice versa: + * + * quot = cyc >> time_shift; + * rem = cyc & (((u64)1 << time_shift) - 1); + * timestamp = time_zero + quot * time_mult + + * ((rem * time_mult) >> time_shift); + */ + __u64 time_zero; + + __u32 size; /* Header size up to __reserved[] fields. */ + __u32 __reserved_1; + + /* + * If cap_usr_time_short, the hardware clock is less than 64bit wide + * and we must compute the 'cyc' value, as used by cap_usr_time, as: + * + * cyc = time_cycles + ((cyc - time_cycles) & time_mask) + * + * NOTE: this form is explicitly chosen such that cap_usr_time_short + * is a correction on top of cap_usr_time, and code that doesn't + * know about cap_usr_time_short still works under the assumption + * the counter doesn't wrap. + */ + __u64 time_cycles; + __u64 time_mask; + + /* + * Hole for extension of the self monitor capabilities + */ + + __u8 __reserved[116*8]; /* align to 1k. */ + + /* + * Control data for the mmap() data buffer. + * + * User-space reading the @data_head value should issue an smp_rmb(), + * after reading this value. + * + * When the mapping is PROT_WRITE the @data_tail value should be + * written by userspace to reflect the last read data, after issueing + * an smp_mb() to separate the data read from the ->data_tail store. + * In this case the kernel will not over-write unread data. + * + * See perf_output_put_handle() for the data ordering. + * + * data_{offset,size} indicate the location and size of the perf record + * buffer within the mmapped area. + */ + __u64 data_head; /* head in the data section */ + __u64 data_tail; /* user-space written tail */ + __u64 data_offset; /* where the buffer starts */ + __u64 data_size; /* data buffer size */ + + /* + * AUX area is defined by aux_{offset,size} fields that should be set + * by the userspace, so that + * + * aux_offset >= data_offset + data_size + * + * prior to mmap()ing it. Size of the mmap()ed area should be aux_size. + * + * Ring buffer pointers aux_{head,tail} have the same semantics as + * data_{head,tail} and same ordering rules apply. + */ + __u64 aux_head; + __u64 aux_tail; + __u64 aux_offset; + __u64 aux_size; +}; + +/* + * The current state of perf_event_header::misc bits usage: + * ('|' used bit, '-' unused bit) + * + * 012 CDEF + * |||---------|||| + * + * Where: + * 0-2 CPUMODE_MASK + * + * C PROC_MAP_PARSE_TIMEOUT + * D MMAP_DATA / COMM_EXEC / FORK_EXEC / SWITCH_OUT + * E MMAP_BUILD_ID / EXACT_IP / SCHED_OUT_PREEMPT + * F (reserved) + */ + +#define PERF_RECORD_MISC_CPUMODE_MASK (7 << 0) +#define PERF_RECORD_MISC_CPUMODE_UNKNOWN (0 << 0) +#define PERF_RECORD_MISC_KERNEL (1 << 0) +#define PERF_RECORD_MISC_USER (2 << 0) +#define PERF_RECORD_MISC_HYPERVISOR (3 << 0) +#define PERF_RECORD_MISC_GUEST_KERNEL (4 << 0) +#define PERF_RECORD_MISC_GUEST_USER (5 << 0) + +/* + * Indicates that /proc/PID/maps parsing are truncated by time out. + */ +#define PERF_RECORD_MISC_PROC_MAP_PARSE_TIMEOUT (1 << 12) +/* + * Following PERF_RECORD_MISC_* are used on different + * events, so can reuse the same bit position: + * + * PERF_RECORD_MISC_MMAP_DATA - PERF_RECORD_MMAP* events + * PERF_RECORD_MISC_COMM_EXEC - PERF_RECORD_COMM event + * PERF_RECORD_MISC_FORK_EXEC - PERF_RECORD_FORK event (perf internal) + * PERF_RECORD_MISC_SWITCH_OUT - PERF_RECORD_SWITCH* events + */ +#define PERF_RECORD_MISC_MMAP_DATA (1 << 13) +#define PERF_RECORD_MISC_COMM_EXEC (1 << 13) +#define PERF_RECORD_MISC_FORK_EXEC (1 << 13) +#define PERF_RECORD_MISC_SWITCH_OUT (1 << 13) +/* + * These PERF_RECORD_MISC_* flags below are safely reused + * for the following events: + * + * PERF_RECORD_MISC_EXACT_IP - PERF_RECORD_SAMPLE of precise events + * PERF_RECORD_MISC_SWITCH_OUT_PREEMPT - PERF_RECORD_SWITCH* events + * PERF_RECORD_MISC_MMAP_BUILD_ID - PERF_RECORD_MMAP2 event + * + * + * PERF_RECORD_MISC_EXACT_IP: + * Indicates that the content of PERF_SAMPLE_IP points to + * the actual instruction that triggered the event. See also + * perf_event_attr::precise_ip. + * + * PERF_RECORD_MISC_SWITCH_OUT_PREEMPT: + * Indicates that thread was preempted in TASK_RUNNING state. + * + * PERF_RECORD_MISC_MMAP_BUILD_ID: + * Indicates that mmap2 event carries build id data. + */ +#define PERF_RECORD_MISC_EXACT_IP (1 << 14) +#define PERF_RECORD_MISC_SWITCH_OUT_PREEMPT (1 << 14) +#define PERF_RECORD_MISC_MMAP_BUILD_ID (1 << 14) +/* + * Reserve the last bit to indicate some extended misc field + */ +#define PERF_RECORD_MISC_EXT_RESERVED (1 << 15) + +struct perf_event_header { + __u32 type; + __u16 misc; + __u16 size; +}; + +struct perf_ns_link_info { + __u64 dev; + __u64 ino; +}; + +enum { + NET_NS_INDEX = 0, + UTS_NS_INDEX = 1, + IPC_NS_INDEX = 2, + PID_NS_INDEX = 3, + USER_NS_INDEX = 4, + MNT_NS_INDEX = 5, + CGROUP_NS_INDEX = 6, + + NR_NAMESPACES, /* number of available namespaces */ +}; + +enum perf_event_type { + + /* + * If perf_event_attr.sample_id_all is set then all event types will + * have the sample_type selected fields related to where/when + * (identity) an event took place (TID, TIME, ID, STREAM_ID, CPU, + * IDENTIFIER) described in PERF_RECORD_SAMPLE below, it will be stashed + * just after the perf_event_header and the fields already present for + * the existing fields, i.e. at the end of the payload. That way a newer + * perf.data file will be supported by older perf tools, with these new + * optional fields being ignored. + * + * struct sample_id { + * { u32 pid, tid; } && PERF_SAMPLE_TID + * { u64 time; } && PERF_SAMPLE_TIME + * { u64 id; } && PERF_SAMPLE_ID + * { u64 stream_id;} && PERF_SAMPLE_STREAM_ID + * { u32 cpu, res; } && PERF_SAMPLE_CPU + * { u64 id; } && PERF_SAMPLE_IDENTIFIER + * } && perf_event_attr::sample_id_all + * + * Note that PERF_SAMPLE_IDENTIFIER duplicates PERF_SAMPLE_ID. The + * advantage of PERF_SAMPLE_IDENTIFIER is that its position is fixed + * relative to header.size. + */ + + /* + * The MMAP events record the PROT_EXEC mappings so that we can + * correlate userspace IPs to code. They have the following structure: + * + * struct { + * struct perf_event_header header; + * + * u32 pid, tid; + * u64 addr; + * u64 len; + * u64 pgoff; + * char filename[]; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_MMAP = 1, + + /* + * struct { + * struct perf_event_header header; + * u64 id; + * u64 lost; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_LOST = 2, + + /* + * struct { + * struct perf_event_header header; + * + * u32 pid, tid; + * char comm[]; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_COMM = 3, + + /* + * struct { + * struct perf_event_header header; + * u32 pid, ppid; + * u32 tid, ptid; + * u64 time; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_EXIT = 4, + + /* + * struct { + * struct perf_event_header header; + * u64 time; + * u64 id; + * u64 stream_id; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_THROTTLE = 5, + PERF_RECORD_UNTHROTTLE = 6, + + /* + * struct { + * struct perf_event_header header; + * u32 pid, ppid; + * u32 tid, ptid; + * u64 time; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_FORK = 7, + + /* + * struct { + * struct perf_event_header header; + * u32 pid, tid; + * + * struct read_format values; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_READ = 8, + + /* + * struct { + * struct perf_event_header header; + * + * # + * # Note that PERF_SAMPLE_IDENTIFIER duplicates PERF_SAMPLE_ID. + * # The advantage of PERF_SAMPLE_IDENTIFIER is that its position + * # is fixed relative to header. + * # + * + * { u64 id; } && PERF_SAMPLE_IDENTIFIER + * { u64 ip; } && PERF_SAMPLE_IP + * { u32 pid, tid; } && PERF_SAMPLE_TID + * { u64 time; } && PERF_SAMPLE_TIME + * { u64 addr; } && PERF_SAMPLE_ADDR + * { u64 id; } && PERF_SAMPLE_ID + * { u64 stream_id;} && PERF_SAMPLE_STREAM_ID + * { u32 cpu, res; } && PERF_SAMPLE_CPU + * { u64 period; } && PERF_SAMPLE_PERIOD + * + * { struct read_format values; } && PERF_SAMPLE_READ + * + * { u64 nr, + * u64 ips[nr]; } && PERF_SAMPLE_CALLCHAIN + * + * # + * # The RAW record below is opaque data wrt the ABI + * # + * # That is, the ABI doesn't make any promises wrt to + * # the stability of its content, it may vary depending + * # on event, hardware, kernel version and phase of + * # the moon. + * # + * # In other words, PERF_SAMPLE_RAW contents are not an ABI. + * # + * + * { u32 size; + * char data[size];}&& PERF_SAMPLE_RAW + * + * { u64 nr; + * { u64 hw_idx; } && PERF_SAMPLE_BRANCH_HW_INDEX + * { u64 from, to, flags } lbr[nr]; + * } && PERF_SAMPLE_BRANCH_STACK + * + * { u64 abi; # enum perf_sample_regs_abi + * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER + * + * { u64 size; + * char data[size]; + * u64 dyn_size; } && PERF_SAMPLE_STACK_USER + * + * { union perf_sample_weight + * { + * u64 full; && PERF_SAMPLE_WEIGHT + * #if defined(__LITTLE_ENDIAN_BITFIELD) + * struct { + * u32 var1_dw; + * u16 var2_w; + * u16 var3_w; + * } && PERF_SAMPLE_WEIGHT_STRUCT + * #elif defined(__BIG_ENDIAN_BITFIELD) + * struct { + * u16 var3_w; + * u16 var2_w; + * u32 var1_dw; + * } && PERF_SAMPLE_WEIGHT_STRUCT + * #endif + * } + * } + * { u64 data_src; } && PERF_SAMPLE_DATA_SRC + * { u64 transaction; } && PERF_SAMPLE_TRANSACTION + * { u64 abi; # enum perf_sample_regs_abi + * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR + * { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR + * { u64 size; + * char data[size]; } && PERF_SAMPLE_AUX + * { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE + * { u64 code_page_size;} && PERF_SAMPLE_CODE_PAGE_SIZE + * }; + */ + PERF_RECORD_SAMPLE = 9, + + /* + * The MMAP2 records are an augmented version of MMAP, they add + * maj, min, ino numbers to be used to uniquely identify each mapping + * + * struct { + * struct perf_event_header header; + * + * u32 pid, tid; + * u64 addr; + * u64 len; + * u64 pgoff; + * union { + * struct { + * u32 maj; + * u32 min; + * u64 ino; + * u64 ino_generation; + * }; + * struct { + * u8 build_id_size; + * u8 __reserved_1; + * u16 __reserved_2; + * u8 build_id[20]; + * }; + * }; + * u32 prot, flags; + * char filename[]; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_MMAP2 = 10, + + /* + * Records that new data landed in the AUX buffer part. + * + * struct { + * struct perf_event_header header; + * + * u64 aux_offset; + * u64 aux_size; + * u64 flags; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_AUX = 11, + + /* + * Indicates that instruction trace has started + * + * struct { + * struct perf_event_header header; + * u32 pid; + * u32 tid; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_ITRACE_START = 12, + + /* + * Records the dropped/lost sample number. + * + * struct { + * struct perf_event_header header; + * + * u64 lost; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_LOST_SAMPLES = 13, + + /* + * Records a context switch in or out (flagged by + * PERF_RECORD_MISC_SWITCH_OUT). See also + * PERF_RECORD_SWITCH_CPU_WIDE. + * + * struct { + * struct perf_event_header header; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_SWITCH = 14, + + /* + * CPU-wide version of PERF_RECORD_SWITCH with next_prev_pid and + * next_prev_tid that are the next (switching out) or previous + * (switching in) pid/tid. + * + * struct { + * struct perf_event_header header; + * u32 next_prev_pid; + * u32 next_prev_tid; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_SWITCH_CPU_WIDE = 15, + + /* + * struct { + * struct perf_event_header header; + * u32 pid; + * u32 tid; + * u64 nr_namespaces; + * { u64 dev, inode; } [nr_namespaces]; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_NAMESPACES = 16, + + /* + * Record ksymbol register/unregister events: + * + * struct { + * struct perf_event_header header; + * u64 addr; + * u32 len; + * u16 ksym_type; + * u16 flags; + * char name[]; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_KSYMBOL = 17, + + /* + * Record bpf events: + * enum perf_bpf_event_type { + * PERF_BPF_EVENT_UNKNOWN = 0, + * PERF_BPF_EVENT_PROG_LOAD = 1, + * PERF_BPF_EVENT_PROG_UNLOAD = 2, + * }; + * + * struct { + * struct perf_event_header header; + * u16 type; + * u16 flags; + * u32 id; + * u8 tag[BPF_TAG_SIZE]; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_BPF_EVENT = 18, + + /* + * struct { + * struct perf_event_header header; + * u64 id; + * char path[]; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_CGROUP = 19, + + /* + * Records changes to kernel text i.e. self-modified code. 'old_len' is + * the number of old bytes, 'new_len' is the number of new bytes. Either + * 'old_len' or 'new_len' may be zero to indicate, for example, the + * addition or removal of a trampoline. 'bytes' contains the old bytes + * followed immediately by the new bytes. + * + * struct { + * struct perf_event_header header; + * u64 addr; + * u16 old_len; + * u16 new_len; + * u8 bytes[]; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_TEXT_POKE = 20, + + /* + * Data written to the AUX area by hardware due to aux_output, may need + * to be matched to the event by an architecture-specific hardware ID. + * This records the hardware ID, but requires sample_id to provide the + * event ID. e.g. Intel PT uses this record to disambiguate PEBS-via-PT + * records from multiple events. + * + * struct { + * struct perf_event_header header; + * u64 hw_id; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_AUX_OUTPUT_HW_ID = 21, + + PERF_RECORD_MAX, /* non-ABI */ +}; + +enum perf_record_ksymbol_type { + PERF_RECORD_KSYMBOL_TYPE_UNKNOWN = 0, + PERF_RECORD_KSYMBOL_TYPE_BPF = 1, + /* + * Out of line code such as kprobe-replaced instructions or optimized + * kprobes or ftrace trampolines. + */ + PERF_RECORD_KSYMBOL_TYPE_OOL = 2, + PERF_RECORD_KSYMBOL_TYPE_MAX /* non-ABI */ +}; + +#define PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER (1 << 0) + +enum perf_bpf_event_type { + PERF_BPF_EVENT_UNKNOWN = 0, + PERF_BPF_EVENT_PROG_LOAD = 1, + PERF_BPF_EVENT_PROG_UNLOAD = 2, + PERF_BPF_EVENT_MAX, /* non-ABI */ +}; + +#define PERF_MAX_STACK_DEPTH 127 +#define PERF_MAX_CONTEXTS_PER_STACK 8 + +enum perf_callchain_context { + PERF_CONTEXT_HV = (__u64)-32, + PERF_CONTEXT_KERNEL = (__u64)-128, + PERF_CONTEXT_USER = (__u64)-512, + + PERF_CONTEXT_GUEST = (__u64)-2048, + PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176, + PERF_CONTEXT_GUEST_USER = (__u64)-2560, + + PERF_CONTEXT_MAX = (__u64)-4095, +}; + +/** + * PERF_RECORD_AUX::flags bits + */ +#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */ +#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */ +#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */ +#define PERF_AUX_FLAG_COLLISION 0x08 /* sample collided with another */ +#define PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK 0xff00 /* PMU specific trace format type */ + +/* CoreSight PMU AUX buffer formats */ +#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */ +#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */ + +#define PERF_FLAG_FD_NO_GROUP (1UL << 0) +#define PERF_FLAG_FD_OUTPUT (1UL << 1) +#define PERF_FLAG_PID_CGROUP (1UL << 2) /* pid=cgroup id, per-cpu mode only */ +#define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */ + +#if defined(__LITTLE_ENDIAN_BITFIELD) +union perf_mem_data_src { + __u64 val; + struct { + __u64 mem_op:5, /* type of opcode */ + mem_lvl:14, /* memory hierarchy level */ + mem_snoop:5, /* snoop mode */ + mem_lock:2, /* lock instr */ + mem_dtlb:7, /* tlb access */ + mem_lvl_num:4, /* memory hierarchy level number */ + mem_remote:1, /* remote */ + mem_snoopx:2, /* snoop mode, ext */ + mem_blk:3, /* access blocked */ + mem_hops:3, /* hop level */ + mem_rsvd:18; + }; +}; +#elif defined(__BIG_ENDIAN_BITFIELD) +union perf_mem_data_src { + __u64 val; + struct { + __u64 mem_rsvd:18, + mem_hops:3, /* hop level */ + mem_blk:3, /* access blocked */ + mem_snoopx:2, /* snoop mode, ext */ + mem_remote:1, /* remote */ + mem_lvl_num:4, /* memory hierarchy level number */ + mem_dtlb:7, /* tlb access */ + mem_lock:2, /* lock instr */ + mem_snoop:5, /* snoop mode */ + mem_lvl:14, /* memory hierarchy level */ + mem_op:5; /* type of opcode */ + }; +}; +#else +#error "Unknown endianness" +#endif + +/* type of opcode (load/store/prefetch,code) */ +#define PERF_MEM_OP_NA 0x01 /* not available */ +#define PERF_MEM_OP_LOAD 0x02 /* load instruction */ +#define PERF_MEM_OP_STORE 0x04 /* store instruction */ +#define PERF_MEM_OP_PFETCH 0x08 /* prefetch */ +#define PERF_MEM_OP_EXEC 0x10 /* code (execution) */ +#define PERF_MEM_OP_SHIFT 0 + +/* + * PERF_MEM_LVL_* namespace being depricated to some extent in the + * favour of newer composite PERF_MEM_{LVLNUM_,REMOTE_,SNOOPX_} fields. + * Supporting this namespace inorder to not break defined ABIs. + * + * memory hierarchy (memory level, hit or miss) + */ +#define PERF_MEM_LVL_NA 0x01 /* not available */ +#define PERF_MEM_LVL_HIT 0x02 /* hit level */ +#define PERF_MEM_LVL_MISS 0x04 /* miss level */ +#define PERF_MEM_LVL_L1 0x08 /* L1 */ +#define PERF_MEM_LVL_LFB 0x10 /* Line Fill Buffer */ +#define PERF_MEM_LVL_L2 0x20 /* L2 */ +#define PERF_MEM_LVL_L3 0x40 /* L3 */ +#define PERF_MEM_LVL_LOC_RAM 0x80 /* Local DRAM */ +#define PERF_MEM_LVL_REM_RAM1 0x100 /* Remote DRAM (1 hop) */ +#define PERF_MEM_LVL_REM_RAM2 0x200 /* Remote DRAM (2 hops) */ +#define PERF_MEM_LVL_REM_CCE1 0x400 /* Remote Cache (1 hop) */ +#define PERF_MEM_LVL_REM_CCE2 0x800 /* Remote Cache (2 hops) */ +#define PERF_MEM_LVL_IO 0x1000 /* I/O memory */ +#define PERF_MEM_LVL_UNC 0x2000 /* Uncached memory */ +#define PERF_MEM_LVL_SHIFT 5 + +#define PERF_MEM_REMOTE_REMOTE 0x01 /* Remote */ +#define PERF_MEM_REMOTE_SHIFT 37 + +#define PERF_MEM_LVLNUM_L1 0x01 /* L1 */ +#define PERF_MEM_LVLNUM_L2 0x02 /* L2 */ +#define PERF_MEM_LVLNUM_L3 0x03 /* L3 */ +#define PERF_MEM_LVLNUM_L4 0x04 /* L4 */ +/* 5-0xa available */ +#define PERF_MEM_LVLNUM_ANY_CACHE 0x0b /* Any cache */ +#define PERF_MEM_LVLNUM_LFB 0x0c /* LFB */ +#define PERF_MEM_LVLNUM_RAM 0x0d /* RAM */ +#define PERF_MEM_LVLNUM_PMEM 0x0e /* PMEM */ +#define PERF_MEM_LVLNUM_NA 0x0f /* N/A */ + +#define PERF_MEM_LVLNUM_SHIFT 33 + +/* snoop mode */ +#define PERF_MEM_SNOOP_NA 0x01 /* not available */ +#define PERF_MEM_SNOOP_NONE 0x02 /* no snoop */ +#define PERF_MEM_SNOOP_HIT 0x04 /* snoop hit */ +#define PERF_MEM_SNOOP_MISS 0x08 /* snoop miss */ +#define PERF_MEM_SNOOP_HITM 0x10 /* snoop hit modified */ +#define PERF_MEM_SNOOP_SHIFT 19 + +#define PERF_MEM_SNOOPX_FWD 0x01 /* forward */ +/* 1 free */ +#define PERF_MEM_SNOOPX_SHIFT 38 + +/* locked instruction */ +#define PERF_MEM_LOCK_NA 0x01 /* not available */ +#define PERF_MEM_LOCK_LOCKED 0x02 /* locked transaction */ +#define PERF_MEM_LOCK_SHIFT 24 + +/* TLB access */ +#define PERF_MEM_TLB_NA 0x01 /* not available */ +#define PERF_MEM_TLB_HIT 0x02 /* hit level */ +#define PERF_MEM_TLB_MISS 0x04 /* miss level */ +#define PERF_MEM_TLB_L1 0x08 /* L1 */ +#define PERF_MEM_TLB_L2 0x10 /* L2 */ +#define PERF_MEM_TLB_WK 0x20 /* Hardware Walker*/ +#define PERF_MEM_TLB_OS 0x40 /* OS fault handler */ +#define PERF_MEM_TLB_SHIFT 26 + +/* Access blocked */ +#define PERF_MEM_BLK_NA 0x01 /* not available */ +#define PERF_MEM_BLK_DATA 0x02 /* data could not be forwarded */ +#define PERF_MEM_BLK_ADDR 0x04 /* address conflict */ +#define PERF_MEM_BLK_SHIFT 40 + +/* hop level */ +#define PERF_MEM_HOPS_0 0x01 /* remote core, same node */ +/* 2-7 available */ +#define PERF_MEM_HOPS_SHIFT 43 + +#define PERF_MEM_S(a, s) \ + (((__u64)PERF_MEM_##a##_##s) << PERF_MEM_##a##_SHIFT) + +/* + * single taken branch record layout: + * + * from: source instruction (may not always be a branch insn) + * to: branch target + * mispred: branch target was mispredicted + * predicted: branch target was predicted + * + * support for mispred, predicted is optional. In case it + * is not supported mispred = predicted = 0. + * + * in_tx: running in a hardware transaction + * abort: aborting a hardware transaction + * cycles: cycles from last branch (or 0 if not supported) + * type: branch type + */ +struct perf_branch_entry { + __u64 from; + __u64 to; + __u64 mispred:1, /* target mispredicted */ + predicted:1,/* target predicted */ + in_tx:1, /* in transaction */ + abort:1, /* transaction abort */ + cycles:16, /* cycle count to last branch */ + type:4, /* branch type */ + reserved:40; +}; + +union perf_sample_weight { + __u64 full; +#if defined(__LITTLE_ENDIAN_BITFIELD) + struct { + __u32 var1_dw; + __u16 var2_w; + __u16 var3_w; + }; +#elif defined(__BIG_ENDIAN_BITFIELD) + struct { + __u16 var3_w; + __u16 var2_w; + __u32 var1_dw; + }; +#else +#error "Unknown endianness" +#endif +}; + +#endif /* _UAPI_LINUX_PERF_EVENT_H */ diff --git a/scripts/sync-kernel.sh b/scripts/sync-kernel.sh index 45b4212..1653c2e 100755 --- a/scripts/sync-kernel.sh +++ b/scripts/sync-kernel.sh @@ -47,6 +47,7 @@ PATH_MAP=( \ [tools/include/uapi/linux/netlink.h]=include/uapi/linux/netlink.h \ [tools/include/uapi/linux/pkt_cls.h]=include/uapi/linux/pkt_cls.h \ [tools/include/uapi/linux/pkt_sched.h]=include/uapi/linux/pkt_sched.h \ + [include/uapi/linux/perf_event.h]=include/uapi/linux/perf_event.h \ [Documentation/bpf/libbpf]=docs \ ) From da06e3efcc3c00f103600ff8282edcab18d68e13 Mon Sep 17 00:00:00 2001 From: Kajol Jain Date: Mon, 6 Dec 2021 14:47:46 +0530 Subject: [PATCH 090/131] perf: Add new macros for mem_hops field Add new macros for mem_hops field which can be used to represent remote-node, socket and board level details. Currently the code had macro for HOPS_0, which corresponds to data coming from another core but same node. Add new macros for HOPS_1 to HOPS_3 to represent remote-node, socket and board level data. For ex: Encodings for mem_hops fields with L2 cache: L2 - local L2 L2 | REMOTE | HOPS_0 - remote core, same node L2 L2 | REMOTE | HOPS_1 - remote node, same socket L2 L2 | REMOTE | HOPS_2 - remote socket, same board L2 L2 | REMOTE | HOPS_3 - remote board L2 Signed-off-by: Kajol Jain Acked-by: Peter Zijlstra (Intel) Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20211206091749.87585-2-kjain@linux.ibm.com --- include/uapi/linux/perf_event.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index bd8860e..1b65042 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -1332,7 +1332,10 @@ union perf_mem_data_src { /* hop level */ #define PERF_MEM_HOPS_0 0x01 /* remote core, same node */ -/* 2-7 available */ +#define PERF_MEM_HOPS_1 0x02 /* remote node, same socket */ +#define PERF_MEM_HOPS_2 0x03 /* remote socket, same board */ +#define PERF_MEM_HOPS_3 0x04 /* remote board */ +/* 5-7 available */ #define PERF_MEM_HOPS_SHIFT 43 #define PERF_MEM_S(a, s) \ From 57df70e180d774726551f70d00d415c39e0c9747 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Fri, 21 Jan 2022 11:09:52 +0100 Subject: [PATCH 091/131] bpf: introduce BPF_F_XDP_HAS_FRAGS flag in prog_flags loading the ebpf program Introduce BPF_F_XDP_HAS_FRAGS and the related field in bpf_prog_aux in order to notify the driver the loaded program support xdp frags. Acked-by: Toke Hoiland-Jorgensen Acked-by: John Fastabend Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/db2e8075b7032a356003f407d1b0deb99adaa0ed.1642758637.git.lorenzo@kernel.org Signed-off-by: Alexei Starovoitov --- include/uapi/linux/bpf.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index fe2272d..945649c 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1113,6 +1113,11 @@ enum bpf_link_type { */ #define BPF_F_SLEEPABLE (1U << 4) +/* If BPF_F_XDP_HAS_FRAGS is used in BPF_PROG_LOAD command, the loaded program + * fully support xdp frags. + */ +#define BPF_F_XDP_HAS_FRAGS (1U << 5) + /* When BPF ldimm64's insn[0].src_reg != 0 then this can have * the following extensions: * From 9af0e1937617d5d1b50db3ce90d0354af3b656e4 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Fri, 21 Jan 2022 11:09:54 +0100 Subject: [PATCH 092/131] bpf: introduce bpf_xdp_get_buff_len helper Introduce bpf_xdp_get_buff_len helper in order to return the xdp buffer total size (linear and paged area) Acked-by: Toke Hoiland-Jorgensen Acked-by: John Fastabend Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/aac9ac3504c84026cf66a3c71b7c5ae89bc991be.1642758637.git.lorenzo@kernel.org Signed-off-by: Alexei Starovoitov --- include/uapi/linux/bpf.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 945649c..5a28772 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -5054,6 +5054,12 @@ union bpf_attr { * This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. + * + * u64 bpf_xdp_get_buff_len(struct xdp_buff *xdp_md) + * Description + * Get the total size of a given xdp buff (linear and paged area) + * Return + * The total size of a given xdp buffer. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5244,6 +5250,7 @@ union bpf_attr { FN(get_func_arg_cnt), \ FN(get_retval), \ FN(set_retval), \ + FN(xdp_get_buff_len), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper From 4ae89237d4d2113b9a7447e976d13d3ddc5f8033 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Fri, 21 Jan 2022 11:10:01 +0100 Subject: [PATCH 093/131] libbpf: Add SEC name for xdp frags programs Introduce support for the following SEC entries for XDP frags property: - SEC("xdp.frags") - SEC("xdp.frags/devmap") - SEC("xdp.frags/cpumap") Acked-by: Toke Hoiland-Jorgensen Acked-by: John Fastabend Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/af23b6e4841c171ad1af01917839b77847a4bc27.1642758637.git.lorenzo@kernel.org Signed-off-by: Alexei Starovoitov --- src/libbpf.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libbpf.c b/src/libbpf.c index fc6d530..a8c7503 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -235,6 +235,8 @@ enum sec_def_flags { SEC_SLEEPABLE = 8, /* allow non-strict prefix matching */ SEC_SLOPPY_PFX = 16, + /* BPF program support non-linear XDP buffer */ + SEC_XDP_FRAGS = 32, }; struct bpf_sec_def { @@ -6570,6 +6572,9 @@ static int libbpf_preload_prog(struct bpf_program *prog, if (def & SEC_SLEEPABLE) opts->prog_flags |= BPF_F_SLEEPABLE; + if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) + opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; + if ((prog->type == BPF_PROG_TYPE_TRACING || prog->type == BPF_PROG_TYPE_LSM || prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) { @@ -8608,8 +8613,11 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("lsm.s/", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm), SEC_DEF("iter/", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter), SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE), + SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS), SEC_DEF("xdp_devmap/", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE), + SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS), SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE), + SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS), SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE | SEC_SLOPPY_PFX), SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE | SEC_SLOPPY_PFX), From fb0b8a7ceab14fab8071a1c5195bae59135a232c Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Fri, 21 Jan 2022 11:10:03 +0100 Subject: [PATCH 094/131] net: xdp: introduce bpf_xdp_pointer utility routine Similar to skb_header_pointer, introduce bpf_xdp_pointer utility routine to return a pointer to a given position in the xdp_buff if the requested area (offset + len) is contained in a contiguous memory area otherwise it will be copied in a bounce buffer provided by the caller. Similar to the tc counterpart, introduce the two following xdp helpers: - bpf_xdp_load_bytes - bpf_xdp_store_bytes Reviewed-by: Eelco Chaudron Acked-by: Toke Hoiland-Jorgensen Acked-by: John Fastabend Acked-by: Jakub Kicinski Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/ab285c1efdd5b7a9d361348b1e7d3ef49f6382b3.1642758637.git.lorenzo@kernel.org Signed-off-by: Alexei Starovoitov --- include/uapi/linux/bpf.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 5a28772..16a7574 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -5060,6 +5060,22 @@ union bpf_attr { * Get the total size of a given xdp buff (linear and paged area) * Return * The total size of a given xdp buffer. + * + * long bpf_xdp_load_bytes(struct xdp_buff *xdp_md, u32 offset, void *buf, u32 len) + * Description + * This helper is provided as an easy way to load data from a + * xdp buffer. It can be used to load *len* bytes from *offset* from + * the frame associated to *xdp_md*, into the buffer pointed by + * *buf*. + * Return + * 0 on success, or a negative error in case of failure. + * + * long bpf_xdp_store_bytes(struct xdp_buff *xdp_md, u32 offset, void *buf, u32 len) + * Description + * Store *len* bytes from buffer *buf* into the frame + * associated to *xdp_md*, at *offset*. + * Return + * 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5251,6 +5267,8 @@ union bpf_attr { FN(get_retval), \ FN(set_retval), \ FN(xdp_get_buff_len), \ + FN(xdp_load_bytes), \ + FN(xdp_store_bytes), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper From a9491bb920e7f5cb6cce6aca99cc552316b936dd Mon Sep 17 00:00:00 2001 From: Kenny Yu Date: Mon, 24 Jan 2022 10:54:01 -0800 Subject: [PATCH 095/131] bpf: Add bpf_copy_from_user_task() helper This adds a helper for bpf programs to read the memory of other tasks. As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process. Signed-off-by: Kenny Yu Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20220124185403.468466-3-kennyyu@fb.com Signed-off-by: Alexei Starovoitov --- include/uapi/linux/bpf.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 16a7574..4a2f704 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -5076,6 +5076,16 @@ union bpf_attr { * associated to *xdp_md*, at *offset*. * Return * 0 on success, or a negative error in case of failure. + * + * long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) + * Description + * Read *size* bytes from user space address *user_ptr* in *tsk*'s + * address space, and stores the data in *dst*. *flags* is not + * used yet and is provided for future extensibility. This helper + * can only be used by sleepable programs. + * Return + * 0 on success, or a negative error in case of failure. On error + * *dst* buffer is zeroed out. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5269,6 +5279,7 @@ union bpf_attr { FN(xdp_get_buff_len), \ FN(xdp_load_bytes), \ FN(xdp_store_bytes), \ + FN(copy_from_user_task), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper From 04aef6ce9bf012e4aaf8bb3def219a5af984745c Mon Sep 17 00:00:00 2001 From: Kenny Yu Date: Mon, 24 Jan 2022 10:54:02 -0800 Subject: [PATCH 096/131] libbpf: Add "iter.s" section for sleepable bpf iterator programs This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable. Signed-off-by: Kenny Yu Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20220124185403.468466-4-kennyyu@fb.com Signed-off-by: Alexei Starovoitov --- src/libbpf.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libbpf.c b/src/libbpf.c index a8c7503..57894f1 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -8612,6 +8612,7 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("lsm/", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm), SEC_DEF("lsm.s/", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm), SEC_DEF("iter/", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter), + SEC_DEF("iter.s/", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter), SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE), SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS), SEC_DEF("xdp_devmap/", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE), From 1af0f62fac06313e02eaf263a4d2aa1aaa5a503a Mon Sep 17 00:00:00 2001 From: Christy Lee Date: Mon, 24 Jan 2022 16:59:22 -0800 Subject: [PATCH 097/131] libbpf: Mark bpf_object__open_buffer() API deprecated Deprecate bpf_object__open_buffer() API in favor of the unified opts-based bpf_object__open_mem() API. Signed-off-by: Christy Lee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220125005923.418339-2-christylee@fb.com --- src/libbpf.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libbpf.h b/src/libbpf.h index 9467006..281cb19 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -180,6 +180,7 @@ bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, const struct bpf_object_open_opts *opts); /* deprecated bpf_object__open variants */ +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__open_mem() instead") LIBBPF_API struct bpf_object * bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, const char *name); From c0c3f46ca6c7ee005c5264580cdf394004663aba Mon Sep 17 00:00:00 2001 From: Christy Lee Date: Mon, 24 Jan 2022 17:09:17 -0800 Subject: [PATCH 098/131] libbpf: Mark bpf_object__open_xattr() deprecated Mark bpf_object__open_xattr() as deprecated, use bpf_object__open_file() instead. [0] Closes: https://github.com/libbpf/libbpf/issues/287 Signed-off-by: Christy Lee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220125010917.679975-1-christylee@fb.com --- src/libbpf.c | 2 +- src/libbpf.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libbpf.c b/src/libbpf.c index 57894f1..9e248ab 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -9460,7 +9460,7 @@ static int bpf_prog_load_xattr2(const struct bpf_prog_load_attr *attr, open_attr.file = attr->file; open_attr.prog_type = attr->prog_type; - obj = bpf_object__open_xattr(&open_attr); + obj = __bpf_object__open_xattr(&open_attr, 0); err = libbpf_get_error(obj); if (err) return libbpf_err(-ENOENT); diff --git a/src/libbpf.h b/src/libbpf.h index 281cb19..1bd0215 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -184,6 +184,7 @@ LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__open_mem() instead") LIBBPF_API struct bpf_object * bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, const char *name); +LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__open_file() instead") LIBBPF_API struct bpf_object * bpf_object__open_xattr(struct bpf_object_open_attr *attr); From d2818a8f2c05f5e5f4f732c1015eced0fb4b2772 Mon Sep 17 00:00:00 2001 From: Kenta Tada Date: Mon, 24 Jan 2022 23:16:21 +0900 Subject: [PATCH 099/131] libbpf: Fix the incorrect register read for syscalls on x86_64 Currently, rcx is read as the fourth parameter of syscall on x86_64. But x86_64 Linux System Call convention uses r10 actually. This commit adds the wrapper for users who want to access to syscall params to analyze the user space. Signed-off-by: Kenta Tada Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220124141622.4378-3-Kenta.Tada@sony.com --- src/bpf_tracing.h | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index 90f56b0..032ba80 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -70,6 +70,7 @@ #define __PT_PARM2_REG si #define __PT_PARM3_REG dx #define __PT_PARM4_REG cx +#define __PT_PARM4_REG_SYSCALL r10 /* syscall uses r10 */ #define __PT_PARM5_REG r8 #define __PT_RET_REG sp #define __PT_FP_REG bp @@ -99,6 +100,7 @@ #define __PT_PARM2_REG rsi #define __PT_PARM3_REG rdx #define __PT_PARM4_REG rcx +#define __PT_PARM4_REG_SYSCALL r10 /* syscall uses r10 */ #define __PT_PARM5_REG r8 #define __PT_RET_REG rsp #define __PT_FP_REG rbp @@ -263,6 +265,26 @@ struct pt_regs; #endif +#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1(x) +#define PT_REGS_PARM2_SYSCALL(x) PT_REGS_PARM2(x) +#define PT_REGS_PARM3_SYSCALL(x) PT_REGS_PARM3(x) +#ifdef __PT_PARM4_REG_SYSCALL +#define PT_REGS_PARM4_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG_SYSCALL) +#else /* __PT_PARM4_REG_SYSCALL */ +#define PT_REGS_PARM4_SYSCALL(x) PT_REGS_PARM4(x) +#endif +#define PT_REGS_PARM5_SYSCALL(x) PT_REGS_PARM5(x) + +#define PT_REGS_PARM1_CORE_SYSCALL(x) PT_REGS_PARM1_CORE(x) +#define PT_REGS_PARM2_CORE_SYSCALL(x) PT_REGS_PARM2_CORE(x) +#define PT_REGS_PARM3_CORE_SYSCALL(x) PT_REGS_PARM3_CORE(x) +#ifdef __PT_PARM4_REG_SYSCALL +#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG_SYSCALL) +#else /* __PT_PARM4_REG_SYSCALL */ +#define PT_REGS_PARM4_CORE_SYSCALL(x) PT_REGS_PARM4_CORE(x) +#endif +#define PT_REGS_PARM5_CORE_SYSCALL(x) PT_REGS_PARM5_CORE(x) + #else /* defined(bpf_target_defined) */ #define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) @@ -290,6 +312,18 @@ struct pt_regs; #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM2_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM3_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM4_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM5_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) + +#define PT_REGS_PARM1_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM2_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM3_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM4_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM5_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) + #endif /* defined(bpf_target_defined) */ #ifndef ___bpf_concat From e72ac682aee17faa50967e8e2982b5697cdaa5ae Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 24 Jan 2022 11:42:48 -0800 Subject: [PATCH 100/131] libbpf: hide and discourage inconsistently named getters Move a bunch of "getters" into libbpf_legacy.h to keep them there in libbpf 1.0. See [0] for discussion of "Discouraged APIs". These getters don't add any maintenance burden and are simple alias, but they are inconsistent in naming. So keep them in libbpf_legacy.h instead of libbpf.h to "hide" them in favor of preferred getters ([1]). Also add two missing getters: bpf_program__type() and bpf_program__expected_attach_type(). [0] https://github.com/libbpf/libbpf/wiki/Libbpf:-the-road-to-v1.0#handling-deprecation-of-apis-and-functionality [1] Closes: https://github.com/libbpf/libbpf/issues/307 Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20220124194254.2051434-2-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- src/btf.h | 5 +---- src/libbpf.c | 21 ++++++++++++--------- src/libbpf.h | 5 ++--- src/libbpf.map | 2 ++ src/libbpf_internal.h | 3 +++ src/libbpf_legacy.h | 17 +++++++++++++++++ 6 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/btf.h b/src/btf.h index 51862fd..96b44d5 100644 --- a/src/btf.h +++ b/src/btf.h @@ -147,8 +147,6 @@ LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id); LIBBPF_API int btf__align_of(const struct btf *btf, __u32 id); LIBBPF_API int btf__fd(const struct btf *btf); LIBBPF_API void btf__set_fd(struct btf *btf, int fd); -LIBBPF_DEPRECATED_SINCE(0, 7, "use btf__raw_data() instead") -LIBBPF_API const void *btf__get_raw_data(const struct btf *btf, __u32 *size); LIBBPF_API const void *btf__raw_data(const struct btf *btf, __u32 *size); LIBBPF_API const char *btf__name_by_offset(const struct btf *btf, __u32 offset); LIBBPF_API const char *btf__str_by_offset(const struct btf *btf, __u32 offset); @@ -159,8 +157,7 @@ LIBBPF_API int btf__get_map_kv_tids(const struct btf *btf, const char *map_name, LIBBPF_API struct btf_ext *btf_ext__new(const __u8 *data, __u32 size); LIBBPF_API void btf_ext__free(struct btf_ext *btf_ext); -LIBBPF_API const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, - __u32 *size); +LIBBPF_API const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size); LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_func_info was never meant as a public API and has wrong assumptions embedded in it; it will be removed in the future libbpf versions") int btf_ext__reloc_func_info(const struct btf *btf, const struct btf_ext *btf_ext, diff --git a/src/libbpf.c b/src/libbpf.c index 9e248ab..4ce94f4 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -7896,10 +7896,8 @@ int bpf_map__set_pin_path(struct bpf_map *map, const char *path) return 0; } -const char *bpf_map__get_pin_path(const struct bpf_map *map) -{ - return map->pin_path; -} +__alias(bpf_map__pin_path) +const char *bpf_map__get_pin_path(const struct bpf_map *map); const char *bpf_map__pin_path(const struct bpf_map *map) { @@ -8464,7 +8462,10 @@ static int bpf_program_nth_fd(const struct bpf_program *prog, int n) return fd; } -enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog) +__alias(bpf_program__type) +enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); + +enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) { return prog->type; } @@ -8508,8 +8509,10 @@ BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS); BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP); -enum bpf_attach_type -bpf_program__get_expected_attach_type(const struct bpf_program *prog) +__alias(bpf_program__expected_attach_type) +enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog); + +enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog) { return prog->expected_attach_type; } @@ -9477,7 +9480,7 @@ static int bpf_prog_load_xattr2(const struct bpf_prog_load_attr *attr, bpf_program__set_expected_attach_type(prog, attach_type); } - if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) { + if (bpf_program__type(prog) == BPF_PROG_TYPE_UNSPEC) { /* * we haven't guessed from section name and user * didn't provide a fallback type, too bad... @@ -10528,7 +10531,7 @@ bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id return libbpf_err_ptr(-ENOMEM); link->detach = &bpf_link__detach_fd; - attach_type = bpf_program__get_expected_attach_type(prog); + attach_type = bpf_program__expected_attach_type(prog); link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts); if (link_fd < 0) { link_fd = -errno; diff --git a/src/libbpf.h b/src/libbpf.h index 1bd0215..56bddb1 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -607,12 +607,12 @@ LIBBPF_API int bpf_program__set_struct_ops(struct bpf_program *prog); LIBBPF_API int bpf_program__set_extension(struct bpf_program *prog); LIBBPF_API int bpf_program__set_sk_lookup(struct bpf_program *prog); -LIBBPF_API enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); +LIBBPF_API enum bpf_prog_type bpf_program__type(const struct bpf_program *prog); LIBBPF_API void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type); LIBBPF_API enum bpf_attach_type -bpf_program__get_expected_attach_type(const struct bpf_program *prog); +bpf_program__expected_attach_type(const struct bpf_program *prog); LIBBPF_API void bpf_program__set_expected_attach_type(struct bpf_program *prog, enum bpf_attach_type type); @@ -760,7 +760,6 @@ LIBBPF_API bool bpf_map__is_offload_neutral(const struct bpf_map *map); */ LIBBPF_API bool bpf_map__is_internal(const struct bpf_map *map); LIBBPF_API int bpf_map__set_pin_path(struct bpf_map *map, const char *path); -LIBBPF_API const char *bpf_map__get_pin_path(const struct bpf_map *map); LIBBPF_API const char *bpf_map__pin_path(const struct bpf_map *map); LIBBPF_API bool bpf_map__is_pinned(const struct bpf_map *map); LIBBPF_API int bpf_map__pin(struct bpf_map *map, const char *path); diff --git a/src/libbpf.map b/src/libbpf.map index e10f082..aef6253 100644 --- a/src/libbpf.map +++ b/src/libbpf.map @@ -424,10 +424,12 @@ LIBBPF_0.6.0 { LIBBPF_0.7.0 { global: bpf_btf_load; + bpf_program__expected_attach_type; bpf_program__log_buf; bpf_program__log_level; bpf_program__set_log_buf; bpf_program__set_log_level; + bpf_program__type; bpf_xdp_attach; bpf_xdp_detach; bpf_xdp_query; diff --git a/src/libbpf_internal.h b/src/libbpf_internal.h index 1565679..bc86b82 100644 --- a/src/libbpf_internal.h +++ b/src/libbpf_internal.h @@ -92,6 +92,9 @@ # define offsetofend(TYPE, FIELD) \ (offsetof(TYPE, FIELD) + sizeof(((TYPE *)0)->FIELD)) #endif +#ifndef __alias +#define __alias(symbol) __attribute__((alias(#symbol))) +#endif /* Check whether a string `str` has prefix `pfx`, regardless if `pfx` is * a string literal known at compilation time or char * pointer known only at diff --git a/src/libbpf_legacy.h b/src/libbpf_legacy.h index 3c2b281..a283cf0 100644 --- a/src/libbpf_legacy.h +++ b/src/libbpf_legacy.h @@ -86,6 +86,23 @@ LIBBPF_API int libbpf_set_strict_mode(enum libbpf_strict_mode mode); #define DECLARE_LIBBPF_OPTS LIBBPF_OPTS +/* "Discouraged" APIs which don't follow consistent libbpf naming patterns. + * They are normally a trivial aliases or wrappers for proper APIs and are + * left to minimize unnecessary disruption for users of libbpf. But they + * shouldn't be used going forward. + */ + +struct bpf_program; +struct bpf_map; +struct btf; +struct btf_ext; + +LIBBPF_API enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); +LIBBPF_API enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog); +LIBBPF_API const char *bpf_map__get_pin_path(const struct bpf_map *map); +LIBBPF_API const void *btf__get_raw_data(const struct btf *btf, __u32 *size); +LIBBPF_API const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size); + #ifdef __cplusplus } /* extern "C" */ #endif From 638311dcb8400b32fdc66b692ea47bc06879bdb0 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 24 Jan 2022 11:42:49 -0800 Subject: [PATCH 101/131] libbpf: deprecate bpf_map__resize() Deprecated bpf_map__resize() in favor of bpf_map__set_max_entries() setter. In addition to having a surprising name (users often don't realize that they need to use bpf_map__resize()), the name also implies some magic way of resizing BPF map after it is created, which is clearly not the case. Another minor annoyance is that bpf_map__resize() disallows 0 value for max_entries, which in some cases is totally acceptable (e.g., like for BPF perf buf case to let libbpf auto-create one buffer per each available CPU core). [0] Closes: https://github.com/libbpf/libbpf/issues/304 Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20220124194254.2051434-3-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- src/libbpf.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libbpf.h b/src/libbpf.h index 56bddb1..5aaf17f 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -718,6 +718,7 @@ LIBBPF_API int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type); /* get/set map size (max_entries) */ LIBBPF_API __u32 bpf_map__max_entries(const struct bpf_map *map); LIBBPF_API int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_map__set_max_entries() instead") LIBBPF_API int bpf_map__resize(struct bpf_map *map, __u32 max_entries); /* get/set map flags */ LIBBPF_API __u32 bpf_map__map_flags(const struct bpf_map *map); From b2a7b1628778b3df1a1604e7bdee4b776a8c6af1 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 24 Jan 2022 11:42:50 -0800 Subject: [PATCH 102/131] libbpf: deprecate bpf_program__is_() and bpf_program__set_() APIs Not sure why these APIs were added in the first place instead of a completely generic (and not requiring constantly adding new APIs with each new BPF program type) bpf_program__type() and bpf_program__set_type() APIs. But as it is right now, there are 13 such specialized is_type/set_type APIs, while latest kernel is already at 30+ BPF program types. Instead of completing the set of APIs and keep chasing kernel's bpf_prog_type enum, deprecate existing subset and recommend generic bpf_program__type() and bpf_program__set_type() APIs. Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20220124194254.2051434-4-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- src/libbpf.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/libbpf.h b/src/libbpf.h index 5aaf17f..5762b57 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -593,18 +593,31 @@ LIBBPF_API int bpf_program__nth_fd(const struct bpf_program *prog, int n); /* * Adjust type of BPF program. Default is kprobe. */ +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_socket_filter(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_tracepoint(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_raw_tracepoint(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_kprobe(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_lsm(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_sched_cls(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_sched_act(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_xdp(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_perf_event(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_tracing(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_struct_ops(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_extension(struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") LIBBPF_API int bpf_program__set_sk_lookup(struct bpf_program *prog); LIBBPF_API enum bpf_prog_type bpf_program__type(const struct bpf_program *prog); @@ -633,18 +646,31 @@ LIBBPF_API int bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd, const char *attach_func_name); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_socket_filter(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_tracepoint(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_raw_tracepoint(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_kprobe(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_lsm(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_sched_cls(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_sched_act(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_xdp(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_perf_event(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_tracing(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_struct_ops(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_extension(const struct bpf_program *prog); +LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") LIBBPF_API bool bpf_program__is_sk_lookup(const struct bpf_program *prog); /* From bad4fa116c31aeffea3a583c664caa7d69d112fa Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 25 Jan 2022 22:21:43 -0800 Subject: [PATCH 103/131] sync: auto-generate latest BPF helpers Latest changes to BPF helper definitions. --- src/bpf_helper_defs.h | 48 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/bpf_helper_defs.h b/src/bpf_helper_defs.h index e2f9541..3d7477d 100644 --- a/src/bpf_helper_defs.h +++ b/src/bpf_helper_defs.h @@ -4247,4 +4247,52 @@ static int (*bpf_get_retval)(void) = (void *) 186; */ static int (*bpf_set_retval)(int retval) = (void *) 187; +/* + * bpf_xdp_get_buff_len + * + * Get the total size of a given xdp buff (linear and paged area) + * + * Returns + * The total size of a given xdp buffer. + */ +static __u64 (*bpf_xdp_get_buff_len)(struct xdp_md *xdp_md) = (void *) 188; + +/* + * bpf_xdp_load_bytes + * + * This helper is provided as an easy way to load data from a + * xdp buffer. It can be used to load *len* bytes from *offset* from + * the frame associated to *xdp_md*, into the buffer pointed by + * *buf*. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_xdp_load_bytes)(struct xdp_md *xdp_md, __u32 offset, void *buf, __u32 len) = (void *) 189; + +/* + * bpf_xdp_store_bytes + * + * Store *len* bytes from buffer *buf* into the frame + * associated to *xdp_md*, at *offset*. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_xdp_store_bytes)(struct xdp_md *xdp_md, __u32 offset, void *buf, __u32 len) = (void *) 190; + +/* + * bpf_copy_from_user_task + * + * Read *size* bytes from user space address *user_ptr* in *tsk*'s + * address space, and stores the data in *dst*. *flags* is not + * used yet and is provided for future extensibility. This helper + * can only be used by sleepable programs. + * + * Returns + * 0 on success, or a negative error in case of failure. On error + * *dst* buffer is zeroed out. + */ +static long (*bpf_copy_from_user_task)(void *dst, __u32 size, const void *user_ptr, struct task_struct *tsk, __u64 flags) = (void *) 191; + From 1e702e8ffe226e81bdf31c2df34392e50115c819 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 25 Jan 2022 22:21:43 -0800 Subject: [PATCH 104/131] sync: latest libbpf changes from kernel Syncing latest libbpf commits from kernel repository. Baseline bpf-next commit: 820e6e227c4053b6b631ae65ef1f65d560cb392b Checkpoint bpf-next commit: c446fdacb10dcb3b9a9ed3b91d91e72d71d94b03 Baseline bpf commit: baa59504c1cd0cca7d41954a45ee0b3dc78e41a0 Checkpoint bpf commit: e2bcbd7769ee8f05e1b3d10848aace98973844e4 Andrii Nakryiko (3): libbpf: hide and discourage inconsistently named getters libbpf: deprecate bpf_map__resize() libbpf: deprecate bpf_program__is_() and bpf_program__set_() APIs Christy Lee (2): libbpf: Mark bpf_object__open_buffer() API deprecated libbpf: Mark bpf_object__open_xattr() deprecated Kajol Jain (1): perf: Add new macros for mem_hops field Kenny Yu (2): bpf: Add bpf_copy_from_user_task() helper libbpf: Add "iter.s" section for sleepable bpf iterator programs Kenta Tada (1): libbpf: Fix the incorrect register read for syscalls on x86_64 Lorenzo Bianconi (4): bpf: introduce BPF_F_XDP_HAS_FRAGS flag in prog_flags loading the ebpf program bpf: introduce bpf_xdp_get_buff_len helper libbpf: Add SEC name for xdp frags programs net: xdp: introduce bpf_xdp_pointer utility routine include/uapi/linux/bpf.h | 41 +++++++++++++++++++++++++++++++++ include/uapi/linux/perf_event.h | 5 +++- src/bpf_tracing.h | 34 +++++++++++++++++++++++++++ src/btf.h | 5 +--- src/libbpf.c | 32 +++++++++++++++++-------- src/libbpf.h | 34 ++++++++++++++++++++++++--- src/libbpf.map | 2 ++ src/libbpf_internal.h | 3 +++ src/libbpf_legacy.h | 17 ++++++++++++++ 9 files changed, 155 insertions(+), 18 deletions(-) -- 2.30.2 --- BPF-CHECKPOINT-COMMIT | 2 +- CHECKPOINT-COMMIT | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BPF-CHECKPOINT-COMMIT b/BPF-CHECKPOINT-COMMIT index af72d6d..86d0b2b 100644 --- a/BPF-CHECKPOINT-COMMIT +++ b/BPF-CHECKPOINT-COMMIT @@ -1 +1 @@ -baa59504c1cd0cca7d41954a45ee0b3dc78e41a0 +e2bcbd7769ee8f05e1b3d10848aace98973844e4 diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index 16a932e..57a6020 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -820e6e227c4053b6b631ae65ef1f65d560cb392b +c446fdacb10dcb3b9a9ed3b91d91e72d71d94b03 From fec0813359a9dd68dff7edb9f221b0f18d94433d Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 25 Jan 2022 22:35:23 -0800 Subject: [PATCH 105/131] sync: regenerate vmlinux.h to include TASK_COMM_LEN constant TASK_COMM_LEN is now part of vmlinux.h on latest kernel. Regenerate vmlinux.h to have it on 5.5 and 4.9 kernels. Signed-off-by: Andrii Nakryiko --- .github/actions/build-selftests/vmlinux.h | 1882 ++++++++++++--------- 1 file changed, 1062 insertions(+), 820 deletions(-) diff --git a/.github/actions/build-selftests/vmlinux.h b/.github/actions/build-selftests/vmlinux.h index 8280fce..7a1dd72 100644 --- a/.github/actions/build-selftests/vmlinux.h +++ b/.github/actions/build-selftests/vmlinux.h @@ -264,26 +264,6 @@ struct raw_spinlock { typedef struct raw_spinlock raw_spinlock_t; -struct spinlock { - union { - struct raw_spinlock rlock; - struct { - u8 __padding[24]; - struct lockdep_map dep_map; - }; - }; -}; - -typedef struct spinlock spinlock_t; - -typedef struct { - arch_rwlock_t raw_lock; - unsigned int magic; - unsigned int owner_cpu; - void *owner; - struct lockdep_map dep_map; -} rwlock_t; - struct ratelimit_state { raw_spinlock_t lock; int interval; @@ -759,6 +739,18 @@ struct syscall_user_dispatch { bool on_dispatch; }; +struct spinlock { + union { + struct raw_spinlock rlock; + struct { + u8 __padding[24]; + struct lockdep_map dep_map; + }; + }; +}; + +typedef struct spinlock spinlock_t; + struct wake_q_node { struct wake_q_node *next; }; @@ -981,8 +973,7 @@ struct fpu { struct fpstate *fpstate; struct fpstate *__task_fpstate; struct fpu_state_perm perm; - long: 64; - long: 64; + struct fpu_state_perm guest_perm; struct fpstate __fpstate; }; @@ -1198,7 +1189,7 @@ struct task_struct { struct completion *vfork_done; int *set_child_tid; int *clear_child_tid; - void *pf_io_worker; + void *worker_private; u64 utime; u64 stime; u64 gtime; @@ -1748,8 +1739,6 @@ struct address_space; struct page_pool; -struct kmem_cache; - struct dev_pagemap; struct page { @@ -1771,26 +1760,6 @@ struct page { atomic_long_t pp_frag_count; }; }; - struct { - union { - struct list_head slab_list; - struct { - struct page *next; - int pages; - }; - }; - struct kmem_cache *slab_cache; - void *freelist; - union { - void *s_mem; - long unsigned int counters; - struct { - unsigned int inuse: 16; - unsigned int objects: 15; - unsigned int frozen: 1; - }; - }; - }; struct { long unsigned int compound_head; unsigned char compound_dtor; @@ -1822,8 +1791,6 @@ struct page { union { atomic_t _mapcount; unsigned int page_type; - unsigned int active; - int units; }; atomic_t _refcount; long unsigned int memcg_data; @@ -2585,7 +2552,7 @@ struct kobject { struct list_head entry; struct kobject *parent; struct kset *kset; - struct kobj_type *ktype; + const struct kobj_type *ktype; struct kernfs_node *sd; struct kref kref; unsigned int state_initialized: 1; @@ -2639,6 +2606,15 @@ struct dev_pm_info { struct dev_pm_qos *qos; }; +struct irq_domain; + +struct msi_device_data; + +struct dev_msi_info { + struct irq_domain *domain; + struct msi_device_data *data; +}; + struct dev_archdata {}; struct dev_iommu; @@ -2660,8 +2636,6 @@ struct device_driver; struct dev_pm_domain; -struct irq_domain; - struct dma_map_ops; struct bus_dma_region; @@ -2695,9 +2669,7 @@ struct device { struct dev_links_info links; struct dev_pm_info power; struct dev_pm_domain *pm_domain; - struct irq_domain *msi_domain; - raw_spinlock_t msi_lock; - struct list_head msi_list; + struct dev_msi_info msi; const struct dma_map_ops *dma_ops; u64 *dma_mask; u64 coherent_dma_mask; @@ -2740,6 +2712,14 @@ struct fwnode_reference_args { u64 args[8]; }; +typedef struct { + arch_rwlock_t raw_lock; + unsigned int magic; + unsigned int owner_cpu; + void *owner; + struct lockdep_map dep_map; +} rwlock_t; + struct vm_struct { struct vm_struct *next; void *addr; @@ -2787,6 +2767,8 @@ struct vm_userfaultfd_ctx { struct userfaultfd_ctx *ctx; }; +struct anon_vma_name; + struct anon_vma; struct vm_operations_struct; @@ -2801,10 +2783,13 @@ struct vm_area_struct { struct mm_struct *vm_mm; pgprot_t vm_page_prot; long unsigned int vm_flags; - struct { - struct rb_node rb; - long unsigned int rb_subtree_last; - } shared; + union { + struct { + struct rb_node rb; + long unsigned int rb_subtree_last; + } shared; + struct anon_vma_name *anon_name; + }; struct list_head anon_vma_chain; struct anon_vma *anon_vma; const struct vm_operations_struct *vm_ops; @@ -2942,7 +2927,7 @@ struct mm_struct { long unsigned int numa_scan_offset; int numa_scan_seq; atomic_t tlb_flush_pending; - bool tlb_flush_batched; + atomic_t tlb_flush_batched; struct uprobes_state uprobes_state; atomic_long_t hugetlb_usage; struct work_struct async_put_work; @@ -3172,11 +3157,11 @@ struct dev_pagemap_ops; struct dev_pagemap { struct vmem_altmap altmap; - struct percpu_ref *ref; - struct percpu_ref internal_ref; + struct percpu_ref ref; struct completion done; enum memory_type type; unsigned int flags; + long unsigned int vmemmap_shift; const struct dev_pagemap_ops *ops; void *owner; int nr_range; @@ -3255,6 +3240,11 @@ struct file { errseq_t f_sb_err; }; +struct anon_vma_name { + struct kref kref; + char name[0]; +}; + typedef unsigned int vm_fault_t; enum page_entry_size { @@ -3737,17 +3727,13 @@ struct x86_platform_ops { struct x86_hyper_runtime hyper; }; -struct pci_dev; - -struct x86_msi_ops { - void (*restore_msi_irqs)(struct pci_dev *); -}; - struct x86_apic_ops { unsigned int (*io_apic_read)(unsigned int, unsigned int); void (*restore)(); }; +struct x86_msi_ops; + struct physid_mask { long unsigned int mask[512]; }; @@ -3803,8 +3789,6 @@ struct resource { struct dev_pagemap_ops { void (*page_free)(struct page *); - void (*kill)(struct dev_pagemap *); - void (*cleanup)(struct dev_pagemap *); vm_fault_t (*migrate_to_ram)(struct vm_fault *); }; @@ -4012,7 +3996,7 @@ struct signal_struct { struct hlist_head multiprocess; int group_exit_code; int notify_count; - struct task_struct *group_exit_task; + struct task_struct *group_exec_task; int group_stop_count; unsigned int flags; struct core_state *core_state; @@ -4084,6 +4068,10 @@ struct rseq { long: 64; }; +enum { + TASK_COMM_LEN = 16, +}; + struct root_domain; enum perf_event_task_context { @@ -4260,18 +4248,10 @@ struct sighand_struct { struct k_sigaction action[64]; }; -struct io_cq; - struct io_context { atomic_long_t refcount; atomic_t active_ref; - atomic_t nr_tasks; - spinlock_t lock; short unsigned int ioprio; - struct xarray icq_tree; - struct io_cq *icq_hint; - struct hlist_head icq_list; - struct work_struct release_work; }; union thread_union { @@ -4694,7 +4674,7 @@ enum migrate_mode { struct exception_table_entry { int insn; int fixup; - int type; + int data; }; struct movsl_mask { @@ -4824,20 +4804,6 @@ typedef struct { __u8 b[16]; } guid_t; -struct io_cq { - struct request_queue *q; - struct io_context *ioc; - union { - struct list_head q_node; - struct kmem_cache *__rcu_icq_cache; - }; - union { - struct hlist_node ioc_node; - struct callback_head __rcu_head; - }; - unsigned int flags; -}; - struct files_stat_struct { long unsigned int nr_files; long unsigned int nr_free_files; @@ -5527,23 +5493,26 @@ enum vm_event_item { THP_SPLIT_PAGE_FAILED = 87, THP_DEFERRED_SPLIT_PAGE = 88, THP_SPLIT_PMD = 89, - THP_SPLIT_PUD = 90, - THP_ZERO_PAGE_ALLOC = 91, - THP_ZERO_PAGE_ALLOC_FAILED = 92, - THP_SWPOUT = 93, - THP_SWPOUT_FALLBACK = 94, - BALLOON_INFLATE = 95, - BALLOON_DEFLATE = 96, - BALLOON_MIGRATE = 97, - SWAP_RA = 98, - SWAP_RA_HIT = 99, - DIRECT_MAP_LEVEL2_SPLIT = 100, - DIRECT_MAP_LEVEL3_SPLIT = 101, - NR_VM_EVENT_ITEMS = 102, + THP_SCAN_EXCEED_NONE_PTE = 90, + THP_SCAN_EXCEED_SWAP_PTE = 91, + THP_SCAN_EXCEED_SHARED_PTE = 92, + THP_SPLIT_PUD = 93, + THP_ZERO_PAGE_ALLOC = 94, + THP_ZERO_PAGE_ALLOC_FAILED = 95, + THP_SWPOUT = 96, + THP_SWPOUT_FALLBACK = 97, + BALLOON_INFLATE = 98, + BALLOON_DEFLATE = 99, + BALLOON_MIGRATE = 100, + SWAP_RA = 101, + SWAP_RA_HIT = 102, + DIRECT_MAP_LEVEL2_SPLIT = 103, + DIRECT_MAP_LEVEL3_SPLIT = 104, + NR_VM_EVENT_ITEMS = 105, }; struct vm_event_state { - long unsigned int event[102]; + long unsigned int event[105]; }; enum memblock_flags { @@ -6687,6 +6656,7 @@ struct kernfs_root { struct kernfs_syscall_ops *syscall_ops; struct list_head supers; wait_queue_head_t deactivate_waitq; + struct rw_semaphore kernfs_rwsem; }; struct kernfs_elem_symlink { @@ -6838,9 +6808,9 @@ struct kobj_uevent_env { }; struct kset_uevent_ops { - int (* const filter)(struct kset *, struct kobject *); - const char * (* const name)(struct kset *, struct kobject *); - int (* const uevent)(struct kset *, struct kobject *, struct kobj_uevent_env *); + int (* const filter)(struct kobject *); + const char * (* const name)(struct kobject *); + int (* const uevent)(struct kobject *, struct kobj_uevent_env *); }; enum cpu_idle_type { @@ -7630,6 +7600,14 @@ struct pollfd { short int revents; }; +enum refcount_saturation_type { + REFCOUNT_ADD_NOT_ZERO_OVF = 0, + REFCOUNT_ADD_OVF = 1, + REFCOUNT_ADD_UAF = 2, + REFCOUNT_SUB_UAF = 3, + REFCOUNT_DEC_LEAK = 4, +}; + struct wait_queue_entry; typedef int (*wait_queue_func_t)(struct wait_queue_entry *, unsigned int, int, void *); @@ -7685,6 +7663,7 @@ struct mempolicy { short unsigned int mode; short unsigned int flags; nodemask_t nodes; + int home_node; union { nodemask_t cpuset_mems_allowed; nodemask_t user_nodemask; @@ -7776,14 +7755,6 @@ struct poll_table_struct { __poll_t _key; }; -enum refcount_saturation_type { - REFCOUNT_ADD_NOT_ZERO_OVF = 0, - REFCOUNT_ADD_OVF = 1, - REFCOUNT_ADD_UAF = 2, - REFCOUNT_SUB_UAF = 3, - REFCOUNT_DEC_LEAK = 4, -}; - struct trace_event_functions; struct trace_event { @@ -7896,10 +7867,10 @@ struct memcg_padding { }; struct memcg_vmstats { - long int state[43]; - long unsigned int events[102]; - long int state_pending[43]; - long unsigned int events_pending[102]; + long int state[44]; + long unsigned int events[105]; + long int state_pending[44]; + long unsigned int events_pending[105]; }; struct fprop_global { @@ -7966,14 +7937,20 @@ struct mem_cgroup { long: 64; struct memcg_padding _pad1_; struct memcg_vmstats vmstats; - atomic_long_t memory_events[8]; - atomic_long_t memory_events_local[8]; + atomic_long_t memory_events[9]; + atomic_long_t memory_events_local[9]; long unsigned int socket_pressure; bool tcpmem_active; int tcpmem_pressure; int kmemcg_id; struct obj_cgroup *objcg; struct list_head objcg_list; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; struct memcg_padding _pad2_; atomic_t moving_account; struct task_struct *move_lock_task; @@ -7989,6 +7966,8 @@ struct mem_cgroup { long: 64; }; +struct kmem_cache; + struct fs_pin; struct pid_namespace { @@ -8215,9 +8194,9 @@ struct perf_event_context { struct task_delay_info { raw_spinlock_t lock; - unsigned int flags; u64 blkio_start; u64 blkio_delay; + u64 swapin_start; u64 swapin_delay; u32 blkio_count; u32 swapin_count; @@ -8225,8 +8204,11 @@ struct task_delay_info { u64 freepages_delay; u64 thrashing_start; u64 thrashing_delay; + u64 compact_start; + u64 compact_delay; u32 freepages_count; u32 thrashing_count; + u32 compact_count; }; struct ftrace_ret_stack { @@ -8236,14 +8218,6 @@ struct ftrace_ret_stack { long unsigned int *retp; }; -struct blk_rq_stat { - u64 mean; - u64 min; - u64 max; - u32 nr_samples; - u64 batch; -}; - enum blk_bounce { BLK_BOUNCE_NONE = 0, BLK_BOUNCE_HIGH = 1, @@ -8335,6 +8309,8 @@ struct gendisk; struct blk_stat_callback; +struct blk_rq_stat; + struct blk_mq_tags; struct blkcg_gq; @@ -8374,7 +8350,7 @@ struct request_queue { unsigned int rq_timeout; int poll_nsec; struct blk_stat_callback *poll_cb; - struct blk_rq_stat poll_stat[16]; + struct blk_rq_stat *poll_stat; struct timer_list timeout; struct work_struct timeout_work; atomic_t nr_active_requests_shared_tags; @@ -8411,6 +8387,7 @@ struct request_queue { bool mq_sysfs_init_done; u64 write_hints[5]; struct blk_independent_access_ranges *ia_ranges; + struct srcu_struct srcu[0]; }; struct cgroup_base_stat { @@ -8529,6 +8506,8 @@ struct taskstats { __u64 thrashing_count; __u64 thrashing_delay_total; __u64 ac_btime64; + __u64 compact_count; + __u64 compact_delay_total; }; struct wait_page_queue { @@ -8554,6 +8533,7 @@ struct writeback_control { unsigned int for_reclaim: 1; unsigned int range_cyclic: 1; unsigned int for_sync: 1; + unsigned int unpinned_fscache_wb: 1; unsigned int no_cgroup_owner: 1; unsigned int punt_to_cgroup: 1; struct bdi_writeback *wb; @@ -9185,13 +9165,6 @@ struct proc_ns_operations { struct ns_common * (*get_parent)(struct ns_common *); }; -struct perf_guest_info_callbacks { - int (*is_in_guest)(); - int (*is_user_mode)(); - long unsigned int (*get_guest_ip)(); - void (*handle_intel_pt_intr)(); -}; - struct perf_cpu_context; struct perf_output_handle; @@ -9672,10 +9645,11 @@ enum { FILTER_OTHER = 0, FILTER_STATIC_STRING = 1, FILTER_DYN_STRING = 2, - FILTER_PTR_STRING = 3, - FILTER_TRACE_FN = 4, - FILTER_COMM = 5, - FILTER_CPU = 6, + FILTER_RDYN_STRING = 3, + FILTER_PTR_STRING = 4, + FILTER_TRACE_FN = 5, + FILTER_COMM = 6, + FILTER_CPU = 7, }; struct property { @@ -10135,6 +10109,14 @@ struct partition_meta_info { u8 volname[64]; }; +struct blk_rq_stat { + u64 mean; + u64 min; + u64 max; + u32 nr_samples; + u64 batch; +}; + struct iovec { void *iov_base; __kernel_size_t iov_len; @@ -10351,7 +10333,8 @@ enum memcg_stat_item { MEMCG_SWAP = 40, MEMCG_SOCK = 41, MEMCG_PERCPU_B = 42, - MEMCG_NR_STAT = 43, + MEMCG_VMALLOC = 43, + MEMCG_NR_STAT = 44, }; enum memcg_memory_event { @@ -10360,10 +10343,11 @@ enum memcg_memory_event { MEMCG_MAX = 2, MEMCG_OOM = 3, MEMCG_OOM_KILL = 4, - MEMCG_SWAP_HIGH = 5, - MEMCG_SWAP_MAX = 6, - MEMCG_SWAP_FAIL = 7, - MEMCG_NR_MEMORY_EVENTS = 8, + MEMCG_OOM_GROUP_KILL = 5, + MEMCG_SWAP_HIGH = 6, + MEMCG_SWAP_MAX = 7, + MEMCG_SWAP_FAIL = 8, + MEMCG_NR_MEMORY_EVENTS = 9, }; enum mem_cgroup_events_target { @@ -10373,10 +10357,10 @@ enum mem_cgroup_events_target { }; struct memcg_vmstats_percpu { - long int state[43]; - long unsigned int events[102]; - long int state_prev[43]; - long unsigned int events_prev[102]; + long int state[44]; + long unsigned int events[105]; + long int state_prev[44]; + long unsigned int events_prev[105]; long unsigned int nr_page_events; long unsigned int targets[2]; }; @@ -10564,6 +10548,7 @@ struct blk_mq_queue_data; struct blk_mq_ops { blk_status_t (*queue_rq)(struct blk_mq_hw_ctx *, const struct blk_mq_queue_data *); void (*commit_rqs)(struct blk_mq_hw_ctx *); + void (*queue_rqs)(struct request **); int (*get_budget)(struct request_queue *); void (*put_budget)(struct request_queue *, int); void (*set_rq_budget_token)(struct request *, int); @@ -11281,8 +11266,8 @@ struct tty_ldisc_ops { void (*flush_buffer)(struct tty_struct *); ssize_t (*read)(struct tty_struct *, struct file *, unsigned char *, size_t, void **, long unsigned int); ssize_t (*write)(struct tty_struct *, struct file *, const unsigned char *, size_t); - int (*ioctl)(struct tty_struct *, struct file *, unsigned int, long unsigned int); - int (*compat_ioctl)(struct tty_struct *, struct file *, unsigned int, long unsigned int); + int (*ioctl)(struct tty_struct *, unsigned int, long unsigned int); + int (*compat_ioctl)(struct tty_struct *, unsigned int, long unsigned int); void (*set_termios)(struct tty_struct *, struct ktermios *); __poll_t (*poll)(struct tty_struct *, struct file *, struct poll_table_struct *); void (*hangup)(struct tty_struct *); @@ -13116,6 +13101,7 @@ struct net_device { netdevice_tracker linkwatch_dev_tracker; netdevice_tracker watchdog_dev_tracker; long: 64; + long: 64; }; struct hh_cache { @@ -13718,7 +13704,7 @@ struct xdp_rxq_info { u32 reg_state; struct xdp_mem_info mem; unsigned int napi_id; - long: 32; + u32 frag_size; long: 64; long: 64; long: 64; @@ -13737,6 +13723,7 @@ struct xdp_buff { struct xdp_rxq_info *rxq; struct xdp_txq_info *txq; u32 frame_sz; + u32 flags; }; struct xdp_frame { @@ -13747,6 +13734,7 @@ struct xdp_frame { u32 frame_sz: 24; struct xdp_mem_info mem; struct net_device *dev_rx; + u32 flags; }; struct nlmsghdr { @@ -16651,6 +16639,16 @@ struct vm_special_mapping { int (*mremap)(const struct vm_special_mapping *, struct vm_area_struct *); }; +struct platform_msi_priv_data; + +struct msi_device_data { + long unsigned int properties; + struct platform_msi_priv_data *platform_data; + struct mutex mutex; + struct xarray __store; + long unsigned int __iter_idx; +}; + struct timens_offsets { struct timespec64 monotonic; struct timespec64 boottime; @@ -17555,10 +17553,80 @@ struct perf_msr { u64 mask; }; +struct pci_device_id { + __u32 vendor; + __u32 device; + __u32 subvendor; + __u32 subdevice; + __u32 class; + __u32 class_mask; + kernel_ulong_t driver_data; + __u32 override_only; +}; + +struct pci_bus; + +struct hotplug_slot; + +struct pci_slot { + struct pci_bus *bus; + struct list_head list; + struct hotplug_slot *hotplug; + unsigned char number; + struct kobject kobj; +}; + +typedef short unsigned int pci_bus_flags_t; + +struct pci_dev; + +struct pci_ops; + +struct pci_bus { + struct list_head node; + struct pci_bus *parent; + struct list_head children; + struct list_head devices; + struct pci_dev *self; + struct list_head slots; + struct resource *resource[4]; + struct list_head resources; + struct resource busn_res; + struct pci_ops *ops; + void *sysdata; + struct proc_dir_entry *procdir; + unsigned char number; + unsigned char primary; + unsigned char max_bus_speed; + unsigned char cur_bus_speed; + char name[48]; + short unsigned int bridge_ctl; + pci_bus_flags_t bus_flags; + struct device *bridge; + struct device dev; + struct bin_attribute *legacy_io; + struct bin_attribute *legacy_mem; + unsigned int is_added: 1; +}; + +enum { + PCI_STD_RESOURCES = 0, + PCI_STD_RESOURCE_END = 5, + PCI_ROM_RESOURCE = 6, + PCI_IOV_RESOURCES = 7, + PCI_IOV_RESOURCE_END = 12, + PCI_BRIDGE_RESOURCES = 13, + PCI_BRIDGE_RESOURCE_END = 16, + PCI_NUM_RESOURCES = 17, + DEVICE_COUNT_RESOURCE = 17, +}; + typedef int pci_power_t; typedef unsigned int pci_channel_state_t; +typedef unsigned int pcie_reset_state_t; + typedef short unsigned int pci_dev_flags_t; struct pci_vpd { @@ -17567,10 +17635,6 @@ struct pci_vpd { u8 cap; }; -struct pci_bus; - -struct pci_slot; - struct rcec_ea; struct pci_driver; @@ -17657,6 +17721,7 @@ struct pci_dev { unsigned int pasid_enabled: 1; unsigned int pri_enabled: 1; unsigned int is_managed: 1; + unsigned int is_msi_managed: 1; unsigned int needs_freset: 1; unsigned int state_saved: 1; unsigned int is_physfn: 1; @@ -17674,6 +17739,7 @@ struct pci_dev { unsigned int link_active_reporting: 1; unsigned int no_vf_scan: 1; unsigned int no_command_memory: 1; + unsigned int rom_bar_overlap: 1; pci_dev_flags_t dev_flags; atomic_t enable_cnt; u32 saved_config_space[16]; @@ -17681,7 +17747,8 @@ struct pci_dev { int rom_attr_enabled; struct bin_attribute *res_attr[17]; struct bin_attribute *res_attr_wc[17]; - const struct attribute_group **msi_irq_groups; + void *msix_base; + raw_spinlock_t msi_lock; struct pci_vpd vpd; union { struct pci_sriov *sriov; @@ -17697,72 +17764,6 @@ struct pci_dev { u8 reset_methods[7]; }; -struct pci_device_id { - __u32 vendor; - __u32 device; - __u32 subvendor; - __u32 subdevice; - __u32 class; - __u32 class_mask; - kernel_ulong_t driver_data; - __u32 override_only; -}; - -struct hotplug_slot; - -struct pci_slot { - struct pci_bus *bus; - struct list_head list; - struct hotplug_slot *hotplug; - unsigned char number; - struct kobject kobj; -}; - -typedef short unsigned int pci_bus_flags_t; - -struct pci_ops; - -struct pci_bus { - struct list_head node; - struct pci_bus *parent; - struct list_head children; - struct list_head devices; - struct pci_dev *self; - struct list_head slots; - struct resource *resource[4]; - struct list_head resources; - struct resource busn_res; - struct pci_ops *ops; - void *sysdata; - struct proc_dir_entry *procdir; - unsigned char number; - unsigned char primary; - unsigned char max_bus_speed; - unsigned char cur_bus_speed; - char name[48]; - short unsigned int bridge_ctl; - pci_bus_flags_t bus_flags; - struct device *bridge; - struct device dev; - struct bin_attribute *legacy_io; - struct bin_attribute *legacy_mem; - unsigned int is_added: 1; -}; - -enum { - PCI_STD_RESOURCES = 0, - PCI_STD_RESOURCE_END = 5, - PCI_ROM_RESOURCE = 6, - PCI_IOV_RESOURCES = 7, - PCI_IOV_RESOURCE_END = 12, - PCI_BRIDGE_RESOURCES = 13, - PCI_BRIDGE_RESOURCE_END = 16, - PCI_NUM_RESOURCES = 17, - DEVICE_COUNT_RESOURCE = 17, -}; - -typedef unsigned int pcie_reset_state_t; - struct pci_dynids { spinlock_t lock; struct list_head list; @@ -21017,55 +21018,37 @@ struct msi_msg { }; }; -struct platform_msi_priv_data; - -struct platform_msi_desc { - struct platform_msi_priv_data *msi_priv_data; - u16 msi_index; -}; - -struct fsl_mc_msi_desc { - u16 msi_index; -}; - -struct ti_sci_inta_msi_desc { - u16 dev_index; +struct pci_msi_desc { + union { + u32 msi_mask; + u32 msix_ctrl; + }; + struct { + u8 is_msix: 1; + u8 multiple: 3; + u8 multi_cap: 3; + u8 can_mask: 1; + u8 is_64: 1; + u8 is_virtual: 1; + unsigned int default_irq; + } msi_attrib; + union { + u8 mask_pos; + void *mask_base; + }; }; struct msi_desc { - struct list_head list; unsigned int irq; unsigned int nvec_used; struct device *dev; struct msi_msg msg; struct irq_affinity_desc *affinity; + struct device_attribute *sysfs_attrs; void (*write_msi_msg)(struct msi_desc *, void *); void *write_msi_msg_data; - union { - struct { - union { - u32 msi_mask; - u32 msix_ctrl; - }; - struct { - u8 is_msix: 1; - u8 multiple: 3; - u8 multi_cap: 3; - u8 can_mask: 1; - u8 is_64: 1; - u8 is_virtual: 1; - u16 entry_nr; - unsigned int default_irq; - } msi_attrib; - union { - u8 mask_pos; - void *mask_base; - }; - }; - struct platform_msi_desc platform; - struct fsl_mc_msi_desc fsl_mc; - struct ti_sci_inta_msi_desc inta; - }; + u16 msi_index; + struct pci_msi_desc pci; }; struct irq_chip_regs { @@ -21290,6 +21273,19 @@ struct hvm_start_info { uint32_t reserved; }; +struct balloon_stats { + long unsigned int current_pages; + long unsigned int target_pages; + long unsigned int target_unpopulated; + long unsigned int balloon_low; + long unsigned int balloon_high; + long unsigned int total_pages; + long unsigned int schedule_delay; + long unsigned int max_schedule_delay; + long unsigned int retry_count; + long unsigned int max_retry_count; +}; + struct setup_data { __u64 next; __u32 type; @@ -21314,6 +21310,7 @@ struct io_tlb_slot; struct io_tlb_mem { phys_addr_t start; phys_addr_t end; + void *vaddr; long unsigned int nslabs; long unsigned int used; unsigned int index; @@ -21378,8 +21375,8 @@ struct hstate { unsigned int free_huge_pages_node[64]; unsigned int surplus_huge_pages_node[64]; unsigned int nr_free_vmemmap_pages; - struct cftype cgroup_files_dfl[7]; - struct cftype cgroup_files_legacy[9]; + struct cftype cgroup_files_dfl[8]; + struct cftype cgroup_files_legacy[10]; char name[32]; }; @@ -21554,7 +21551,8 @@ typedef struct { struct text_poke_loc { s32 rel_addr; - s32 rel32; + s32 disp; + u8 len; u8 opcode; const u8 text[5]; u8 old; @@ -22225,6 +22223,14 @@ struct pkru_state { u32 pad; }; +struct fpu_guest { + u64 xfeatures; + u64 perm; + u64 xfd_err; + unsigned int uabi_size; + struct fpstate *fpstate; +}; + struct user_regset_view { const char *name; const struct user_regset *regsets; @@ -22642,6 +22648,7 @@ enum mf_flags { MF_ACTION_REQUIRED = 2, MF_MUST_KILL = 4, MF_SOFT_OFFLINE = 8, + MF_UNPOISON = 16, }; enum mce_notifier_prios { @@ -22988,6 +22995,12 @@ struct acpi_table_boot { u8 reserved[3]; }; +struct acpi_cedt_header { + u8 type; + u8 reserved; + u16 length; +}; + struct acpi_hmat_structure { u16 type; u16 reserved; @@ -23108,13 +23121,18 @@ union acpi_subtable_headers { struct acpi_subtable_header common; struct acpi_hmat_structure hmat; struct acpi_prmt_module_header prmt; + struct acpi_cedt_header cedt; }; typedef int (*acpi_tbl_entry_handler)(union acpi_subtable_headers *, const long unsigned int); +typedef int (*acpi_tbl_entry_handler_arg)(union acpi_subtable_headers *, void *, const long unsigned int); + struct acpi_subtable_proc { int id; acpi_tbl_entry_handler handler; + acpi_tbl_entry_handler_arg handler_arg; + void *arg; int count; }; @@ -23704,6 +23722,8 @@ typedef void (*nmi_shootdown_cb)(int, struct pt_regs *); struct pci_ops___2; +struct pci_dev___2; + enum intercept_words { INTERCEPT_CR = 0, INTERCEPT_DR = 1, @@ -24197,9 +24217,7 @@ struct msi_domain_ops { void (*msi_free)(struct irq_domain *, struct msi_domain_info *, unsigned int); int (*msi_check)(struct irq_domain *, struct msi_domain_info *, struct device *); int (*msi_prepare)(struct irq_domain *, struct device *, int, msi_alloc_info_t *); - void (*msi_finish)(msi_alloc_info_t *, int); void (*set_desc)(msi_alloc_info_t *, struct msi_desc *); - int (*handle_error)(struct irq_domain *, struct msi_desc *, int); int (*domain_alloc_irqs)(struct irq_domain *, struct device *, int); void (*domain_free_irqs)(struct irq_domain *, struct device *); }; @@ -24223,6 +24241,10 @@ enum { MSI_FLAG_ACTIVATE_EARLY = 16, MSI_FLAG_MUST_REACTIVATE = 32, MSI_FLAG_LEVEL_CAPABLE = 64, + MSI_FLAG_DEV_SYSFS = 128, + MSI_FLAG_MSIX_CONTIGUOUS = 256, + MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS = 512, + MSI_FLAG_FREE_MSI_DESCS = 1024, }; enum { @@ -25280,7 +25302,12 @@ struct bpf_map { struct work_struct work; struct mutex freeze_mutex; atomic64_t writecnt; - long: 64; + struct { + spinlock_t lock; + enum bpf_prog_type type; + bool jited; + bool xdp_has_frags; + } owner; long: 64; }; @@ -25695,7 +25722,11 @@ enum bpf_func_id { BPF_FUNC_get_func_arg_cnt = 185, BPF_FUNC_get_retval = 186, BPF_FUNC_set_retval = 187, - __BPF_FUNC_MAX_ID = 188, + BPF_FUNC_xdp_get_buff_len = 188, + BPF_FUNC_xdp_load_bytes = 189, + BPF_FUNC_xdp_store_bytes = 190, + BPF_FUNC_copy_from_user_task = 191, + __BPF_FUNC_MAX_ID = 192, }; struct bpf_func_info { @@ -25880,6 +25911,7 @@ struct bpf_prog_aux { bool func_proto_unreliable; bool sleepable; bool tail_call_reachable; + bool xdp_has_frags; struct hlist_node tramp_hlist; const struct btf_type *attach_func_proto; const char *attach_func_name; @@ -25958,7 +25990,7 @@ enum bpf_arg_type { ARG_PTR_TO_SOCKET_OR_NULL = 271, ARG_PTR_TO_ALLOC_MEM_OR_NULL = 273, ARG_PTR_TO_STACK_OR_NULL = 278, - __BPF_ARG_TYPE_LIMIT = 1023, + __BPF_ARG_TYPE_LIMIT = 2047, }; enum bpf_return_type { @@ -25976,9 +26008,9 @@ enum bpf_return_type { RET_PTR_TO_SOCKET_OR_NULL = 259, RET_PTR_TO_TCP_SOCK_OR_NULL = 260, RET_PTR_TO_SOCK_COMMON_OR_NULL = 261, - RET_PTR_TO_ALLOC_MEM_OR_NULL = 262, + RET_PTR_TO_ALLOC_MEM_OR_NULL = 1286, RET_PTR_TO_BTF_ID_OR_NULL = 264, - __BPF_RET_TYPE_LIMIT = 1023, + __BPF_RET_TYPE_LIMIT = 2047, }; struct bpf_func_proto { @@ -26043,7 +26075,7 @@ enum bpf_reg_type { PTR_TO_SOCK_COMMON_OR_NULL = 268, PTR_TO_TCP_SOCK_OR_NULL = 269, PTR_TO_BTF_ID_OR_NULL = 272, - __BPF_REG_TYPE_LIMIT = 1023, + __BPF_REG_TYPE_LIMIT = 2047, }; struct bpf_verifier_log; @@ -26189,11 +26221,6 @@ struct btf_mod_pair { }; struct bpf_array_aux { - struct { - spinlock_t lock; - enum bpf_prog_type type; - bool jited; - } owner; struct list_head poke_progs; struct bpf_map *map; struct mutex poke_mutex; @@ -26553,13 +26580,6 @@ struct multiprocess_signals { typedef int (*proc_visitor)(struct task_struct *, void *); -enum { - IOPRIO_CLASS_NONE = 0, - IOPRIO_CLASS_RT = 1, - IOPRIO_CLASS_BE = 2, - IOPRIO_CLASS_IDLE = 3, -}; - typedef struct poll_table_struct poll_table; enum { @@ -26592,7 +26612,8 @@ struct proc_fs_info { enum bpf_type_flag { PTR_MAYBE_NULL = 256, MEM_RDONLY = 512, - __BPF_TYPE_LAST_FLAG = 512, + MEM_ALLOC = 1024, + __BPF_TYPE_LAST_FLAG = 1024, }; enum bpf_cgroup_storage_type { @@ -26652,6 +26673,12 @@ enum con_flush_mode { CONSOLE_REPLAY_ALL = 1, }; +enum error_detector { + ERROR_DETECTOR_KFENCE = 0, + ERROR_DETECTOR_KASAN = 1, + ERROR_DETECTOR_WARN = 2, +}; + struct warn_args { const char *fmt; va_list args; @@ -27320,6 +27347,7 @@ struct worker_pool { int id; unsigned int flags; long unsigned int watchdog_ts; + atomic_t nr_running; struct list_head worklist; int nr_workers; int nr_idle; @@ -27334,16 +27362,7 @@ struct worker_pool { struct workqueue_attrs *attrs; struct hlist_node hash_node; int refcnt; - long: 32; - long: 64; - long: 64; - atomic_t nr_running; struct callback_head rcu; - long: 64; - long: 64; - long: 64; - long: 64; - long: 64; }; enum { @@ -27549,12 +27568,14 @@ struct kthread_create_info { struct kthread { long unsigned int flags; unsigned int cpu; + int result; int (*threadfn)(void *); void *data; mm_segment_t oldfs; struct completion parked; struct completion exited; struct cgroup_subsys_state *blkcg_css; + char *full_name; }; enum KTHREAD_BITS { @@ -28638,13 +28659,9 @@ enum cpuacct_stat_index { CPUACCT_STAT_NSTATS = 2, }; -struct cpuacct_usage { - u64 usages[2]; -}; - struct cpuacct { struct cgroup_subsys_state css; - struct cpuacct_usage *cpuusage; + u64 *cpuusage; struct kernel_cpustat *cpustat; }; @@ -29193,6 +29210,12 @@ enum { EFFECTIVE_LIST = 3, }; +enum msi_desc_filter { + MSI_DESC_ALL = 0, + MSI_DESC_NOTASSOCIATED = 1, + MSI_DESC_ASSOCIATED = 2, +}; + struct irq_affinity { unsigned int pre_vectors; unsigned int post_vectors; @@ -29302,6 +29325,12 @@ typedef void (*btf_trace_irq_matrix_free)(void *, int, unsigned int, struct irq_ typedef void (*call_rcu_func_t)(struct callback_head *, rcu_callback_t); +struct rcu_cblist { + struct callback_head *head; + struct callback_head **tail; + long int len; +}; + struct rcu_synchronize { struct callback_head head; struct completion completion; @@ -29346,17 +29375,17 @@ typedef void (*holdouts_func_t)(struct list_head *, bool, bool *); typedef void (*postgp_func_t)(struct rcu_tasks *); +struct rcu_tasks_percpu; + struct rcu_tasks { - struct callback_head *cbs_head; - struct callback_head **cbs_tail; struct wait_queue_head cbs_wq; - raw_spinlock_t cbs_lock; + raw_spinlock_t cbs_gbl_lock; int gp_state; int gp_sleep; int init_fract; long unsigned int gp_jiffies; long unsigned int gp_start; - long unsigned int n_gps; + long unsigned int tasks_gp_seq; long unsigned int n_ipis; long unsigned int n_ipis_fails; struct task_struct *kthread_ptr; @@ -29367,10 +29396,37 @@ struct rcu_tasks { holdouts_func_t holdouts_func; postgp_func_t postgp_func; call_rcu_func_t call_func; + struct rcu_tasks_percpu *rtpcpu; + int percpu_enqueue_shift; + int percpu_enqueue_lim; + int percpu_dequeue_lim; + long unsigned int percpu_dequeue_gpseq; + struct mutex barrier_q_mutex; + atomic_t barrier_q_count; + struct completion barrier_q_completion; + long unsigned int barrier_q_seq; char *name; char *kname; }; +struct rcu_tasks_percpu { + struct rcu_segcblist cblist; + raw_spinlock_t lock; + long unsigned int rtp_jiffies; + long unsigned int rtp_n_lock_retries; + struct work_struct rtp_work; + struct irq_work rtp_irq_work; + struct callback_head barrier_q_head; + int cpu; + struct rcu_tasks *rtpp; +}; + +struct trc_stall_chk_rdr { + int nesting; + int ipi_to_cpu; + u8 needqs; +}; + struct rcu_tasks_test_desc { struct callback_head rh; const char *name; @@ -29385,12 +29441,6 @@ enum { GP_REPLAY = 4, }; -struct rcu_cblist { - struct callback_head *head; - struct callback_head **tail; - long int len; -}; - enum rcutorture_type { RCU_FLAVOR = 0, RCU_TASKS_FLAVOR = 1, @@ -29481,7 +29531,6 @@ struct rcu_data { bool core_needs_qs; bool beenonline; bool gpwrap; - bool exp_deferred_qs; bool cpu_started; struct rcu_node *mynode; long unsigned int grpmask; @@ -29656,11 +29705,6 @@ typedef void (*btf_trace_sys_enter)(void *, struct pt_regs *, long int); typedef void (*btf_trace_sys_exit)(void *, struct pt_regs *, long int); -enum profile_type { - PROFILE_TASK_EXIT = 0, - PROFILE_MUNMAP = 1, -}; - struct profile_hit { u32 pc; u32 hits; @@ -29995,6 +30039,12 @@ struct tk_fast { typedef s64 int64_t; +enum wd_read_status { + WD_READ_SUCCESS = 0, + WD_READ_UNSTABLE = 1, + WD_READ_SKIP = 2, +}; + enum tick_nohz_mode { NOHZ_MODE_INACTIVE = 0, NOHZ_MODE_LOWRES = 1, @@ -30885,8 +30935,8 @@ struct trace_event_raw_cgroup_root { struct trace_event_raw_cgroup { struct trace_entry ent; int root; - int id; int level; + u64 id; u32 __data_loc_path; char __data[0]; }; @@ -30894,8 +30944,8 @@ struct trace_event_raw_cgroup { struct trace_event_raw_cgroup_migrate { struct trace_entry ent; int dst_root; - int dst_id; int dst_level; + u64 dst_id; int pid; u32 __data_loc_dst_path; u32 __data_loc_comm; @@ -30905,8 +30955,8 @@ struct trace_event_raw_cgroup_migrate { struct trace_event_raw_cgroup_event { struct trace_entry ent; int root; - int id; int level; + u64 id; u32 __data_loc_path; int val; char __data[0]; @@ -31609,13 +31659,13 @@ enum audit_nfcfgop { AUDIT_NFT_OP_INVALID = 19, }; -enum fsnotify_obj_type { - FSNOTIFY_OBJ_TYPE_INODE = 0, - FSNOTIFY_OBJ_TYPE_PARENT = 1, - FSNOTIFY_OBJ_TYPE_VFSMOUNT = 2, - FSNOTIFY_OBJ_TYPE_SB = 3, - FSNOTIFY_OBJ_TYPE_COUNT = 4, - FSNOTIFY_OBJ_TYPE_DETACHED = 4, +enum fsnotify_iter_type { + FSNOTIFY_ITER_TYPE_INODE = 0, + FSNOTIFY_ITER_TYPE_VFSMOUNT = 1, + FSNOTIFY_ITER_TYPE_SB = 2, + FSNOTIFY_ITER_TYPE_PARENT = 3, + FSNOTIFY_ITER_TYPE_INODE2 = 4, + FSNOTIFY_ITER_TYPE_COUNT = 5, }; struct audit_aux_data { @@ -31712,7 +31762,7 @@ struct fsnotify_group { }; struct fsnotify_iter_info { - struct fsnotify_mark *marks[4]; + struct fsnotify_mark *marks[5]; unsigned int report_mask; int srcu_idx; }; @@ -31733,6 +31783,15 @@ struct fsnotify_event { struct list_head list; }; +enum fsnotify_obj_type { + FSNOTIFY_OBJ_TYPE_ANY = 4294967295, + FSNOTIFY_OBJ_TYPE_INODE = 0, + FSNOTIFY_OBJ_TYPE_VFSMOUNT = 1, + FSNOTIFY_OBJ_TYPE_SB = 2, + FSNOTIFY_OBJ_TYPE_COUNT = 3, + FSNOTIFY_OBJ_TYPE_DETACHED = 3, +}; + struct audit_parent { struct list_head watches; struct fsnotify_mark mark; @@ -32267,8 +32326,8 @@ struct trace_array { int buffer_disabled; int sys_refcount_enter; int sys_refcount_exit; - struct trace_event_file *enter_syscall_files[450]; - struct trace_event_file *exit_syscall_files[450]; + struct trace_event_file *enter_syscall_files[451]; + struct trace_event_file *exit_syscall_files[451]; int stop_count; int clock_id; int nr_topts; @@ -32774,6 +32833,7 @@ enum trace_flag_type { TRACE_FLAG_SOFTIRQ = 16, TRACE_FLAG_PREEMPT_RESCHED = 32, TRACE_FLAG_NMI = 64, + TRACE_FLAG_BH_OFF = 128, }; enum event_trigger_type { @@ -33091,17 +33151,20 @@ enum { FLAGS_FILL_END = 805306368, }; -struct disk_stats { - u64 nsecs[4]; - long unsigned int sectors[4]; - long unsigned int ios[4]; - long unsigned int merges[4]; - long unsigned int io_ticks; - local_t in_flight[2]; +struct io_cq { + struct request_queue *q; + struct io_context *ioc; + union { + struct list_head q_node; + struct kmem_cache *__rcu_icq_cache; + }; + union { + struct hlist_node ioc_node; + struct callback_head __rcu_head; + }; + unsigned int flags; }; -typedef __u32 blk_mq_req_flags_t; - enum req_opf { REQ_OP_READ = 0, REQ_OP_WRITE = 1, @@ -33226,7 +33289,6 @@ struct request { struct list_head queuelist; struct request *rq_next; }; - struct gendisk *rq_disk; struct block_device *part; u64 start_time_ns; u64 io_start_time_ns; @@ -33235,7 +33297,7 @@ struct request { short unsigned int write_hint; short unsigned int ioprio; enum mq_rq_state state; - refcount_t ref; + atomic_t ref; long unsigned int deadline; union { struct hlist_node hash; @@ -33266,35 +33328,6 @@ struct request { void *end_io_data; }; -struct elevator_type; - -struct elevator_queue { - struct elevator_type *type; - void *elevator_data; - struct kobject kobj; - struct mutex sysfs_lock; - unsigned int registered: 1; - struct hlist_head hash[64]; -}; - -struct blk_mq_ctxs; - -struct blk_mq_ctx { - struct { - spinlock_t lock; - struct list_head rq_lists[3]; - long: 64; - long: 64; - }; - unsigned int cpu; - short unsigned int index_hw[3]; - struct blk_mq_hw_ctx *hctxs[3]; - struct request_queue *queue; - struct blk_mq_ctxs *ctxs; - struct kobject kobj; - long: 64; -}; - struct blk_mq_hw_ctx { struct { spinlock_t lock; @@ -33337,19 +33370,6 @@ struct blk_mq_hw_ctx { struct dentry *debugfs_dir; struct dentry *sched_debugfs_dir; struct list_head hctx_list; - struct srcu_struct srcu[0]; -}; - -struct blk_stat_callback { - struct list_head list; - struct timer_list timer; - struct blk_rq_stat *cpu_stat; - int (*bucket_fn)(const struct request *); - unsigned int buckets; - struct blk_rq_stat *stat; - void (*timer_fn)(struct blk_stat_callback *); - void *data; - struct callback_head rcu; }; struct blk_mq_tags { @@ -33510,89 +33530,6 @@ struct blk_crypto_mode { unsigned int ivsize; }; -struct blk_mq_ctxs { - struct kobject kobj; - struct blk_mq_ctx *queue_ctx; -}; - -struct blk_mq_alloc_data { - struct request_queue *q; - blk_mq_req_flags_t flags; - unsigned int shallow_depth; - unsigned int cmd_flags; - req_flags_t rq_flags; - unsigned int nr_tags; - struct request **cached_rq; - struct blk_mq_ctx *ctx; - struct blk_mq_hw_ctx *hctx; -}; - -enum elv_merge { - ELEVATOR_NO_MERGE = 0, - ELEVATOR_FRONT_MERGE = 1, - ELEVATOR_BACK_MERGE = 2, - ELEVATOR_DISCARD_MERGE = 3, -}; - -struct elevator_mq_ops { - int (*init_sched)(struct request_queue *, struct elevator_type *); - void (*exit_sched)(struct elevator_queue *); - int (*init_hctx)(struct blk_mq_hw_ctx *, unsigned int); - void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); - void (*depth_updated)(struct blk_mq_hw_ctx *); - bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); - bool (*bio_merge)(struct request_queue *, struct bio *, unsigned int); - int (*request_merge)(struct request_queue *, struct request **, struct bio *); - void (*request_merged)(struct request_queue *, struct request *, enum elv_merge); - void (*requests_merged)(struct request_queue *, struct request *, struct request *); - void (*limit_depth)(unsigned int, struct blk_mq_alloc_data *); - void (*prepare_request)(struct request *); - void (*finish_request)(struct request *); - void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); - struct request * (*dispatch_request)(struct blk_mq_hw_ctx *); - bool (*has_work)(struct blk_mq_hw_ctx *); - void (*completed_request)(struct request *, u64); - void (*requeue_request)(struct request *); - struct request * (*former_request)(struct request_queue *, struct request *); - struct request * (*next_request)(struct request_queue *, struct request *); - void (*init_icq)(struct io_cq *); - void (*exit_icq)(struct io_cq *); -}; - -struct elv_fs_entry; - -struct blk_mq_debugfs_attr; - -struct elevator_type { - struct kmem_cache *icq_cache; - struct elevator_mq_ops ops; - size_t icq_size; - size_t icq_align; - struct elv_fs_entry *elevator_attrs; - const char *elevator_name; - const char *elevator_alias; - const unsigned int elevator_features; - struct module *elevator_owner; - const struct blk_mq_debugfs_attr *queue_debugfs_attrs; - const struct blk_mq_debugfs_attr *hctx_debugfs_attrs; - char icq_cache_name[22]; - struct list_head list; -}; - -struct elv_fs_entry { - struct attribute attr; - ssize_t (*show)(struct elevator_queue *, char *); - ssize_t (*store)(struct elevator_queue *, const char *, size_t); -}; - -struct blk_mq_debugfs_attr { - const char *name; - umode_t mode; - int (*show)(void *, struct seq_file *); - ssize_t (*write)(void *, const char *, size_t, loff_t *); - const struct seq_operations *seq_ops; -}; - typedef void blk_log_action_t(struct trace_iterator *, const char *, bool); struct ftrace_event_field { @@ -33722,6 +33659,10 @@ enum { PROCESS_OR = 4, }; +struct ustring_buffer { + char buffer[1024]; +}; + enum { TOO_MANY_CLOSE = 4294967295, TOO_MANY_OPEN = 4294967294, @@ -33765,7 +33706,7 @@ struct event_trigger_data { }; struct event_trigger_ops { - void (*func)(struct event_trigger_data *, struct trace_buffer *, void *, struct ring_buffer_event *); + void (*trigger)(struct event_trigger_data *, struct trace_buffer *, void *, struct ring_buffer_event *); int (*init)(struct event_trigger_ops *, struct event_trigger_data *); void (*free)(struct event_trigger_ops *, struct event_trigger_data *); int (*print)(struct seq_file *, struct event_trigger_ops *, struct event_trigger_data *); @@ -33776,9 +33717,9 @@ struct event_command { char *name; enum event_trigger_type trigger_type; int flags; - int (*func)(struct event_command *, struct trace_event_file *, char *, char *, char *); - int (*reg)(char *, struct event_trigger_ops *, struct event_trigger_data *, struct trace_event_file *); - void (*unreg)(char *, struct event_trigger_ops *, struct event_trigger_data *, struct trace_event_file *); + int (*parse)(struct event_command *, struct trace_event_file *, char *, char *, char *); + int (*reg)(char *, struct event_trigger_data *, struct trace_event_file *); + void (*unreg)(char *, struct event_trigger_data *, struct trace_event_file *); void (*unreg_all)(struct trace_event_file *); int (*set_filter)(char *, struct event_trigger_data *, struct trace_event_file *); struct event_trigger_ops * (*get_trigger_ops)(char *, char *); @@ -34133,7 +34074,6 @@ struct security_hook_heads { struct hlist_head sb_pivotroot; struct hlist_head sb_set_mnt_opts; struct hlist_head sb_clone_mnt_opts; - struct hlist_head sb_add_mnt_opt; struct hlist_head move_mount; struct hlist_head dentry_init_security; struct hlist_head dentry_create_files_as; @@ -34201,7 +34141,7 @@ struct security_hook_heads { struct hlist_head task_setpgid; struct hlist_head task_getpgid; struct hlist_head task_getsid; - struct hlist_head task_getsecid_subj; + struct hlist_head current_getsecid_subj; struct hlist_head task_getsecid_obj; struct hlist_head task_setnice; struct hlist_head task_setioprio; @@ -34509,11 +34449,6 @@ struct trace_kprobe { struct trace_probe tp; }; -enum error_detector { - ERROR_DETECTOR_KFENCE = 0, - ERROR_DETECTOR_KASAN = 1, -}; - struct trace_event_raw_error_report_template { struct trace_entry ent; enum error_detector error_detector; @@ -35817,6 +35752,8 @@ typedef u64 (*btf_bpf_event_output_data)(void *, struct bpf_map *, u64, void *, typedef u64 (*btf_bpf_copy_from_user)(void *, u32, const void *); +typedef u64 (*btf_bpf_copy_from_user_task)(void *, u32, const void *, struct task_struct *, u64); + typedef u64 (*btf_bpf_per_cpu_ptr)(const void *, u32); typedef u64 (*btf_bpf_this_cpu_ptr)(const void *); @@ -39898,6 +39835,8 @@ enum mapping_flags { AS_LARGE_FOLIO_SUPPORT = 6, }; +typedef int filler_t(void *, struct page *); + struct wait_page_key { struct folio *folio; int bit_nr; @@ -39910,6 +39849,12 @@ struct pagevec { struct page *pages[15]; }; +struct folio_batch { + unsigned char nr; + bool percpu_pvec_drained; + struct folio *folios[15]; +}; + struct fid { union { struct { @@ -39936,6 +39881,7 @@ struct trace_event_raw_mm_filemap_op_page_cache { long unsigned int i_ino; long unsigned int index; dev_t s_dev; + unsigned char order; char __data[0]; }; @@ -39963,9 +39909,9 @@ struct trace_event_data_offsets_filemap_set_wb_err {}; struct trace_event_data_offsets_file_check_and_advance_wb_err {}; -typedef void (*btf_trace_mm_filemap_delete_from_page_cache)(void *, struct page *); +typedef void (*btf_trace_mm_filemap_delete_from_page_cache)(void *, struct folio *); -typedef void (*btf_trace_mm_filemap_add_to_page_cache)(void *, struct page *); +typedef void (*btf_trace_mm_filemap_add_to_page_cache)(void *, struct folio *); typedef void (*btf_trace_filemap_set_wb_err)(void *, struct address_space *, errseq_t); @@ -40792,11 +40738,7 @@ struct pcpu_alloc_info { struct pcpu_group_info groups[0]; }; -typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int, size_t, size_t); - -typedef void (*pcpu_fc_free_fn_t)(void *, size_t); - -typedef void (*pcpu_fc_populate_pte_fn_t)(long unsigned int); +typedef int pcpu_fc_cpu_to_node_fn_t(int); typedef int pcpu_fc_cpu_distance_fn_t(unsigned int, unsigned int); @@ -41177,7 +41119,7 @@ typedef void (*btf_trace_mm_compaction_isolate_migratepages)(void *, long unsign typedef void (*btf_trace_mm_compaction_isolate_freepages)(void *, long unsigned int, long unsigned int, long unsigned int, long unsigned int); -typedef void (*btf_trace_mm_compaction_migratepages)(void *, long unsigned int, int, struct list_head *); +typedef void (*btf_trace_mm_compaction_migratepages)(void *, long unsigned int, unsigned int); typedef void (*btf_trace_mm_compaction_begin)(void *, long unsigned int, long unsigned int, long unsigned int, long unsigned int, bool); @@ -41273,11 +41215,6 @@ struct memcg_path { typedef unsigned int pgtbl_mod_mask; -struct zap_details { - struct address_space *zap_mapping; - struct page *single_page; -}; - typedef int (*pte_fn_t)(pte_t *, long unsigned int, void *); enum { @@ -41297,6 +41234,11 @@ enum { SWP_SCANNING = 16384, }; +struct zap_details { + struct address_space *zap_mapping; + struct folio *single_folio; +}; + struct copy_subpage_arg { struct page *dst; struct page *src; @@ -41449,6 +41391,13 @@ struct madvise_walk_private { bool pageout; }; +enum { + IOPRIO_CLASS_NONE = 0, + IOPRIO_CLASS_RT = 1, + IOPRIO_CLASS_BE = 2, + IOPRIO_CLASS_IDLE = 3, +}; + enum { BIO_NO_PAGE_REF = 0, BIO_CLONED = 1, @@ -41592,6 +41541,10 @@ enum hugetlb_memory_event { HUGETLB_NR_MEMORY_EVENTS = 1, }; +struct hugetlb_cgroup_per_node { + long unsigned int usage[2]; +}; + struct hugetlb_cgroup { struct cgroup_subsys_state css; struct page_counter hugepage[2]; @@ -41600,6 +41553,7 @@ struct hugetlb_cgroup { atomic_long_t events_local[2]; struct cgroup_file events_file[2]; struct cgroup_file events_local_file[2]; + struct hugetlb_cgroup_per_node *nodeinfo[0]; }; enum vma_resv_mode { @@ -41896,6 +41850,11 @@ typedef void (*btf_trace_mm_migrate_pages)(void *, long unsigned int, long unsig typedef void (*btf_trace_mm_migrate_pages_start)(void *, enum migrate_mode, int); +struct demotion_nodes { + short unsigned int nr; + short int nodes[15]; +}; + enum scan_result { SCAN_FAIL = 0, SCAN_SUCCEED = 1, @@ -42169,25 +42128,23 @@ enum mf_action_page_type { MF_MSG_KERNEL_HIGH_ORDER = 1, MF_MSG_SLAB = 2, MF_MSG_DIFFERENT_COMPOUND = 3, - MF_MSG_POISONED_HUGE = 4, - MF_MSG_HUGE = 5, - MF_MSG_FREE_HUGE = 6, - MF_MSG_NON_PMD_HUGE = 7, - MF_MSG_UNMAP_FAILED = 8, - MF_MSG_DIRTY_SWAPCACHE = 9, - MF_MSG_CLEAN_SWAPCACHE = 10, - MF_MSG_DIRTY_MLOCKED_LRU = 11, - MF_MSG_CLEAN_MLOCKED_LRU = 12, - MF_MSG_DIRTY_UNEVICTABLE_LRU = 13, - MF_MSG_CLEAN_UNEVICTABLE_LRU = 14, - MF_MSG_DIRTY_LRU = 15, - MF_MSG_CLEAN_LRU = 16, - MF_MSG_TRUNCATED_LRU = 17, - MF_MSG_BUDDY = 18, - MF_MSG_BUDDY_2ND = 19, - MF_MSG_DAX = 20, - MF_MSG_UNSPLIT_THP = 21, - MF_MSG_UNKNOWN = 22, + MF_MSG_HUGE = 4, + MF_MSG_FREE_HUGE = 5, + MF_MSG_NON_PMD_HUGE = 6, + MF_MSG_UNMAP_FAILED = 7, + MF_MSG_DIRTY_SWAPCACHE = 8, + MF_MSG_CLEAN_SWAPCACHE = 9, + MF_MSG_DIRTY_MLOCKED_LRU = 10, + MF_MSG_CLEAN_MLOCKED_LRU = 11, + MF_MSG_DIRTY_UNEVICTABLE_LRU = 12, + MF_MSG_CLEAN_UNEVICTABLE_LRU = 13, + MF_MSG_DIRTY_LRU = 14, + MF_MSG_CLEAN_LRU = 15, + MF_MSG_TRUNCATED_LRU = 16, + MF_MSG_BUDDY = 17, + MF_MSG_DAX = 18, + MF_MSG_UNSPLIT_THP = 19, + MF_MSG_UNKNOWN = 20, }; typedef long unsigned int dax_entry_t; @@ -42873,6 +42830,12 @@ struct simple_transaction_argresp { char data[0]; }; +enum utf8_normalization { + UTF8_NFDI = 0, + UTF8_NFDICF = 1, + UTF8_NMAX = 2, +}; + struct simple_attr { int (*get)(void *, u64 *); int (*set)(void *, u64); @@ -43324,6 +43287,8 @@ enum fsconfig_command { typedef int get_block_t(struct inode *, sector_t, struct buffer_head *, int); +typedef __u32 blk_mq_req_flags_t; + struct dax_device; struct iomap_page_ops; @@ -44037,6 +44002,7 @@ struct io_uring_task { atomic_t in_idle; spinlock_t task_lock; struct io_wq_work_list task_list; + struct io_wq_work_list prior_task_list; struct callback_head task_work; bool task_running; }; @@ -44141,6 +44107,7 @@ enum { IOSQE_IO_HARDLINK_BIT = 3, IOSQE_ASYNC_BIT = 4, IOSQE_BUFFER_SELECT_BIT = 5, + IOSQE_CQE_SKIP_SUCCESS_BIT = 6, }; enum { @@ -44676,6 +44643,7 @@ struct io_submit_state { struct io_submit_link link; bool plug_started; bool need_plug; + bool flush_cqes; short unsigned int submit_nr; struct blk_plug plug; }; @@ -44701,7 +44669,8 @@ struct io_ring_ctx { unsigned int restricted: 1; unsigned int off_timeout_used: 1; unsigned int drain_active: 1; - long: 26; + unsigned int drain_disabled: 1; + long: 25; long: 64; long: 64; long: 64; @@ -44843,8 +44812,6 @@ struct io_poll_iocb { struct file *file; struct wait_queue_head *head; __poll_t events; - bool done; - bool canceled; struct wait_queue_entry wait; }; @@ -45091,6 +45058,7 @@ struct io_kiocb { struct io_wq_work work; const struct cred *creds; struct io_buffer *kbuf; + atomic_t poll_refs; }; struct io_timeout_data { @@ -45133,6 +45101,7 @@ enum { REQ_F_HARDLINK_BIT = 3, REQ_F_FORCE_ASYNC_BIT = 4, REQ_F_BUFFER_SELECT_BIT = 5, + REQ_F_CQE_SKIP_BIT = 6, REQ_F_FAIL_BIT = 8, REQ_F_INFLIGHT_BIT = 9, REQ_F_CUR_POS_BIT = 10, @@ -45147,9 +45116,10 @@ enum { REQ_F_REFCOUNT_BIT = 19, REQ_F_ARM_LTIMEOUT_BIT = 20, REQ_F_ASYNC_DATA_BIT = 21, - REQ_F_SUPPORT_NOWAIT_BIT = 22, - REQ_F_ISREG_BIT = 23, - __REQ_F_LAST_BIT = 24, + REQ_F_SKIP_LINK_CQES_BIT = 22, + REQ_F_SUPPORT_NOWAIT_BIT = 23, + REQ_F_ISREG_BIT = 24, + __REQ_F_LAST_BIT = 25, }; enum { @@ -45159,6 +45129,7 @@ enum { REQ_F_HARDLINK = 8, REQ_F_FORCE_ASYNC = 16, REQ_F_BUFFER_SELECT = 32, + REQ_F_CQE_SKIP = 64, REQ_F_FAIL = 256, REQ_F_INFLIGHT = 512, REQ_F_CUR_POS = 1024, @@ -45169,12 +45140,13 @@ enum { REQ_F_BUFFER_SELECTED = 32768, REQ_F_COMPLETE_INLINE = 65536, REQ_F_REISSUE = 131072, - REQ_F_SUPPORT_NOWAIT = 4194304, - REQ_F_ISREG = 8388608, + REQ_F_SUPPORT_NOWAIT = 8388608, + REQ_F_ISREG = 16777216, REQ_F_CREDS = 262144, REQ_F_REFCOUNT = 524288, REQ_F_ARM_LTIMEOUT = 1048576, REQ_F_ASYNC_DATA = 2097152, + REQ_F_SKIP_LINK_CQES = 4194304, }; struct async_poll { @@ -45667,17 +45639,25 @@ typedef void (*btf_trace_iomap_iter_srcmap)(void *, struct inode *, struct iomap typedef void (*btf_trace_iomap_iter)(void *, struct iomap_iter *, const void *, long unsigned int); +struct iomap_ops { + int (*iomap_begin)(struct inode *, loff_t, loff_t, unsigned int, struct iomap___2 *, struct iomap___2 *); + int (*iomap_end)(struct inode *, loff_t, loff_t, ssize_t, unsigned int, struct iomap___2 *); +}; + +struct folio_iter { + struct folio *folio; + size_t offset; + size_t length; + size_t _seg_count; + int _i; +}; + enum { BIOSET_NEED_BVECS = 1, BIOSET_NEED_RESCUER = 2, BIOSET_PERCPU_CACHE = 4, }; -struct iomap_ops { - int (*iomap_begin)(struct inode *, loff_t, loff_t, unsigned int, struct iomap___2 *, struct iomap___2 *); - int (*iomap_end)(struct inode *, loff_t, loff_t, ssize_t, unsigned int, struct iomap___2 *); -}; - struct iomap_ioend { struct list_head io_list; u16 io_type; @@ -45694,7 +45674,7 @@ struct iomap_writepage_ctx; struct iomap_writeback_ops { int (*map_blocks)(struct iomap_writepage_ctx *, struct inode *, loff_t); int (*prepare_ioend)(struct iomap_ioend *, int); - void (*discard_page)(struct page *, loff_t); + void (*discard_folio)(struct folio *, loff_t); }; struct iomap_writepage_ctx { @@ -45711,8 +45691,8 @@ struct iomap_page { }; struct iomap_readpage_ctx { - struct page *cur_page; - bool cur_page_in_bio; + struct folio *cur_folio; + bool cur_folio_in_bio; struct bio *bio; struct readahead_control *rac; }; @@ -45992,17 +45972,24 @@ struct netfs_cache_resources { void *cache_priv; void *cache_priv2; unsigned int debug_id; + unsigned int inval_counter; +}; + +enum netfs_read_from_hole { + NETFS_READ_HOLE_IGNORE = 0, + NETFS_READ_HOLE_CLEAR = 1, + NETFS_READ_HOLE_FAIL = 2, }; struct netfs_read_subrequest; struct netfs_cache_ops { void (*end_operation)(struct netfs_cache_resources *); - int (*read)(struct netfs_cache_resources *, loff_t, struct iov_iter *, bool, netfs_io_terminated_t, void *); + int (*read)(struct netfs_cache_resources *, loff_t, struct iov_iter *, enum netfs_read_from_hole, netfs_io_terminated_t, void *); int (*write)(struct netfs_cache_resources *, loff_t, struct iov_iter *, netfs_io_terminated_t, void *); void (*expand_readahead)(struct netfs_cache_resources *, loff_t *, size_t *, loff_t); enum netfs_read_source (*prepare_read)(struct netfs_read_subrequest *, loff_t); - int (*prepare_write)(struct netfs_cache_resources *, loff_t *, size_t *, loff_t); + int (*prepare_write)(struct netfs_cache_resources *, loff_t *, size_t *, loff_t, bool); }; struct netfs_read_request; @@ -46101,6 +46088,7 @@ struct trace_event_raw_netfs_read { loff_t start; size_t len; enum netfs_read_trace what; + unsigned int netfs_inode; char __data[0]; }; @@ -46606,7 +46594,10 @@ struct ext4_fc_stats { unsigned int fc_ineligible_reason_count[10]; long unsigned int fc_num_commits; long unsigned int fc_ineligible_commits; + long unsigned int fc_failed_commits; + long unsigned int fc_skipped_commits; long unsigned int fc_numblks; + u64 s_fc_avg_commit_time; }; struct ext4_fc_alloc_region { @@ -46930,7 +46921,7 @@ struct ext4_sb_info { struct ext4_li_request *s_li_request; unsigned int s_li_wait_mult; struct task_struct *s_mmp_tsk; - atomic_t s_last_trim_minblks; + long unsigned int s_last_trim_minblks; struct crypto_shash *s_chksum_driver; __u32 s_csum_seed; struct shrinker s_es_shrinker; @@ -46950,6 +46941,7 @@ struct ext4_sb_info { struct fscrypt_dummy_policy s_dummy_enc_policy; struct percpu_rw_semaphore s_writepages_rwsem; struct dax_device *s_daxdev; + u64 s_dax_part_off; errseq_t s_bdev_wb_err; spinlock_t s_bdev_wb_lock; spinlock_t s_error_lock; @@ -46968,15 +46960,18 @@ struct ext4_sb_info { time64_t s_last_error_time; struct work_struct s_error_work; atomic_t s_fc_subtid; - atomic_t s_fc_ineligible_updates; struct list_head s_fc_q[2]; struct list_head s_fc_dentry_q[2]; unsigned int s_fc_bytes; spinlock_t s_fc_lock; struct buffer_head *s_fc_bh; struct ext4_fc_stats s_fc_stats; - u64 s_fc_avg_commit_time; struct ext4_fc_replay_state s_fc_replay_state; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; }; struct ext4_group_info { @@ -47072,11 +47067,6 @@ enum { }; enum { - EXT4_FC_REASON_OK = 0, - EXT4_FC_REASON_INELIGIBLE = 1, - EXT4_FC_REASON_ALREADY_COMMITTED = 2, - EXT4_FC_REASON_FC_START_FAILED = 3, - EXT4_FC_REASON_FC_FAILED = 4, EXT4_FC_REASON_XATTR = 0, EXT4_FC_REASON_CROSS_RENAME = 1, EXT4_FC_REASON_JOURNAL_FLAG_CHANGE = 2, @@ -47131,11 +47121,6 @@ struct fname { char name[0]; }; -enum { - BLK_RW_ASYNC = 0, - BLK_RW_SYNC = 1, -}; - enum SHIFT_DIRECTION { SHIFT_LEFT = 0, SHIFT_RIGHT = 1, @@ -47519,6 +47504,8 @@ struct fsmap_head { struct fsmap fmh_recs[0]; }; +typedef void ext4_update_sb_callback(struct ext4_super_block *, const void *); + struct getfsmap_info { struct super_block *gi_sb; struct fsmap_head *gi_data; @@ -47776,6 +47763,15 @@ struct ext4_new_flex_group_data { ext4_group_t count; }; +struct disk_stats { + u64 nsecs[4]; + long unsigned int sectors[4]; + long unsigned int ios[4]; + long unsigned int merges[4]; + long unsigned int io_ticks; + local_t in_flight[2]; +}; + enum stat_group { STAT_READ = 0, STAT_WRITE = 1, @@ -48678,6 +48674,14 @@ struct trace_event_raw_ext4_fc_track_range { char __data[0]; }; +struct trace_event_raw_ext4_update_sb { + struct trace_entry ent; + dev_t dev; + ext4_fsblk_t fsblk; + unsigned int flags; + char __data[0]; +}; + struct trace_event_data_offsets_ext4_other_inode_update_time {}; struct trace_event_data_offsets_ext4_free_inode {}; @@ -48856,6 +48860,8 @@ struct trace_event_data_offsets_ext4_fc_track_inode {}; struct trace_event_data_offsets_ext4_fc_track_range {}; +struct trace_event_data_offsets_ext4_update_sb {}; + typedef void (*btf_trace_ext4_other_inode_update_time)(void *, struct inode *, ino_t); typedef void (*btf_trace_ext4_free_inode)(void *, struct inode *); @@ -49078,6 +49084,8 @@ typedef void (*btf_trace_ext4_fc_track_inode)(void *, struct inode *, int); typedef void (*btf_trace_ext4_fc_track_range)(void *, struct inode *, long int, long int, int); +typedef void (*btf_trace_ext4_update_sb)(void *, struct super_block *, ext4_fsblk_t, unsigned int); + struct ext4_err_translation { int code; int errno; @@ -49091,79 +49099,74 @@ enum { Opt_resgid = 4, Opt_resuid = 5, Opt_sb = 6, - Opt_err_cont = 7, - Opt_err_panic = 8, - Opt_err_ro = 9, - Opt_nouid32 = 10, - Opt_debug = 11, - Opt_removed = 12, - Opt_user_xattr = 13, - Opt_nouser_xattr = 14, - Opt_acl = 15, - Opt_noacl = 16, - Opt_auto_da_alloc = 17, - Opt_noauto_da_alloc = 18, - Opt_noload = 19, - Opt_commit = 20, - Opt_min_batch_time = 21, - Opt_max_batch_time = 22, - Opt_journal_dev = 23, - Opt_journal_path = 24, - Opt_journal_checksum = 25, - Opt_journal_async_commit = 26, - Opt_abort = 27, - Opt_data_journal = 28, - Opt_data_ordered = 29, - Opt_data_writeback = 30, - Opt_data_err_abort = 31, - Opt_data_err_ignore = 32, - Opt_test_dummy_encryption = 33, - Opt_inlinecrypt = 34, - Opt_usrjquota = 35, - Opt_grpjquota = 36, - Opt_offusrjquota = 37, - Opt_offgrpjquota = 38, - Opt_jqfmt_vfsold = 39, - Opt_jqfmt_vfsv0 = 40, - Opt_jqfmt_vfsv1 = 41, - Opt_quota = 42, - Opt_noquota = 43, - Opt_barrier = 44, - Opt_nobarrier = 45, - Opt_err___2 = 46, - Opt_usrquota = 47, - Opt_grpquota = 48, - Opt_prjquota = 49, - Opt_i_version = 50, - Opt_dax = 51, - Opt_dax_always = 52, - Opt_dax_inode = 53, - Opt_dax_never = 54, - Opt_stripe = 55, - Opt_delalloc = 56, - Opt_nodelalloc = 57, - Opt_warn_on_error = 58, - Opt_nowarn_on_error = 59, - Opt_mblk_io_submit = 60, - Opt_lazytime = 61, - Opt_nolazytime = 62, - Opt_debug_want_extra_isize = 63, - Opt_nomblk_io_submit = 64, - Opt_block_validity = 65, - Opt_noblock_validity = 66, - Opt_inode_readahead_blks = 67, - Opt_journal_ioprio = 68, - Opt_dioread_nolock = 69, - Opt_dioread_lock = 70, - Opt_discard = 71, - Opt_nodiscard = 72, - Opt_init_itable = 73, - Opt_noinit_itable = 74, - Opt_max_dir_size_kb = 75, - Opt_nojournal_checksum = 76, - Opt_nombcache = 77, - Opt_no_prefetch_block_bitmaps = 78, - Opt_mb_optimize_scan = 79, + Opt_nouid32 = 7, + Opt_debug = 8, + Opt_removed = 9, + Opt_user_xattr = 10, + Opt_nouser_xattr = 11, + Opt_acl = 12, + Opt_noacl = 13, + Opt_auto_da_alloc = 14, + Opt_noauto_da_alloc = 15, + Opt_noload = 16, + Opt_commit = 17, + Opt_min_batch_time = 18, + Opt_max_batch_time = 19, + Opt_journal_dev = 20, + Opt_journal_path = 21, + Opt_journal_checksum = 22, + Opt_journal_async_commit = 23, + Opt_abort = 24, + Opt_data_journal = 25, + Opt_data_ordered = 26, + Opt_data_writeback = 27, + Opt_data_err_abort = 28, + Opt_data_err_ignore = 29, + Opt_test_dummy_encryption = 30, + Opt_inlinecrypt = 31, + Opt_usrjquota = 32, + Opt_grpjquota = 33, + Opt_quota = 34, + Opt_noquota = 35, + Opt_barrier = 36, + Opt_nobarrier = 37, + Opt_err___2 = 38, + Opt_usrquota = 39, + Opt_grpquota = 40, + Opt_prjquota = 41, + Opt_i_version = 42, + Opt_dax = 43, + Opt_dax_always = 44, + Opt_dax_inode = 45, + Opt_dax_never = 46, + Opt_stripe = 47, + Opt_delalloc = 48, + Opt_nodelalloc = 49, + Opt_warn_on_error = 50, + Opt_nowarn_on_error = 51, + Opt_mblk_io_submit = 52, + Opt_debug_want_extra_isize = 53, + Opt_nomblk_io_submit = 54, + Opt_block_validity = 55, + Opt_noblock_validity = 56, + Opt_inode_readahead_blks = 57, + Opt_journal_ioprio = 58, + Opt_dioread_nolock = 59, + Opt_dioread_lock = 60, + Opt_discard = 61, + Opt_nodiscard = 62, + Opt_init_itable = 63, + Opt_noinit_itable = 64, + Opt_max_dir_size_kb = 65, + Opt_nojournal_checksum = 66, + Opt_nombcache = 67, + Opt_no_prefetch_block_bitmaps = 68, + Opt_mb_optimize_scan = 69, + Opt_errors = 70, + Opt_data = 71, + Opt_data_err = 72, + Opt_jqfmt = 73, + Opt_dax_type = 74, }; struct mount_opts { @@ -49172,10 +49175,35 @@ struct mount_opts { int flags; }; -struct ext4_parsed_options { - long unsigned int journal_devnum; - unsigned int journal_ioprio; +struct ext4_fs_context { + char *s_qf_names[3]; + char *test_dummy_enc_arg; + int s_jquota_fmt; int mb_optimize_scan; + short unsigned int qname_spec; + long unsigned int vals_s_flags; + long unsigned int mask_s_flags; + long unsigned int journal_devnum; + long unsigned int s_commit_interval; + long unsigned int s_stripe; + unsigned int s_inode_readahead_blks; + unsigned int s_want_extra_isize; + unsigned int s_li_wait_mult; + unsigned int s_max_dir_size_kb; + unsigned int journal_ioprio; + unsigned int vals_s_mount_opt; + unsigned int mask_s_mount_opt; + unsigned int vals_s_mount_opt2; + unsigned int mask_s_mount_opt2; + unsigned int vals_s_mount_flags; + unsigned int mask_s_mount_flags; + unsigned int opt_flags; + unsigned int spec; + u32 s_max_batch_time; + u32 s_min_batch_time; + kuid_t s_resuid; + kgid_t s_resgid; + ext4_fsblk_t s_sb_block; }; struct ext4_mount_options { @@ -49277,6 +49305,13 @@ struct ext4_fc_tail { __le32 fc_crc; }; +enum { + EXT4_FC_STATUS_OK = 0, + EXT4_FC_STATUS_INELIGIBLE = 1, + EXT4_FC_STATUS_SKIPPED = 2, + EXT4_FC_STATUS_FAILED = 3, +}; + struct ext4_fc_dentry_update { int fcd_op; int fcd_parent; @@ -49683,6 +49718,70 @@ struct utf8_table { long int lval; }; +enum fscache_cookie_state { + FSCACHE_COOKIE_STATE_QUIESCENT = 0, + FSCACHE_COOKIE_STATE_LOOKING_UP = 1, + FSCACHE_COOKIE_STATE_CREATING = 2, + FSCACHE_COOKIE_STATE_ACTIVE = 3, + FSCACHE_COOKIE_STATE_INVALIDATING = 4, + FSCACHE_COOKIE_STATE_FAILED = 5, + FSCACHE_COOKIE_STATE_LRU_DISCARDING = 6, + FSCACHE_COOKIE_STATE_WITHDRAWING = 7, + FSCACHE_COOKIE_STATE_RELINQUISHING = 8, + FSCACHE_COOKIE_STATE_DROPPED = 9, +}; + +struct fscache_cache; + +struct fscache_volume { + refcount_t ref; + atomic_t n_cookies; + atomic_t n_accesses; + unsigned int debug_id; + unsigned int key_hash; + char *key; + struct list_head proc_link; + struct hlist_bl_node hash_link; + struct work_struct work; + struct fscache_cache *cache; + void *cache_priv; + spinlock_t lock; + long unsigned int flags; + u8 coherency_len; + u8 coherency[0]; +}; + +struct fscache_cookie { + refcount_t ref; + atomic_t n_active; + atomic_t n_accesses; + unsigned int debug_id; + unsigned int inval_counter; + spinlock_t lock; + struct fscache_volume *volume; + void *cache_priv; + struct hlist_bl_node hash_link; + struct list_head proc_link; + struct list_head commit_link; + struct work_struct work; + loff_t object_size; + long unsigned int unused_at; + long unsigned int flags; + enum fscache_cookie_state state; + u8 advice; + u8 key_len; + u8 aux_len; + u32 key_hash; + union { + void *key; + u8 inline_key[16]; + }; + union { + void *aux; + u8 inline_aux[8]; + }; +}; + struct p9_qid { u8 type; u32 version; @@ -49882,53 +49981,6 @@ struct dotl_iattr_map { int p9_iattr_valid; }; -enum fscache_checkaux { - FSCACHE_CHECKAUX_OKAY = 0, - FSCACHE_CHECKAUX_NEEDS_UPDATE = 1, - FSCACHE_CHECKAUX_OBSOLETE = 2, -}; - -struct fscache_cache_tag; - -struct fscache_cookie_def { - char name[16]; - uint8_t type; - struct fscache_cache_tag * (*select_cache)(const void *, const void *); - enum fscache_checkaux (*check_aux)(void *, const void *, uint16_t, loff_t); - void (*get_context)(void *, void *); - void (*put_context)(void *, void *); - void (*mark_page_cached)(void *, struct address_space *, struct page *); -}; - -struct fscache_cookie { - refcount_t ref; - atomic_t n_children; - atomic_t n_active; - unsigned int debug_id; - spinlock_t lock; - spinlock_t stores_lock; - struct hlist_head backing_objects; - const struct fscache_cookie_def *def; - struct fscache_cookie *parent; - struct hlist_bl_node hash_link; - struct list_head proc_link; - void *netfs_data; - struct xarray stores; - long unsigned int flags; - u8 type; - u8 key_len; - u8 aux_len; - u32 key_hash; - union { - void *key; - u8 inline_key[16]; - }; - union { - void *aux; - u8 inline_aux[8]; - }; -}; - struct p9_flock { u8 type; u32 flags; @@ -50658,7 +50710,6 @@ union security_list_options { int (*sb_pivotroot)(const struct path *, const struct path *); int (*sb_set_mnt_opts)(struct super_block *, void *, long unsigned int, long unsigned int *); int (*sb_clone_mnt_opts)(const struct super_block *, struct super_block *, long unsigned int, long unsigned int *); - int (*sb_add_mnt_opt)(const char *, const char *, int, void **); int (*move_mount)(const struct path *, const struct path *); int (*dentry_init_security)(struct dentry *, int, const struct qstr *, const char **, void **, u32 *); int (*dentry_create_files_as)(struct dentry *, int, struct qstr *, const struct cred *, struct cred *); @@ -50726,7 +50777,7 @@ union security_list_options { int (*task_setpgid)(struct task_struct *, pid_t); int (*task_getpgid)(struct task_struct *); int (*task_getsid)(struct task_struct *); - void (*task_getsecid_subj)(struct task_struct *, u32 *); + void (*current_getsecid_subj)(u32 *); void (*task_getsecid_obj)(struct task_struct *, u32 *); int (*task_setnice)(struct task_struct *, int); int (*task_setioprio)(struct task_struct *, int); @@ -51307,7 +51358,7 @@ struct ethtool_link_ext_state_info { enum ethtool_link_ext_substate_bad_signal_integrity bad_signal_integrity; enum ethtool_link_ext_substate_cable_issue cable_issue; enum ethtool_link_ext_substate_module module; - u8 __link_ext_substate; + u32 __link_ext_substate; }; }; @@ -55758,12 +55809,6 @@ struct crypto_report_rng { unsigned int seedsize; }; -struct random_ready_callback { - struct list_head list; - void (*func)(struct random_ready_callback *); - struct module *owner; -}; - struct drbg_string { const unsigned char *buf; size_t len; @@ -55789,6 +55834,12 @@ struct drbg_state_ops { int (*crypto_fini)(struct drbg_state *); }; +enum drbg_seed_state { + DRBG_SEED_STATE_UNSEEDED = 0, + DRBG_SEED_STATE_PARTIAL = 1, + DRBG_SEED_STATE_FULL = 2, +}; + struct drbg_state { struct mutex drbg_mutex; unsigned char *V; @@ -55807,16 +55858,15 @@ struct drbg_state { struct crypto_wait ctr_wait; struct scatterlist sg_in; struct scatterlist sg_out; - bool seeded; + enum drbg_seed_state seeded; + long unsigned int last_seed_time; bool pr; bool fips_primed; unsigned char *prev; - struct work_struct seed_work; struct crypto_rng *jent; const struct drbg_state_ops *d_ops; const struct drbg_core *core; struct drbg_string test_data; - struct random_ready_callback random_ready; }; enum drbg_prefixes { @@ -56006,7 +56056,7 @@ struct asymmetric_key_id { }; struct asymmetric_key_ids { - void *id[2]; + void *id[3]; }; struct public_key_signature; @@ -56023,7 +56073,7 @@ struct asymmetric_key_subtype___2 { }; struct public_key_signature { - struct asymmetric_key_id *auth_ids[2]; + struct asymmetric_key_id *auth_ids[3]; u8 *s; u8 *digest; u32 s_size; @@ -56309,6 +56359,12 @@ enum { DISK_EVENT_EJECT_REQUEST = 2, }; +enum { + DISK_EVENT_FLAG_POLL = 1, + DISK_EVENT_FLAG_UEVENT = 2, + DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE = 4, +}; + struct blk_integrity_profile; struct blk_integrity { @@ -56373,7 +56429,7 @@ struct blkdev_dio { }; struct bio_alloc_cache { - struct bio_list free_list; + struct bio *free_list; unsigned int nr; }; @@ -56394,6 +56450,14 @@ struct rq_qos { struct dentry *debugfs_dir; }; +struct blk_mq_debugfs_attr { + const char *name; + umode_t mode; + int (*show)(void *, struct seq_file *); + ssize_t (*write)(void *, const char *, size_t, loff_t *); + const struct seq_operations *seq_ops; +}; + struct rq_qos_ops { void (*throttle)(struct rq_qos *, struct bio *); void (*track)(struct rq_qos *, struct request *, struct bio *); @@ -56421,6 +56485,47 @@ struct bio_slab { char name[8]; }; +struct elevator_type; + +struct elevator_queue { + struct elevator_type *type; + void *elevator_data; + struct kobject kobj; + struct mutex sysfs_lock; + unsigned int registered: 1; + struct hlist_head hash[64]; +}; + +struct blk_mq_ctxs; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_lists[3]; + long: 64; + long: 64; + }; + unsigned int cpu; + short unsigned int index_hw[3]; + struct blk_mq_hw_ctx *hctxs[3]; + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; + long: 64; +}; + +struct blk_stat_callback { + struct list_head list; + struct timer_list timer; + struct blk_rq_stat *cpu_stat; + int (*bucket_fn)(const struct request *); + unsigned int buckets; + struct blk_rq_stat *stat; + void (*timer_fn)(struct blk_stat_callback *); + void *data; + struct callback_head rcu; +}; + enum { BLK_MQ_F_SHOULD_MERGE = 1, BLK_MQ_F_TAG_QUEUE_SHARED = 2, @@ -56439,6 +56544,81 @@ enum { BLK_MQ_CPU_WORK_BATCH = 8, }; +enum elv_merge { + ELEVATOR_NO_MERGE = 0, + ELEVATOR_FRONT_MERGE = 1, + ELEVATOR_BACK_MERGE = 2, + ELEVATOR_DISCARD_MERGE = 3, +}; + +struct blk_mq_alloc_data; + +struct elevator_mq_ops { + int (*init_sched)(struct request_queue *, struct elevator_type *); + void (*exit_sched)(struct elevator_queue *); + int (*init_hctx)(struct blk_mq_hw_ctx *, unsigned int); + void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); + void (*depth_updated)(struct blk_mq_hw_ctx *); + bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); + bool (*bio_merge)(struct request_queue *, struct bio *, unsigned int); + int (*request_merge)(struct request_queue *, struct request **, struct bio *); + void (*request_merged)(struct request_queue *, struct request *, enum elv_merge); + void (*requests_merged)(struct request_queue *, struct request *, struct request *); + void (*limit_depth)(unsigned int, struct blk_mq_alloc_data *); + void (*prepare_request)(struct request *); + void (*finish_request)(struct request *); + void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); + struct request * (*dispatch_request)(struct blk_mq_hw_ctx *); + bool (*has_work)(struct blk_mq_hw_ctx *); + void (*completed_request)(struct request *, u64); + void (*requeue_request)(struct request *); + struct request * (*former_request)(struct request_queue *, struct request *); + struct request * (*next_request)(struct request_queue *, struct request *); + void (*init_icq)(struct io_cq *); + void (*exit_icq)(struct io_cq *); +}; + +struct elv_fs_entry; + +struct elevator_type { + struct kmem_cache *icq_cache; + struct elevator_mq_ops ops; + size_t icq_size; + size_t icq_align; + struct elv_fs_entry *elevator_attrs; + const char *elevator_name; + const char *elevator_alias; + const unsigned int elevator_features; + struct module *elevator_owner; + const struct blk_mq_debugfs_attr *queue_debugfs_attrs; + const struct blk_mq_debugfs_attr *hctx_debugfs_attrs; + char icq_cache_name[22]; + struct list_head list; +}; + +struct blk_mq_alloc_data { + struct request_queue *q; + blk_mq_req_flags_t flags; + unsigned int shallow_depth; + unsigned int cmd_flags; + req_flags_t rq_flags; + unsigned int nr_tags; + struct request **cached_rq; + struct blk_mq_ctx *ctx; + struct blk_mq_hw_ctx *hctx; +}; + +struct elv_fs_entry { + struct attribute attr; + ssize_t (*show)(struct elevator_queue *, char *); + ssize_t (*store)(struct elevator_queue *, const char *, size_t); +}; + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx *queue_ctx; +}; + enum { WBT_RWQ_BG = 0, WBT_RWQ_KSWAPD = 1, @@ -56680,12 +56860,6 @@ typedef void (*btf_trace_block_bio_remap)(void *, struct bio *, dev_t, sector_t) typedef void (*btf_trace_block_rq_remap)(void *, struct request *, dev_t, sector_t); -enum { - BLK_MQ_NO_TAG = 4294967295, - BLK_MQ_TAG_MIN = 1, - BLK_MQ_TAG_MAX = 4294967294, -}; - enum blkg_rwstat_type { BLKG_RWSTAT_READ = 0, BLKG_RWSTAT_WRITE = 1, @@ -56769,6 +56943,12 @@ struct queue_sysfs_entry { ssize_t (*store)(struct request_queue *, const char *, size_t); }; +enum { + BLK_MQ_NO_TAG = 4294967295, + BLK_MQ_TAG_MIN = 1, + BLK_MQ_TAG_MAX = 4294967294, +}; + enum { REQ_FSEQ_PREFLUSH = 1, REQ_FSEQ_DATA = 2, @@ -56778,11 +56958,6 @@ enum { FLUSH_PENDING_TIMEOUT = 5000, }; -enum { - ICQ_EXITED = 4, - ICQ_DESTROYED = 8, -}; - struct rq_map_data { struct page **pages; int page_order; @@ -56854,13 +57029,12 @@ struct sbq_wait { struct wait_queue_entry wait; }; -typedef bool busy_iter_fn(struct blk_mq_hw_ctx *, struct request *, void *, bool); - typedef bool busy_tag_iter_fn(struct request *, void *, bool); struct bt_iter_data { struct blk_mq_hw_ctx *hctx; - busy_iter_fn *fn; + struct request_queue *q; + busy_tag_iter_fn *fn; void *data; bool reserved; }; @@ -56875,7 +57049,7 @@ struct bt_tags_iter_data { struct blk_queue_stats { struct list_head callbacks; spinlock_t lock; - bool enable_accounting; + int accounting; }; struct blk_mq_hw_ctx_sysfs_entry { @@ -56957,6 +57131,12 @@ struct class_dev_iter { const struct device_type *type; }; +enum { + GENHD_FL_REMOVABLE = 1, + GENHD_FL_HIDDEN = 2, + GENHD_FL_NO_PART = 4, +}; + struct badblocks { struct device *dev; int count; @@ -57483,11 +57663,6 @@ struct rq_qos_wait_data { bool got_token; }; -enum { - DISK_EVENT_FLAG_POLL = 1, - DISK_EVENT_FLAG_UEVENT = 2, -}; - struct disk_events { struct list_head node; struct gendisk *disk; @@ -57922,6 +58097,12 @@ typedef void (*swap_func_t)(void *, void *, int); typedef int (*cmp_r_func_t)(const void *, const void *, const void *); +struct random_ready_callback { + struct list_head list; + void (*func)(struct random_ready_callback *); + struct module *owner; +}; + struct siprand_state { long unsigned int v0; long unsigned int v1; @@ -58031,6 +58212,11 @@ struct genradix_node { }; }; +struct strarray { + char **array; + size_t n; +}; + struct reciprocal_value_adv { u32 m; u8 sh; @@ -58057,6 +58243,8 @@ struct blake2s_state { unsigned int outlen; }; +typedef void (*blake2s_compress_t)(struct blake2s_state *, const u8 *, size_t, u32); + enum blake2s_iv { BLAKE2S_IV0 = 1779033703, BLAKE2S_IV1 = 3144134277, @@ -58068,8 +58256,6 @@ enum blake2s_iv { BLAKE2S_IV7 = 1541459225, }; -typedef void (*blake2s_compress_t)(struct blake2s_state *, const u8 *, size_t, u32); - enum devm_ioremap_type { DEVM_IOREMAP = 0, DEVM_IOREMAP_UC = 1, @@ -59872,6 +60058,7 @@ struct pci_host_bridge { long: 64; long: 64; long: 64; + long: 64; long unsigned int private[0]; }; @@ -60024,6 +60211,7 @@ struct acpi_driver; struct acpi_gpio_mapping; struct acpi_device { + u32 pld_crc; int device_type; acpi_handle handle; struct fwnode_handle fwnode; @@ -60358,11 +60546,6 @@ struct walk_rcec_data { void *user_data; }; -struct aspm_latency { - u32 l0s; - u32 l1; -}; - struct pcie_link_state { struct pci_dev *pdev; struct pci_dev *downstream; @@ -60379,9 +60562,6 @@ struct pcie_link_state { u32 clkpm_enabled: 1; u32 clkpm_default: 1; u32 clkpm_disable: 1; - struct aspm_latency latency_up; - struct aspm_latency latency_dw; - struct aspm_latency acceptable[8]; }; struct pci_filp_private { @@ -61283,6 +61463,12 @@ struct acpi_table_desc { u16 validation_count; }; +enum acpi_cedt_type { + ACPI_CEDT_TYPE_CHBS = 0, + ACPI_CEDT_TYPE_CFMWS = 1, + ACPI_CEDT_TYPE_RESERVED = 2, +}; + struct acpi_madt_io_sapic { struct acpi_subtable_header header; u8 id; @@ -61370,6 +61556,7 @@ enum acpi_subtable_type { ACPI_SUBTABLE_COMMON = 0, ACPI_SUBTABLE_HMAT = 1, ACPI_SUBTABLE_PRMT = 2, + ACPI_SUBTABLE_CEDT = 3, }; struct acpi_subtable_entry { @@ -64193,6 +64380,20 @@ struct container_dev { int (*offline)(struct container_dev *); }; +struct acpi_cedt_cfmws { + struct acpi_cedt_header header; + u32 reserved1; + u64 base_hpa; + u64 window_size; + u8 interleave_ways; + u8 interleave_arithmetic; + u16 reserved2; + u32 granularity; + u16 restrictions; + u16 qtg_id; + u32 interleave_targets[0]; +} __attribute__((packed)); + struct acpi_table_slit { struct acpi_table_header header; u64 locality_count; @@ -65715,8 +65916,6 @@ struct uart_8250_port { unsigned char ier; unsigned char lcr; unsigned char mcr; - unsigned char mcr_mask; - unsigned char mcr_force; unsigned char cur_iotype; unsigned int rpm_tx_active; unsigned char canary; @@ -65779,7 +65978,6 @@ struct dma_slave_config { u32 src_port_window_size; u32 dst_port_window_size; bool device_fc; - unsigned int slave_id; void *peripheral_config; size_t peripheral_size; }; @@ -65820,22 +66018,23 @@ enum dma_status { enum dma_transaction_type { DMA_MEMCPY = 0, - DMA_XOR = 1, - DMA_PQ = 2, - DMA_XOR_VAL = 3, - DMA_PQ_VAL = 4, - DMA_MEMSET = 5, - DMA_MEMSET_SG = 6, - DMA_INTERRUPT = 7, - DMA_PRIVATE = 8, - DMA_ASYNC_TX = 9, - DMA_SLAVE = 10, - DMA_CYCLIC = 11, - DMA_INTERLEAVE = 12, - DMA_COMPLETION_NO_ORDER = 13, - DMA_REPEAT = 14, - DMA_LOAD_EOT = 15, - DMA_TX_TYPE_END = 16, + DMA_MEMCPY_SG = 1, + DMA_XOR = 2, + DMA_PQ = 3, + DMA_XOR_VAL = 4, + DMA_PQ_VAL = 5, + DMA_MEMSET = 6, + DMA_MEMSET_SG = 7, + DMA_INTERRUPT = 8, + DMA_PRIVATE = 9, + DMA_ASYNC_TX = 10, + DMA_SLAVE = 11, + DMA_CYCLIC = 12, + DMA_INTERLEAVE = 13, + DMA_COMPLETION_NO_ORDER = 14, + DMA_REPEAT = 15, + DMA_LOAD_EOT = 16, + DMA_TX_TYPE_END = 17, }; struct data_chunk { @@ -65987,6 +66186,7 @@ struct dma_device { int (*device_router_config)(struct dma_chan___2 *); void (*device_free_chan_resources)(struct dma_chan___2 *); struct dma_async_tx_descriptor * (*device_prep_dma_memcpy)(struct dma_chan___2 *, dma_addr_t, dma_addr_t, size_t, long unsigned int); + struct dma_async_tx_descriptor * (*device_prep_dma_memcpy_sg)(struct dma_chan___2 *, struct scatterlist *, unsigned int, struct scatterlist *, unsigned int, long unsigned int); struct dma_async_tx_descriptor * (*device_prep_dma_xor)(struct dma_chan___2 *, dma_addr_t, dma_addr_t *, unsigned int, size_t, long unsigned int); struct dma_async_tx_descriptor * (*device_prep_dma_xor_val)(struct dma_chan___2 *, dma_addr_t *, unsigned int, size_t, enum sum_check_flags *, long unsigned int); struct dma_async_tx_descriptor * (*device_prep_dma_pq)(struct dma_chan___2 *, dma_addr_t *, dma_addr_t *, unsigned int, const unsigned char *, size_t, long unsigned int); @@ -66266,22 +66466,18 @@ enum pci_board_num_t { pbn_wch382_2 = 102, pbn_wch384_4 = 103, pbn_wch384_8 = 104, - pbn_pericom_PI7C9X7951 = 105, - pbn_pericom_PI7C9X7952 = 106, - pbn_pericom_PI7C9X7954 = 107, - pbn_pericom_PI7C9X7958 = 108, - pbn_sunix_pci_1s = 109, - pbn_sunix_pci_2s = 110, - pbn_sunix_pci_4s = 111, - pbn_sunix_pci_8s = 112, - pbn_sunix_pci_16s = 113, - pbn_titan_1_4000000 = 114, - pbn_titan_2_4000000 = 115, - pbn_titan_4_4000000 = 116, - pbn_titan_8_4000000 = 117, - pbn_moxa8250_2p = 118, - pbn_moxa8250_4p = 119, - pbn_moxa8250_8p = 120, + pbn_sunix_pci_1s = 105, + pbn_sunix_pci_2s = 106, + pbn_sunix_pci_4s = 107, + pbn_sunix_pci_8s = 108, + pbn_sunix_pci_16s = 109, + pbn_titan_1_4000000 = 110, + pbn_titan_2_4000000 = 111, + pbn_titan_4_4000000 = 112, + pbn_titan_8_4000000 = 113, + pbn_moxa8250_2p = 114, + pbn_moxa8250_4p = 115, + pbn_moxa8250_8p = 116, }; struct software_node { @@ -66312,6 +66508,12 @@ struct exar8250 { int line[0]; }; +struct pericom8250 { + void *virt; + unsigned int nr; + int line[0]; +}; + struct memdev { const char *name; umode_t mode; @@ -66341,7 +66543,6 @@ struct trace_event_raw_add_device_randomness { struct trace_event_raw_random__mix_pool_bytes { struct trace_entry ent; - const char *pool_name; int bytes; long unsigned int IP; char __data[0]; @@ -66349,7 +66550,6 @@ struct trace_event_raw_random__mix_pool_bytes { struct trace_event_raw_credit_entropy_bits { struct trace_entry ent; - const char *pool_name; int bits; int entropy_count; long unsigned int IP; @@ -66358,7 +66558,6 @@ struct trace_event_raw_credit_entropy_bits { struct trace_event_raw_debit_entropy { struct trace_entry ent; - const char *pool_name; int debit_bits; char __data[0]; }; @@ -66385,7 +66584,6 @@ struct trace_event_raw_random__get_random_bytes { struct trace_event_raw_random__extract_entropy { struct trace_entry ent; - const char *pool_name; int nbytes; int entropy_count; long unsigned int IP; @@ -66428,13 +66626,13 @@ struct trace_event_data_offsets_prandom_u32 {}; typedef void (*btf_trace_add_device_randomness)(void *, int, long unsigned int); -typedef void (*btf_trace_mix_pool_bytes)(void *, const char *, int, long unsigned int); +typedef void (*btf_trace_mix_pool_bytes)(void *, int, long unsigned int); -typedef void (*btf_trace_mix_pool_bytes_nolock)(void *, const char *, int, long unsigned int); +typedef void (*btf_trace_mix_pool_bytes_nolock)(void *, int, long unsigned int); -typedef void (*btf_trace_credit_entropy_bits)(void *, const char *, int, int, long unsigned int); +typedef void (*btf_trace_credit_entropy_bits)(void *, int, int, long unsigned int); -typedef void (*btf_trace_debit_entropy)(void *, const char *, int); +typedef void (*btf_trace_debit_entropy)(void *, int); typedef void (*btf_trace_add_input_randomness)(void *, int); @@ -66444,47 +66642,39 @@ typedef void (*btf_trace_get_random_bytes)(void *, int, long unsigned int); typedef void (*btf_trace_get_random_bytes_arch)(void *, int, long unsigned int); -typedef void (*btf_trace_extract_entropy)(void *, const char *, int, int, long unsigned int); +typedef void (*btf_trace_extract_entropy)(void *, int, int, long unsigned int); typedef void (*btf_trace_urandom_read)(void *, int, int, int); typedef void (*btf_trace_prandom_u32)(void *, unsigned int); -struct poolinfo { - int poolbitshift; - int poolwords; - int poolbytes; - int poolfracbits; - int tap1; - int tap2; - int tap3; - int tap4; - int tap5; +enum poolinfo { + POOL_WORDS = 128, + POOL_WORDMASK = 127, + POOL_BYTES = 512, + POOL_BITS = 4096, + POOL_BITSHIFT = 12, + POOL_ENTROPY_SHIFT = 3, + POOL_FRACBITS = 32768, + POOL_TAP1 = 104, + POOL_TAP2 = 76, + POOL_TAP3 = 51, + POOL_TAP4 = 25, + POOL_TAP5 = 1, + EXTRACT_SIZE = 16, }; struct crng_state { - __u32 state[16]; + u32 state[16]; long unsigned int init_time; spinlock_t lock; }; -struct entropy_store { - const struct poolinfo *poolinfo; - __u32 *pool; - const char *name; - spinlock_t lock; - short unsigned int add_ptr; - short unsigned int input_rotate; - int entropy_count; - unsigned int last_data_init: 1; - __u8 last_data[16]; -}; - struct fast_pool { - __u32 pool[4]; + u32 pool[4]; long unsigned int last; - short unsigned int reg_idx; - unsigned char count; + u16 reg_idx; + u8 count; }; struct batched_entropy { @@ -66864,6 +67054,7 @@ enum tpm_chip_flags { TPM_CHIP_FLAG_HAVE_TIMEOUTS = 16, TPM_CHIP_FLAG_ALWAYS_POWERED = 32, TPM_CHIP_FLAG_FIRMWARE_POWER_MANAGED = 64, + TPM_CHIP_FLAG_FIRMWARE_UPGRADE = 128, }; enum tpm2_structures { @@ -66878,6 +67069,7 @@ enum tpm2_return_codes { TPM2_RC_INITIALIZE = 256, TPM2_RC_FAILURE = 257, TPM2_RC_DISABLED = 288, + TPM2_RC_UPGRADE = 301, TPM2_RC_COMMAND_CODE = 323, TPM2_RC_TESTING = 2314, TPM2_RC_REFERENCE_H0 = 2320, @@ -68147,7 +68339,6 @@ typedef void (*irq_write_msi_msg_t)(struct msi_desc *, struct msi_msg *); struct platform_msi_priv_data { struct device *dev; void *host_data; - const struct attribute_group **msi_irq_groups; msi_alloc_info_t arg; irq_write_msi_msg_t write_msg; int devid; @@ -68259,6 +68450,7 @@ struct loop_device { struct gendisk *lo_disk; struct mutex lo_mutex; bool idr_visible; + struct work_struct rundown_work; }; struct loop_cmd { @@ -68800,6 +68992,7 @@ struct skb_shared_info { unsigned int gso_type; u32 tskey; atomic_t dataref; + unsigned int xdp_frags_size; void *destructor_arg; skb_frag_t frags[17]; }; @@ -70789,6 +70982,7 @@ struct wpan_phy { possible_net_t _net; long: 64; long: 64; + long: 64; char priv[0]; }; @@ -73646,44 +73840,45 @@ enum power_supply_property { POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT_MAX = 34, POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD = 35, POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD = 36, - POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT = 37, - POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT = 38, - POWER_SUPPLY_PROP_INPUT_POWER_LIMIT = 39, - POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN = 40, - POWER_SUPPLY_PROP_ENERGY_EMPTY_DESIGN = 41, - POWER_SUPPLY_PROP_ENERGY_FULL = 42, - POWER_SUPPLY_PROP_ENERGY_EMPTY = 43, - POWER_SUPPLY_PROP_ENERGY_NOW = 44, - POWER_SUPPLY_PROP_ENERGY_AVG = 45, - POWER_SUPPLY_PROP_CAPACITY = 46, - POWER_SUPPLY_PROP_CAPACITY_ALERT_MIN = 47, - POWER_SUPPLY_PROP_CAPACITY_ALERT_MAX = 48, - POWER_SUPPLY_PROP_CAPACITY_ERROR_MARGIN = 49, - POWER_SUPPLY_PROP_CAPACITY_LEVEL = 50, - POWER_SUPPLY_PROP_TEMP = 51, - POWER_SUPPLY_PROP_TEMP_MAX = 52, - POWER_SUPPLY_PROP_TEMP_MIN = 53, - POWER_SUPPLY_PROP_TEMP_ALERT_MIN = 54, - POWER_SUPPLY_PROP_TEMP_ALERT_MAX = 55, - POWER_SUPPLY_PROP_TEMP_AMBIENT = 56, - POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MIN = 57, - POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MAX = 58, - POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW = 59, - POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG = 60, - POWER_SUPPLY_PROP_TIME_TO_FULL_NOW = 61, - POWER_SUPPLY_PROP_TIME_TO_FULL_AVG = 62, - POWER_SUPPLY_PROP_TYPE = 63, - POWER_SUPPLY_PROP_USB_TYPE = 64, - POWER_SUPPLY_PROP_SCOPE = 65, - POWER_SUPPLY_PROP_PRECHARGE_CURRENT = 66, - POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT = 67, - POWER_SUPPLY_PROP_CALIBRATE = 68, - POWER_SUPPLY_PROP_MANUFACTURE_YEAR = 69, - POWER_SUPPLY_PROP_MANUFACTURE_MONTH = 70, - POWER_SUPPLY_PROP_MANUFACTURE_DAY = 71, - POWER_SUPPLY_PROP_MODEL_NAME = 72, - POWER_SUPPLY_PROP_MANUFACTURER = 73, - POWER_SUPPLY_PROP_SERIAL_NUMBER = 74, + POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR = 37, + POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT = 38, + POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT = 39, + POWER_SUPPLY_PROP_INPUT_POWER_LIMIT = 40, + POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN = 41, + POWER_SUPPLY_PROP_ENERGY_EMPTY_DESIGN = 42, + POWER_SUPPLY_PROP_ENERGY_FULL = 43, + POWER_SUPPLY_PROP_ENERGY_EMPTY = 44, + POWER_SUPPLY_PROP_ENERGY_NOW = 45, + POWER_SUPPLY_PROP_ENERGY_AVG = 46, + POWER_SUPPLY_PROP_CAPACITY = 47, + POWER_SUPPLY_PROP_CAPACITY_ALERT_MIN = 48, + POWER_SUPPLY_PROP_CAPACITY_ALERT_MAX = 49, + POWER_SUPPLY_PROP_CAPACITY_ERROR_MARGIN = 50, + POWER_SUPPLY_PROP_CAPACITY_LEVEL = 51, + POWER_SUPPLY_PROP_TEMP = 52, + POWER_SUPPLY_PROP_TEMP_MAX = 53, + POWER_SUPPLY_PROP_TEMP_MIN = 54, + POWER_SUPPLY_PROP_TEMP_ALERT_MIN = 55, + POWER_SUPPLY_PROP_TEMP_ALERT_MAX = 56, + POWER_SUPPLY_PROP_TEMP_AMBIENT = 57, + POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MIN = 58, + POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MAX = 59, + POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW = 60, + POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG = 61, + POWER_SUPPLY_PROP_TIME_TO_FULL_NOW = 62, + POWER_SUPPLY_PROP_TIME_TO_FULL_AVG = 63, + POWER_SUPPLY_PROP_TYPE = 64, + POWER_SUPPLY_PROP_USB_TYPE = 65, + POWER_SUPPLY_PROP_SCOPE = 66, + POWER_SUPPLY_PROP_PRECHARGE_CURRENT = 67, + POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT = 68, + POWER_SUPPLY_PROP_CALIBRATE = 69, + POWER_SUPPLY_PROP_MANUFACTURE_YEAR = 70, + POWER_SUPPLY_PROP_MANUFACTURE_MONTH = 71, + POWER_SUPPLY_PROP_MANUFACTURE_DAY = 72, + POWER_SUPPLY_PROP_MODEL_NAME = 73, + POWER_SUPPLY_PROP_MANUFACTURER = 74, + POWER_SUPPLY_PROP_SERIAL_NUMBER = 75, }; enum power_supply_type { @@ -73974,6 +74169,7 @@ enum { POWER_SUPPLY_HEALTH_WARM = 11, POWER_SUPPLY_HEALTH_COOL = 12, POWER_SUPPLY_HEALTH_HOT = 13, + POWER_SUPPLY_HEALTH_NO_BATTERY = 14, }; enum { @@ -73991,6 +74187,12 @@ enum { POWER_SUPPLY_SCOPE_DEVICE = 2, }; +enum power_supply_charge_behaviour { + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO = 0, + POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE = 1, + POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE = 2, +}; + struct power_supply_attr { const char *prop_name; char attr_name[31]; @@ -75097,10 +75299,12 @@ struct a4tech_sc { }; struct apple_sc { + struct hid_device *hdev; long unsigned int quirks; unsigned int fn_on; unsigned int fn_found; long unsigned int pressed_numlock[12]; + struct timer_list battery_timer; }; struct apple_key_translation { @@ -75137,6 +75341,17 @@ struct zpff_device { struct hid_report *report; }; +enum simatic_ipc_station_ids { + SIMATIC_IPC_INVALID_STATION_ID = 0, + SIMATIC_IPC_IPC227D = 1281, + SIMATIC_IPC_IPC427D = 1793, + SIMATIC_IPC_IPC227E = 2305, + SIMATIC_IPC_IPC277E = 2306, + SIMATIC_IPC_IPC427E = 2561, + SIMATIC_IPC_IPC477E = 2562, + SIMATIC_IPC_IPC127E = 3329, +}; + struct pmc_bit_map { const char *name; u32 bit_mask; @@ -76621,6 +76836,7 @@ struct packet_type { int (*func)(struct sk_buff *, struct net_device *, struct packet_type *, struct net_device *); void (*list_func)(struct list_head *, struct packet_type *, struct net_device *); bool (*id_match)(struct packet_type *, struct sock *); + struct net *af_packet_net; void *af_packet_priv; struct list_head list; }; @@ -77197,6 +77413,11 @@ struct seg6_pernet_data { struct in6_addr *tun_src; }; +enum xdp_buff_flags { + XDP_FLAGS_HAS_FRAGS = 1, + XDP_FLAGS_FRAGS_PF_MEMALLOC = 2, +}; + enum rt_class_t { RT_TABLE_UNSPEC = 0, RT_TABLE_COMPAT = 252, @@ -77884,8 +78105,14 @@ typedef u64 (*btf_bpf_skb_change_head)(struct sk_buff *, u32, u64); typedef u64 (*btf_sk_skb_change_head)(struct sk_buff *, u32, u64); +typedef u64 (*btf_bpf_xdp_get_buff_len)(struct xdp_buff *); + typedef u64 (*btf_bpf_xdp_adjust_head)(struct xdp_buff *, int); +typedef u64 (*btf_bpf_xdp_load_bytes)(struct xdp_buff *, u32, void *, u32); + +typedef u64 (*btf_bpf_xdp_store_bytes)(struct xdp_buff *, u32, void *, u32); + typedef u64 (*btf_bpf_xdp_adjust_tail)(struct xdp_buff *, int); typedef u64 (*btf_bpf_xdp_adjust_meta)(struct xdp_buff *, int); @@ -81794,6 +82021,7 @@ struct psched_ratecfg { u64 rate_bytes_ps; u32 mult; u16 overhead; + u16 mpu; u8 linklayer; u8 shift; }; @@ -89306,6 +89534,13 @@ struct packet_fanout { long: 64; struct packet_type prot_hook; struct sock *arr[0]; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; }; struct packet_rollover { @@ -89353,6 +89588,13 @@ struct packet_sock { long: 64; long: 64; struct packet_type prot_hook; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; + long: 64; atomic_t tp_drops; long: 32; long: 64; From ae220adbb22010f109c8e54bdb949416ec424df1 Mon Sep 17 00:00:00 2001 From: Evgeny Vereshchagin Date: Mon, 31 Jan 2022 00:08:34 +0000 Subject: [PATCH 106/131] ci: no longer remove elfutils while building the fuzzer Without it coverage reports can't be built ``` [2022-01-31 00:05:36,094 DEBUG] Generating file view html index file as: "/out/report/linux/file_view_index.html". Traceback (most recent call last): File "/opt/code_coverage/coverage_utils.py", line 829, in sys.exit(Main()) File "/opt/code_coverage/coverage_utils.py", line 823, in Main return _CmdPostProcess(args) File "/opt/code_coverage/coverage_utils.py", line 780, in _CmdPostProcess processor.PrepareHtmlReport() File "/opt/code_coverage/coverage_utils.py", line 577, in PrepareHtmlReport self.GenerateFileViewHtmlIndexFile(per_file_coverage_summary, File "/opt/code_coverage/coverage_utils.py", line 450, in GenerateFileViewHtmlIndexFile self.GetCoverageHtmlReportPathForFile(file_path), File "/opt/code_coverage/coverage_utils.py", line 422, in GetCoverageHtmlReportPathForFile assert os.path.isfile( AssertionError: "/tmp/tmp.UYax4l19Gh/lib/system.h" is not a file. ``` It's a follow-up to 393a058d061d49d Signed-off-by: Evgeny Vereshchagin --- scripts/build-fuzzers.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/build-fuzzers.sh b/scripts/build-fuzzers.sh index 0591451..22637ae 100755 --- a/scripts/build-fuzzers.sh +++ b/scripts/build-fuzzers.sh @@ -22,10 +22,10 @@ export LIB_FUZZING_ENGINE=${LIB_FUZZING_ENGINE:--fsanitize=fuzzer} # fail to compile (for example, elfutils-0.185 fails to compile with LDFLAGS enabled # due to https://bugs.gentoo.org/794601) so let's just point the script to # commits referring to versions of libelf that actually can be built -elfutils=$(mktemp -d) -git clone git://sourceware.org/git/elfutils.git "$elfutils" +rm -rf elfutils +git clone git://sourceware.org/git/elfutils.git ( -cd "$elfutils" +cd elfutils git checkout 983e86fd89e8bf02f2d27ba5dce5bf078af4ceda git log --oneline -1 @@ -50,11 +50,9 @@ make -C libelf -j$(nproc) V=1 ) make -C src BUILD_STATIC_ONLY=y V=1 clean -make -C src -j$(nproc) CFLAGS="-I$elfutils/libelf $CFLAGS" BUILD_STATIC_ONLY=y V=1 +make -C src -j$(nproc) CFLAGS="-I$(pwd)/elfutils/libelf $CFLAGS" BUILD_STATIC_ONLY=y V=1 $CC $CFLAGS -Isrc -Iinclude -Iinclude/uapi -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -c fuzz/bpf-object-fuzzer.c -o bpf-object-fuzzer.o -$CXX $CXXFLAGS $LIB_FUZZING_ENGINE bpf-object-fuzzer.o src/libbpf.a "$elfutils/libelf/libelf.a" -l:libz.a -o "$OUT/bpf-object-fuzzer" +$CXX $CXXFLAGS $LIB_FUZZING_ENGINE bpf-object-fuzzer.o src/libbpf.a "$(pwd)/elfutils/libelf/libelf.a" -l:libz.a -o "$OUT/bpf-object-fuzzer" cp fuzz/bpf-object-fuzzer_seed_corpus.zip "$OUT" - -rm -rf "$elfutils" From 9a6406573367245e32ca9eda8c4ffd2e5b4d7487 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 31 Jan 2022 23:12:08 -0800 Subject: [PATCH 107/131] sync: move bpf-next checkpoint to include selftests fixes There are no relevant libbpf commits, but new checkpoint commit contains important BPF selftests commits fixing CI failures from kernel repo side. Signed-off-by: Andrii Nakryiko --- CHECKPOINT-COMMIT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index 57a6020..b5b0d3d 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -c446fdacb10dcb3b9a9ed3b91d91e72d71d94b03 +b3dddab2ff10853aa3ef70483415d07fee3034ba From 9f276b240b2e4a9e0a1446fdc02ce7e593d90242 Mon Sep 17 00:00:00 2001 From: Jakub Sitnicki Date: Sun, 30 Jan 2022 12:55:18 +0100 Subject: [PATCH 108/131] selftests/bpf: Extend verifier and bpf_sock tests for dst_port loads Add coverage to the verifier tests and tests for reading bpf_sock fields to ensure that 32-bit, 16-bit, and 8-bit loads from dst_port field are allowed only at intended offsets and produce expected values. While 16-bit and 8-bit access to dst_port field is straight-forward, 32-bit wide loads need be allowed and produce a zero-padded 16-bit value for backward compatibility. Signed-off-by: Jakub Sitnicki Link: https://lore.kernel.org/r/20220130115518.213259-3-jakub@cloudflare.com Signed-off-by: Alexei Starovoitov --- include/uapi/linux/bpf.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 4a2f704..a7f0dde 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -5574,7 +5574,8 @@ struct bpf_sock { __u32 src_ip4; __u32 src_ip6[4]; __u32 src_port; /* host byte order */ - __u32 dst_port; /* network byte order */ + __be16 dst_port; /* network byte order */ + __u16 :16; /* zero padding */ __u32 dst_ip4; __u32 dst_ip6[4]; __u32 state; From 74dcd1bf6ad0286a2466401a2fb2694d8dc381ba Mon Sep 17 00:00:00 2001 From: Dave Marchevsky Date: Mon, 31 Jan 2022 17:46:10 -0800 Subject: [PATCH 109/131] libbpf: Deprecate btf_ext rec_size APIs btf_ext__{func,line}_info_rec_size functions are used in conjunction with already-deprecated btf_ext__reloc_{func,line}_info functions. Since struct btf_ext is opaque to the user it was necessary to expose rec_size getters in the past. btf_ext__reloc_{func,line}_info were deprecated in commit 8505e8709b5ee ("libbpf: Implement generalized .BTF.ext func/line info adjustment") as they're not compatible with support for multiple programs per section. It was decided[0] that users of these APIs should implement their own .btf.ext parsing to access this data, in which case the rec_size getters are unnecessary. So deprecate them from libbpf 0.7.0 onwards. [0] Closes: https://github.com/libbpf/libbpf/issues/277 Signed-off-by: Dave Marchevsky Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220201014610.3522985-1-davemarchevsky@fb.com --- src/btf.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/btf.h b/src/btf.h index 96b44d5..b10729f 100644 --- a/src/btf.h +++ b/src/btf.h @@ -168,8 +168,10 @@ int btf_ext__reloc_line_info(const struct btf *btf, const struct btf_ext *btf_ext, const char *sec_name, __u32 insns_cnt, void **line_info, __u32 *cnt); -LIBBPF_API __u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext); -LIBBPF_API __u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext); +LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_func_info is deprecated; write custom func_info parsing to fetch rec_size") +__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext); +LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_line_info is deprecated; write custom line_info parsing to fetch rec_size") +__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext); LIBBPF_API int btf__find_str(struct btf *btf, const char *s); LIBBPF_API int btf__add_str(struct btf *btf, const char *s); From c245b0eeafcf33b05afbea7c58f0655fb1c096a6 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Tue, 1 Feb 2022 15:58:08 +0100 Subject: [PATCH 110/131] libbpf: Deprecate xdp_cpumap, xdp_devmap and classifier sec definitions Deprecate xdp_cpumap, xdp_devmap and classifier sec definitions. Introduce xdp/devmap and xdp/cpumap definitions according to the standard for SEC("") in libbpf: - prog_type.prog_flags/attach_place Signed-off-by: Lorenzo Bianconi Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/5c7bd9426b3ce6a31d9a4b1f97eb299e1467fc52.1643727185.git.lorenzo@kernel.org --- src/libbpf.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index 4ce94f4..1b0936b 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -237,6 +237,8 @@ enum sec_def_flags { SEC_SLOPPY_PFX = 16, /* BPF program support non-linear XDP buffer */ SEC_XDP_FRAGS = 32, + /* deprecated sec definitions not supposed to be used */ + SEC_DEPRECATED = 64, }; struct bpf_sec_def { @@ -6575,6 +6577,10 @@ static int libbpf_preload_prog(struct bpf_program *prog, if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; + if (def & SEC_DEPRECATED) + pr_warn("SEC(\"%s\") is deprecated, please see https://github.com/libbpf/libbpf/wiki/Libbpf-1.0-migration-guide#bpf-program-sec-annotation-deprecations for details\n", + prog->sec_name); + if ((prog->type == BPF_PROG_TYPE_TRACING || prog->type == BPF_PROG_TYPE_LSM || prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) { @@ -8596,7 +8602,7 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("kretprobe/", KPROBE, 0, SEC_NONE, attach_kprobe), SEC_DEF("uretprobe/", KPROBE, 0, SEC_NONE), SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), - SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE | SEC_SLOPPY_PFX), + SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE | SEC_SLOPPY_PFX | SEC_DEPRECATED), SEC_DEF("action", SCHED_ACT, 0, SEC_NONE | SEC_SLOPPY_PFX), SEC_DEF("tracepoint/", TRACEPOINT, 0, SEC_NONE, attach_tp), SEC_DEF("tp/", TRACEPOINT, 0, SEC_NONE, attach_tp), @@ -8618,9 +8624,11 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("iter.s/", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter), SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE), SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS), - SEC_DEF("xdp_devmap/", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE), + SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE), + SEC_DEF("xdp_devmap/", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE | SEC_DEPRECATED), SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS), - SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE), + SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE), + SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE | SEC_DEPRECATED), SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS), SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE | SEC_SLOPPY_PFX), From bd402dccaf275b0c429d33a8ded77f8ca23b6c7d Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Mon, 31 Jan 2022 14:05:23 -0800 Subject: [PATCH 111/131] libbpf: Open code low level bpf commands. Open code low level bpf commands used by light skeleton to be able to avoid full libbpf eventually. Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/bpf/20220131220528.98088-3-alexei.starovoitov@gmail.com --- src/skel_internal.h | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/skel_internal.h b/src/skel_internal.h index 0b84d8e..57507f1 100644 --- a/src/skel_internal.h +++ b/src/skel_internal.h @@ -70,19 +70,59 @@ static inline int skel_closenz(int fd) return -EINVAL; } +#ifndef offsetofend +#define offsetofend(TYPE, MEMBER) \ + (offsetof(TYPE, MEMBER) + sizeof((((TYPE *)0)->MEMBER))) +#endif + +static inline int skel_map_create(enum bpf_map_type map_type, + const char *map_name, + __u32 key_size, + __u32 value_size, + __u32 max_entries) +{ + const size_t attr_sz = offsetofend(union bpf_attr, map_extra); + union bpf_attr attr; + + memset(&attr, 0, attr_sz); + + attr.map_type = map_type; + strncpy(attr.map_name, map_name, sizeof(attr.map_name)); + attr.key_size = key_size; + attr.value_size = value_size; + attr.max_entries = max_entries; + + return skel_sys_bpf(BPF_MAP_CREATE, &attr, attr_sz); +} + +static inline int skel_map_update_elem(int fd, const void *key, + const void *value, __u64 flags) +{ + const size_t attr_sz = offsetofend(union bpf_attr, flags); + union bpf_attr attr; + + memset(&attr, 0, attr_sz); + attr.map_fd = fd; + attr.key = (long) key; + attr.value = (long) value; + attr.flags = flags; + + return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); +} + static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) { int map_fd = -1, prog_fd = -1, key = 0, err; union bpf_attr attr; - map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1, NULL); + map_fd = skel_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1); if (map_fd < 0) { opts->errstr = "failed to create loader map"; err = -errno; goto out; } - err = bpf_map_update_elem(map_fd, &key, opts->data, 0); + err = skel_map_update_elem(map_fd, &key, opts->data, 0); if (err < 0) { opts->errstr = "failed to update loader map"; err = -errno; From a7a3a8811c145a3c440496a03f79833b0f03f9bd Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Mon, 31 Jan 2022 14:05:24 -0800 Subject: [PATCH 112/131] libbpf: Open code raw_tp_open and link_create commands. Open code raw_tracepoint_open and link_create used by light skeleton to be able to avoid full libbpf eventually. Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/bpf/20220131220528.98088-4-alexei.starovoitov@gmail.com --- src/skel_internal.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/skel_internal.h b/src/skel_internal.h index 57507f1..dcd3336 100644 --- a/src/skel_internal.h +++ b/src/skel_internal.h @@ -110,6 +110,32 @@ static inline int skel_map_update_elem(int fd, const void *key, return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); } +static inline int skel_raw_tracepoint_open(const char *name, int prog_fd) +{ + const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd); + union bpf_attr attr; + + memset(&attr, 0, attr_sz); + attr.raw_tracepoint.name = (long) name; + attr.raw_tracepoint.prog_fd = prog_fd; + + return skel_sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz); +} + +static inline int skel_link_create(int prog_fd, int target_fd, + enum bpf_attach_type attach_type) +{ + const size_t attr_sz = offsetofend(union bpf_attr, link_create.iter_info_len); + union bpf_attr attr; + + memset(&attr, 0, attr_sz); + attr.link_create.prog_fd = prog_fd; + attr.link_create.target_fd = target_fd; + attr.link_create.attach_type = attach_type; + + return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz); +} + static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) { int map_fd = -1, prog_fd = -1, key = 0, err; From 352c13cdeea07e9c28958c31a4370a209b303ec8 Mon Sep 17 00:00:00 2001 From: Delyan Kratunov Date: Wed, 2 Feb 2022 15:54:23 -0800 Subject: [PATCH 113/131] libbpf: Deprecate bpf_prog_test_run_xattr and bpf_prog_test_run Deprecate non-extendable bpf_prog_test_run{,_xattr} in favor of OPTS-based bpf_prog_test_run_opts ([0]). [0] Closes: https://github.com/libbpf/libbpf/issues/286 Signed-off-by: Delyan Kratunov Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220202235423.1097270-5-delyank@fb.com --- src/bpf.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bpf.h b/src/bpf.h index c2e8327..16b2175 100644 --- a/src/bpf.h +++ b/src/bpf.h @@ -453,12 +453,14 @@ struct bpf_prog_test_run_attr { * out: length of cxt_out */ }; +LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_test_run_opts() instead") LIBBPF_API int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr); /* * bpf_prog_test_run does not check that data_out is large enough. Consider - * using bpf_prog_test_run_xattr instead. + * using bpf_prog_test_run_opts instead. */ +LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_test_run_opts() instead") LIBBPF_API int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size, void *data_out, __u32 *size_out, __u32 *retval, __u32 *duration); From a9cd83ae259696d647d5cd07d764a9bcbe591af6 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 2 Feb 2022 14:59:11 -0800 Subject: [PATCH 114/131] libbpf: Stop using deprecated bpf_map__is_offload_neutral() Open-code bpf_map__is_offload_neutral() logic in one place in to-be-deprecated bpf_prog_load_xattr2. Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Reviewed-by: Quentin Monnet Link: https://lore.kernel.org/bpf/20220202225916.3313522-2-andrii@kernel.org --- src/libbpf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libbpf.c b/src/libbpf.c index 1b0936b..81605de 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -9505,7 +9505,7 @@ static int bpf_prog_load_xattr2(const struct bpf_prog_load_attr *attr, } bpf_object__for_each_map(map, obj) { - if (!bpf_map__is_offload_neutral(map)) + if (map->def.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) map->map_ifindex = attr->ifindex; } From 20442822c076f9ab934f926d496ba5f1c53637da Mon Sep 17 00:00:00 2001 From: Delyan Kratunov Date: Thu, 3 Feb 2022 10:00:32 -0800 Subject: [PATCH 115/131] libbpf: Deprecate priv/set_priv storage Arbitrary storage via bpf_*__set_priv/__priv is being deprecated without a replacement ([1]). perf uses this capability, but most of that is going away with the removal of prologue generation ([2]). perf is already suppressing deprecation warnings, so the remaining cleanup will happen separately. [1]: Closes: https://github.com/libbpf/libbpf/issues/294 [2]: https://lore.kernel.org/bpf/20220123221932.537060-1-jolsa@kernel.org/ Signed-off-by: Delyan Kratunov Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220203180032.1921580-1-delyank@fb.com --- src/libbpf.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libbpf.h b/src/libbpf.h index 5762b57..c8d8daa 100644 --- a/src/libbpf.h +++ b/src/libbpf.h @@ -246,8 +246,10 @@ struct bpf_object *bpf_object__next(struct bpf_object *prev); (pos) = (tmp), (tmp) = bpf_object__next(tmp)) typedef void (*bpf_object_clear_priv_t)(struct bpf_object *, void *); +LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") LIBBPF_API int bpf_object__set_priv(struct bpf_object *obj, void *priv, bpf_object_clear_priv_t clear_priv); +LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") LIBBPF_API void *bpf_object__priv(const struct bpf_object *prog); LIBBPF_API int @@ -279,9 +281,10 @@ bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *prog) typedef void (*bpf_program_clear_priv_t)(struct bpf_program *, void *); +LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") LIBBPF_API int bpf_program__set_priv(struct bpf_program *prog, void *priv, bpf_program_clear_priv_t clear_priv); - +LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") LIBBPF_API void *bpf_program__priv(const struct bpf_program *prog); LIBBPF_API void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex); @@ -769,8 +772,10 @@ LIBBPF_API __u64 bpf_map__map_extra(const struct bpf_map *map); LIBBPF_API int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra); typedef void (*bpf_map_clear_priv_t)(struct bpf_map *, void *); +LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") LIBBPF_API int bpf_map__set_priv(struct bpf_map *map, void *priv, bpf_map_clear_priv_t clear_priv); +LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") LIBBPF_API void *bpf_map__priv(const struct bpf_map *map); LIBBPF_API int bpf_map__set_initial_value(struct bpf_map *map, const void *data, size_t size); From 47673bd255ec205788e4946c7fdf125492f1eaa5 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 3 Feb 2022 14:50:17 -0800 Subject: [PATCH 116/131] libbpf: Deprecate forgotten btf__get_map_kv_tids() btf__get_map_kv_tids() is in the same group of APIs as btf_ext__reloc_func_info()/btf_ext__reloc_line_info() which were only used by BCC. It was missed to be marked as deprecated in [0]. Fixing that to complete [1]. [0] https://patchwork.kernel.org/project/netdevbpf/patch/20220201014610.3522985-1-davemarchevsky@fb.com/ [1] Closes: https://github.com/libbpf/libbpf/issues/277 Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20220203225017.1795946-1-andrii@kernel.org --- src/btf.h | 1 + src/libbpf.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/btf.h b/src/btf.h index b10729f..951ac74 100644 --- a/src/btf.h +++ b/src/btf.h @@ -150,6 +150,7 @@ LIBBPF_API void btf__set_fd(struct btf *btf, int fd); LIBBPF_API const void *btf__raw_data(const struct btf *btf, __u32 *size); LIBBPF_API const char *btf__name_by_offset(const struct btf *btf, __u32 offset); LIBBPF_API const char *btf__str_by_offset(const struct btf *btf, __u32 offset); +LIBBPF_DEPRECATED_SINCE(0, 7, "this API is not necessary when BTF-defined maps are used") LIBBPF_API int btf__get_map_kv_tids(const struct btf *btf, const char *map_name, __u32 expected_key_size, __u32 expected_value_size, diff --git a/src/libbpf.c b/src/libbpf.c index 81605de..904cdf8 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -4202,9 +4202,12 @@ static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) if (!bpf_map__is_internal(map)) { pr_warn("Use of BPF_ANNOTATE_KV_PAIR is deprecated, use BTF-defined maps in .maps section instead\n"); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, def->value_size, &key_type_id, &value_type_id); +#pragma GCC diagnostic pop } else { /* * LLVM annotates global data differently in BTF, that is, From fd181bc349bd6d479cc0a9fdf122885ad352eaf8 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 3 Feb 2022 16:50:28 -0800 Subject: [PATCH 117/131] sync: latest libbpf changes from kernel Syncing latest libbpf commits from kernel repository. Baseline bpf-next commit: b3dddab2ff10853aa3ef70483415d07fee3034ba Checkpoint bpf-next commit: 227a0713b319e7a8605312dee1c97c97a719a9fc Baseline bpf commit: e2bcbd7769ee8f05e1b3d10848aace98973844e4 Checkpoint bpf commit: 77b1b8b43ec3c060ecf7e926a92b0f8772171046 Alexei Starovoitov (2): libbpf: Open code low level bpf commands. libbpf: Open code raw_tp_open and link_create commands. Andrii Nakryiko (2): libbpf: Stop using deprecated bpf_map__is_offload_neutral() libbpf: Deprecate forgotten btf__get_map_kv_tids() Dave Marchevsky (1): libbpf: Deprecate btf_ext rec_size APIs Delyan Kratunov (2): libbpf: Deprecate bpf_prog_test_run_xattr and bpf_prog_test_run libbpf: Deprecate priv/set_priv storage Jakub Sitnicki (1): selftests/bpf: Extend verifier and bpf_sock tests for dst_port loads Lorenzo Bianconi (1): libbpf: Deprecate xdp_cpumap, xdp_devmap and classifier sec definitions include/uapi/linux/bpf.h | 3 +- src/bpf.h | 4 ++- src/btf.h | 7 ++-- src/libbpf.c | 19 ++++++++--- src/libbpf.h | 7 +++- src/skel_internal.h | 70 ++++++++++++++++++++++++++++++++++++++-- 6 files changed, 99 insertions(+), 11 deletions(-) -- 2.30.2 --- BPF-CHECKPOINT-COMMIT | 2 +- CHECKPOINT-COMMIT | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BPF-CHECKPOINT-COMMIT b/BPF-CHECKPOINT-COMMIT index 86d0b2b..3044fc9 100644 --- a/BPF-CHECKPOINT-COMMIT +++ b/BPF-CHECKPOINT-COMMIT @@ -1 +1 @@ -e2bcbd7769ee8f05e1b3d10848aace98973844e4 +77b1b8b43ec3c060ecf7e926a92b0f8772171046 diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index b5b0d3d..b548895 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -b3dddab2ff10853aa3ef70483415d07fee3034ba +227a0713b319e7a8605312dee1c97c97a719a9fc From cdef8257a897b3bb8ba5c10603af4a61a324ec69 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 4 Feb 2022 15:28:06 -0800 Subject: [PATCH 118/131] ci: bump LLVM_VER to 15 Another bump of latest clang version to be used during BPF selftests build. Signed-off-by: Andrii Nakryiko --- .github/actions/build-selftests/build_selftests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-selftests/build_selftests.sh b/.github/actions/build-selftests/build_selftests.sh index 05e3d39..7baca39 100755 --- a/.github/actions/build-selftests/build_selftests.sh +++ b/.github/actions/build-selftests/build_selftests.sh @@ -8,7 +8,7 @@ source ${THISDIR}/helpers.sh travis_fold start prepare_selftests "Building selftests" -LLVM_VER=14 +LLVM_VER=15 LIBBPF_PATH="${REPO_ROOT}" PREPARE_SELFTESTS_SCRIPT=${THISDIR}/prepare_selftests-${KERNEL}.sh From d6783c28b40e8355d2e3bd4a8141b88da7704f6d Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 7 Feb 2022 09:37:38 -0800 Subject: [PATCH 119/131] README: update Ubuntu package link Point to libbpf package in Ubuntu Impish, so it doesn't expire soon. Closes: https://github.com/libbpf/libbpf/issues/451 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a76315..b5b7f51 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Distributions packaging libbpf from this mirror: - [Gentoo](https://packages.gentoo.org/packages/dev-libs/libbpf) - [Debian](https://packages.debian.org/source/sid/libbpf) - [Arch](https://www.archlinux.org/packages/extra/x86_64/libbpf/) - - [Ubuntu](https://packages.ubuntu.com/source/groovy/libbpf) + - [Ubuntu](https://packages.ubuntu.com/source/impish/libbpf) - [Alpine](https://pkgs.alpinelinux.org/packages?name=libbpf) Benefits of packaging from the mirror over packaging from kernel sources: From 1ceec54cb06f164428f2ec872f8eef4d2cb5768b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mauricio=20V=C3=A1squez?= Date: Mon, 7 Feb 2022 09:50:50 -0500 Subject: [PATCH 120/131] libbpf: Remove mode check in libbpf_set_strict_mode() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libbpf_set_strict_mode() checks that the passed mode doesn't contain extra bits for LIBBPF_STRICT_* flags that don't exist yet. It makes it difficult for applications to disable some strict flags as something like "LIBBPF_STRICT_ALL & ~LIBBPF_STRICT_MAP_DEFINITIONS" is rejected by this check and they have to use a rather complicated formula to calculate it.[0] One possibility is to change LIBBPF_STRICT_ALL to only contain the bits of all existing LIBBPF_STRICT_* flags instead of 0xffffffff. However it's not possible because the idea is that applications compiled against older libbpf_legacy.h would still be opting into latest LIBBPF_STRICT_ALL features.[1] The other possibility is to remove that check so something like "LIBBPF_STRICT_ALL & ~LIBBPF_STRICT_MAP_DEFINITIONS" is allowed. It's what this commit does. [0]: https://lore.kernel.org/bpf/20220204220435.301896-1-mauricio@kinvolk.io/ [1]: https://lore.kernel.org/bpf/CAEf4BzaTWa9fELJLh+bxnOb0P1EMQmaRbJVG0L+nXZdy0b8G3Q@mail.gmail.com/ Fixes: 93b8952d223a ("libbpf: deprecate legacy BPF map definitions") Signed-off-by: Mauricio Vásquez Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220207145052.124421-2-mauricio@kinvolk.io --- src/libbpf.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libbpf.c b/src/libbpf.c index 904cdf8..2262bcd 100644 --- a/src/libbpf.c +++ b/src/libbpf.c @@ -156,14 +156,6 @@ enum libbpf_strict_mode libbpf_mode = LIBBPF_STRICT_NONE; int libbpf_set_strict_mode(enum libbpf_strict_mode mode) { - /* __LIBBPF_STRICT_LAST is the last power-of-2 value used + 1, so to - * get all possible values we compensate last +1, and then (2*x - 1) - * to get the bit mask - */ - if (mode != LIBBPF_STRICT_ALL - && (mode & ~((__LIBBPF_STRICT_LAST - 1) * 2 - 1))) - return errno = EINVAL, -EINVAL; - libbpf_mode = mode; return 0; } From f5bd7054f9083dc512b3b438993764b737314af0 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 8 Feb 2022 10:15:52 +0300 Subject: [PATCH 121/131] libbpf: Fix signedness bug in btf_dump_array_data() The btf__resolve_size() function returns negative error codes so "elem_size" must be signed for the error handling to work. Fixes: 920d16af9b42 ("libbpf: BTF dumper support for typed data") Signed-off-by: Dan Carpenter Signed-off-by: Daniel Borkmann Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20220208071552.GB10495@kili --- src/btf_dump.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/btf_dump.c b/src/btf_dump.c index b9a3260..55aed9e 100644 --- a/src/btf_dump.c +++ b/src/btf_dump.c @@ -1861,14 +1861,15 @@ static int btf_dump_array_data(struct btf_dump *d, { const struct btf_array *array = btf_array(t); const struct btf_type *elem_type; - __u32 i, elem_size = 0, elem_type_id; + __u32 i, elem_type_id; + __s64 elem_size; bool is_array_member; elem_type_id = array->type; elem_type = skip_mods_and_typedefs(d->btf, elem_type_id, NULL); elem_size = btf__resolve_size(d->btf, elem_type_id); if (elem_size <= 0) { - pr_warn("unexpected elem size %d for array type [%u]\n", elem_size, id); + pr_warn("unexpected elem size %lld for array type [%u]\n", elem_size, id); return -EINVAL; } From 50b4d99bbc48b21fef19cb4255d41290b80f786e Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 9 Feb 2022 03:17:37 +0100 Subject: [PATCH 122/131] libbpf: Add PT_REGS_SYSCALL_REGS macro Architectures that select ARCH_HAS_SYSCALL_WRAPPER pass a pointer to struct pt_regs to syscall handlers, others unpack it into individual function parameters. Introduce a macro to describe what a particular arch does. Signed-off-by: Ilya Leoshkevich Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220209021745.2215452-3-iii@linux.ibm.com --- src/bpf_tracing.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index 032ba80..a5e9265 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -326,6 +326,16 @@ struct pt_regs; #endif /* defined(bpf_target_defined) */ +/* + * When invoked from a syscall handler kprobe, returns a pointer to a + * struct pt_regs containing syscall arguments and suitable for passing to + * PT_REGS_PARMn_SYSCALL() and PT_REGS_PARMn_CORE_SYSCALL(). + */ +#ifndef PT_REGS_SYSCALL_REGS +/* By default, assume that the arch selects ARCH_HAS_SYSCALL_WRAPPER. */ +#define PT_REGS_SYSCALL_REGS(ctx) ((struct pt_regs *)PT_REGS_PARM1(ctx)) +#endif + #ifndef ___bpf_concat #define ___bpf_concat(a, b) a ## b #endif From 8a28842a20788208ca60cebad2f48754df5d30ee Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 9 Feb 2022 03:17:39 +0100 Subject: [PATCH 123/131] libbpf: Fix accessing syscall arguments on powerpc powerpc does not select ARCH_HAS_SYSCALL_WRAPPER, so its syscall handlers take "unpacked" syscall arguments. Indicate this to libbpf using PT_REGS_SYSCALL_REGS macro. Reported-by: Heiko Carstens Signed-off-by: Ilya Leoshkevich Signed-off-by: Andrii Nakryiko Tested-by: Naveen N. Rao Link: https://lore.kernel.org/bpf/20220209021745.2215452-5-iii@linux.ibm.com --- src/bpf_tracing.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index a5e9265..20bc637 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -180,6 +180,8 @@ #define __PT_RC_REG gpr[3] #define __PT_SP_REG sp #define __PT_IP_REG nip +/* powerpc does not select ARCH_HAS_SYSCALL_WRAPPER. */ +#define PT_REGS_SYSCALL_REGS(ctx) ctx #elif defined(bpf_target_sparc) From 497ec1d35ca3bb824f9345e32b665157077b1746 Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 9 Feb 2022 03:17:40 +0100 Subject: [PATCH 124/131] libbpf: Fix riscv register names riscv registers are accessed via struct user_regs_struct, not struct pt_regs. The program counter member in this struct is called pc, not epc. The frame pointer is called s0, not fp. Fixes: 3cc31d794097 ("libbpf: Normalize PT_REGS_xxx() macro definitions") Signed-off-by: Ilya Leoshkevich Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220209021745.2215452-6-iii@linux.ibm.com --- src/bpf_tracing.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index 20bc637..03e501a 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -210,10 +210,10 @@ #define __PT_PARM4_REG a3 #define __PT_PARM5_REG a4 #define __PT_RET_REG ra -#define __PT_FP_REG fp +#define __PT_FP_REG s0 #define __PT_RC_REG a5 #define __PT_SP_REG sp -#define __PT_IP_REG epc +#define __PT_IP_REG pc #endif From 32c19d8505ff32fe84d16277232bb7f0b645d63a Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 9 Feb 2022 03:17:41 +0100 Subject: [PATCH 125/131] libbpf: Fix accessing syscall arguments on riscv riscv does not select ARCH_HAS_SYSCALL_WRAPPER, so its syscall handlers take "unpacked" syscall arguments. Indicate this to libbpf using PT_REGS_SYSCALL_REGS macro. Reported-by: Heiko Carstens Signed-off-by: Ilya Leoshkevich Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220209021745.2215452-7-iii@linux.ibm.com --- src/bpf_tracing.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index 03e501a..41a015e 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -214,6 +214,8 @@ #define __PT_RC_REG a5 #define __PT_SP_REG sp #define __PT_IP_REG pc +/* riscv does not select ARCH_HAS_SYSCALL_WRAPPER. */ +#define PT_REGS_SYSCALL_REGS(ctx) ctx #endif From f1a756d7935340abb489207686ad852911d26c41 Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 9 Feb 2022 03:17:43 +0100 Subject: [PATCH 126/131] libbpf: Allow overriding PT_REGS_PARM1{_CORE}_SYSCALL arm64 and s390 need a special way to access the first syscall argument. Signed-off-by: Ilya Leoshkevich Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220209021745.2215452-9-iii@linux.ibm.com --- src/bpf_tracing.h | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index 41a015e..276130d 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -70,13 +70,15 @@ #define __PT_PARM2_REG si #define __PT_PARM3_REG dx #define __PT_PARM4_REG cx -#define __PT_PARM4_REG_SYSCALL r10 /* syscall uses r10 */ #define __PT_PARM5_REG r8 #define __PT_RET_REG sp #define __PT_FP_REG bp #define __PT_RC_REG ax #define __PT_SP_REG sp #define __PT_IP_REG ip +/* syscall uses r10 for PARM4 */ +#define PT_REGS_PARM4_SYSCALL(x) ((x)->r10) +#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(x, r10) #else @@ -100,13 +102,15 @@ #define __PT_PARM2_REG rsi #define __PT_PARM3_REG rdx #define __PT_PARM4_REG rcx -#define __PT_PARM4_REG_SYSCALL r10 /* syscall uses r10 */ #define __PT_PARM5_REG r8 #define __PT_RET_REG rsp #define __PT_FP_REG rbp #define __PT_RC_REG rax #define __PT_SP_REG rsp #define __PT_IP_REG rip +/* syscall uses r10 for PARM4 */ +#define PT_REGS_PARM4_SYSCALL(x) ((x)->r10) +#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(x, r10) #endif /* __i386__ */ @@ -269,22 +273,22 @@ struct pt_regs; #endif +#ifndef PT_REGS_PARM1_SYSCALL #define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1(x) +#endif #define PT_REGS_PARM2_SYSCALL(x) PT_REGS_PARM2(x) #define PT_REGS_PARM3_SYSCALL(x) PT_REGS_PARM3(x) -#ifdef __PT_PARM4_REG_SYSCALL -#define PT_REGS_PARM4_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG_SYSCALL) -#else /* __PT_PARM4_REG_SYSCALL */ +#ifndef PT_REGS_PARM4_SYSCALL #define PT_REGS_PARM4_SYSCALL(x) PT_REGS_PARM4(x) #endif #define PT_REGS_PARM5_SYSCALL(x) PT_REGS_PARM5(x) +#ifndef PT_REGS_PARM1_CORE_SYSCALL #define PT_REGS_PARM1_CORE_SYSCALL(x) PT_REGS_PARM1_CORE(x) +#endif #define PT_REGS_PARM2_CORE_SYSCALL(x) PT_REGS_PARM2_CORE(x) #define PT_REGS_PARM3_CORE_SYSCALL(x) PT_REGS_PARM3_CORE(x) -#ifdef __PT_PARM4_REG_SYSCALL -#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG_SYSCALL) -#else /* __PT_PARM4_REG_SYSCALL */ +#ifndef PT_REGS_PARM4_CORE_SYSCALL #define PT_REGS_PARM4_CORE_SYSCALL(x) PT_REGS_PARM4_CORE(x) #endif #define PT_REGS_PARM5_CORE_SYSCALL(x) PT_REGS_PARM5_CORE(x) From 9f6e3a7a59894af6ff9221083800556e68ffcd8a Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 9 Feb 2022 03:17:44 +0100 Subject: [PATCH 127/131] libbpf: Fix accessing the first syscall argument on arm64 On arm64, the first syscall argument should be accessed via orig_x0 (see arch/arm64/include/asm/syscall.h). Currently regs[0] is used instead, leading to bpf_syscall_macro test failure. orig_x0 cannot be added to struct user_pt_regs, since its layout is a part of the ABI. Therefore provide access to it only through PT_REGS_PARM1_CORE_SYSCALL() by using a struct pt_regs flavor. Reported-by: Heiko Carstens Signed-off-by: Ilya Leoshkevich Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220209021745.2215452-10-iii@linux.ibm.com --- src/bpf_tracing.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index 276130d..b645310 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -146,6 +146,10 @@ #elif defined(bpf_target_arm64) +struct pt_regs___arm64 { + unsigned long orig_x0; +}; + /* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */ #define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x)) #define __PT_PARM1_REG regs[0] @@ -158,6 +162,8 @@ #define __PT_RC_REG regs[0] #define __PT_SP_REG sp #define __PT_IP_REG pc +#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma("GCC error \"use PT_REGS_PARM1_CORE_SYSCALL() instead\""); 0l; }) +#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ((const struct pt_regs___arm64 *)(x), orig_x0) #elif defined(bpf_target_mips) From f7e08b4a8f24ee23dc596710ff85fbcd2ad6f10f Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 9 Feb 2022 03:17:45 +0100 Subject: [PATCH 128/131] libbpf: Fix accessing the first syscall argument on s390 On s390, the first syscall argument should be accessed via orig_gpr2 (see arch/s390/include/asm/syscall.h). Currently gpr[2] is used instead, leading to bpf_syscall_macro test failure. orig_gpr2 cannot be added to user_pt_regs, since its layout is a part of the ABI. Therefore provide access to it only through PT_REGS_PARM1_CORE_SYSCALL() by using a struct pt_regs flavor. Reported-by: Andrii Nakryiko Signed-off-by: Ilya Leoshkevich Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220209021745.2215452-11-iii@linux.ibm.com --- src/bpf_tracing.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index b645310..eb6eb3b 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -118,6 +118,10 @@ #elif defined(bpf_target_s390) +struct pt_regs___s390 { + unsigned long orig_gpr2; +}; + /* s390 provides user_pt_regs instead of struct pt_regs to userspace */ #define __PT_REGS_CAST(x) ((const user_pt_regs *)(x)) #define __PT_PARM1_REG gprs[2] @@ -130,6 +134,8 @@ #define __PT_RC_REG gprs[2] #define __PT_SP_REG gprs[15] #define __PT_IP_REG psw.addr +#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma("GCC error \"use PT_REGS_PARM1_CORE_SYSCALL() instead\""); 0l; }) +#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ((const struct pt_regs___s390 *)(x), orig_gpr2) #elif defined(bpf_target_arm) From b0a3b9e8fe16f7ef8cdb2a526cf28b3e419db6b8 Mon Sep 17 00:00:00 2001 From: Hengqi Chen Date: Mon, 7 Feb 2022 22:31:33 +0800 Subject: [PATCH 129/131] libbpf: Add BPF_KPROBE_SYSCALL macro Add syscall-specific variant of BPF_KPROBE named BPF_KPROBE_SYSCALL ([0]). The new macro hides the underlying way of getting syscall input arguments. With the new macro, the following code: SEC("kprobe/__x64_sys_close") int BPF_KPROBE(do_sys_close, struct pt_regs *regs) { int fd; fd = PT_REGS_PARM1_CORE(regs); /* do something with fd */ } can be written as: SEC("kprobe/__x64_sys_close") int BPF_KPROBE_SYSCALL(do_sys_close, int fd) { /* do something with fd */ } [0] Closes: https://github.com/libbpf/libbpf/issues/425 Signed-off-by: Hengqi Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220207143134.2977852-2-hengqi.chen@gmail.com --- src/bpf_tracing.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/bpf_tracing.h b/src/bpf_tracing.h index eb6eb3b..e3a8c94 100644 --- a/src/bpf_tracing.h +++ b/src/bpf_tracing.h @@ -470,4 +470,39 @@ typeof(name(0)) name(struct pt_regs *ctx) \ } \ static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) +#define ___bpf_syscall_args0() ctx +#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs) +#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs) +#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs) +#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs) +#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs) +#define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args) + +/* + * BPF_KPROBE_SYSCALL is a variant of BPF_KPROBE, which is intended for + * tracing syscall functions, like __x64_sys_close. It hides the underlying + * platform-specific low-level way of getting syscall input arguments from + * struct pt_regs, and provides a familiar typed and named function arguments + * syntax and semantics of accessing syscall input parameters. + * + * Original struct pt_regs* context is preserved as 'ctx' argument. This might + * be necessary when using BPF helpers like bpf_perf_event_output(). + * + * This macro relies on BPF CO-RE support. + */ +#define BPF_KPROBE_SYSCALL(name, args...) \ +name(struct pt_regs *ctx); \ +static __attribute__((always_inline)) typeof(name(0)) \ +____##name(struct pt_regs *ctx, ##args); \ +typeof(name(0)) name(struct pt_regs *ctx) \ +{ \ + struct pt_regs *regs = PT_REGS_SYSCALL_REGS(ctx); \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ + return ____##name(___bpf_syscall_args(args)); \ + _Pragma("GCC diagnostic pop") \ +} \ +static __attribute__((always_inline)) typeof(name(0)) \ +____##name(struct pt_regs *ctx, ##args) + #endif From 37493e639f9add8c06604c493657e533c835e0a1 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 8 Feb 2022 22:39:09 -0800 Subject: [PATCH 130/131] libbpf: Fix compilation warning due to mismatched printf format On ppc64le architecture __s64 is long int and requires %ld. Cast to ssize_t and use %zd to avoid architecture-specific specifiers. Fixes: 4172843ed4a3 ("libbpf: Fix signedness bug in btf_dump_array_data()") Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20220209063909.1268319-1-andrii@kernel.org --- src/btf_dump.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/btf_dump.c b/src/btf_dump.c index 55aed9e..07ebe70 100644 --- a/src/btf_dump.c +++ b/src/btf_dump.c @@ -1869,7 +1869,8 @@ static int btf_dump_array_data(struct btf_dump *d, elem_type = skip_mods_and_typedefs(d->btf, elem_type_id, NULL); elem_size = btf__resolve_size(d->btf, elem_type_id); if (elem_size <= 0) { - pr_warn("unexpected elem size %lld for array type [%u]\n", elem_size, id); + pr_warn("unexpected elem size %zd for array type [%u]\n", + (ssize_t)elem_size, id); return -EINVAL; } From 528094c0c1465499d9cca1da30a28e89254b4be7 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 9 Feb 2022 08:10:37 -0800 Subject: [PATCH 131/131] sync: latest libbpf changes from kernel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Syncing latest libbpf commits from kernel repository. Baseline bpf-next commit: 227a0713b319e7a8605312dee1c97c97a719a9fc Checkpoint bpf-next commit: dc37dc617fabfb1c3a16d49f5d8cc20e9e3608ca Baseline bpf commit: 77b1b8b43ec3c060ecf7e926a92b0f8772171046 Checkpoint bpf commit: fe68195daf34d5dddacd3f93dd3eafc4beca3a0e Andrii Nakryiko (1): libbpf: Fix compilation warning due to mismatched printf format Dan Carpenter (1): libbpf: Fix signedness bug in btf_dump_array_data() Hengqi Chen (1): libbpf: Add BPF_KPROBE_SYSCALL macro Ilya Leoshkevich (7): libbpf: Add PT_REGS_SYSCALL_REGS macro libbpf: Fix accessing syscall arguments on powerpc libbpf: Fix riscv register names libbpf: Fix accessing syscall arguments on riscv libbpf: Allow overriding PT_REGS_PARM1{_CORE}_SYSCALL libbpf: Fix accessing the first syscall argument on arm64 libbpf: Fix accessing the first syscall argument on s390 Mauricio Vásquez (1): libbpf: Remove mode check in libbpf_set_strict_mode() src/bpf_tracing.h | 85 +++++++++++++++++++++++++++++++++++++++++------ src/btf_dump.c | 6 ++-- src/libbpf.c | 8 ----- 3 files changed, 79 insertions(+), 20 deletions(-) -- 2.30.2 --- BPF-CHECKPOINT-COMMIT | 2 +- CHECKPOINT-COMMIT | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BPF-CHECKPOINT-COMMIT b/BPF-CHECKPOINT-COMMIT index 3044fc9..24171b2 100644 --- a/BPF-CHECKPOINT-COMMIT +++ b/BPF-CHECKPOINT-COMMIT @@ -1 +1 @@ -77b1b8b43ec3c060ecf7e926a92b0f8772171046 +fe68195daf34d5dddacd3f93dd3eafc4beca3a0e diff --git a/CHECKPOINT-COMMIT b/CHECKPOINT-COMMIT index b548895..af1a0c8 100644 --- a/CHECKPOINT-COMMIT +++ b/CHECKPOINT-COMMIT @@ -1 +1 @@ -227a0713b319e7a8605312dee1c97c97a719a9fc +dc37dc617fabfb1c3a16d49f5d8cc20e9e3608ca