mirror of
https://github.com/json-c/json-c.git
synced 2026-03-24 15:39:07 +08:00
Add an initial version of a benchmarking harness for json-c, to be able to more easily compare the performance of different library versions.
This commit is contained in:
Eric Haszlakiewicz
80
bench/README.bench.md
Normal file
80
bench/README.bench.md
Normal file
@@ -0,0 +1,80 @@
|
||||
|
||||
Benchmark tests for json-c
|
||||
|
||||
General strategy:
|
||||
-------------------
|
||||
|
||||
* Identify "after" commit hash
|
||||
* Use provided directory
|
||||
* Use provided commit hash
|
||||
* Local changes in current working directory
|
||||
* ${cur_branch}
|
||||
* Identify "before" commit hash, in order of preference
|
||||
* Use provided directory
|
||||
* Use provided commit hash
|
||||
* Use origin/${cur_branch}, if different from ${after_commit}
|
||||
* Use previous release
|
||||
|
||||
* If not using existing dir, clone to src-${after_commit}
|
||||
* or, checkout appropriate commit in existing src-${after_commit}
|
||||
* Create build & install dirs for ${after_commit}
|
||||
* Build & install ${after_commit}
|
||||
* Compile benchmark programs against install-${after_commit}
|
||||
|
||||
* If not using existing dir, clone to src-${before_commit}
|
||||
* or, checkout appropriate commit in existing src-${before_commit}
|
||||
* Create build & install dirs for ${before_commit}
|
||||
* Build & install ${before_commit}
|
||||
* Compile benchmark programs against install-${before_commit}
|
||||
|
||||
* Run benchmark in each location
|
||||
* Compare results
|
||||
|
||||
heaptrack memory profiler
|
||||
---------------------------
|
||||
|
||||
https://milianw.de/blog/heaptrack-a-heap-memory-profiler-for-linux.html
|
||||
|
||||
|
||||
```
|
||||
yum install libdwarf-devel elfutils boost-devel libunwind-devel
|
||||
|
||||
git clone git://anongit.kde.org/heaptrack
|
||||
cd heaptrack
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \
|
||||
-DCMAKE_INSTALL_PREFIX=$HOME/heaptrack-install ..
|
||||
make install
|
||||
```
|
||||
|
||||
|
||||
Issues
|
||||
--------
|
||||
|
||||
* jc-bench.sh is incomplete.
|
||||
|
||||
* valgrind massif misreports "extra-heap" bytes?
|
||||
|
||||
"json_parse -n canada.json" shows 38640 KB maxrss.
|
||||
|
||||
Using valgrind --tool=massif, a large amount of memory is listed as
|
||||
wasted "extra-heap" bytes. (~5.6MB)
|
||||
|
||||
```
|
||||
valgrind --tool=massif --massif-out-file=massif.out ./json_parse -n ~/canada.json
|
||||
ms_print massif.out
|
||||
```
|
||||
|
||||
|
||||
Using heaptrack, and analyzing the histogram, only shows ~2.6MB
|
||||
```
|
||||
heaptrack ./json_parse -n canada.json
|
||||
heaptrack --analyze heaptrack*gz -H histgram.out
|
||||
awk ' { s=$1; count=$2; ru=(int((s+ 15) / 16)) * 16; wasted = ((ru-s)*count); print s, count, ru-s, wasted; total=total+wasted} END { print "Total: ", total }' histogram.out
|
||||
```
|
||||
|
||||
With the (unreleased) arraylist trimming changes, maxrss reported by
|
||||
getrusage() goes down, but massif claims *more* total usage, and a HUGE
|
||||
extra-heap amount (50% of total).
|
||||
|
||||
220
bench/jc-bench.sh
Executable file
220
bench/jc-bench.sh
Executable file
@@ -0,0 +1,220 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Benchmarking harness for json-c
|
||||
#
|
||||
# Use this to compare the behavior of two different versions of the library,
|
||||
# such as json-c-0.14 release vs master, master vs local changes, etc...
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
trap 'echo "FAILED!"' EXIT
|
||||
|
||||
RUNDIR=$(dirname "$0")
|
||||
RUNDIR=$(cd "$RUNDIR" && pwd)
|
||||
|
||||
TOP=$(cd "$RUNDIR/.." && pwd)
|
||||
|
||||
usage()
|
||||
{
|
||||
exitval=$1
|
||||
errmsg=$2
|
||||
if [ $exitval -ne 0 ] ; then
|
||||
exec 1>&2
|
||||
fi
|
||||
if [ ! -z "$errmsg" ] ; then
|
||||
echo "ERROR: $errmsg" 1>&2
|
||||
fi
|
||||
cat <<EOF
|
||||
Usage: $0 [-h] [-v] [--build] [--run] [--compare] ...XAX...
|
||||
EOF
|
||||
|
||||
exit $extival
|
||||
}
|
||||
|
||||
before_arg=
|
||||
after_arg=
|
||||
do_all=1
|
||||
do_build=0
|
||||
do_run=0
|
||||
do_compare=0
|
||||
|
||||
while [ $# -gt 0 ] ; do
|
||||
case "$1" in
|
||||
--before)
|
||||
before_arg=$2
|
||||
shift
|
||||
;;
|
||||
--after)
|
||||
after_arg=$2
|
||||
shift
|
||||
;;
|
||||
--build)
|
||||
do_all=0
|
||||
do_build=1
|
||||
;;
|
||||
--run)
|
||||
do_all=0
|
||||
do_run=1
|
||||
;;
|
||||
--compare)
|
||||
do_all=0
|
||||
do_compare=1
|
||||
;;
|
||||
-h)
|
||||
usage 0 ""
|
||||
;;
|
||||
-v)
|
||||
set -x
|
||||
;;
|
||||
*)
|
||||
usage 1 "Unknown args: $*"
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
WORK="${RUNDIR}/work"
|
||||
mkdir -p "${WORK}"
|
||||
|
||||
# XAX use a different data dir
|
||||
if [ ! -r "${WORK}/../canada.json" ] ; then
|
||||
curl -L -o "${WORK}/../canada.json" 'https://github.com/mloskot/json_benchmark/raw/master/data/canada.json'
|
||||
fi
|
||||
|
||||
# Identify "after" commit hash
|
||||
after_src_dir=$TOP
|
||||
after_commit=
|
||||
if [ ! -z "$after_arg" ] ; then
|
||||
# XXX decode this in more detail.
|
||||
# XXX for now, just assume it's a path
|
||||
after_src_dir=$after_arg
|
||||
after_commit=
|
||||
fi
|
||||
|
||||
# Identify "before" commit hash
|
||||
before_src_dir=
|
||||
#before_commit=origin/master
|
||||
before_commit=origin/json-c-0.14
|
||||
if [ ! -z "$before_arg" ] ; then
|
||||
# XXX decode this in more detail
|
||||
before_src_dir="$before_arg"
|
||||
before_commit=
|
||||
fi
|
||||
|
||||
compile_benchmark()
|
||||
{
|
||||
local bname=$1
|
||||
local src_dir="$2"
|
||||
local src_commit="$3"
|
||||
|
||||
local build_dir="${WORK}/$bname/build"
|
||||
local inst_dir="${WORK}/$bname/install"
|
||||
local bench_dir="${WORK}/$bname/bench"
|
||||
|
||||
echo
|
||||
echo "=========== $bname ==========="
|
||||
echo
|
||||
|
||||
mkdir -p "${build_dir}"
|
||||
mkdir -p "${inst_dir}"
|
||||
mkdir -p "${bench_dir}"
|
||||
|
||||
if [ ! -z "$src_commit" ] ; then
|
||||
# Resolve the short hash, tag or branch name to full hash
|
||||
src_commit=$(git rev-parse $src_commit)
|
||||
fi
|
||||
|
||||
# No src dir specified, clone and checkout $src_commit
|
||||
if [ -z "$src_dir" ] ; then
|
||||
src_dir="${WORK}/$bname/src"
|
||||
echo "=== Using sources in $src_dir"
|
||||
mkdir -p "$src_dir"
|
||||
at_commit=$(git --git-dir="$src_dir/.git" rev-parse HEAD 2> /dev/null || true)
|
||||
echo "at_commit: $at_commit"
|
||||
if [ -z "$at_commit" ] ; then
|
||||
# Assume it's an empty dir
|
||||
git clone -n "$TOP" "$src_dir"
|
||||
fi
|
||||
git -C "$src_dir" --git-dir="$src_dir/.git" checkout "$src_commit"
|
||||
fi
|
||||
# else, use the provided $src_dir
|
||||
|
||||
cd "${build_dir}"
|
||||
cmake -DCMAKE_INSTALL_PREFIX="${inst_dir}" "${src_dir}"
|
||||
make all install
|
||||
|
||||
cd "${bench_dir}"
|
||||
cmake -DCMAKE_PREFIX_PATH="${inst_dir}" "${TOP}/apps"
|
||||
make all
|
||||
}
|
||||
|
||||
# XXX TODO: name "after" and "before" uniquely using the dir & commit
|
||||
|
||||
if [ $do_all -ne 0 -o $do_build -ne 0 ] ; then
|
||||
compile_benchmark "after" "${after_src_dir}" "${after_commit}"
|
||||
compile_benchmark "before" "${before_src_dir}" "${before_commit}"
|
||||
fi
|
||||
|
||||
run_benchmark()
|
||||
{
|
||||
local bname=$1
|
||||
local inst_dir="${WORK}/$bname/install"
|
||||
local bench_dir="${WORK}/$bname/bench"
|
||||
|
||||
local INPUT=${WORK}/../canada.json
|
||||
|
||||
cd "${bench_dir}"
|
||||
mkdir -p results
|
||||
(time ./json_parse -n "${INPUT}") > results/basic_timing.out 2>&1
|
||||
valgrind --tool=massif --massif-out-file=massif.out ./json_parse -n "${INPUT}"
|
||||
ms_print massif.out > results/ms_print.out
|
||||
heaptrack -o heaptrack_out ./json_parse -n "${INPUT}"
|
||||
heaptrack --analyze heaptrack_out.gz -H histogram.out > results/heaptrack.out
|
||||
awk ' { s=$1; count=$2; ru=(int((s+ 15) / 16)) * 16; wasted = ((ru-s)*count); print s, count, ru-s, wasted; total=total+wasted} END { print "Total: ", total }' histogram.out > results/histogram2.out
|
||||
|
||||
# XXX stamp some info about what was built & run into ./results/.
|
||||
|
||||
echo "DONE with $bname"
|
||||
}
|
||||
|
||||
if [ $do_all -ne 0 -o $do_run -ne 0 ] ; then
|
||||
run_benchmark "after"
|
||||
run_benchmark "before"
|
||||
fi
|
||||
|
||||
if [ $do_compare -ne 0 ] ; then
|
||||
# XXX this needs better analysis
|
||||
cd "${WORK}"
|
||||
diff -udr before/bench/results after/bench/results || true
|
||||
else
|
||||
echo "To compare results, run:"
|
||||
echo "$0 --compare"
|
||||
fi
|
||||
|
||||
trap '' EXIT
|
||||
|
||||
:<<=cut
|
||||
|
||||
Benchmarks to run:
|
||||
|
||||
* Parse JSON strings, of various sizes and characteristics
|
||||
* Flags: STRICT vs. non-STRICT, validate UTF8
|
||||
|
||||
* Serialization time
|
||||
* plain, spaces, pretty
|
||||
|
||||
* json_c_visit tests
|
||||
* JSON pointer tests
|
||||
|
||||
Things to record and compare:
|
||||
|
||||
* Running time
|
||||
* Peak memory usage
|
||||
* Useful bytes vs. overhead for memory allocations
|
||||
* Total number of allocations
|
||||
* Average allocation size
|
||||
* Log of all allocation sizes
|
||||
|
||||
=cut
|
||||
|
||||
Reference in New Issue
Block a user