Reformat the json_object-split branch with clang-format

Remove all of the temporary defines, structures, old compat fuctions, extra fields, etc... that were needed during the conversion to a split set of json_object_* structures.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,23 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Note: for general questions and comments, please use the forums at:
+https://groups.google.com/forum/#!forum/json-c
+
+**Describe the bug**
+A clear and concise description of what the bug is, and any information about where you're running into the bug that you feel might be relevant.
+
+**Steps To Reproduce**
+List the steps to reproduce the behavior.
+If possible, please attach a sample json file and/or a minimal code example.
+
+**Version and Platform**
+- json-c version: [e.g. json-c-0.14, or a specific commit hash]
+- OS: [e.g. Ubuntu 20.04, Debian Buster, NetBSD 9, etc...]
+- Custom cmake/build flags, if any

.gitignore

@@ -83,7 +83,11 @@
 /Testing/
 
 # ...and build artifacts.
-/doc
+/doc/html
 /libjson-c.a
 /libjson-c.so
 /libjson-c.so.*
+
+# Benchmarking input and output
+/bench/data
+/bench/work

CMakeLists.txt

@@ -7,7 +7,7 @@ if(POLICY CMP0048)
 endif()
 
 # JSON-C library is C only project.
-project(json-c LANGUAGES C VERSION 0.13.99)
+project(json-c LANGUAGES C VERSION 0.14.99)
 
 # If we've got 3.0 then it's good, let's provide support. Otherwise, leave it be.
 if(POLICY CMP0038)
@@ -40,6 +40,10 @@ if (CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND BUILD_TESTING AND
 add_subdirectory(tests)
 endif()
 
+if (NOT MSVC)  # cmd line apps don't built on Windows currently.
+add_subdirectory(apps)
+endif()
+
 # Set some packaging variables.
 set(CPACK_PACKAGE_NAME              "${PROJECT_NAME}")
 set(CPACK_PACKAGE_VERSION_MAJOR     "${PROJECT_VERSION_MAJOR}")
@@ -65,6 +69,7 @@ include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
 
 option(BUILD_SHARED_LIBS  "Default to building shared libraries" ON)
+option(BUILD_STATIC_LIBS  "Default to building static libraries" ON)
 
 # Generate a release merge and test it to verify the correctness of republishing the package.
 ADD_CUSTOM_TARGET(distcheck
@@ -78,10 +83,11 @@ COMMAND make package_source
 )
 
 # Enable or disable features. By default, all features are turned off.
-option(ENABLE_RDRAND                "Enable RDRAND Hardware RNG Hash Seed"          OFF)
-option(ENABLE_THREADING             "Enable partial threading support."             OFF)
-option(DISABLE_WERROR               "Avoid treating compiler warnings as fatal errors"    OFF)
-option(DISABLE_BSYMBOLIC            "Avoid linking with -Bsymbolic-function"   OFF)
+option(DISABLE_BSYMBOLIC              "Avoid linking with -Bsymbolic-function."               OFF)
+option(DISABLE_THREAD_LOCAL_STORAGE   "Disable using Thread-Local Storage (HAVE___THREAD)."   OFF)
+option(DISABLE_WERROR                 "Avoid treating compiler warnings as fatal errors."     OFF)
+option(ENABLE_RDRAND                  "Enable RDRAND Hardware RNG Hash Seed."                 OFF)
+option(ENABLE_THREADING               "Enable partial threading support."                     OFF)
 
 if (UNIX OR MINGW OR CYGWIN)
     list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
@@ -108,6 +114,7 @@ check_include_files("stdlib.h;stdarg.h;string.h;float.h" STDC_HEADERS)
 
 check_include_file(unistd.h         HAVE_UNISTD_H)
 check_include_file(sys/types.h      HAVE_SYS_TYPES_H)
+check_include_file(sys/resource.h   HAVE_SYS_RESOURCE_H) # for getrusage
 
 check_include_file("dlfcn.h"        HAVE_DLFCN_H)
 check_include_file("endian.h"       HAVE_ENDIAN_H)
@@ -165,6 +172,9 @@ endif()
 if (HAVE_SYSLOG_H)
     check_symbol_exists(vsyslog     "syslog.h" HAVE_VSYSLOG)
 endif()
+if (HAVE_SYS_RESOURCE_H)
+    check_symbol_exists(getrusage   "sys/resource.h" HAVE_GETRUSAGE)
+endif()
 
 check_symbol_exists(strtoll     "stdlib.h" HAVE_STRTOLL)
 check_symbol_exists(strtoull    "stdlib.h" HAVE_STRTOULL)
@@ -195,6 +205,12 @@ check_type_size(int64_t             SIZEOF_INT64_T)
 check_type_size(long                SIZEOF_LONG)
 check_type_size("long long"         SIZEOF_LONG_LONG)
 check_type_size("size_t"            SIZEOF_SIZE_T)
+if (MSVC)
+list(APPEND CMAKE_EXTRA_INCLUDE_FILES BaseTsd.h)
+check_type_size("SSIZE_T"           SIZEOF_SSIZE_T)
+else()
+check_type_size("ssize_t"           SIZEOF_SSIZE_T)
+endif()
 
 check_c_source_compiles(
 [=[
@@ -208,14 +224,16 @@ check_c_source_compiles(
   "int main() { int i, x = 0; i = __sync_add_and_fetch(&x,1); return x; }"
   HAVE_ATOMIC_BUILTINS)
 
-check_c_source_compiles(
-  "__thread int x = 0; int main() { return 0; }"
-  HAVE___THREAD)
+if (NOT DISABLE_THREAD_LOCAL_STORAGE)
+  check_c_source_compiles(
+    "__thread int x = 0; int main() { return 0; }"
+    HAVE___THREAD)
 
-if (HAVE___THREAD)
-    set(SPEC___THREAD __thread)
-elseif (MSVC)
-    set(SPEC___THREAD __declspec(thread))
+  if (HAVE___THREAD)
+      set(SPEC___THREAD __thread)
+  elseif (MSVC)
+      set(SPEC___THREAD __declspec(thread))
+  endif()
 endif()
 
 # Hardware random number is not available on Windows? Says, config.h.win32. Best to preserve compatibility.
@@ -225,9 +243,9 @@ endif()
 
 # Once we've done basic symbol/header searches let's add them in.
 configure_file(${PROJECT_SOURCE_DIR}/cmake/config.h.in        ${PROJECT_BINARY_DIR}/config.h)
-message(STATUS "Written ${PROJECT_BINARY_DIR}/config.h")
+message(STATUS "Wrote ${PROJECT_BINARY_DIR}/config.h")
 configure_file(${PROJECT_SOURCE_DIR}/cmake/json_config.h.in   ${PROJECT_BINARY_DIR}/json_config.h)
-message(STATUS "Written ${PROJECT_BINARY_DIR}/json_config.h")
+message(STATUS "Wrote ${PROJECT_BINARY_DIR}/json_config.h")
 
 if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
     set(CMAKE_C_FLAGS           "${CMAKE_C_FLAGS} -ffunction-sections -fdata-sections")
@@ -299,7 +317,7 @@ if ($ENV{VALGRIND})
 endif()
 
 set(JSON_C_PUBLIC_HEADERS
-    ${PROJECT_BINARY_DIR}/config.h
+    # Note: config.h is _not_ included here
     ${PROJECT_BINARY_DIR}/json_config.h
 
     ${PROJECT_SOURCE_DIR}/json.h
@@ -349,23 +367,7 @@ set(JSON_C_SOURCES
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_BINARY_DIR})
 
-# generate doxygen documentation for json-c API
-
-find_package(Doxygen)
-option(BUILD_DOCUMENTATION "Create and install the HTML based API documentation(requires Doxygen)" ${DOXYGEN_FOUND})
-
-if (DOXYGEN_FOUND)
-
-	add_custom_target(doc
-	COMMAND ${DOXYGEN_EXECUTABLE} ${PROJECT_SOURCE_DIR}/Doxyfile
-		WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
-
-	# request to configure the file
-	configure_file(Doxyfile Doxyfile)
-
-else (DOXYGEN_FOUND)
-	message("Warning: doxygen not found, the 'doc' target will not be included")
-endif(DOXYGEN_FOUND)
+add_subdirectory(doc)
 
 # uninstall
 add_custom_target(uninstall
@@ -381,9 +383,9 @@ add_library(${PROJECT_NAME}
     ${JSON_C_HEADERS}
 )
 set_target_properties(${PROJECT_NAME} PROPERTIES
-    VERSION 4.0.0
-    SOVERSION 4)
-
+    VERSION 5.0.0
+    SOVERSION 5)
+list(APPEND CMAKE_TARGETS ${PROJECT_NAME})
 # If json-c is used as subroject it set to target correct interface -I flags and allow
 # to build external target without extra include_directories(...)
 target_include_directories(${PROJECT_NAME}
@@ -392,7 +394,35 @@ target_include_directories(${PROJECT_NAME}
         $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}>
 )
 
-install(TARGETS ${PROJECT_NAME}
+# Allow to build static and shared libraries at the same time
+if (BUILD_STATIC_LIBS AND BUILD_SHARED_LIBS)
+    set(STATIC_LIB ${PROJECT_NAME}-static)
+    add_library(${STATIC_LIB} STATIC
+        ${JSON_C_SOURCES}
+        ${JSON_C_HEADERS}
+    )
+
+    # rename the static library
+    if (NOT MSVC)
+    set_target_properties(${STATIC_LIB} PROPERTIES
+        OUTPUT_NAME ${PROJECT_NAME}
+    )
+    endif()
+    list(APPEND CMAKE_TARGETS ${STATIC_LIB})
+endif ()
+
+# Always create new install dirs with 0755 permissions, regardless of umask
+set(CMAKE_INSTALL_DEFAULT_DIRECTORY_PERMISSIONS
+	OWNER_READ
+	OWNER_WRITE
+	OWNER_EXECUTE
+	GROUP_READ
+	GROUP_EXECUTE
+	WORLD_READ
+	WORLD_EXECUTE
+   )
+
+install(TARGETS ${CMAKE_TARGETS}
     EXPORT ${PROJECT_NAME}-targets
     RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

ChangeLog

@@ -1,4 +1,27 @@
 
+Next Release 0.15
+=====================
+
+Deprecated and removed features:
+--------------------------------
+...none yet...
+
+Other changes
+--------------
+* Add a json_parse binary, for use in testing changes (not installed).
+* Issue #471: always create directories with mode 0755, regardless of umask.
+* Added a JSON_TOKENER_ALLOW_TRAILING_CHARS flag to allow multiple objects
+  to be parsed even when JSON_TOKENER_STRICT is set.
+* Split the internal json_object structure into several sub-types, one for
+   each json_type (json_object_object, json_object_string, etc...).
+  This improves memory usage and speed, with the benchmark under
+   bench/ report 5.8% faster test time and 6%(max RSS)-12%(peak heap)
+   less memory usage.
+  Memory used just for json_object structures decreased 27%, so use cases
+   with fewer arrays and/or strings would benefit more.
+
+
+***
 
 0.14 (up to commit 9ed00a6, 2020/04/14)
 =========================================

README.json_object-split.md

@@ -0,0 +1,15 @@
+
+This branch implements the changes briefly described at:
+
+https://github.com/json-c/json-c/wiki/Proposal:-struct-json_object-split
+
+These were originally mentioned in:
+Issue #535 - short string optimization: excessive array length
+
+The changes are expected to impact and possibly invalidate:
+Issue #552 - Some anlysis about memory allocator in json-c
+
+and will likely cause notable conflicts in any other significant un-merged
+changes, such as PR#620 - Introduce json_object_new_string_{ext,noalloc}()
+
+

README.md

@@ -93,20 +93,23 @@ CMake Options <a name="CMake"></a>
 The json-c library is built with [CMake](https://cmake.org/cmake-tutorial/),
 which can take a few options.
 
-Variable             | Type   | Description
----------------------|--------|--------------
-CMAKE_INSTALL_PREFIX | String | The install location.
-CMAKE_BUILD_TYPE     | String | Defaults to "debug"
-BUILD_SHARED_LIBS    | Bool   | The default build generates a dynamic (dll/so) library.  Set this to OFF to create a static library instead.
-ENABLE_RDRAND        | Bool   | Enable RDRAND Hardware RNG Hash Seed
-ENABLE_THREADING     | Bool   | Enable partial threading support
-DISABLE_WERROR       | Bool   | Disable use of -Werror
-DISABLE_BSYMBOLIC    | Bool   | Disable use of -Bsymbolic-functions
+Variable                     | Type   | Description
+-----------------------------|--------|--------------
+CMAKE_INSTALL_PREFIX         | String | The install location.
+CMAKE_BUILD_TYPE             | String | Defaults to "debug".
+BUILD_SHARED_LIBS            | Bool   | The default build generates a dynamic (dll/so) library.  Set this to OFF to create a static library only.
+BUILD_STATIC_LIBS            | Bool   | The default build generates a static (lib/a) library.  Set this to OFF to create a shared library only.
+DISABLE_BSYMBOLIC            | Bool   | Disable use of -Bsymbolic-functions.
+DISABLE_THREAD_LOCAL_STORAGE | Bool   | Disable use of Thread-Local Storage (HAVE___THREAD).
+DISABLE_WERROR               | Bool   | Disable use of -Werror.
+ENABLE_RDRAND                | Bool   | Enable RDRAND Hardware RNG Hash Seed.
+ENABLE_THREADING             | Bool   | Enable partial threading support.
 
 Pass these options as `-D` on CMake's command-line.
 
 ```sh
-cmake -DBUILD_SHARED_LIBS=OFF ...
+# build a static library only
+cmake -DBUILD_SHARED_LIBS=OFF ..
 ```
 
 ### Building with partial threading support

RELEASE_CHECKLIST.txt

@@ -31,7 +31,7 @@
 ## Release creation
 
 Start creating the new release:
-    release=0.14
+    release=0.15
     git clone https://github.com/json-c/json-c json-c-${release}
 
     mkdir distcheck
@@ -52,7 +52,6 @@ Make any fixes/changes *before* branching.
 
 Using ${release}:
 	Update the version in json_c_version.h
-	Update the version in Doxyfile (PROJECT_NUMBER)
 	Update the version in CMakeLists.txt (VERSION in the project(...) line)
 	Update the version in config.h.win32 (several places)
 
@@ -99,7 +98,7 @@ Go to Amazon S3 service at:
     https://console.aws.amazon.com/s3/
 
 Upload the two tarballs in the json-c_releases folder.
-	When uploading, use "Reduced Redundancy", and make the uploaded files publicly accessible.
+	When uploading, use "Standard" storage class, and make the uploaded files publicly accessible.
 
 Logout of Amazon S3, and verify that the files are visible.
     https://s3.amazonaws.com/json-c_releases/releases/index.html
@@ -114,7 +113,6 @@ Add new section to ChangeLog for ${release}+1
 
 Use ${release}.99 to indicate a version "newer" than anything on the branch:
 	Update the version in json_c_version.h
-	Update the version in Doxyfile
 	Update the version in CMakeLists.txt
 	Update the version in config.h.win32
 
@@ -122,6 +120,9 @@ Update RELEASE_CHECKLIST.txt, set release=${release}+1
 
 Update the set_target_properties() line in CmakeLists.txt to match the release branch.
 
+    git commit -a -m "Update the master branch to version 0.${release}.99"
+    git push
+
 ------------
 
 Update the gh-pages branch with new docs:
@@ -135,12 +136,12 @@ Update the gh-pages branch with new docs:
     mkdir json-c-${release}
     cp -R ../json-c-${release}/doc json-c-${release}/.
     git add json-c-${release}
-    git commit
+    git commit -a -m "Add the ${release} docs."
 
     vi index.html
     # Add/change links to current release.
 
-    git commit index.html
+    git commit -a -m "Update the doc links to point at ${release}"
 
     git push
 

apps/CMakeLists.txt

@@ -0,0 +1,122 @@
+
+cmake_minimum_required(VERSION 2.8)  # see ../CMakeLists.txt for why 2.8
+
+if(POLICY CMP0075)
+    cmake_policy(SET CMP0075 NEW)
+endif()
+
+include(CheckSymbolExists)
+include(CheckIncludeFile)
+include(CMakePackageConfigHelpers)
+
+# First, sort out whether we're running inside a json-c build,
+#  or standalone, such as part of a benchmark build.
+
+if ("${PROJECT_NAME}" STREQUAL "json-c")
+# Part of an overall json-c build
+set(APPS_LINK_LIBS "${PROJECT_NAME}")
+
+# We know we have this in our current sources:
+set(HAVE_JSON_TOKENER_GET_PARSE_END)
+
+else()
+
+# Standalone mode, using an already installed json-c library, somewhere.
+# The path to the json-c install must be specified with -DCMAKE_PREFIX_PATH=...
+
+project(apps)
+find_package(PkgConfig)
+
+# PkgConfig is supposed to include CMAKE_PREFIX_PATH in the PKG_CONFIG_PATH
+#  that's used when running pkg-config, but it just doesn't work :(
+# https://gitlab.kitware.com/cmake/cmake/issues/18150
+#set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH True)
+
+# Instead, we handle it explicitly here and update PKG_CONFIG_PATH ourselves.
+if (NOT CMAKE_PREFIX_PATH)
+	message(FATAL_ERROR "Please specify -DCMAKE_PREFIX_PATH=... when running cmake.")
+endif()
+
+# Note: find_file isn't recursive :(
+find_file(PC_FILE_PATH "json-c.pc"
+	PATHS "${CMAKE_PREFIX_PATH}/lib64" "${CMAKE_PREFIX_PATH}/lib"
+	PATH_SUFFIXES "pkgconfig"
+	NO_DEFAULT_PATH)
+get_filename_component(PC_DIR_PATH "${PC_FILE_PATH}" DIRECTORY)
+set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${PC_DIR_PATH}")
+message(STATUS "PC_FILE_PATH=${PC_FILE_PATH}")
+message(STATUS "PC_DIR_PATH=${PC_DIR_PATH}")
+
+pkg_check_modules(PC_JSONC json-c)
+if (PC_JSONC_FOUND)
+	message(STATUS "Found json-c using pkg-config: ${PC_JSONC_PREFIX}")
+	message(STATUS " PC_JSONC_INCLUDE_DIRS=${PC_JSONC_INCLUDE_DIRS}")
+	message(STATUS " PC_JSONC_LIBRARIES=${PC_JSONC_LIBRARIES}")
+	message(STATUS " PC_JSONC_LIBRARY_DIRS=${PC_JSONC_LIBRARY_DIRS}")
+	link_directories(${PC_JSONC_LIBRARY_DIRS})
+	include_directories(${PC_JSONC_INCLUDE_DIRS})
+	# for target_link_libraries(...)
+	set(APPS_INCLUDE_DIRS ${PC_JSONC_INCLUDE_DIRS})
+	set(APPS_LINK_DIRS ${PC_JSONC_LIBRARY_DIRS})
+	set(APPS_LINK_LIBS ${PC_JSONC_LIBRARIES})
+else()
+	message(STATUS "Using find_package to locate json-c")
+
+	# Note: find_package needs CMAKE_PREFIX_PATH set appropriately.
+	# XXX json-c's installed cmake files don't actually set up what's
+	#  needed to use find_package() by itself, so we're just using it
+	#  to confirm the top of the install location.
+	find_package(json-c CONFIG)  # sets json-c_DIR
+
+	# Assume json-c-config.cmake is in lib64/cmake/json-c/
+	get_filename_component(json-c_TOP "${json-c_DIR}/../../.." ABSOLUTE)
+	get_filename_component(json-c_LIBDIR "${json-c_DIR}/../.." ABSOLUTE)
+
+	message(STATUS " json-c_TOP=${json-c_TOP}")
+	message(STATUS " json-c_DIR=${json-c_DIR}")
+	message(STATUS " json-c_LIBDIR=${json-c_LIBDIR}")
+
+	link_directories(${json-c_LIBDIR})
+	include_directories(${json-c_TOP}/include)
+	include_directories(${json-c_TOP}/include/json-c)
+	set(APPS_LINK_DIRS "${json-c_LIBDIR}")
+	set(APPS_INCLUDE_DIRS "${json-c_TOP}/include;${json-c_TOP}/include/json-c")
+
+	set(APPS_LINK_LIBS json-c)
+endif()
+
+set(CMAKE_REQUIRED_LINK_OPTIONS "-L${APPS_LINK_DIRS}")
+set(CMAKE_REQUIRED_LIBRARIES ${APPS_LINK_LIBS})
+set(CMAKE_REQUIRED_INCLUDES ${APPS_INCLUDE_DIRS})
+check_symbol_exists(json_tokener_get_parse_end "json_tokener.h" HAVE_JSON_TOKENER_GET_PARSE_END)
+
+endif() # end "standalone mode" block
+
+# ---------------------------------
+
+check_include_file(sys/resource.h   HAVE_SYS_RESOURCE_H) # for getrusage
+if (HAVE_SYS_RESOURCE_H)
+    check_symbol_exists(getrusage   "sys/resource.h" HAVE_GETRUSAGE)
+endif()
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/apps_config.h.in
+               ${PROJECT_BINARY_DIR}/apps_config.h)
+message(STATUS "Wrote ${PROJECT_BINARY_DIR}/apps_config.h")
+
+# ---------------------------------
+
+include_directories(PUBLIC ${CMAKE_SOURCE_DIR})
+include_directories(${PROJECT_SOURCE_DIR})
+include_directories(${PROJECT_BINARY_DIR})
+
+# ---------------------------------
+
+# Now, finally, the actual executables we're building:
+
+add_executable(json_parse json_parse.c)
+target_link_libraries(json_parse PRIVATE ${APPS_LINK_LIBS})
+
+# Note: it is intentional that there are no install instructions here yet.
+# When/if the interface of the app(s) listed here settles down enough to
+# publish as part of a regular build that will be added.
+

apps/cmake/apps_config.h.in

@@ -0,0 +1,8 @@
+
+/* Define to 1 if you have the <sys/resource.h> header file. */
+#cmakedefine HAVE_SYS_RESOURCE_H
+
+/* Define if you have the `getrusage' function. */
+#cmakedefine HAVE_GETRUSAGE
+
+#cmakedefine HAVE_JSON_TOKENER_GET_PARSE_END

apps/json_parse.c

@@ -0,0 +1,187 @@
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "apps_config.h"
+
+/* XXX for a regular program, these should be <json-c/foo.h>
+ * but that's inconvenient when building in the json-c source tree.
+ */
+#include "json_object.h"
+#include "json_tokener.h"
+#include "json_util.h"
+
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#include <sys/time.h>
+#endif
+
+static int formatted_output = 0;
+static int show_output = 1;
+static int strict_mode = 0;
+static const char *fname = NULL;
+
+#ifndef HAVE_JSON_TOKENER_GET_PARSE_END
+#define json_tokener_get_parse_end(tok) ((tok)->char_offset)
+#endif
+
+static void usage(const char *argv0, int exitval, const char *errmsg);
+static void showmem(void);
+static int parseit(int fd, int (*callback)(struct json_object *));
+static int showobj(struct json_object *new_obj);
+
+static void showmem(void)
+{
+#ifdef HAVE_GETRUSAGE
+	struct rusage rusage;
+	memset(&rusage, 0, sizeof(rusage));
+	getrusage(RUSAGE_SELF, &rusage);
+	printf("maxrss: %ld KB\n", rusage.ru_maxrss);
+#endif
+}
+
+static int parseit(int fd, int (*callback)(struct json_object *))
+{
+	struct json_object *obj;
+	char buf[32768];
+	int ret;
+	int depth = JSON_TOKENER_DEFAULT_DEPTH;
+	json_tokener *tok;
+
+	tok = json_tokener_new_ex(depth);
+	if (!tok)
+	{
+		fprintf(stderr, "unable to allocate json_tokener: %s\n", strerror(errno));
+		return 1;
+	}
+	json_tokener_set_flags(tok, JSON_TOKENER_STRICT
+#ifdef JSON_TOKENER_ALLOW_TRAILING_CHARS
+		 | JSON_TOKENER_ALLOW_TRAILING_CHARS
+#endif
+	);
+
+	// XXX push this into some kind of json_tokener_parse_fd API?
+	//  json_object_from_fd isn't flexible enough, and mirroring
+	//   everything you can do with a tokener into json_util.c seems
+	//   like the wrong approach.
+	size_t total_read = 0;
+	while ((ret = read(fd, buf, sizeof(buf))) > 0)
+	{
+		total_read += ret;
+		int start_pos = 0;
+		while (start_pos != ret)
+		{
+			obj = json_tokener_parse_ex(tok, &buf[start_pos], ret - start_pos);
+			enum json_tokener_error jerr = json_tokener_get_error(tok);
+			int parse_end = json_tokener_get_parse_end(tok);
+			if (obj == NULL && jerr != json_tokener_continue)
+			{
+				char *aterr = &buf[start_pos + parse_end];
+				fflush(stdout);
+				int fail_offset = total_read - ret + start_pos + parse_end;
+				fprintf(stderr, "Failed at offset %d: %s %c\n", fail_offset,
+				        json_tokener_error_desc(jerr), aterr[0]);
+				json_tokener_free(tok);
+				return 1;
+			}
+			if (obj != NULL)
+			{
+				int cb_ret = callback(obj);
+				json_object_put(obj);
+				if (cb_ret != 0)
+				{
+					json_tokener_free(tok);
+					return 1;
+				}
+			}
+			start_pos += json_tokener_get_parse_end(tok);
+			assert(start_pos <= ret);
+		}
+	}
+	if (ret < 0)
+	{
+		fprintf(stderr, "error reading fd %d: %s\n", fd, strerror(errno));
+	}
+
+	json_tokener_free(tok);
+	return 0;
+}
+
+static int showobj(struct json_object *new_obj)
+{
+	if (new_obj == NULL)
+	{
+		fprintf(stderr, "%s: Failed to parse\n", fname);
+		return 1;
+	}
+
+	printf("Successfully parsed object from %s\n", fname);
+
+	if (show_output)
+	{
+		const char *output;
+		if (formatted_output)
+			output = json_object_to_json_string(new_obj);
+		else
+			output = json_object_to_json_string_ext(new_obj, JSON_C_TO_STRING_PRETTY);
+		printf("%s\n", output);
+	}
+
+	showmem();
+	return 0;
+}
+
+static void usage(const char *argv0, int exitval, const char *errmsg)
+{
+	FILE *fp = stdout;
+	if (exitval != 0)
+		fp = stderr;
+	if (errmsg != NULL)
+		fprintf(fp, "ERROR: %s\n\n", errmsg);
+	fprintf(fp, "Usage: %s [-f] [-n] [-s]\n", argv0);
+	fprintf(fp, "  -f - Format the output with JSON_C_TO_STRING_PRETTY\n");
+	fprintf(fp, "  -n - No output\n");
+	fprintf(fp, "  -s - Parse in strict mode, flags:\n");
+	fprintf(fp, "       JSON_TOKENER_STRICT|JSON_TOKENER_ALLOW_TRAILING_CHARS\n");
+
+	fprintf(fp, "\nWARNING WARNING WARNING\n");
+	fprintf(fp, "This is a prototype, it may change or be removed at any time!\n");
+	exit(exitval);
+}
+
+int main(int argc, char **argv)
+{
+	json_object *new_obj;
+	int opt;
+
+	while ((opt = getopt(argc, argv, "fhns")) != -1)
+	{
+		switch (opt)
+		{
+		case 'f': formatted_output = 1; break;
+		case 'n': show_output = 0; break;
+		case 's': strict_mode = 1; break;
+		case 'h': usage(argv[0], 0, NULL);
+		default: /* '?' */ usage(argv[0], EXIT_FAILURE, "Unknown arguments");
+		}
+	}
+	if (optind >= argc)
+	{
+		usage(argv[0], EXIT_FAILURE, "Expected argument after options");
+	}
+	fname = argv[optind];
+
+	int fd = open(argv[optind], O_RDONLY, 0);
+	showmem();
+	if (parseit(fd, showobj) != 0)
+		exit(EXIT_FAILURE);
+	showmem();
+
+	exit(EXIT_SUCCESS);
+}

arraylist.c

@@ -40,13 +40,13 @@ struct array_list *array_list_new(array_list_free_fn *free_fn)
 {
 	struct array_list *arr;
 
-	arr = (struct array_list *)calloc(1, sizeof(struct array_list));
+	arr = (struct array_list *)malloc(sizeof(struct array_list));
 	if (!arr)
 		return NULL;
 	arr->size = ARRAY_LIST_DEFAULT_SIZE;
 	arr->length = 0;
 	arr->free_fn = free_fn;
-	if (!(arr->array = (void **)calloc(arr->size, sizeof(void *))))
+	if (!(arr->array = (void **)malloc(arr->size * sizeof(void *))))
 	{
 		free(arr);
 		return NULL;
@@ -92,11 +92,11 @@ static int array_list_expand_internal(struct array_list *arr, size_t max)
 	if (!(t = realloc(arr->array, new_size * sizeof(void *))))
 		return -1;
 	arr->array = (void **)t;
-	(void)memset(arr->array + arr->size, 0, (new_size - arr->size) * sizeof(void *));
 	arr->size = new_size;
 	return 0;
 }
 
+//static inline int _array_list_put_idx(struct array_list *arr, size_t idx, void *data)
 int array_list_put_idx(struct array_list *arr, size_t idx, void *data)
 {
 	if (idx > SIZE_T_MAX - 1)
@@ -106,6 +106,17 @@ int array_list_put_idx(struct array_list *arr, size_t idx, void *data)
 	if (idx < arr->length && arr->array[idx])
 		arr->free_fn(arr->array[idx]);
 	arr->array[idx] = data;
+	if (idx > arr->length)
+	{
+		/* Zero out the arraylist slots in between the old length
+		   and the newly added entry so we know those entries are
+		   empty.
+		   e.g. when setting array[7] in an array that used to be 
+		   only 5 elements longs, array[5] and array[6] need to be
+		   set to 0.
+		 */
+		memset(arr->array + arr->length, 0, (idx - arr->length) * sizeof(void *));
+	}
 	if (arr->length <= idx)
 		arr->length = idx + 1;
 	return 0;
@@ -113,7 +124,17 @@ int array_list_put_idx(struct array_list *arr, size_t idx, void *data)
 
 int array_list_add(struct array_list *arr, void *data)
 {
-	return array_list_put_idx(arr, arr->length, data);
+	/* Repeat some of array_list_put_idx() so we can skip several
+	   checks that we know are unnecessary when appending at the end
+	 */
+	size_t idx = arr->length;
+	if (idx > SIZE_T_MAX - 1)
+		return -1;
+	if (array_list_expand_internal(arr, idx + 1))
+		return -1;
+	arr->array[idx] = data;
+	arr->length++;
+	return 0;
 }
 
 void array_list_sort(struct array_list *arr, int (*compar)(const void *, const void *))
@@ -136,6 +157,9 @@ int array_list_del_idx(struct array_list *arr, size_t idx, size_t count)
 {
 	size_t i, stop;
 
+	/* Avoid overflow in calculation with large indices. */
+	if (idx > SIZE_T_MAX - count)
+		return -1;
 	stop = idx + count;
 	if (idx >= arr->length || stop > arr->length)
 		return -1;

bench/README.bench.md

@@ -0,0 +1,80 @@
+
+Benchmark tests for json-c
+
+General strategy:
+-------------------
+
+* Identify "after" commit hash
+    * Use provided directory
+    * Use provided commit hash
+    * Local changes in current working directory
+    * ${cur_branch}
+* Identify "before" commit hash, in order of preference
+    * Use provided directory
+    * Use provided commit hash
+    * Use origin/${cur_branch}, if different from ${after_commit}
+    * Use previous release
+
+* If not using existing dir, clone to src-${after_commit}
+    * or, checkout appropriate commit in existing src-${after_commit}
+* Create build & install dirs for ${after_commit}
+* Build & install ${after_commit}
+* Compile benchmark programs against install-${after_commit}
+
+* If not using existing dir, clone to src-${before_commit}
+    * or, checkout appropriate commit in existing src-${before_commit}
+* Create build & install dirs for ${before_commit}
+* Build & install ${before_commit}
+* Compile benchmark programs against install-${before_commit}
+
+* Run benchmark in each location
+* Compare results
+
+heaptrack memory profiler
+---------------------------
+
+https://milianw.de/blog/heaptrack-a-heap-memory-profiler-for-linux.html
+
+
+```
+yum install libdwarf-devel elfutils boost-devel libunwind-devel
+
+git clone git://anongit.kde.org/heaptrack
+cd heaptrack
+mkdir build
+cd build
+cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+	-DCMAKE_INSTALL_PREFIX=$HOME/heaptrack-install ..
+make install
+```
+
+
+Issues
+--------
+
+* jc-bench.sh is incomplete.
+
+* valgrind massif misreports "extra-heap" bytes?
+
+    "json_parse -n canada.json" shows 38640 KB maxrss.
+
+    Using valgrind --tool=massif, a large amount of memory is listed as
+     wasted "extra-heap" bytes.  (~5.6MB)
+
+    ```
+    valgrind --tool=massif --massif-out-file=massif.out ./json_parse -n ~/canada.json
+    ms_print massif.out
+    ```
+
+
+    Using heaptrack, and analyzing the histogram, only shows ~2.6MB
+    ```
+    heaptrack ./json_parse -n canada.json
+    heaptrack --analyze heaptrack*gz -H histgram.out
+    awk ' { s=$1; count=$2; ru=(int((s+ 15) / 16)) * 16; wasted = ((ru-s)*count); print s, count, ru-s, wasted; total=total+wasted} END { print "Total: ", total }' histogram.out
+    ```
+
+ With the (unreleased) arraylist trimming changes, maxrss reported by
+  getrusage() goes down, but massif claims *more* total usage, and a HUGE 
+  extra-heap amount (50% of total).
+

bench/jc-bench.sh

@@ -0,0 +1,284 @@
+#!/bin/sh
+#
+# Benchmarking harness for json-c
+#
+# Use this to compare the behavior of two different versions of the library,
+# such as json-c-0.14 release vs master, master vs local changes, etc...
+#
+
+set -e
+
+trap 'echo "FAILED!"' EXIT
+
+RUNDIR=$(dirname "$0")
+RUNDIR=$(cd "$RUNDIR" && pwd)
+
+TOP=$(cd "$RUNDIR/.." && pwd)
+
+usage()
+{
+	exitval=$1
+	errmsg=$2
+	if [ $exitval -ne 0 ] ; then
+		exec 1>&2
+	fi
+	if [ ! -z "$errmsg" ] ; then
+		echo "ERROR: $errmsg" 1>&2
+	fi
+	cat <<EOF
+Usage: $0 [-h] [-v] [--build] [--run] [--compare] ...XAX...
+EOF
+
+	exit $extival
+}
+
+before_arg=
+after_arg=
+do_all=1
+do_build=0
+do_run=0
+do_compare=0
+
+while [ $# -gt 0 ] ; do
+	case "$1" in
+	--before)
+		before_arg=$2
+		shift
+		;;
+	--after)
+		after_arg=$2
+		shift
+		;;
+	--build)
+		do_all=0
+		do_build=1
+		;;
+	--run)
+		do_all=0
+		do_run=1
+		;;
+	--compare)
+		do_all=0
+		do_compare=1
+		;;
+	-h)
+		usage 0 ""
+		;;
+	-v)
+		set -x
+		;;
+	*)
+		usage 1 "Unknown args: $*"
+		;;
+	esac
+	shift
+done
+
+WORK="${RUNDIR}/work"
+mkdir -p "${WORK}"
+
+DATA="${RUNDIR}/data"
+mkdir -p "${DATA}"
+
+for file in citm_catalog.json twitter.json canada.json ; do
+	if [ ! -r "${DATA}/${file}" ] ; then
+		echo "Fetching ${file} from github.com/mloskot/json_benchmark"
+		URL="https://github.com/mloskot/json_benchmark/raw/master/data/${file}"
+		curl -s -L -o "${DATA}/${file}" "$URL"
+	fi
+done
+echo
+
+# Identify "after" commit hash, in order of preference
+if [ ! -z "$after_arg" -a -d "$after_arg" ] ; then
+	# Use provided directory
+	after_src_dir="$after_arg"
+	after_commit=
+	echo "Using provided directory [$after_arg] as 'after'"
+else
+	_commit=
+	if [ ! -z "$after_arg" ] ; then
+		# Use provided commit hash
+		_commit=$(git rev-parse --verify "$after_arg")
+	fi
+	if [ ! -z "$_commit" ] ;then
+		after_src_dir=  # i.e. current tree
+		after_commit="$_commit"
+		echo "Using provided commit [$after_arg => $_commit] as 'after'"
+	else
+		# Local changes in current working directory
+		# ${cur_branch}
+		after_src_dir=$TOP
+		after_commit=
+		echo "Using local changes in $TOP as 'after'"
+	fi
+fi
+
+# Identify "before" commit hash, in order of preference
+if [ ! -z "$before_arg" -a -d "$before_arg" ] ; then
+   	# Use provided directory
+	before_src_dir="$before_arg"
+	before_commit=
+	echo "Using provided directory [$before_arg] as 'before'"
+else
+	_commit=
+	if [ ! -z "$before_arg" ] ; then
+		# Use provided commit hash
+		_commit=$(git rev-parse --verify "$before_arg")
+	fi
+	if [ ! -z "$_commit" ] ;then
+		before_src_dir=  # i.e. current tree
+		before_commit="$_commit"
+		echo "Using provided commit [$before_arg => $_commit] as 'before'"
+	else
+		# Use origin/${cur_branch}, if different from ${after_commit}
+		_cur_branch=$(git rev-parse --abbrev-ref HEAD)
+		_commit=
+		if [ ! -z "${_cur_branch}" ] ; then
+			_commit=$(git rev-parse --verify "origin/${_cur_branch}")
+			echo "Using origin/${_cur_branch} [$_commit] as 'before'"
+		fi
+		if [ "$_commit" = "${after_commit}" ] ; then
+			_commit=
+		fi
+	fi
+
+	if [ ! -z "$_commit" ] ; then
+		before_src_dir=  # i.e. current tree
+		before_commit="$_commit"
+	else
+		# Use previous release
+		before_src_dir=  # i.e. current tree
+		before_commit="$(git tag | sort | tail -1)"
+		echo "Using previous release [$before_commit] as 'before'"
+	fi
+fi
+
+echo
+
+compile_benchmark()
+{
+	local bname=$1
+	local src_dir="$2"
+	local src_commit="$3"
+
+	local build_dir="${WORK}/$bname/build"
+	local inst_dir="${WORK}/$bname/install"
+	local bench_dir="${WORK}/$bname/bench"
+
+	echo
+	echo "=========== $bname ==========="
+	echo
+
+	mkdir -p "${build_dir}"
+	mkdir -p "${inst_dir}"
+	mkdir -p "${bench_dir}"
+
+	if [ ! -z "$src_commit" ] ; then
+		# Resolve the short hash, tag or branch name to full hash
+		src_commit=$(git rev-parse $src_commit)
+	fi
+
+	# No src dir specified, clone and checkout $src_commit
+	if [ -z "$src_dir" ] ; then
+		src_dir="${WORK}/$bname/src"
+		echo "=== Using sources in $src_dir"
+		mkdir -p "$src_dir"
+		at_commit=$(git --git-dir="$src_dir/.git" rev-parse HEAD 2> /dev/null || true)
+		echo "at_commit: $at_commit"
+		if [ -z "$at_commit" ] ; then
+			# Assume it's an empty dir
+			git clone -n "$TOP" "$src_dir"
+		fi
+		git -C "$src_dir" --git-dir="$src_dir/.git" checkout "$src_commit"
+	fi
+	# else, use the provided $src_dir
+
+	if [ -e "${src_dir}/CMakeLists.txt" ] ; then
+		cd "${build_dir}"
+		cmake -DCMAKE_INSTALL_PREFIX="${inst_dir}" "${src_dir}"
+	else
+		# Old versions of json-c used automake/autoconf
+		cd "${src_dir}"
+		sh autogen.sh   # always run it, configure doesn't always work
+		cd "${build_dir}"
+		"${src_dir}/configure" --prefix="${inst_dir}"
+	fi
+	make all install
+
+	cd "${bench_dir}"
+	cmake -DCMAKE_PREFIX_PATH="${inst_dir}" "${TOP}/apps"
+	make all
+}
+
+# XXX TODO: name "after" and "before" uniquely using the dir & commit
+
+if [ $do_all -ne 0 -o $do_build -ne 0 ] ; then
+	sleep 5   # Wait slightly, to allow the human to read the message
+	          #  about what exactly we're doing to benchmark.
+	compile_benchmark "after" "${after_src_dir}" "${after_commit}"
+	compile_benchmark "before" "${before_src_dir}" "${before_commit}"
+fi
+
+run_benchmark()
+{
+	local bname=$1
+	local inst_dir="${WORK}/$bname/install"
+	local bench_dir="${WORK}/$bname/bench"
+
+	local INPUT=${DATA}/canada.json
+
+	cd "${bench_dir}"
+	mkdir -p results
+	(time ./json_parse -n "${INPUT}") > results/basic_timing.out 2>&1
+	valgrind --tool=massif --massif-out-file=massif.out ./json_parse -n "${INPUT}"
+	ms_print massif.out > results/ms_print.out
+	heaptrack -o heaptrack_out ./json_parse -n "${INPUT}"
+	heaptrack --analyze heaptrack_out.gz -H histogram.out > results/heaptrack.out
+	awk ' { s=$1; count=$2; ru=(int((s+ 15) / 16)) * 16; wasted = ((ru-s)*count); print s, count, ru-s, wasted; total=total+wasted} END { print "Total: ", total }' histogram.out > results/histogram2.out
+
+	# XXX stamp some info about what was built & run into ./results/.
+
+	echo "DONE with $bname"
+}
+
+if [ $do_all -ne 0 -o $do_run -ne 0 ] ; then
+	run_benchmark "after"
+	run_benchmark "before"
+fi
+
+if [ $do_compare -ne 0 ] ; then
+	# XXX this needs better analysis
+	cd "${WORK}"
+	diff -udr before/bench/results after/bench/results || true
+else
+	echo "To compare results, run:"
+	echo "$0 --compare"
+fi
+
+trap '' EXIT
+
+:<<=cut
+
+Benchmarks to run:
+
+* Parse JSON strings, of various sizes and characteristics
+    * Flags: STRICT vs. non-STRICT, validate UTF8
+
+* Serialization time
+    * plain, spaces, pretty
+
+* json_c_visit tests
+* JSON pointer tests
+
+Things to record and compare:
+
+* Running time
+* Peak memory usage
+* Useful bytes vs. overhead for memory allocations
+* Total number of allocations
+* Average allocation size
+* Log of all allocation sizes
+
+=cut
+

cmake-configure

@@ -52,6 +52,9 @@ while [ $# -gt 0 ] ; do
 		FLAGS+=(-DCMAKE_INSTALL_PREFIX="$2")
 		shift
 		;;
+	--prefix=*)
+		FLAGS+=(-DCMAKE_INSTALL_PREFIX="${1##--prefix=}")
+		;;
 	--enable-threading)
 		FLAGS+=(-DENABLE_THREADING=ON)
 		;;
@@ -62,7 +65,7 @@ while [ $# -gt 0 ] ; do
 		FLAGS+=(-DBUILD_SHARED_LIBS=ON)
 		;;
 	--enable-static)
-		FLAGS+=(-DBUILD_SHARED_LIBS=OFF)
+		FLAGS+=(-DBUILD_STATIC_LIBS=ON)
 		;;
 	--disable-Bsymbolic)
 		FLAGS+=(-DDISABLE_BSYMBOLIC=ON)

cmake/config.h.in

@@ -1,4 +1,3 @@
-/* config.h.in.  Generated from configure.ac by autoheader.  */
 
 /* Enable RDRAND Hardware RNG Hash Seed */
 #cmakedefine ENABLE_RDRAND "@ENABLE_RDRAND@"
@@ -54,6 +53,9 @@
 /* Define to 1 if you have the <sys/param.h> header file. */
 #cmakedefine HAVE_SYS_PARAM_H @HAVE_SYS_PARAM_H@
 
+/* Define to 1 if you have the <sys/resource.h> header file. */
+#cmakedefine HAVE_SYS_RESOURCE_H
+
 /* Define to 1 if you have the <sys/stat.h> header file. */
 #cmakedefine HAVE_SYS_STAT_H
 
@@ -135,6 +137,9 @@
 /* Define to 1 if you have the `vsyslog' function. */
 #cmakedefine HAVE_VSYSLOG @HAVE_VSYSLOG@
 
+/* Define if you have the `getrusage' function. */
+#cmakedefine HAVE_GETRUSAGE
+
 #cmakedefine HAVE_STRTOLL
 #if !defined(HAVE_STRTOLL)
 #define strtoll @json_c_strtoll@
@@ -189,6 +194,9 @@
 /* The number of bytes in type size_t */
 #cmakedefine SIZEOF_SIZE_T @SIZEOF_SIZE_T@
 
+/* The number of bytes in type ssize_t */
+#cmakedefine SIZEOF_SSIZE_T @SIZEOF_SSIZE_T@
+
 /* Specifier for __thread */
 #cmakedefine SPEC___THREAD @SPEC___THREAD@
 

config.h.win32

@@ -177,7 +177,7 @@
 #define PACKAGE_NAME "JSON C Library"
 
 /* Define to the full name and version of this package. */
-#define PACKAGE_STRING "JSON C Library 0.13.99"
+#define PACKAGE_STRING "JSON C Library 0.14.99"
 
 /* Define to the one symbol short name of this package. */
 #define PACKAGE_TARNAME "json-c"
@@ -186,13 +186,13 @@
 #define PACKAGE_URL "https://github.com/json-c/json-c"
 
 /* Define to the version of this package. */
-#define PACKAGE_VERSION "0.13.99"
+#define PACKAGE_VERSION "0.14.99"
 
 /* Define to 1 if you have the ANSI C header files. */
 #define STDC_HEADERS 1
 
 /* Version number of package */
-#define VERSION "0.13.99"
+#define VERSION "0.14.99"
 
 /* Define to empty if `const' does not conform to ANSI C. */
 /* #undef const */

doc/CMakeLists.txt

@@ -0,0 +1,16 @@
+# generate doxygen documentation for json-c API
+
+find_package(Doxygen)
+
+if (DOXYGEN_FOUND)
+
+	configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in
+	  ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
+	message(STATUS "Wrote ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile")
+
+	add_custom_target(doc
+	COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
+
+else (DOXYGEN_FOUND)
+	message("Warning: doxygen not found, the 'doc' target will not be included")
+endif(DOXYGEN_FOUND)

doc/Doxyfile.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = json-c
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 0.13.99
+PROJECT_NUMBER         = @PROJECT_VERSION@
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -58,7 +58,7 @@ PROJECT_LOGO           =
 # entered, it will be relative to the location where doxygen was started. If
 # left blank the current directory will be used.
 
-OUTPUT_DIRECTORY       = doc
+OUTPUT_DIRECTORY       = @CMAKE_CURRENT_BINARY_DIR@
 
 # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub-
 # directories (in 2 levels) under the output directory of each output format and
@@ -753,7 +753,7 @@ WARN_LOGFILE           =
 # spaces.
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  =
+INPUT                  = @CMAKE_SOURCE_DIR@ @CMAKE_BINARY_DIR@
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

json_c_version.h

@@ -12,12 +12,16 @@
 #ifndef _json_c_version_h_
 #define _json_c_version_h_
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define JSON_C_MAJOR_VERSION 0
-#define JSON_C_MINOR_VERSION 13
+#define JSON_C_MINOR_VERSION 14
 #define JSON_C_MICRO_VERSION 99
 #define JSON_C_VERSION_NUM \
 	((JSON_C_MAJOR_VERSION << 16) | (JSON_C_MINOR_VERSION << 8) | JSON_C_MICRO_VERSION)
-#define JSON_C_VERSION "0.13.99"
+#define JSON_C_VERSION "0.14.99"
 
 #ifndef JSON_EXPORT
 #if defined(_MSC_VER)
@@ -44,4 +48,8 @@ JSON_EXPORT const char *json_c_version(void); /* Returns JSON_C_VERSION */
  */
 JSON_EXPORT int json_c_version_num(void); /* Returns JSON_C_VERSION_NUM */
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif

json_object_iterator.c

@@ -9,6 +9,7 @@
 *
 *******************************************************************************
 */
+#include "config.h"
 
 #include <stddef.h>
 

json_object_private.h

@@ -20,14 +20,18 @@
 extern "C" {
 #endif
 
-/**< how many bytes are directly stored in json_object for strings? */
-#define LEN_DIRECT_STRING_DATA 32
-
 struct json_object;
 #include "json_inttypes.h"
 #include "json_types.h"
 
-typedef void(json_object_private_delete_fn)(struct json_object *o);
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif /* HAVE_UNISTD_H */
+
+#ifdef _MSC_VER
+#include <BaseTsd.h>
+typedef SSIZE_T ssize_t;
+#endif
 
 /* json object int type, support extension*/
 typedef enum json_object_int_type
@@ -40,39 +44,56 @@ struct json_object
 {
 	enum json_type o_type;
 	uint32_t _ref_count;
-	json_object_private_delete_fn *_delete;
 	json_object_to_json_string_fn *_to_json_string;
 	struct printbuf *_pb;
-	union data
-	{
-		json_bool c_boolean;
-		double c_double;
-		struct
-		{
-			union
-			{
-				int64_t c_int64;
-				uint64_t c_uint64;
-			} cint;
-			enum json_object_int_type cint_type;
-		} c_int;
-		struct lh_table *c_object;
-		struct array_list *c_array;
-		struct
-		{
-			union
-			{
-				/* optimize: if we have small strings, we can store them
-				 * directly. This saves considerable CPU cycles AND memory.
-				 */
-				char *ptr;
-				char data[LEN_DIRECT_STRING_DATA];
-			} str;
-			int len;
-		} c_string;
-	} o;
 	json_object_delete_fn *_user_delete;
 	void *_userdata;
+	// Actually longer, always malloc'd as some more-specific type.
+	// The rest of a struct json_object_${o_type} follows
+};
+
+struct json_object_object
+{
+	struct json_object base;
+	struct lh_table *c_object;
+};
+struct json_object_array
+{
+	struct json_object base;
+	struct array_list *c_array;
+};
+
+struct json_object_boolean
+{
+	struct json_object base;
+	json_bool c_boolean;
+};
+struct json_object_double
+{
+	struct json_object base;
+	double c_double;
+};
+struct json_object_int
+{
+	struct json_object base;
+	enum json_object_int_type cint_type;
+	union
+	{
+		int64_t c_int64;
+		uint64_t c_uint64;
+	} cint;
+};
+struct json_object_string
+{
+	struct json_object base;
+	ssize_t len; // Signed b/c negative lengths indicate data is a pointer
+	// Consider adding an "alloc" field here, if json_object_set_string calls
+	// to expand the length of a string are common operations to perform.
+	union
+	{
+		char idata[1]; // Immediate data.  Actually longer
+		char *pdata;   // Only when len < 0
+	} c_string;
 };
 
 void _json_c_set_last_err(const char *err_fmt, ...);

json_tokener.c

@@ -25,7 +25,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "arraylist.h"
 #include "debug.h"
 #include "json_inttypes.h"
 #include "json_object.h"
@@ -41,6 +40,9 @@
 #ifdef HAVE_XLOCALE_H
 #include <xlocale.h>
 #endif
+#ifdef HAVE_STRINGS_H
+#include <strings.h>
+#endif /* HAVE_STRINGS_H */
 
 #define jt_hexdigit(x) (((x) <= '9') ? (x) - '0' : ((x)&7) + 9)
 
@@ -1130,7 +1132,7 @@ out:
 		tok->err = json_tokener_error_parse_utf8_string;
 	}
 	if (c && (state == json_tokener_state_finish) && (tok->depth == 0) &&
-	    (tok->flags & JSON_TOKENER_STRICT))
+	    (tok->flags & (JSON_TOKENER_STRICT|JSON_TOKENER_ALLOW_TRAILING_CHARS)) == JSON_TOKENER_STRICT)
 	{
 		/* unexpected char after JSON data */
 		tok->err = json_tokener_error_parse_unexpected;

json_tokener.h

@@ -144,12 +144,25 @@ typedef struct json_tokener json_tokener;
  * restrictive from one release to the next, causing your
  * code to fail on previously working input.
  *
+ * Note that setting this will also effectively disable parsing
+ * of multiple json objects in a single character stream
+ * (e.g. {"foo":123}{"bar":234}); if you want to allow that
+ * also set JSON_TOKENER_ALLOW_TRAILING_CHARS
+ *
  * This flag is not set by default.
  *
  * @see json_tokener_set_flags()
  */
 #define JSON_TOKENER_STRICT 0x01
 
+/**
+ * Use with JSON_TOKENER_STRICT to allow trailing characters after the
+ * first parsed object.
+ *
+ * @see json_tokener_set_flags()
+ */
+#define JSON_TOKENER_ALLOW_TRAILING_CHARS 0x02
+
 /**
  * Cause json_tokener_parse_ex() to validate that input is UTF8.
  * If this flag is specified and validation fails, then

json_util.c

@@ -245,19 +245,17 @@ int json_parse_uint64(const char *buf, uint64_t *retval)
 {
 	char *end = NULL;
 	uint64_t val;
-	errno = 1;
 
+	errno = 0;
 	while (*buf == ' ')
-	{
 		buf++;
-	}
 	if (*buf == '-')
-		errno = 0;
+		return 1;  /* error: uint cannot be negative */
 
 	val = strtoull(buf, &end, 10);
 	if (end != buf)
 		*retval = val;
-	return ((errno == 0) || (end == buf)) ? 1 : 0;
+	return ((val == 0 && errno != 0) || (end == buf)) ? 1 : 0;
 }
 
 #ifndef HAVE_REALLOC

json_util.h

@@ -100,6 +100,7 @@ JSON_EXPORT int json_object_to_fd(int fd, struct json_object *obj, int flags);
  */
 JSON_EXPORT const char *json_util_get_last_err(void);
 
+/* these parsing helpers return zero on success */
 JSON_EXPORT int json_parse_int64(const char *buf, int64_t *retval);
 JSON_EXPORT int json_parse_uint64(const char *buf, uint64_t *retval);
 JSON_EXPORT int json_parse_double(const char *buf, double *retval);

json_visit.h

@@ -8,6 +8,10 @@
  */
 #include "json_object.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef int(json_c_visit_userfunc)(json_object *jso, int flags, json_object *parent_jso,
                                    const char *jso_key, size_t *jso_index, void *userarg);
 
@@ -90,4 +94,8 @@ JSON_EXPORT int json_c_visit(json_object *jso, int future_flags, json_c_visit_us
  */
 #define JSON_C_VISIT_RETURN_ERROR -1
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif /* _json_c_json_visit_h_ */

linkhash.c

@@ -12,6 +12,7 @@
 
 #include "config.h"
 
+#include <assert.h>
 #include <limits.h>
 #include <stdarg.h>
 #include <stddef.h>
@@ -499,6 +500,8 @@ struct lh_table *lh_table_new(int size, lh_entry_free_fn *free_fn, lh_hash_fn *h
 	int i;
 	struct lh_table *t;
 
+	/* Allocate space for elements to avoid divisions by zero. */
+	assert(size > 0);
 	t = (struct lh_table *)calloc(1, sizeof(struct lh_table));
 	if (!t)
 		return NULL;
@@ -578,8 +581,12 @@ int lh_table_insert_w_hash(struct lh_table *t, const void *k, const void *v, con
 	unsigned long n;
 
 	if (t->count >= t->size * LH_LOAD_FACTOR)
-		if (lh_table_resize(t, t->size * 2) != 0)
+	{
+		/* Avoid signed integer overflow with large tables. */
+		int new_size = (t->size > INT_MAX / 2) ? INT_MAX : (t->size * 2);
+		if (t->size == INT_MAX || lh_table_resize(t, new_size) != 0)
 			return -1;
+	}
 
 	n = h % t->size;
 

math_compat.h

@@ -12,6 +12,9 @@
 #ifdef HAVE_DECL__ISNAN
 #include <float.h>
 #define isnan(x) _isnan(x)
+#else
+/* On platforms like AIX and "IBM i" we need to provide our own isnan */
+#define isnan(x) ((x) != (x))
 #endif
 #endif
 
@@ -19,6 +22,10 @@
 #ifdef HAVE_DECL__FINITE
 #include <float.h>
 #define isinf(x) (!_finite(x))
+#else
+#include <float.h>
+/* On platforms like AIX and "IBM i" we need to provide our own isinf */
+#define isinf(x) ((x) < -DBL_MAX || (x) > DBL_MAX)
 #endif
 #endif
 

printbuf.c

@@ -15,6 +15,7 @@
 
 #include "config.h"
 
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -65,10 +66,16 @@ static int printbuf_extend(struct printbuf *p, int min_size)
 
 	if (p->size >= min_size)
 		return 0;
-
-	new_size = p->size * 2;
-	if (new_size < min_size + 8)
+	/* Prevent signed integer overflows with large buffers. */
+	if (min_size > INT_MAX - 8)
+		return -1;
+	if (p->size > INT_MAX / 2)
 		new_size = min_size + 8;
+	else {
+		new_size = p->size * 2;
+		if (new_size < min_size + 8)
+			new_size = min_size + 8;
+	}
 #ifdef PRINTBUF_DEBUG
 	MC_DEBUG("printbuf_memappend: realloc "
 	         "bpos=%d min_size=%d old_size=%d new_size=%d\n",
@@ -83,6 +90,9 @@ static int printbuf_extend(struct printbuf *p, int min_size)
 
 int printbuf_memappend(struct printbuf *p, const char *buf, int size)
 {
+	/* Prevent signed integer overflows with large buffers. */
+	if (size > INT_MAX - p->bpos - 1)
+		return -1;
 	if (p->size <= p->bpos + size + 1)
 	{
 		if (printbuf_extend(p, p->bpos + size + 1) < 0)
@@ -100,6 +110,9 @@ int printbuf_memset(struct printbuf *pb, int offset, int charvalue, int len)
 
 	if (offset == -1)
 		offset = pb->bpos;
+	/* Prevent signed integer overflows with large buffers. */
+	if (len > INT_MAX - offset)
+		return -1;
 	size_needed = offset + len;
 	if (pb->size < size_needed)
 	{

random_seed.c

@@ -26,19 +26,8 @@
 static void do_cpuid(int regs[], int h)
 {
 	/* clang-format off */
-    __asm__ __volatile__(
-#if defined __x86_64__
-                         "pushq %%rbx;\n"
-#else
-                         "pushl %%ebx;\n"
-#endif
-                         "cpuid;\n"
-#if defined __x86_64__
-                         "popq %%rbx;\n"
-#else
-                         "popl %%ebx;\n"
-#endif
-                         : "=a"(regs[0]), [ebx] "=r"(regs[1]), "=c"(regs[2]), "=d"(regs[3])
+    __asm__ __volatile__("cpuid"
+                         : "=a"(regs[0]), "=b"(regs[1]), "=c"(regs[2]), "=d"(regs[3])
                          : "a"(h));
 	/* clang-format on */
 }
@@ -54,12 +43,51 @@ static void do_cpuid(int regs[], int h)
 
 #if HAS_X86_CPUID
 
+static int get_rdrand_seed(void);
+
+/* Valid values are -1 (haven't tested), 0 (no), and 1 (yes). */
+static int _has_rdrand = -1;
+
 static int has_rdrand(void)
 {
-	// CPUID.01H:ECX.RDRAND[bit 30] == 1
+	if (_has_rdrand != -1)
+	{
+		return _has_rdrand;
+	}
+
+	/* CPUID.01H:ECX.RDRAND[bit 30] == 1 */
 	int regs[4];
 	do_cpuid(regs, 1);
-	return (regs[2] & (1 << 30)) != 0;
+	if (!(regs[2] & (1 << 30)))
+	{
+		_has_rdrand = 0;
+		return 0;
+	}
+
+	/*
+	 * Some CPUs advertise RDRAND in CPUID, but return 0xFFFFFFFF
+	 * unconditionally. To avoid locking up later, test RDRAND here. If over
+	 * 3 trials RDRAND has returned the same value, declare it broken.
+	 * Example CPUs are AMD Ryzen 3000 series
+	 * and much older AMD APUs, such as the E1-1500
+	 * https://github.com/systemd/systemd/issues/11810
+	 * https://linuxreviews.org/RDRAND_stops_returning_random_values_on_older_AMD_CPUs_after_suspend
+	 */
+	_has_rdrand = 0;
+	int prev = get_rdrand_seed();
+	for (int i = 0; i < 3; i++)
+	{
+		int temp = get_rdrand_seed();
+		if (temp != prev)
+		{
+			_has_rdrand = 1;
+			break;
+		}
+
+		prev = temp;
+	}
+
+	return _has_rdrand;
 }
 
 #endif
@@ -74,7 +102,7 @@ static int get_rdrand_seed(void)
 {
 	DEBUG_SEED("get_rdrand_seed");
 	int _eax;
-	// rdrand eax
+	/* rdrand eax */
 	/* clang-format off */
 	__asm__ __volatile__("1: .byte 0x0F\n"
 	                     "   .byte 0xC7\n"
@@ -114,7 +142,7 @@ static int get_rdrand_seed(void)
 	DEBUG_SEED("get_rdrand_seed");
 	int _eax;
 retry:
-	// rdrand eax
+	/* rdrand eax */
 	__asm _emit 0x0F __asm _emit 0xC7 __asm _emit 0xF0
 	__asm jnc retry
 	__asm mov _eax, eax
@@ -188,6 +216,10 @@ static int get_dev_random_seed(void)
 
 /* clang-format off */
 #include <windows.h>
+
+/* Caution: these blank lines must remain so clang-format doesn't reorder
+   includes to put windows.h after wincrypt.h */
+
 #include <wincrypt.h>
 /* clang-format on */
 #ifndef __GNUC__

snprintf_compat.h

@@ -13,7 +13,7 @@
 
 #include <stdarg.h>
 
-#if !defined(HAVE_SNPRINTF) && defined(_MSC_VER)
+#if !defined(HAVE_SNPRINTF) && (defined(_MSC_VER) || defined(__MINGW32__))
 static int json_c_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 {
 	int ret;

strerror_override.c

@@ -20,7 +20,9 @@ static struct
 	ENTRY(EIO),
 	ENTRY(ENXIO),
 	ENTRY(E2BIG),
+#ifdef ENOEXEC
 	ENTRY(ENOEXEC),
+#endif
 	ENTRY(EBADF),
 	ENTRY(ECHILD),
 	ENTRY(EDEADLK),

tests/test4.c

@@ -3,12 +3,15 @@
  */
 
 #include "config.h"
+#include <assert.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
 #include "json_inttypes.h"
 #include "json_object.h"
 #include "json_tokener.h"
+#include "snprintf_compat.h"
 
 void print_hex(const char *s)
 {
@@ -24,6 +27,29 @@ void print_hex(const char *s)
 	putchar('\n');
 }
 
+static void test_lot_of_adds(void);
+static void test_lot_of_adds()
+{
+	int ii;
+	char key[50];
+	json_object *jobj = json_object_new_object();
+	assert(jobj != NULL);
+	for (ii = 0; ii < 500; ii++)
+	{
+		snprintf(key, sizeof(key), "k%d", ii);
+		json_object *iobj = json_object_new_int(ii);
+		assert(iobj != NULL);
+		if (json_object_object_add(jobj, key, iobj))
+		{
+			fprintf(stderr, "FAILED to add object #%d\n", ii);
+			abort();
+		}
+	}
+	printf("%s\n", json_object_to_json_string(jobj));
+	assert(json_object_object_length(jobj) == 500);
+	json_object_put(jobj);
+}
+
 int main(void)
 {
 	const char *input = "\"\\ud840\\udd26,\\ud840\\udd27,\\ud800\\udd26,\\ud800\\udd27\"";
@@ -52,5 +78,8 @@ int main(void)
 		retval = 1;
 	}
 	json_object_put(parse_result);
+
+	test_lot_of_adds();
+
 	return retval;
 }

tests/test_deep_copy.c

@@ -126,7 +126,7 @@ int main(int argc, char **argv)
 	src3 = json_tokener_parse(json_str3);
 
 	assert(src1 != NULL);
-	assert(src1 != NULL);
+	assert(src2 != NULL);
 	assert(src3 != NULL);
 
 	printf("PASSED - loaded input data\n");

tests/test_double_serializer.c

@@ -59,6 +59,7 @@ int main()
 #else
 	// Just fake it up, so the output matches.
 	printf("obj.to_string(with thread format)=%s\n", "T0.52X");
+	printf("obj.to_string(long thread format)=%s\n", "Ttttttttttttt0.52xxxxxxxxxxxxxxxxxxX");
 	printf("obj.to_string(back to global format)=%s\n", "x0.524y");
 #endif
 	if (json_c_set_serialization_double_format(NULL, JSON_C_OPTION_GLOBAL) < 0)

tests/test_json_pointer.c

@@ -60,7 +60,7 @@ static const char *rec_input_json_str =
 /* clang-format on */
 
 /* Example from RFC */
-static void test_example_get()
+static void test_example_get(void)
 {
 	int i;
 	struct json_object *jo1, *jo2, *jo3;
@@ -126,7 +126,7 @@ static void test_example_get()
 }
 
 /* I'm not too happy with the RFC example to test the recusion of the json_pointer_get() function */
-static void test_recursion_get()
+static void test_recursion_get(void)
 {
 	struct json_object *jo2, *jo1 = json_tokener_parse(rec_input_json_str);
 
@@ -161,7 +161,7 @@ static void test_recursion_get()
 	json_object_put(jo1);
 }
 
-static void test_wrong_inputs_get()
+static void test_wrong_inputs_get(void)
 {
 	struct json_object *jo2, *jo1 = json_tokener_parse(input_json_str);
 
@@ -231,7 +231,7 @@ static void test_wrong_inputs_get()
 	json_object_put(jo1);
 }
 
-static void test_example_set()
+static void test_example_set(void)
 {
 	struct json_object *jo2, *jo1 = json_tokener_parse(input_json_str);
 
@@ -272,7 +272,7 @@ static void test_example_set()
 	json_object_put(jo1);
 }
 
-static void test_wrong_inputs_set()
+static void test_wrong_inputs_set(void)
 {
 	struct json_object *jo2, *jo1 = json_tokener_parse(input_json_str);
 

tests/test_parse.c

@@ -149,8 +149,8 @@ static void test_utf8_parse()
 	// json_tokener_parse doesn't support checking for byte order marks.
 	// It's the responsibility of the caller to detect and skip a BOM.
 	// Both of these checks return null.
-	char utf8_bom[] = {0xEF, 0xBB, 0xBF, 0x00};
-	char utf8_bom_and_chars[] = {0xEF, 0xBB, 0xBF, '{', '}', 0x00};
+	char* utf8_bom = "\xEF\xBB\xBF";
+	char* utf8_bom_and_chars = "\xEF\xBB\xBF{}";
 	single_basic_parse(utf8_bom, 0);
 	single_basic_parse(utf8_bom_and_chars, 0);
 }
@@ -200,7 +200,8 @@ struct incremental_step
 	int length;
 	int char_offset;
 	enum json_tokener_error expected_error;
-	int reset_tokener;
+	int reset_tokener; /* Set to 1 to call json_tokener_reset() after parsing */
+	int tok_flags; /* JSON_TOKENER_* flags to pass to json_tokener_set_flags() */
 } incremental_steps[] = {
 
     /* Check that full json messages can be parsed, both w/ and w/o a reset */
@@ -237,13 +238,20 @@ struct incremental_step
     {"{\"x\": 123 }\"X\"", -1, 11, json_tokener_success, 0},
     {"\"Y\"", -1, -1, json_tokener_success, 1},
 
+    /* Trailing characters should cause a failure in strict mode */
+    {"{\"foo\":9}{\"bar\":8}", -1, 9, json_tokener_error_parse_unexpected, 1, JSON_TOKENER_STRICT },
+
+    /* ... unless explicitly allowed. */
+    {"{\"foo\":9}{\"bar\":8}", -1, 9, json_tokener_success, 0, JSON_TOKENER_STRICT|JSON_TOKENER_ALLOW_TRAILING_CHARS },
+    {"{\"b\":8}ignored garbage", -1, 7, json_tokener_success, 1, JSON_TOKENER_STRICT|JSON_TOKENER_ALLOW_TRAILING_CHARS },
+
     /* To stop parsing a number we need to reach a non-digit, e.g. a \0 */
     {"1", 1, 1, json_tokener_continue, 0},
     /* This should parse as the number 12, since it continues the "1" */
     {"2", 2, 1, json_tokener_success, 0},
     {"12{", 3, 2, json_tokener_success, 1},
     /* Parse number in strict model */
-    {"[02]", -1, 3, json_tokener_error_parse_number, 3},
+    {"[02]", -1, 3, json_tokener_error_parse_number, 1, JSON_TOKENER_STRICT },
 
     /* Similar tests for other kinds of objects: */
     /* These could all return success immediately, since regardless of
@@ -259,8 +267,8 @@ struct incremental_step
     {"Infinity", 9, 8, json_tokener_success, 1},
     {"infinity", 9, 8, json_tokener_success, 1},
     {"-infinity", 10, 9, json_tokener_success, 1},
-    {"infinity", 9, 0, json_tokener_error_parse_unexpected, 3},
-    {"-infinity", 10, 1, json_tokener_error_parse_unexpected, 3},
+    {"infinity", 9, 0, json_tokener_error_parse_unexpected, 1, JSON_TOKENER_STRICT },
+    {"-infinity", 10, 1, json_tokener_error_parse_unexpected, 1, JSON_TOKENER_STRICT },
 
     {"inf", 3, 3, json_tokener_continue, 0},
     {"inity", 6, 5, json_tokener_success, 1},
@@ -342,7 +350,7 @@ struct incremental_step
     {"\"\\a\"", -1, 2, json_tokener_error_parse_string, 1},
 
     /* Check '\'' in strict model */
-    {"\'foo\'", -1, 0, json_tokener_error_parse_unexpected, 3},
+    {"\'foo\'", -1, 0, json_tokener_error_parse_unexpected, 1, JSON_TOKENER_STRICT },
 
     /* Parse array/object */
     {"[1,2,3]", -1, -1, json_tokener_success, 0},
@@ -364,42 +372,42 @@ struct incremental_step
     {"[1,2,3,]", -1, -1, json_tokener_success, 0},
     {"[1,2,,3,]", -1, 5, json_tokener_error_parse_unexpected, 0},
 
-    {"[1,2,3,]", -1, 7, json_tokener_error_parse_unexpected, 3},
-    {"{\"a\":1,}", -1, 7, json_tokener_error_parse_unexpected, 3},
+    {"[1,2,3,]", -1, 7, json_tokener_error_parse_unexpected, 1, JSON_TOKENER_STRICT },
+    {"{\"a\":1,}", -1, 7, json_tokener_error_parse_unexpected, 1, JSON_TOKENER_STRICT },
 
     // utf-8 test
     // acsll encoding
-    {"\x22\x31\x32\x33\x61\x73\x63\x24\x25\x26\x22", -1, -1, json_tokener_success, 5},
+    {"\x22\x31\x32\x33\x61\x73\x63\x24\x25\x26\x22", -1, -1, json_tokener_success, 1, JSON_TOKENER_VALIDATE_UTF8 },
     {"\x22\x31\x32\x33\x61\x73\x63\x24\x25\x26\x22", -1, -1, json_tokener_success, 1},
     // utf-8 encoding
-    {"\x22\xe4\xb8\x96\xe7\x95\x8c\x22", -1, -1, json_tokener_success, 5},
-    {"\x22\xe4\xb8", -1, 3, json_tokener_error_parse_utf8_string, 4},
-    {"\x96\xe7\x95\x8c\x22", -1, 0, json_tokener_error_parse_utf8_string, 5},
+    {"\x22\xe4\xb8\x96\xe7\x95\x8c\x22", -1, -1, json_tokener_success, 1, JSON_TOKENER_VALIDATE_UTF8 },
+    {"\x22\xe4\xb8", -1, 3, json_tokener_error_parse_utf8_string, 0, JSON_TOKENER_VALIDATE_UTF8 },
+    {"\x96\xe7\x95\x8c\x22", -1, 0, json_tokener_error_parse_utf8_string, 1, JSON_TOKENER_VALIDATE_UTF8 },
     {"\x22\xe4\xb8\x96\xe7\x95\x8c\x22", -1, -1, json_tokener_success, 1},
-    {"\x22\xcf\x80\xcf\x86\x22", -1, -1, json_tokener_success, 5},
-    {"\x22\xf0\xa5\x91\x95\x22", -1, -1, json_tokener_success, 5},
+    {"\x22\xcf\x80\xcf\x86\x22", -1, -1, json_tokener_success, 1, JSON_TOKENER_VALIDATE_UTF8 },
+    {"\x22\xf0\xa5\x91\x95\x22", -1, -1, json_tokener_success, 1, JSON_TOKENER_VALIDATE_UTF8 },
     // wrong utf-8 encoding
-    {"\x22\xe6\x9d\x4e\x22", -1, 3, json_tokener_error_parse_utf8_string, 5},
+    {"\x22\xe6\x9d\x4e\x22", -1, 3, json_tokener_error_parse_utf8_string, 1, JSON_TOKENER_VALIDATE_UTF8 },
     {"\x22\xe6\x9d\x4e\x22", -1, 5, json_tokener_success, 1},
     // GBK encoding
-    {"\x22\xc0\xee\xc5\xf4\x22", -1, 2, json_tokener_error_parse_utf8_string, 5},
+    {"\x22\xc0\xee\xc5\xf4\x22", -1, 2, json_tokener_error_parse_utf8_string, 1, JSON_TOKENER_VALIDATE_UTF8 },
     {"\x22\xc0\xee\xc5\xf4\x22", -1, 6, json_tokener_success, 1},
     // char after space
-    {"\x20\x20\x22\xe4\xb8\x96\x22", -1, -1, json_tokener_success, 5},
-    {"\x20\x20\x81\x22\xe4\xb8\x96\x22", -1, 2, json_tokener_error_parse_utf8_string, 5},
-    {"\x5b\x20\x81\x31\x5d", -1, 2, json_tokener_error_parse_utf8_string, 5},
+    {"\x20\x20\x22\xe4\xb8\x96\x22", -1, -1, json_tokener_success, 1, JSON_TOKENER_VALIDATE_UTF8 },
+    {"\x20\x20\x81\x22\xe4\xb8\x96\x22", -1, 2, json_tokener_error_parse_utf8_string, 1, JSON_TOKENER_VALIDATE_UTF8 },
+    {"\x5b\x20\x81\x31\x5d", -1, 2, json_tokener_error_parse_utf8_string, 1, JSON_TOKENER_VALIDATE_UTF8 },
     // char in state inf
     {"\x49\x6e\x66\x69\x6e\x69\x74\x79", 9, 8, json_tokener_success, 1},
-    {"\x49\x6e\x66\x81\x6e\x69\x74\x79", -1, 3, json_tokener_error_parse_utf8_string, 5},
+    {"\x49\x6e\x66\x81\x6e\x69\x74\x79", -1, 3, json_tokener_error_parse_utf8_string, 1, JSON_TOKENER_VALIDATE_UTF8 },
     // char in escape unicode
-    {"\x22\x5c\x75\x64\x38\x35\x35\x5c\x75\x64\x63\x35\x35\x22", 15, 14, json_tokener_success, 5},
+    {"\x22\x5c\x75\x64\x38\x35\x35\x5c\x75\x64\x63\x35\x35\x22", 15, 14, json_tokener_success, 1, JSON_TOKENER_VALIDATE_UTF8 },
     {"\x22\x5c\x75\x64\x38\x35\x35\xc0\x75\x64\x63\x35\x35\x22", -1, 8,
-     json_tokener_error_parse_utf8_string, 5},
-    {"\x22\x5c\x75\x64\x30\x30\x33\x31\xc0\x22", -1, 9, json_tokener_error_parse_utf8_string, 5},
+     json_tokener_error_parse_utf8_string, 1, JSON_TOKENER_VALIDATE_UTF8 },
+    {"\x22\x5c\x75\x64\x30\x30\x33\x31\xc0\x22", -1, 9, json_tokener_error_parse_utf8_string, 1, JSON_TOKENER_VALIDATE_UTF8 },
     // char in number
-    {"\x31\x31\x81\x31\x31", -1, 2, json_tokener_error_parse_utf8_string, 5},
+    {"\x31\x31\x81\x31\x31", -1, 2, json_tokener_error_parse_utf8_string, 1, JSON_TOKENER_VALIDATE_UTF8 },
     // char in object
-    {"\x7b\x22\x31\x81\x22\x3a\x31\x7d", -1, 3, json_tokener_error_parse_utf8_string, 5},
+    {"\x7b\x22\x31\x81\x22\x3a\x31\x7d", -1, 3, json_tokener_error_parse_utf8_string, 1, JSON_TOKENER_VALIDATE_UTF8 },
 
     {NULL, -1, -1, json_tokener_success, 0},
 };
@@ -435,23 +443,10 @@ static void test_incremental_parse()
 		int length = step->length;
 		size_t expected_char_offset;
 
-		if (step->reset_tokener & 2)
-		{
-			if (step->reset_tokener & 4)
-				json_tokener_set_flags(tok, 3);
-			else
-				json_tokener_set_flags(tok, JSON_TOKENER_STRICT);
-		}
-		else
-		{
-			if (step->reset_tokener & 4)
-				json_tokener_set_flags(tok, JSON_TOKENER_VALIDATE_UTF8);
-			else
-				json_tokener_set_flags(tok, 0);
-		}
+		json_tokener_set_flags(tok, step->tok_flags);
 
 		if (length == -1)
-			length = strlen(step->string_to_parse);
+			length = (int)strlen(step->string_to_parse);
 		if (step->char_offset == -1)
 			expected_char_offset = length;
 		else

tests/test_parse.expected

@@ -108,6 +108,9 @@ json_tokener_parse_ex(tok, ":13}}XXXX  ,  10) ... OK: got object of type [object
 json_tokener_parse_ex(tok, XXXX        ,   4) ... OK: got correct error: unexpected character
 json_tokener_parse_ex(tok, {"x": 123 }"X",  14) ... OK: got object of type [object]: { "x": 123 }
 json_tokener_parse_ex(tok, "Y"         ,   3) ... OK: got object of type [string]: "Y"
+json_tokener_parse_ex(tok, {"foo":9}{"bar":8},  18) ... OK: got correct error: unexpected character
+json_tokener_parse_ex(tok, {"foo":9}{"bar":8},  18) ... OK: got object of type [object]: { "foo": 9 }
+json_tokener_parse_ex(tok, {"b":8}ignored garbage,  22) ... OK: got object of type [object]: { "b": 8 }
 json_tokener_parse_ex(tok, 1           ,   1) ... OK: got correct error: continue
 json_tokener_parse_ex(tok, 2           ,   2) ... OK: got object of type [int]: 12
 json_tokener_parse_ex(tok, 12{         ,   3) ... OK: got object of type [int]: 12
@@ -213,5 +216,5 @@ json_tokener_parse_ex(tok, "\ud855
 json_tokener_parse_ex(tok, "\ud0031<33>"  ,  10) ... OK: got correct error: invalid utf-8 string
 json_tokener_parse_ex(tok, 11<31>11       ,   5) ... OK: got correct error: invalid utf-8 string
 json_tokener_parse_ex(tok, {"1<>":1}    ,   8) ... OK: got correct error: invalid utf-8 string
-End Incremental Tests OK=127 ERROR=0
+End Incremental Tests OK=130 ERROR=0
 ==================================

tests/test_parse_int64.expected

@@ -34,13 +34,13 @@ buf=123 parseit=0, value=123
 ==========json_parse_uint64() test===========
 buf=x parseit=1, value=666 
 buf=0 parseit=0, value=0 
-buf=-0 parseit=1, value=0 
+buf=-0 parseit=1, value=666 
 buf=00000000 parseit=0, value=0 
-buf=-00000000 parseit=1, value=0 
+buf=-00000000 parseit=1, value=666 
 buf=1 parseit=0, value=1 
 buf=2147483647 parseit=0, value=2147483647 
-buf=-1 parseit=1, value=18446744073709551615 
-buf=-9223372036854775808 parseit=1, value=9223372036854775808 
+buf=-1 parseit=1, value=666 
+buf=-9223372036854775808 parseit=1, value=666 
 buf=   1 parseit=0, value=1 
 buf=00001234 parseit=0, value=1234 
 buf=0001234x parseit=0, value=1234 

tests/test_set_serializer.c

@@ -26,7 +26,7 @@ static int custom_serializer(struct json_object *o, struct printbuf *pb, int lev
 
 int main(int argc, char **argv)
 {
-	json_object *my_object;
+	json_object *my_object, *my_sub_object;
 
 	MC_SET_DEBUG(1);
 
@@ -67,5 +67,17 @@ int main(int argc, char **argv)
 	json_object_put(my_object);
 	assert(freeit_was_called);
 
+	// ============================================
+
+	my_object = json_object_new_object();
+	my_sub_object = json_object_new_double(1.0);
+	json_object_object_add(my_object, "double", my_sub_object);
+	printf("Check that the custom serializer does not include nul byte:\n");
+	json_object_set_serializer(my_sub_object, json_object_double_to_json_string, "%125.0f,", NULL);
+	printf("my_object.to_string(custom serializer)=%s\n",
+	       json_object_to_json_string_ext(my_object, JSON_C_TO_STRING_NOZERO));
+
+	json_object_put(my_object);
+
 	return 0;
 }

tests/test_set_serializer.expected

@@ -8,3 +8,5 @@ Check that the custom serializer isn't free'd until the last json_object_put:
 my_object.to_string(custom serializer)=Custom Output
 Next line of output should be from the custom freeit function:
 freeit, value=123
+Check that the custom serializer does not include nul byte:
+my_object.to_string(custom serializer)={"double":                                                                                                                            1.}

tests/test_set_value.c

@@ -56,14 +56,18 @@ int main(int argc, char **argv)
 #define MID "A MID STRING"
 //             12345678901234567890123456789012....
 #define HUGE "A string longer than 32 chars as to check non local buf codepath"
-	tmp = json_object_new_string(SHORT);
-	assert(strcmp(json_object_get_string(tmp), SHORT) == 0);
-	json_object_set_string(tmp, MID);
+	tmp = json_object_new_string(MID);
 	assert(strcmp(json_object_get_string(tmp), MID) == 0);
-	json_object_set_string(tmp, HUGE);
-	assert(strcmp(json_object_get_string(tmp), HUGE) == 0);
+	assert(strcmp(json_object_to_json_string(tmp), "\"" MID "\"") == 0);
 	json_object_set_string(tmp, SHORT);
 	assert(strcmp(json_object_get_string(tmp), SHORT) == 0);
+	assert(strcmp(json_object_to_json_string(tmp), "\"" SHORT "\"") == 0);
+	json_object_set_string(tmp, HUGE);
+	assert(strcmp(json_object_get_string(tmp), HUGE) == 0);
+	assert(strcmp(json_object_to_json_string(tmp), "\"" HUGE "\"") == 0);
+	json_object_set_string(tmp, SHORT);
+	assert(strcmp(json_object_get_string(tmp), SHORT) == 0);
+	assert(strcmp(json_object_to_json_string(tmp), "\"" SHORT "\"") == 0);
 	json_object_put(tmp);
 	printf("STRING PASSED\n");
 

tests/test_util_file.c

@@ -24,10 +24,10 @@
 
 static void test_read_valid_with_fd(const char *testdir);
 static void test_read_valid_nested_with_fd(const char *testdir);
-static void test_read_nonexistant();
+static void test_read_nonexistant(void);
 static void test_read_closed(void);
 
-static void test_write_to_file();
+static void test_write_to_file(void);
 static void stat_and_cat(const char *file);
 static void test_read_fd_equal(const char *testdir);