Add a JSON_TOKENER_ALLOW_TRAILING_CHARS flag for json_tokener_set_flags() to allow multiple objects to be parsed from input even when JSON_TOKENER_STRICT is set.

ChangeLog

@@ -8,7 +8,9 @@ Deprecated and removed features:
 
 Other changes
 --------------
-...none yet...
+* Added a JSON_TOKENER_ALLOW_TRAILING_CHARS flag to allow multiple objects
+  to be parsed even when JSON_TOKENER_STRICT is set.
+
 
 ***
 

json_tokener.c

@@ -25,7 +25,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "arraylist.h"
 #include "debug.h"
 #include "json_inttypes.h"
 #include "json_object.h"
@@ -1130,7 +1129,7 @@ out:
 		tok->err = json_tokener_error_parse_utf8_string;
 	}
 	if (c && (state == json_tokener_state_finish) && (tok->depth == 0) &&
-	    (tok->flags & JSON_TOKENER_STRICT))
+	    (tok->flags & (JSON_TOKENER_STRICT|JSON_TOKENER_ALLOW_TRAILING_CHARS)) == JSON_TOKENER_STRICT)
 	{
 		/* unexpected char after JSON data */
 		tok->err = json_tokener_error_parse_unexpected;

json_tokener.h

@@ -144,12 +144,25 @@ typedef struct json_tokener json_tokener;
  * restrictive from one release to the next, causing your
  * code to fail on previously working input.
  *
+ * Note that setting this will also effectively disable parsing
+ * of multiple json objects in a single character stream
+ * (e.g. {"foo":123}{"bar":234}); if you want to allow that
+ * also set JSON_TOKENER_ALLOW_TRAILING_CHARS
+ *
  * This flag is not set by default.
  *
  * @see json_tokener_set_flags()
  */
 #define JSON_TOKENER_STRICT 0x01
 
+/**
+ * Use with JSON_TOKENER_STRICT to allow trailing characters after the
+ * first parsed object.
+ *
+ * @see json_tokener_set_flags()
+ */
+#define JSON_TOKENER_ALLOW_TRAILING_CHARS 0x02
+
 /**
  * Cause json_tokener_parse_ex() to validate that input is UTF8.
  * If this flag is specified and validation fails, then

tests/test_parse.c

@@ -238,6 +238,13 @@ struct incremental_step
     {"{\"x\": 123 }\"X\"", -1, 11, json_tokener_success, 0},
     {"\"Y\"", -1, -1, json_tokener_success, 1},
 
+    /* Trailing characters should cause a failure in strict mode */
+    {"{\"foo\":9}{\"bar\":8}", -1, 9, json_tokener_error_parse_unexpected, 1, JSON_TOKENER_STRICT },
+
+    /* ... unless explicitly allowed. */
+    {"{\"foo\":9}{\"bar\":8}", -1, 9, json_tokener_success, 0, JSON_TOKENER_STRICT|JSON_TOKENER_ALLOW_TRAILING_CHARS },
+    {"{\"b\":8}ignored garbage", -1, 7, json_tokener_success, 1, JSON_TOKENER_STRICT|JSON_TOKENER_ALLOW_TRAILING_CHARS },
+
     /* To stop parsing a number we need to reach a non-digit, e.g. a \0 */
     {"1", 1, 1, json_tokener_continue, 0},
     /* This should parse as the number 12, since it continues the "1" */

tests/test_parse.expected

@@ -108,6 +108,9 @@ json_tokener_parse_ex(tok, ":13}}XXXX  ,  10) ... OK: got object of type [object
 json_tokener_parse_ex(tok, XXXX        ,   4) ... OK: got correct error: unexpected character
 json_tokener_parse_ex(tok, {"x": 123 }"X",  14) ... OK: got object of type [object]: { "x": 123 }
 json_tokener_parse_ex(tok, "Y"         ,   3) ... OK: got object of type [string]: "Y"
+json_tokener_parse_ex(tok, {"foo":9}{"bar":8},  18) ... OK: got correct error: unexpected character
+json_tokener_parse_ex(tok, {"foo":9}{"bar":8},  18) ... OK: got object of type [object]: { "foo": 9 }
+json_tokener_parse_ex(tok, {"b":8}ignored garbage,  22) ... OK: got object of type [object]: { "b": 8 }
 json_tokener_parse_ex(tok, 1           ,   1) ... OK: got correct error: continue
 json_tokener_parse_ex(tok, 2           ,   2) ... OK: got object of type [int]: 12
 json_tokener_parse_ex(tok, 12{         ,   3) ... OK: got object of type [int]: 12
@@ -213,5 +216,5 @@ json_tokener_parse_ex(tok, "\ud855
 json_tokener_parse_ex(tok, "\ud0031<33>"  ,  10) ... OK: got correct error: invalid utf-8 string
 json_tokener_parse_ex(tok, 11<31>11       ,   5) ... OK: got correct error: invalid utf-8 string
 json_tokener_parse_ex(tok, {"1<>":1}    ,   8) ... OK: got correct error: invalid utf-8 string
-End Incremental Tests OK=127 ERROR=0
+End Incremental Tests OK=130 ERROR=0
 ==================================