From 7662139501603434eba79aa7b916d1497d3d9cf4 Mon Sep 17 00:00:00 2001 From: qeqx <284521809+anon5376@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:06:11 +0900 Subject: [PATCH] tokener: clear pending surrogate on reset Discard the previous document's high surrogate when resetting the tokener. Otherwise the next Unicode escape can add a replacement character or combine with a surrogate from the abandoned document. Extend the existing incremental parse tests for incomplete input and a fatal error, using independent next-document string expectations. Fixes #984. Reported-by: GeekQAQ (GitHub) Assisted-by: OpenAI Codex (AI) Reviewed-by: GPT-6 Luna (AI) --- json_tokener.c | 1 + tests/test_parse.c | 8 ++++++++ tests/test_parse.expected | 8 +++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/json_tokener.c b/json_tokener.c index 516142a..e9b7d4d 100644 --- a/json_tokener.c +++ b/json_tokener.c @@ -214,6 +214,7 @@ void json_tokener_reset(struct json_tokener *tok) json_tokener_reset_level(tok, i); tok->depth = 0; tok->err = json_tokener_success; + tok->high_surrogate = 0; } struct json_object *json_tokener_parse(const char *str) diff --git a/tests/test_parse.c b/tests/test_parse.c index 220ce5e..e9763c4 100644 --- a/tests/test_parse.c +++ b/tests/test_parse.c @@ -335,6 +335,14 @@ struct incremental_step {"{ \"foo", -1, -1, json_tokener_continue, 1, 0}, {": \"bar\"}", -1, 0, json_tokener_error_parse_unexpected, 1, 0}, + /* Reset must discard a pending high surrogate from the previous document. */ + {"\"\\uD834", -1, -1, json_tokener_continue, 1, 0}, + {"\"\\u0041\"", -1, -1, json_tokener_success, 1, 0}, + {"\"\\uD834", -1, -1, json_tokener_continue, 1, 0}, + {"\"\\uDD1E\"", -1, -1, json_tokener_success, 1, 0}, + {"\"\\uD834\\uZZZZ\"", -1, 9, json_tokener_error_parse_string, 1, 0}, + {"\"\\u0041\"", -1, -1, json_tokener_success, 1, 0}, + /* Check a supplemental code point that looks like a high surrogate */ {"\"\\uD836", -1, -1, json_tokener_continue, 0, 0}, {"\\uDE87", -1, -1, json_tokener_continue, 0, 0}, diff --git a/tests/test_parse.expected b/tests/test_parse.expected index b409a36..ca35dce 100644 --- a/tests/test_parse.expected +++ b/tests/test_parse.expected @@ -140,6 +140,12 @@ json_tokener_parse_ex(tok, "ä" , 4) ... OK: got object of type [string json_tokener_parse_ex(tok, "ä" , 4) ... OK: got object of type [string]: "ä" json_tokener_parse_ex(tok, { "foo , 6) ... OK: got correct error: continue json_tokener_parse_ex(tok, : "bar"} , 8) ... OK: got correct error: unexpected character +json_tokener_parse_ex(tok, "\uD834 , 7) ... OK: got correct error: continue +json_tokener_parse_ex(tok, "\u0041" , 8) ... OK: got object of type [string]: "A" +json_tokener_parse_ex(tok, "\uD834 , 7) ... OK: got correct error: continue +json_tokener_parse_ex(tok, "\uDD1E" , 8) ... OK: got object of type [string]: "�" +json_tokener_parse_ex(tok, "\uD834\uZZZZ", 14) ... OK: got correct error: invalid string sequence +json_tokener_parse_ex(tok, "\u0041" , 8) ... OK: got object of type [string]: "A" json_tokener_parse_ex(tok, "\uD836 , 7) ... OK: got correct error: continue json_tokener_parse_ex(tok, \uDE87 , 6) ... OK: got correct error: continue json_tokener_parse_ex(tok, " , 1) ... OK: got object of type [string]: "𝪇" @@ -388,5 +394,5 @@ json_tokener_parse_ex(tok, {"":1} , 7) ... OK: got correct error: invalid json_tokener_parse_ex(tok, {"":1} , 7) ... OK: got correct error: invalid string sequence json_tokener_parse_ex(tok, {"":1} , 7) ... OK: got correct error: invalid string sequence json_tokener_parse_ex(tok, {"":1} , 7) ... OK: got correct error: invalid string sequence -End Incremental Tests OK=292 ERROR=0 +End Incremental Tests OK=298 ERROR=0 ==================================